No OneTemporary
Actions

Size

353 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/crmd/lrm.c b/crmd/lrm.c
	index bb8af67ede..11ea90a9d5 100644
	--- a/crmd/lrm.c
	+++ b/crmd/lrm.c
	@@ -1,2404 +1,2406 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/services.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	#define START_DELAY_THRESHOLD 5 * 60 * 1000
	#define MAX_LRM_REG_FAILS 30

	struct delete_event_s {
	int rc;
	const char *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
	static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
	static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
	static int delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
	const char *user_name);

	static lrmd_event_data_t construct_op(lrm_state_t lrm_state, xmlNode * rsc_op,
	const char rsc_id, const char operation);
	static void do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operation,
	xmlNode * msg, xmlNode * request);

	void send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id);

	static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
	int log_level);
	static int do_update_resource(const char node_name, lrmd_rsc_info_t rsc, lrmd_event_data_t * op);

	static void
	lrm_connection_destroy(void)
	{
	if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	crm_crit("LRM Connection failed");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	clear_bit(fsa_input_register, R_LRM_CONNECTED);

	} else {
	crm_info("LRM Connection disconnected");
	}

	}

	static char *
	make_stop_id(const char *rsc, int call_id)
	{
	char *op_id = NULL;

	op_id = calloc(1, strlen(rsc) + 34);
	if (op_id != NULL) {
	snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id);
	}
	return op_id;
	}

	static void
	copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") == NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") != NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	/*
	* \internal
	* \brief Remove a recurring operation from a resource's history
	*
	* \param[in,out] history Resource history to modify
	* \param[in] op Operation to remove
	*
	* \return TRUE if the operation was found and removed, FALSE otherwise
	*/
	static gboolean
	history_remove_recurring_op(rsc_history_t history, const lrmd_event_data_t op)
	{
	GList *iter;

	for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
	lrmd_event_data_t *existing = iter->data;

	if ((op->interval == existing->interval)
	&& crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
	&& safe_str_eq(op->op_type, existing->op_type)) {

	history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
	lrmd_free_event(existing);
	return TRUE;
	}
	}
	return FALSE;
	}

	/*
	* \internal
	* \brief Free all recurring operations in resource history
	*
	* \param[in,out] history Resource history to modify
	*/
	static void
	history_free_recurring_ops(rsc_history_t *history)
	{
	GList *iter;

	for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
	lrmd_free_event(iter->data);
	}
	g_list_free(history->recurring_op_list);
	history->recurring_op_list = NULL;
	}

	/*
	* \internal
	* \brief Free resource history
	*
	* \param[in,out] history Resource history to free
	*/
	void
	history_free(gpointer data)
	{
	rsc_history_t history = (rsc_history_t)data;

	if (history->stop_params) {
	g_hash_table_destroy(history->stop_params);
	}

	/* Don't need to free history->rsc.id because it's set to history->id */
	free(history->rsc.type);
	free(history->rsc.class);
	free(history->rsc.provider);

	lrmd_free_event(history->failed);
	lrmd_free_event(history->last);
	free(history->id);
	history_free_recurring_ops(history);
	free(history);
	}

	static void
	update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int target_rc = 0;
	rsc_history_t *entry = NULL;

	if (op->rsc_deleted) {
	crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
	delete_rsc_status(lrm_state, op->rsc_id, cib_quorum_override, NULL);
	return;
	}

	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	return;
	}

	crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);

	entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
	if (entry == NULL && rsc) {
	entry = calloc(1, sizeof(rsc_history_t));
	entry->id = strdup(op->rsc_id);
	g_hash_table_insert(lrm_state->resource_history, entry->id, entry);

	entry->rsc.id = entry->id;
	entry->rsc.type = strdup(rsc->type);
	entry->rsc.class = strdup(rsc->class);
	if (rsc->provider) {
	entry->rsc.provider = strdup(rsc->provider);
	} else {
	entry->rsc.provider = NULL;
	}

	} else if (entry == NULL) {
	crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
	return;
	}

	entry->last_callid = op->call_id;
	target_rc = rsc_op_expected_rc(op);
	if (op->op_status == PCMK_LRM_OP_CANCELLED) {
	if (op->interval > 0) {
	crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type,
	op->interval);
	history_remove_recurring_op(entry, op);
	return;
	} else {
	crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
	op->rc, op->op_status);
	}

	} else if (did_rsc_op_fail(op, target_rc)) {
	/* We must store failed monitors here
	* - otherwise the block below will cause them to be forgetten them when a stop happens
	*/
	if (entry->failed) {
	lrmd_free_event(entry->failed);
	}
	entry->failed = lrmd_copy_event(op);

	} else if (op->interval == 0) {
	if (entry->last) {
	lrmd_free_event(entry->last);
	}
	entry->last = lrmd_copy_event(op);

	if (op->params &&
	(safe_str_eq(CRMD_ACTION_START, op->op_type) \|\|
	safe_str_eq("reload", op->op_type) \|\|
	safe_str_eq(CRMD_ACTION_STATUS, op->op_type))) {

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}
	entry->stop_params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str,
	g_hash_destroy_str);

	g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
	}
	}

	if (op->interval > 0) {
	/* Ensure there are no duplicates */
	history_remove_recurring_op(entry, op);

	crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
	entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));

	} else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
	crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
	g_list_length(entry->recurring_op_list), op->rsc_id,
	op->op_type, op->interval);
	history_free_recurring_ops(entry);
	}
	}

	void
	lrm_op_callback(lrmd_event_data_t * op)
	{
	const char *nodename = NULL;
	lrm_state_t *lrm_state = NULL;

	CRM_CHECK(op != NULL, return);

	/* determine the node name for this connection. */
	nodename = op->remote_nodename ? op->remote_nodename : fsa_our_uname;

	if (op->type == lrmd_event_disconnect && (safe_str_eq(nodename, fsa_our_uname))) {
	/* if this is the local lrmd ipc connection, set the right bits in the
	* crmd when the connection goes down */
	lrm_connection_destroy();
	return;
	} else if (op->type != lrmd_event_exec_complete) {
	/* we only need to process execution results */
	return;
	}

	lrm_state = lrm_state_find(nodename);
	CRM_ASSERT(lrm_state != NULL);

	process_lrm_event(lrm_state, op, NULL);
	}

	/* A_LRM_CONNECT */
	void
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* This only pertains to local lrmd connections. Remote connections are handled as
	* resources within the pengine. Connecting and disconnecting from remote lrmd instances
	* handled differently than the local. */

	lrm_state_t *lrm_state = NULL;

	if(fsa_our_uname == NULL) {
	return; /* Nothing to do */
	}
	lrm_state = lrm_state_find_or_create(fsa_our_uname);
	if (lrm_state == NULL) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	if (action & A_LRM_DISCONNECT) {
	if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
	if (action == A_LRM_DISCONNECT) {
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	clear_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_info("Disconnecting from the LRM");
	lrm_state_disconnect(lrm_state);
	lrm_state_reset_tables(lrm_state);
	crm_notice("Disconnected from the LRM");
	}

	if (action & A_LRM_CONNECT) {
	int ret = pcmk_ok;

	crm_debug("Connecting to the LRM");
	ret = lrm_state_ipc_connect(lrm_state);

	if (ret != pcmk_ok) {
	if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
	crm_warn("Failed to sign on to the LRM %d"
	" (%d max) times", lrm_state->num_lrm_register_fails, MAX_LRM_REG_FAILS);

	crm_timer_start(wait_timer);
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	if (ret != pcmk_ok) {
	crm_err("Failed to sign on to the LRM %d" " (max) times",
	lrm_state->num_lrm_register_fails);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	set_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_info("LRM connection established");
	}

	if (action & ~(A_LRM_CONNECT \| A_LRM_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
	}
	}

	static gboolean
	lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
	{
	int counter = 0;
	gboolean rc = TRUE;
	const char *when = "lrm disconnect";

	GHashTableIter gIter;
	const char *key = NULL;
	rsc_history_t *entry = NULL;
	struct recurring_op_s *pending = NULL;

	crm_debug("Checking for active resources before exit");

	if (cur_state == S_TERMINATE) {
	log_level = LOG_ERR;
	when = "shutdown";

	} else if (is_set(fsa_input_register, R_SHUTDOWN)) {
	when = "shutdown... waiting";
	}

	if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
	guint removed = g_hash_table_foreach_remove(
	lrm_state->pending_ops, stop_recurring_actions, lrm_state);
	guint nremaining = g_hash_table_size(lrm_state->pending_ops);

	if (removed \|\| nremaining) {
	crm_notice("Stopped %u recurring operations at %s (%u operations remaining)",
	removed, when, nremaining);
	}
	}

	if (lrm_state->pending_ops) {
	g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
	/* Ignore recurring actions in the shutdown calculations */
	if (pending->interval == 0) {
	counter++;
	}
	}
	}

	if (counter > 0) {
	do_crm_log(log_level, "%d pending LRM operations at %s", counter, when);

	if (cur_state == S_TERMINATE \|\| !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
	g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&gIter, (gpointer)&key, (gpointer)&pending)) {
	do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
	}

	} else {
	rc = FALSE;
	}
	return rc;
	}

	if (lrm_state->resource_history == NULL) {
	return rc;
	}

	if (cur_state == S_TERMINATE \|\| is_set(fsa_input_register, R_SHUTDOWN)) {
	/* At this point we're not waiting, we're just shutting down */
	when = "shutdown";
	}

	counter = 0;
	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
	if (is_rsc_active(lrm_state, entry->id) == FALSE) {
	continue;
	}

	counter++;
	crm_trace("Found %s active", entry->id);
	if (lrm_state->pending_ops) {
	GHashTableIter hIter;

	g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&hIter, (gpointer)&key, (gpointer)&pending)) {
	if (crm_str_eq(entry->id, pending->rsc_id, TRUE)) {
	crm_notice("%sction %s (%s) incomplete at %s",
	pending->interval == 0 ? "A" : "Recurring a",
	key, pending->op_key, when);
	}
	}
	}
	}

	if (counter) {
	crm_err("%d resources were active at %s.", counter, when);
	}

	return rc;
	}

	GHashTable *metadata_hash = NULL;

	static char *
	get_rsc_metadata(const char type, const char rclass, const char *provider, bool force)
	{
	int rc = pcmk_ok;
	int len = 0;
	char *key = NULL;
	char *metadata = NULL;

	/* Always use a local connection for this operation */
	lrm_state_t *lrm_state = lrm_state_find(fsa_our_uname);

	CRM_CHECK(type != NULL, return NULL);
	CRM_CHECK(rclass != NULL, return NULL);
	CRM_CHECK(lrm_state != NULL, return NULL);

	if (provider == NULL) {
	provider = "heartbeat";
	}

	if (metadata_hash == NULL) {
	metadata_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	}

	len = strlen(type) + strlen(rclass) + strlen(provider) + 4;
	key = malloc(len);
	if(key == NULL) {
	return NULL;
	}

	snprintf(key, len, "%s::%s:%s", type, rclass, provider);
	if(force == FALSE) {
	- crm_trace("Retreiving cached metadata for %s", key);
	metadata = g_hash_table_lookup(metadata_hash, key);
	+ if (metadata) {
	+ crm_trace("Retrieved cached metadata for %s", key);
	+ }
	}

	if(metadata == NULL) {
	rc = lrm_state_get_metadata(lrm_state, rclass, provider, type, &metadata, 0);
	crm_trace("Retrieved live metadata for %s: %s (%d)", key, pcmk_strerror(rc), rc);
	if(rc == pcmk_ok) {
	CRM_LOG_ASSERT(metadata != NULL);
	g_hash_table_insert(metadata_hash, key, metadata);
	key = NULL;
	} else {
	CRM_LOG_ASSERT(metadata == NULL);
	metadata = NULL;
	}
	}

	if (metadata == NULL) {
	crm_warn("No metadata found for %s: %s (%d)", key, pcmk_strerror(rc), rc);
	}

	free(key);
	return metadata;
	}

	static char *
	build_parameter_list(lrmd_event_data_t op, xmlNode metadata, xmlNode *result,
	const char *criteria, bool target, bool invert_for_xml)
	{
	int len = 0;
	int max = 0;
	char *list = NULL;

	xmlNode *param = NULL;
	xmlNode *params = NULL;

	const char *secure_terms[] = {
	"password",
	"passwd",
	"user",
	};

	if(safe_str_eq("private", criteria)) {
	/* It will take time for the agents to be updated
	* Check for some common terms
	*/
	max = DIMOF(secure_terms);
	}

	params = find_xml_node(metadata, "parameters", TRUE);
	for (param = __xml_first_child(params); param != NULL; param = __xml_next(param)) {
	if (crm_str_eq((const char *)param->name, "parameter", TRUE)) {
	bool accept = FALSE;
	const char *name = crm_element_value(param, "name");
	const char *value = crm_element_value(param, criteria);

	if(max && value) {
	/* Turn off the compatibility logic once an agent has been updated to know about 'private' */
	max = 0;
	}

	if (name == NULL) {
	crm_err("Invalid parameter in %s metadata", op->rsc_id);

	} else if(target == crm_is_true(value)) {
	accept = TRUE;

	} else if(max) {
	int lpc = 0;
	bool found = FALSE;

	for(lpc = 0; found == FALSE && lpc < max; lpc++) {
	if(safe_str_eq(secure_terms[lpc], name)) {
	found = TRUE;
	}
	}

	if(found == target) {
	accept = TRUE;
	}
	}

	if(accept) {
	int start = len;

	crm_trace("Attr %s is %s%s", name, target?"":"not ", criteria);

	len += strlen(name) + 2;
	list = realloc_safe(list, len + 1);
	sprintf(list + start, " %s ", name);

	} else {
	crm_trace("Rejecting %s for %s", name, criteria);
	}

	if(invert_for_xml) {
	crm_trace("Inverting %s match for %s xml", name, criteria);
	accept = !accept;
	}

	if(result && accept) {
	value = g_hash_table_lookup(op->params, name);
	if(value != NULL) {
	crm_trace("Adding attr %s=%s to the xml result", name, value);
	crm_xml_add(result, name, value);
	}
	}
	}
	}

	return list;
	}

	static bool
	resource_supports_action(xmlNode metadata, const char name)
	{
	const char *value = NULL;

	xmlNode *action = NULL;
	xmlNode *actions = NULL;

	actions = find_xml_node(metadata, "actions", TRUE);
	for (action = __xml_first_child(actions); action != NULL; action = __xml_next(action)) {
	if (crm_str_eq((const char *)action->name, "action", TRUE)) {
	value = crm_element_value(action, "name");
	if (safe_str_eq(name, value)) {
	return TRUE;
	}
	}
	}

	return FALSE;
	}

	static void
	append_restart_list(lrmd_event_data_t op, xmlNode metadata, xmlNode * update, const char *version)
	{
	char *list = NULL;
	char *digest = NULL;
	xmlNode *restart = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	if (op->interval > 0) {
	/* monitors are not reloadable */
	return;
	}

	if(resource_supports_action(metadata, "reload")) {
	restart = create_xml_node(NULL, XML_TAG_PARAMS);
	/* Any parameters with unique="1" should be added into the "op-force-restart" list. */
	list = build_parameter_list(op, metadata, restart, "unique", TRUE, FALSE);

	} else {
	/* Resource does not support reloads */
	return;
	}

	digest = calculate_operation_digest(restart, version);
	/* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
	* no matter if it actually supports any parameters with unique="1"). */
	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list? list: "");
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	crm_trace("%s: %s, %s", op->rsc_id, digest, list);
	crm_log_xml_trace(restart, "restart digest source");

	free_xml(restart);
	free(digest);
	free(list);
	}

	static void
	append_secure_list(lrmd_event_data_t op, xmlNode metadata, xmlNode * update, const char *version)
	{
	char *list = NULL;
	char *digest = NULL;
	xmlNode *secure = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	/*
	* To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
	* secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
	* the insecure ones
	*/
	secure = create_xml_node(NULL, XML_TAG_PARAMS);
	list = build_parameter_list(op, metadata, secure, "private", TRUE, TRUE);

	if (list != NULL) {
	digest = calculate_operation_digest(secure, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, list);
	crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);

	crm_trace("%s: %s, %s", op->rsc_id, digest, list);
	crm_log_xml_trace(secure, "secure digest source");
	} else {
	crm_trace("%s: no secure parameters", op->rsc_id);
	}

	free_xml(secure);
	free(digest);
	free(list);
	}

	static gboolean
	build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op,
	const char *src)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	xmlNode *metadata = NULL;
	const char *m_string = NULL;
	const char *caller_version = NULL;

	if (op == NULL) {
	return FALSE;
	}

	target_rc = rsc_op_expected_rc(op);

	/* there is a small risk in formerly mixed clusters that it will
	* be sub-optimal.
	*
	* however with our upgrade policy, the update we send should
	* still be completely supported anyway
	*/
	caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
	CRM_LOG_ASSERT(caller_version != NULL);

	if(caller_version == NULL) {
	caller_version = CRM_FEATURE_SET;
	}

	crm_trace("Building %s operation update with originator version: %s", op->rsc_id, caller_version);
	xml_op = create_operation_update(parent, op, caller_version, target_rc, fsa_our_uname, src, LOG_DEBUG);
	if (xml_op == NULL) {
	return TRUE;
	}

	if (rsc == NULL \|\| op->params == NULL \|\| crm_str_eq(CRMD_ACTION_STOP, op->op_type, TRUE)) {
	/* Stopped resources don't need the digest logic */
	crm_trace("No digests needed for %s %p %p %s", op->rsc_id, op->params, rsc, op->op_type);
	return TRUE;
	}

	m_string = get_rsc_metadata(rsc->type, rsc->class, rsc->provider, safe_str_eq(op->op_type, RSC_START));
	if(m_string == NULL) {
	crm_err("No metadata for %s::%s:%s", rsc->provider, rsc->class, rsc->type);
	return TRUE;
	}

	metadata = string2xml(m_string);
	if(metadata == NULL) {
	crm_err("Metadata for %s::%s:%s is not valid XML", rsc->provider, rsc->class, rsc->type);
	return TRUE;
	}

	- crm_trace("Includind additional digests for %s::%s:%s", rsc->provider, rsc->class, rsc->type);
	+ crm_trace("Including additional digests for %s::%s:%s", rsc->provider, rsc->class, rsc->type);
	append_restart_list(op, metadata, xml_op, caller_version);
	append_secure_list(op, metadata, xml_op, caller_version);

	free_xml(metadata);
	return TRUE;
	}

	static gboolean
	is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
	{
	rsc_history_t *entry = NULL;

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	if (entry == NULL \|\| entry->last == NULL) {
	return FALSE;
	}

	crm_trace("Processing %s: %s.%d=%d",
	rsc_id, entry->last->op_type, entry->last->interval, entry->last->rc);
	if (entry->last->rc == PCMK_OCF_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) {
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_OK
	&& safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) {
	/* a stricter check is too complex...
	* leave that to the PE
	*/
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
	return FALSE;

	} else if (entry->last->interval == 0 && entry->last->rc == PCMK_OCF_NOT_CONFIGURED) {
	/* Badly configured resources can't be reliably stopped */
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
	{
	GHashTableIter iter;
	rsc_history_t *entry = NULL;

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {

	GList *gIter = NULL;
	xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);

	crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.class);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);

	if (entry->last && entry->last->params) {
	const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
	if (container) {
	crm_trace("Resource %s is a part of container resource %s", entry->id, container);
	crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
	}
	}
	build_operation_update(xml_rsc, &(entry->rsc), entry->failed, __FUNCTION__);
	build_operation_update(xml_rsc, &(entry->rsc), entry->last, __FUNCTION__);
	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
	build_operation_update(xml_rsc, &(entry->rsc), gIter->data, __FUNCTION__);
	}
	}

	return FALSE;
	}

	static xmlNode *
	do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags)
	{
	xmlNode *xml_state = NULL;
	xmlNode *xml_data = NULL;
	xmlNode *rsc_list = NULL;
	crm_node_t *peer = NULL;

	peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
	CRM_CHECK(peer != NULL, return NULL);

	xml_state = do_update_node_cib(peer, update_flags, NULL, __FUNCTION__);

	xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
	crm_xml_add(xml_data, XML_ATTR_ID, peer->uuid);
	rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);

	/* Build a list of active (not always running) resources */
	build_active_RAs(lrm_state, rsc_list);

	crm_log_xml_trace(xml_state, "Current state of the LRM");

	return xml_state;
	}

	xmlNode *
	do_lrm_query(gboolean is_replace, const char *node_name)
	{
	lrm_state_t *lrm_state = lrm_state_find(node_name);
	xmlNode *xml_state;

	if (!lrm_state) {
	crm_err("Could not query lrm state for lrmd node %s", node_name);
	return NULL;
	}
	xml_state = do_lrm_query_internal(lrm_state,
	node_update_cluster\|node_update_peer);

	/* In case this function is called to generate a join confirmation to
	* send to the DC, force the current and expected join state to member.
	* This isn't necessary for newer DCs but is backward compatible.
	*/
	crm_xml_add(xml_state, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
	crm_xml_add(xml_state, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);

	return xml_state;
	}

	static void
	notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, from_host, rsc_id, rc == pcmk_ok ? "" : " not");

	op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
	CRM_ASSERT(op != NULL);

	if (rc == pcmk_ok) {
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	} else {
	op->op_status = PCMK_LRM_OP_ERROR;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	}

	send_direct_ack(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);

	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	/* this isn't expected - trigger a new transition */
	time_t now = time(NULL);
	char *now_s = crm_itoa(now);

	crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"last-lrm-refresh", now_s, FALSE, NULL, NULL);

	free(now_s);
	}
	}

	static gboolean
	lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	struct delete_event_s *event = user_data;
	struct pending_deletion_op_s *op = value;

	if (crm_str_eq(event->rsc, op->rsc, TRUE)) {
	notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
	return TRUE;
	}
	return FALSE;
	}

	static gboolean
	lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
	{
	const char *rsc = user_data;
	struct recurring_op_s *pending = value;

	if (crm_str_eq(rsc, pending->rsc_id, TRUE)) {
	crm_info("Removing op %s:%d for deleted resource %s",
	pending->op_key, pending->call_id, rsc);
	return TRUE;
	}
	return FALSE;
	}

	/*
	* Remove the rsc from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/
	#define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"

	static int
	delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
	const char *user_name)
	{
	char *rsc_xpath = NULL;
	int max = 0;
	int rc = pcmk_ok;

	CRM_CHECK(rsc_id != NULL, return -ENXIO);

	max = strlen(rsc_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + 1;
	rsc_xpath = calloc(1, max);
	snprintf(rsc_xpath, max, rsc_template, lrm_state->node_name, rsc_id);

	rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
	NULL, NULL, call_options \| cib_xpath, user_name);

	free(rsc_xpath);
	return rc;
	}

	static void
	delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
	GHashTableIter * rsc_gIter, int rc, const char *user_name)
	{
	struct delete_event_s event;

	CRM_CHECK(rsc_id != NULL, return);

	if (rc == pcmk_ok) {
	char *rsc_id_copy = strdup(rsc_id);

	if (rsc_gIter)
	g_hash_table_iter_remove(rsc_gIter);
	else
	g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
	crm_debug("sync: Sending delete op for %s", rsc_id_copy);
	delete_rsc_status(lrm_state, rsc_id_copy, cib_quorum_override, user_name);

	g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
	free(rsc_id_copy);
	}

	if (input) {
	notify_deleted(lrm_state, input, rsc_id, rc);
	}

	event.rc = rc;
	event.rsc = rsc_id;
	event.lrm_state = lrm_state;
	g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
	}

	/*
	* Remove the op from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/

	#define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
	#define op_call_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']"

	static void
	delete_op_entry(lrm_state_t * lrm_state, lrmd_event_data_t * op, const char *rsc_id,
	const char *key, int call_id)
	{
	xmlNode *xml_top = NULL;

	if (op != NULL) {
	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);

	if (op->interval > 0) {
	char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval);

	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, XML_ATTR_ID, op_id);
	free(op_id);
	}

	crm_debug("async: Sending delete op for %s_%s_%d (call=%d)",
	op->rsc_id, op->op_type, op->interval, op->call_id);

	fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, cib_quorum_override);

	} else if (rsc_id != NULL && key != NULL) {
	int max = 0;
	char *op_xpath = NULL;

	if (call_id > 0) {
	max =
	strlen(op_call_template) + strlen(rsc_id) + strlen(lrm_state->node_name) +
	strlen(key) + 10;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_call_template, lrm_state->node_name, rsc_id, key, call_id);

	} else {
	max =
	strlen(op_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + strlen(key) +
	1;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_template, lrm_state->node_name, rsc_id, key);
	}

	crm_debug("sync: Sending delete op for %s (call=%d)", rsc_id, call_id);
	fsa_cib_conn->cmds->delete(fsa_cib_conn, op_xpath, NULL, cib_quorum_override \| cib_xpath);

	free(op_xpath);

	} else {
	crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key);
	return;
	}

	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	void
	lrm_clear_last_failure(const char rsc_id, const char node_name)
	{
	char *attr = NULL;
	GHashTableIter iter;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;
	rsc_history_t *entry = NULL;

	attr = generate_op_key(rsc_id, "last_failure", 0);

	/* This clears last failure for every lrm state that has this rsc.*/
	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (node_name != NULL) {
	if (strcmp(node_name, lrm_state->node_name) != 0) {
	/* filter by node_name if node_name is present */
	continue;
	}
	}

	delete_op_entry(lrm_state, NULL, rsc_id, attr, 0);

	if (!lrm_state->resource_history) {
	continue;
	}

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
	if (crm_str_eq(rsc_id, entry->id, TRUE)) {
	lrmd_free_event(entry->failed);
	entry->failed = NULL;
	}
	}
	}
	free(attr);
	g_list_free(lrm_state_list);
	}

	/* Returns: gboolean - cancellation is in progress */
	static gboolean
	cancel_op(lrm_state_t * lrm_state, const char rsc_id, const char key, int op, gboolean remove)
	{
	int rc = pcmk_ok;
	char *local_key = NULL;
	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != 0, return FALSE);
	CRM_CHECK(rsc_id != NULL, return FALSE);
	if (key == NULL) {
	local_key = make_stop_id(rsc_id, op);
	key = local_key;
	}
	pending = g_hash_table_lookup(lrm_state->pending_ops, key);

	if (pending) {
	if (remove && pending->remove == FALSE) {
	pending->remove = TRUE;
	crm_debug("Scheduling %s for removal", key);
	}

	if (pending->cancelled) {
	crm_debug("Operation %s already cancelled", key);
	free(local_key);
	return FALSE;
	}

	pending->cancelled = TRUE;

	} else {
	crm_info("No pending op found for %s", key);
	free(local_key);
	return FALSE;
	}

	crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
	rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type, pending->interval);
	if (rc == pcmk_ok) {
	crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
	free(local_key);
	return TRUE;
	}

	crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
	/* The caller needs to make sure the entry is
	* removed from the pending_ops list
	*
	* Usually by returning TRUE inside the worker function
	* supplied to g_hash_table_foreach_remove()
	*
	* Not removing the entry from pending_ops will block
	* the node from shutting down
	*/
	free(local_key);
	return FALSE;
	}

	struct cancel_data {
	gboolean done;
	gboolean remove;
	const char *key;
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct cancel_data *data = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (crm_str_eq(op->op_key, data->key, TRUE)) {
	data->done = TRUE;
	remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
	}
	return remove;
	}

	static gboolean
	cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
	{
	guint removed = 0;
	struct cancel_data data;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(key != NULL, return FALSE);

	data.key = key;
	data.rsc = rsc;
	data.done = FALSE;
	data.remove = remove;
	data.lrm_state = lrm_state;

	removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
	crm_trace("Removed %u op cache entries, new size: %u",
	removed, g_hash_table_size(lrm_state->pending_ops));
	return data.done;
	}

	static lrmd_rsc_info_t *
	get_lrm_resource(lrm_state_t * lrm_state, xmlNode * resource, xmlNode * op_msg, gboolean do_create)
	{
	lrmd_rsc_info_t *rsc = NULL;
	const char *id = ID(resource);
	const char *type = crm_element_value(resource, XML_ATTR_TYPE);
	const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER);
	const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG);

	crm_trace("Retrieving %s from the LRM.", id);
	CRM_CHECK(id != NULL, return NULL);

	rsc = lrm_state_get_rsc_info(lrm_state, id, 0);

	if (!rsc && long_id) {
	rsc = lrm_state_get_rsc_info(lrm_state, long_id, 0);
	}

	if (!rsc && do_create) {
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(type != NULL, return NULL);

	crm_trace("Adding rsc %s before operation", id);

	lrm_state_register_rsc(lrm_state, id, class, provider, type, lrmd_opt_drop_recurring);

	rsc = lrm_state_get_rsc_info(lrm_state, id, 0);

	if (!rsc) {
	fsa_data_t *msg_data = NULL;

	crm_err("Could not add resource %s to LRM %s", id, lrm_state->node_name);
	/* only register this as a internal error if this involves the local
	* lrmd. Otherwise we're likely dealing with an unresponsive remote-node
	* which is not a FSA failure. */
	if (lrm_state_is_local(lrm_state) == TRUE) {
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}
	}

	return rsc;
	}

	static void
	delete_resource(lrm_state_t * lrm_state,
	const char *id,
	lrmd_rsc_info_t * rsc,
	GHashTableIter * gIter,
	const char *sys,
	const char *host,
	const char *user,
	ha_msg_input_t * request,
	gboolean unregister)
	{
	int rc = pcmk_ok;

	crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host);

	if (rsc && unregister) {
	rc = lrm_state_unregister_rsc(lrm_state, id, 0);
	}

	if (rc == pcmk_ok) {
	crm_trace("Resource '%s' deleted", id);
	} else if (rc == -EINPROGRESS) {
	crm_info("Deletion of resource '%s' pending", id);
	if (request) {
	struct pending_deletion_op_s *op = NULL;
	char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);

	op = calloc(1, sizeof(struct pending_deletion_op_s));
	op->rsc = strdup(rsc->id);
	op->input = copy_ha_msg_input(request);
	g_hash_table_insert(lrm_state->deletion_ops, ref, op);
	}
	return;
	} else {
	crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d",
	id, sys, user ? user : "internal", host, rc);
	}

	delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
	}

	static int
	get_fake_call_id(lrm_state_t lrm_state, const char rsc_id)
	{
	int call_id = 999999999;
	rsc_history_t *entry = NULL;

	if(lrm_state) {
	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	}

	/* Make sure the call id is greater than the last successful operation,
	* otherwise the failure will not result in a possible recovery of the resource
	* as it could appear the failure occurred before the successful start */
	if (entry) {
	call_id = entry->last_callid + 1;
	}

	if (call_id < 0) {
	call_id = 1;
	}
	return call_id;
	}

	static void
	force_reprobe(lrm_state_t lrm_state, const char from_sys, const char from_host, const char user_name, gboolean is_remote_node)
	{
	GHashTableIter gIter;
	rsc_history_t *entry = NULL;


	crm_info("clearing resource history on node %s", lrm_state->node_name);
	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	/* only unregister the resource during a reprobe if it is not a remote connection
	* resource. otherwise unregistering the connection will terminate remote-node
	* membership */
	gboolean unregister = TRUE;

	if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
	lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
	if (remote_lrm_state) {
	/* when forcing a reprobe, make sure to clear remote node before
	* clearing the remote node's connection resource */
	force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE);
	}
	unregister = FALSE;
	}

	delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, from_host,
	user_name, NULL, unregister);
	}

	/* Now delete the copy in the CIB */
	erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);

	/* And finally, _delete_ the value in attrd
	* Setting it to FALSE results in the PE sending us back here again
	*/
	update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
	}

	static void
	synthesize_lrmd_failure(lrm_state_t lrm_state, xmlNode action, int rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
	const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
	xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);

	if(xml_rsc == NULL) {
	/* Do something else? driect_ack? */
	crm_info("Skipping %s=%d on %s (%p): no resource",
	crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc, target_node, lrm_state);
	return;

	} else if(operation == NULL) {
	/* This probably came from crm_resource -C, nothing to do */
	crm_info("Skipping %s=%d on %s (%p): no operation",
	crm_element_value(action, XML_ATTR_TRANSITION_KEY), rc, target_node, lrm_state);
	return;
	}

	op = construct_op(lrm_state, action, ID(xml_rsc), operation);
	CRM_ASSERT(op != NULL);

	op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
	if(safe_str_eq(operation, RSC_NOTIFY)) {
	/* Notifications can't fail yet */
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;

	} else {
	op->op_status = PCMK_LRM_OP_ERROR;
	op->rc = rc;
	}
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	crm_info("Faking result %d for %s_%s_%d on %s (%p)", op->rc, op->rsc_id, op->op_type, op->interval, target_node, lrm_state);

	if(lrm_state) {
	process_lrm_event(lrm_state, op, NULL);

	} else {
	lrmd_rsc_info_t rsc;

	rsc.id = strdup(op->rsc_id);
	rsc.type = crm_element_value_copy(xml_rsc, XML_ATTR_TYPE);
	rsc.class = crm_element_value_copy(xml_rsc, XML_AGENT_ATTR_CLASS);
	rsc.provider = crm_element_value_copy(xml_rsc, XML_AGENT_ATTR_PROVIDER);

	do_update_resource(target_node, &rsc, op);

	free(rsc.id);
	free(rsc.type);
	free(rsc.class);
	free(rsc.provider);
	}
	lrmd_free_event(op);
	}


	/* A_LRM_INVOKE */
	void
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean create_rsc = TRUE;
	lrm_state_t *lrm_state = NULL;
	const char *crm_op = NULL;
	const char *from_sys = NULL;
	const char *from_host = NULL;
	const char *operation = NULL;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *user_name = NULL;
	const char *target_node = NULL;
	gboolean is_remote_node = FALSE;
	gboolean crm_rsc_delete = FALSE;

	if (input->xml != NULL) {
	/* Remote node operations are routed here to their remote connections */
	target_node = crm_element_value(input->xml, XML_LRM_ATTR_TARGET);
	}
	if (target_node == NULL) {
	target_node = fsa_our_uname;
	} else if (safe_str_neq(target_node, fsa_our_uname)) {
	is_remote_node = TRUE;
	}

	lrm_state = lrm_state_find(target_node);

	if (lrm_state == NULL && is_remote_node) {
	crm_err("no lrmd connection for remote node %s found on cluster node %s. Can not process request.",
	target_node, fsa_our_uname);

	/* The action must be recorded here and in the CIB as failed */
	synthesize_lrmd_failure(NULL, input->xml, PCMK_OCF_CONNECTION_DIED);
	return;
	}

	CRM_ASSERT(lrm_state != NULL);

	#if ENABLE_ACL
	user_name = crm_acl_get_set_user(input->msg, F_CRM_USER, NULL);
	crm_trace("LRM command from user '%s'", user_name);
	#endif

	crm_op = crm_element_value(input->msg, F_CRM_TASK);
	from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
	}

	crm_trace("LRM command from: %s", from_sys);

	if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
	/* remember this delete op came from crm_resource */
	crm_rsc_delete = TRUE;
	operation = CRMD_ACTION_DELETE;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	operation = CRM_OP_LRM_REFRESH;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
	lrmd_event_data_t *op = NULL;
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* The lrmd can not fail a resource, it does not understand the
	* concept of success or failure in relation to a resource, it simply
	* executes operations and reports the results. We determine what a failure is.
	* Becaues of this, if we want to fail a resource we have to fake what we
	* understand a failure to look like.
	*
	* To do this we create a fake lrmd operation event for the resource
	* we want to fail. We then pass that event to the lrmd client callback
	* so it will be processed as if it actually came from the lrmd. */
	op = construct_op(lrm_state, input->xml, ID(xml_rsc), "asyncmon");
	CRM_ASSERT(op != NULL);

	free((char *)op->user_data);
	op->user_data = NULL;
	op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
	op->interval = 0;
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	#if ENABLE_ACL
	if (user_name && is_privileged(user_name) == FALSE) {
	crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;
	}
	#endif

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);
	if (rsc) {
	crm_info("Failing resource %s...", rsc->id);
	process_lrm_event(lrm_state, op, NULL);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	lrmd_free_rsc_info(rsc);
	} else {
	crm_info("Cannot find/create resource in order to fail it...");
	crm_log_xml_warn(input->msg, "bad input");
	}

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;

	} else if (input->xml != NULL) {
	operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
	}

	if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	int rc = pcmk_ok;
	xmlNode *fragment = do_lrm_query_internal(lrm_state, node_update_all);

	fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
	crm_info("Forced a local LRM refresh: call=%d", rc);

	if(strcmp(CRM_SYSTEM_CRMD, from_sys) != 0) {
	xmlNode *reply = create_request(
	CRM_OP_INVOKE_LRM, fragment,
	from_host, from_sys, CRM_SYSTEM_LRMD, fsa_our_uuid);

	crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}
	free_xml(reply);
	}

	free_xml(fragment);

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
	xmlNode *data = do_lrm_query_internal(lrm_state, node_update_all);
	xmlNode *reply = create_reply(input->msg, data);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_err("Unable to route reply");
	crm_log_xml_err(reply, "reply");
	}
	free_xml(reply);
	free_xml(data);

	} else if (safe_str_eq(operation, CRM_OP_PROBED)) {
	update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE, user_name, is_remote_node);

	} else if (safe_str_eq(operation, CRM_OP_REPROBE) \|\| safe_str_eq(crm_op, CRM_OP_REPROBE)) {
	crm_notice("Forcing the status of all resources to be redetected");

	force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node);

	if(strcmp(CRM_SYSTEM_TENGINE, from_sys) != 0
	&& strcmp(CRM_SYSTEM_TENGINE, from_sys) != 0) {
	xmlNode *reply = create_request(
	CRM_OP_INVOKE_LRM, NULL,
	from_host, from_sys, CRM_SYSTEM_LRMD, fsa_our_uuid);

	crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}
	free_xml(reply);
	}

	} else if (operation != NULL) {
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *params = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* only the first 16 chars are used by the LRM */
	params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);

	if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	create_rsc = FALSE;
	}

	if(lrm_state_is_connected(lrm_state) == FALSE) {
	synthesize_lrmd_failure(lrm_state, input->xml, PCMK_OCF_CONNECTION_DIED);
	return;
	}

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);
	if (rsc == NULL && create_rsc) {
	crm_err("Invalid resource definition for %s", ID(xml_rsc));
	crm_log_xml_warn(input->msg, "bad input");

	/* if the operation couldn't complete because we can't register
	* the resource, return a generic error */
	synthesize_lrmd_failure(lrm_state, input->xml, PCMK_OCF_NOT_CONFIGURED);

	} else if (rsc == NULL) {
	lrmd_event_data_t *op = NULL;

	crm_notice("Not creating resource for a %s event: %s", operation, ID(input->xml));
	delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok, user_name);

	op = construct_op(lrm_state, input->xml, ID(xml_rsc), operation);

	/* Deleting something that does not exist is a success */
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	CRM_ASSERT(op != NULL);

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);

	} else if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
	char *op_key = NULL;
	char *meta_key = NULL;
	int call = 0;
	const char *call_id = NULL;
	const char *op_task = NULL;
	const char *op_interval = NULL;
	gboolean in_progress = FALSE;

	CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command");
	lrmd_free_rsc_info(rsc); return);

	meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	op_interval = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
	op_task = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
	call_id = crm_element_value(params, meta_key);
	free(meta_key);

	CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command");
	lrmd_free_rsc_info(rsc); return);
	CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command");
	lrmd_free_rsc_info(rsc); return);

	op_key = generate_op_key(rsc->id, op_task, crm_parse_int(op_interval, "0"));

	crm_debug("PE requested op %s (call=%s) be cancelled",
	op_key, call_id ? call_id : "NA");
	call = crm_parse_int(call_id, "0");
	if (call == 0) {
	/* the normal case when the PE cancels a recurring op */
	in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);

	} else {
	/* the normal case when the PE cancels an orphan op */
	in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
	}

	/* Acknowledge the cancellation operation if it's for a remote connection resource */
	if (in_progress == FALSE \|\| is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
	lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc->id, op_task);
	char *op_id = make_stop_id(rsc->id, call);

	if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
	crm_info("Nothing known about operation %d for %s", call, op_key);
	}
	delete_op_entry(lrm_state, NULL, rsc->id, op_key, call);

	CRM_ASSERT(op != NULL);

	op->rc = PCMK_OCF_OK;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(from_host, from_sys, rsc, op, rsc->id);
	lrmd_free_event(op);

	/* needed?? yes for the cancellation operation of a remote connection resource */
	g_hash_table_remove(lrm_state->pending_ops, op_id);
	free(op_id);
	}

	free(op_key);

	} else if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	gboolean unregister = TRUE;

	#if ENABLE_ACL
	int cib_rc = delete_rsc_status(lrm_state, rsc->id, cib_dryrun \| cib_sync_call, user_name);
	if (cib_rc != pcmk_ok) {
	lrmd_event_data_t *op = NULL;

	crm_err
	("Attempted deletion of resource status '%s' from CIB for %s (user=%s) on %s failed: (rc=%d) %s",
	rsc->id, from_sys, user_name ? user_name : "unknown", from_host, cib_rc,
	pcmk_strerror(cib_rc));

	op = construct_op(lrm_state, input->xml, rsc->id, operation);
	op->op_status = PCMK_LRM_OP_ERROR;

	if (cib_rc == -EACCES) {
	op->rc = PCMK_OCF_INSUFFICIENT_PRIV;
	} else {
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	}
	send_direct_ack(from_host, from_sys, NULL, op, rsc->id);
	lrmd_free_event(op);
	lrmd_free_rsc_info(rsc);
	return;
	}
	#endif
	if (crm_rsc_delete == TRUE && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
	unregister = FALSE;
	}

	delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys, from_host, user_name, input, unregister);

	} else {
	do_lrm_rsc_op(lrm_state, rsc, operation, input->xml, input->msg);
	}

	lrmd_free_rsc_info(rsc);

	} else {
	crm_err("Operation was neither a lrm_query, nor a rsc op. %s", crm_str(crm_op));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	static lrmd_event_data_t *
	construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, const char rsc_id, const char operation)
	{
	lrmd_event_data_t *op = NULL;
	const char *op_delay = NULL;
	const char *op_timeout = NULL;
	const char *op_interval = NULL;
	GHashTable *params = NULL;

	const char *transition = NULL;

	CRM_ASSERT(rsc_id != NULL);

	op = calloc(1, sizeof(lrmd_event_data_t));
	op->type = lrmd_event_exec_complete;
	op->op_type = strdup(operation);
	op->op_status = PCMK_LRM_OP_PENDING;
	op->rc = -1;
	op->rsc_id = strdup(rsc_id);
	op->interval = 0;
	op->timeout = 0;
	op->start_delay = 0;

	if (rsc_op == NULL) {
	CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
	op->user_data = NULL;
	/* the stop_all_resources() case
	* by definition there is no DC (or they'd be shutting
	* us down).
	* So we should put our version here.
	*/
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));

	crm_trace("Constructed %s op for %s", operation, rsc_id);
	return op;
	}

	params = xml2list(rsc_op);
	g_hash_table_remove(params, CRM_META "_op_target_rc");

	op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
	op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
	op_interval = crm_meta_value(params, XML_LRM_ATTR_INTERVAL);

	op->interval = crm_parse_int(op_interval, "0");
	op->timeout = crm_parse_int(op_timeout, "0");
	op->start_delay = crm_parse_int(op_delay, "0");

	if (safe_str_neq(operation, RSC_STOP)) {
	op->params = params;

	} else {
	rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);

	/* If we do not have stop parameters cached, use
	* whatever we are given */
	if (!entry \|\| !entry->stop_params) {
	op->params = params;
	} else {
	/* Copy the cached parameter list so that we stop the resource
	* with the old attributes, not the new ones */
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_foreach(params, copy_meta_keys, op->params);
	g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
	g_hash_table_destroy(params);
	params = NULL;
	}
	}

	/* sanity */
	if (op->interval < 0) {
	op->interval = 0;
	}
	if (op->timeout <= 0) {
	op->timeout = op->interval;
	}
	if (op->start_delay < 0) {
	op->start_delay = 0;
	}

	transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
	CRM_CHECK(transition != NULL, return op);

	op->user_data = strdup(transition);

	if (op->interval != 0) {
	if (safe_str_eq(operation, CRMD_ACTION_START)
	\|\| safe_str_eq(operation, CRMD_ACTION_STOP)) {
	crm_err("Start and Stop actions cannot have an interval: %d", op->interval);
	op->interval = 0;
	}
	}

	crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval);

	return op;
	}

	void
	send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id)
	{
	xmlNode *reply = NULL;
	xmlNode update, iter;
	crm_node_t *peer = NULL;

	CRM_CHECK(op != NULL, return);
	if (op->rsc_id == NULL) {
	CRM_ASSERT(rsc_id != NULL);
	op->rsc_id = strdup(rsc_id);
	}
	if (to_sys == NULL) {
	to_sys = CRM_SYSTEM_TENGINE;
	}

	peer = crm_get_peer(0, fsa_our_uname);
	update = do_update_node_cib(peer, node_update_none, NULL, __FUNCTION__);

	iter = create_xml_node(update, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);

	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);
	reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);

	crm_log_xml_trace(update, "ACK Update");

	crm_debug("ACK'ing resource op %s_%s_%d from %s: %s",
	op->rsc_id, op->op_type, op->interval, op->user_data,
	crm_element_value(reply, XML_ATTR_REFERENCE));

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}

	free_xml(update);
	free_xml(reply);
	}

	gboolean
	verify_stopped(enum crmd_fsa_state cur_state, int log_level)
	{
	gboolean res = TRUE;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;

	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
	/* keep iterating through all even when false is returned */
	res = FALSE;
	}
	}

	set_bit(fsa_input_register, R_SENT_RSC_STOP);
	g_list_free(lrm_state_list); lrm_state_list = NULL;
	return res;
	}

	struct stop_recurring_action_s {
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct stop_recurring_action_s *event = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0 && crm_str_eq(op->rsc_id, event->rsc->id, TRUE)) {
	crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
	remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static gboolean
	stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	lrm_state_t *lrm_state = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0) {
	crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, key);
	remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static void
	do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char operation, xmlNode msg,
	xmlNode * request)
	{
	int call_id = 0;
	char *op_id = NULL;
	lrmd_event_data_t *op = NULL;
	lrmd_key_value_t *params = NULL;
	fsa_data_t *msg_data = NULL;
	const char *transition = NULL;
	gboolean stop_recurring = FALSE;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(operation != NULL, return);

	if (msg != NULL) {
	transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
	if (transition == NULL) {
	crm_log_xml_err(msg, "Missing transition number");
	}
	}

	op = construct_op(lrm_state, msg, rsc->id, operation);
	CRM_CHECK(op != NULL, return);

	if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
	&& op->interval == 0
	&& strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {

	/* pcmk remote connections are a special use case.
	* We never ever want to stop monitoring a connection resource until
	* the entire migration has completed. If the connection is ever unexpected
	* severed, even during a migration, this is an event we must detect.*/
	stop_recurring = FALSE;

	} else if (op->interval == 0
	&& strcmp(operation, CRMD_ACTION_STATUS) != 0
	&& strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {

	/* stop any previous monitor operations before changing the resource state */
	stop_recurring = TRUE;
	}

	if (stop_recurring == TRUE) {
	guint removed = 0;
	struct stop_recurring_action_s data;

	data.rsc = rsc;
	data.lrm_state = lrm_state;
	removed = g_hash_table_foreach_remove(
	lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);

	if (removed) {
	crm_debug("Stopped %u recurring operations in preparation for %s_%s_%d",
	removed, rsc->id, operation, op->interval);
	}
	}

	/* now do the op */
	crm_info("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval);

	if (fsa_state != S_NOT_DC && fsa_state != S_POLICY_ENGINE && fsa_state != S_TRANSITION_ENGINE) {
	if (safe_str_neq(operation, "fail")
	&& safe_str_neq(operation, CRMD_ACTION_STOP)) {
	crm_info("Discarding attempt to perform action %s on %s in state %s",
	operation, rsc->id, fsa_state2string(fsa_state));
	op->rc = CRM_DIRECT_NACK_RC;
	op->op_status = PCMK_LRM_OP_ERROR;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	lrmd_free_event(op);
	free(op_id);
	return;
	}
	}

	op_id = generate_op_key(rsc->id, op->op_type, op->interval);

	if (op->interval > 0) {
	/* cancel it so we can then restart it without conflict */
	cancel_op_key(lrm_state, rsc, op_id, FALSE);
	}

	if (op->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, op->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	params = lrmd_key_value_add(params, key, value);
	}
	}

	call_id = lrm_state_exec(lrm_state,
	rsc->id,
	op->op_type,
	op->user_data, op->interval, op->timeout, op->start_delay, params);

	if (call_id <= 0 && lrm_state_is_local(lrm_state)) {
	crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);

	} else if (call_id <= 0) {

	crm_err("Operation %s on resource %s failed to execute on remote node %s: %d", operation, rsc->id, lrm_state->node_name, call_id);
	op->call_id = get_fake_call_id(lrm_state, rsc->id);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;
	process_lrm_event(lrm_state, op, NULL);

	} else {
	/* record all operations so we can wait
	* for them to complete during shutdown
	*/
	char *call_id_s = make_stop_id(rsc->id, call_id);
	struct recurring_op_s *pending = NULL;

	pending = calloc(1, sizeof(struct recurring_op_s));
	crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);

	pending->call_id = call_id;
	pending->interval = op->interval;
	pending->op_type = strdup(operation);
	pending->op_key = strdup(op_id);
	pending->rsc_id = strdup(rsc->id);
	pending->start_time = time(NULL);
	pending->user_data = strdup(op->user_data);
	g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);

	if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) {
	char *uuid = NULL;
	int dummy = 0, target_rc = 0;

	crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc);
	free(uuid);

	op->rc = target_rc;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	}

	pending->params = op->params;
	op->params = NULL;
	}

	free(op_id);
	lrmd_free_event(op);
	return;
	}

	int last_resource_update = 0;

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource update %d complete: rc=%d", call_id, rc);
	break;
	default:
	crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
	}

	if (call_id == last_resource_update) {
	last_resource_update = 0;
	trigger_fsa(fsa_source);
	}
	}

	static int
	do_update_resource(const char node_name, lrmd_rsc_info_t rsc, lrmd_event_data_t * op)
	{
	/*
	<status>
	<nodes_status id=uname>
	<lrm>
	<lrm_resources>
	<lrm_resource id=...>
	</...>
	*/
	int rc = pcmk_ok;
	xmlNode update, iter = NULL;
	int call_opt = crmd_cib_smart_opt();
	const char *uuid = NULL;

	CRM_CHECK(op != NULL, return 0);

	iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
	update = iter;
	iter = create_xml_node(iter, XML_CIB_TAG_STATE);

	if (safe_str_eq(node_name, fsa_our_uname)) {
	uuid = fsa_our_uuid;

	} else {
	/* remote nodes uuid and uname are equal */
	uuid = node_name;
	crm_xml_add(iter, XML_NODE_IS_REMOTE, "true");
	}

	CRM_LOG_ASSERT(uuid != NULL);
	if(uuid == NULL) {
	rc = -EINVAL;
	goto done;
	}

	crm_xml_add(iter, XML_ATTR_UUID, uuid);
	crm_xml_add(iter, XML_ATTR_UNAME, node_name);
	crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);

	iter = create_xml_node(iter, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, uuid);

	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);

	if (rsc) {
	const char *container = NULL;

	crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
	crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class);
	crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);

	if (op->params) {
	container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
	}
	if (container) {
	crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
	crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
	}

	} else {
	crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id);
	send_direct_ack(NULL, NULL, rsc, op, op->rsc_id);
	goto cleanup;
	}

	crm_log_xml_trace(update, __FUNCTION__);

	/* make it an asynchronous call and be done with it
	*
	* Best case:
	* the resource state will be discovered during
	* the next signup or election.
	*
	* Bad case:
	* we are shutting down and there is no DC at the time,
	* but then why were we shutting down then anyway?
	* (probably because of an internal error)
	*
	* Worst case:
	* we get shot for having resources "running" when the really weren't
	*
	* the alternative however means blocking here for too long, which
	* isn't acceptable
	*/
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);

	if (rc > 0) {
	last_resource_update = rc;
	}
	done:
	/* the return code is a call number, not an error code */
	crm_trace("Sent resource state update message: %d for %s=%d on %s", rc,
	op->op_type, op->interval, op->rsc_id);
	fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);

	cleanup:
	free_xml(update);
	return rc;
	}

	void
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
	{
	CRM_CHECK(FALSE, return);
	}

	gboolean
	process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending)
	{
	char *op_id = NULL;
	char *op_key = NULL;

	int update_id = 0;
	gboolean remove = FALSE;
	gboolean removed = FALSE;
	lrmd_rsc_info_t *rsc = NULL;

	CRM_CHECK(op != NULL, return FALSE);
	CRM_CHECK(op->rsc_id != NULL, return FALSE);

	op_id = make_stop_id(op->rsc_id, op->call_id);
	op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
	rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
	if(pending == NULL) {
	remove = TRUE;
	pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
	}

	if (op->op_status == PCMK_LRM_OP_ERROR) {
	switch(op->rc) {
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_MASTER:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_MASTER:
	/* Leave it up to the TE/PE to decide if this is an error */
	op->op_status = PCMK_LRM_OP_DONE;
	break;
	default:
	/* Nothing to do */
	break;
	}
	}

	if (op->op_status != PCMK_LRM_OP_CANCELLED) {
	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	/* Keep notify ops out of the CIB */
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
	} else {
	update_id = do_update_resource(lrm_state->node_name, rsc, op);
	}
	} else if (op->interval == 0) {
	/* This will occur when "crm resource cleanup" is called while actions are in-flight */
	crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else if (pending == NULL) {
	/* We don't need to do anything for cancelled ops
	* that are not in our pending op list. There are no
	* transition actions waiting on these operations. */

	} else if (op->user_data == NULL) {
	/* At this point we have a pending entry, but no transition
	* key present in the user_data field. report this */
	crm_err("Op %s (call=%d): No user data", op_key, op->call_id);

	} else if (pending->remove) {
	/* The tengine canceled this op, we have been waiting for the cancel to finish. */
	delete_op_entry(lrm_state, op, op->rsc_id, op_key, op->call_id);

	} else if (pending && op->rsc_deleted) {
	/* The tengine initiated this op, but it was cancelled outside of the
	* tengine's control during a resource cleanup/re-probe request. The tengine
	* must be alerted that this operation completed, otherwise the tengine
	* will continue waiting for this update to occur until it is timed out.
	* We don't want this update going to the cib though, so use a direct ack. */
	crm_trace("Op %s (call=%d): cancelled due to rsc deletion", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else {
	/* Before a stop is called, no need to direct ack */
	crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id);
	}

	if(remove == FALSE) {
	/* The caller will do this afterwards, but keep the logging consistent */
	removed = TRUE;

	} else if ((op->interval == 0) && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
	removed = TRUE;
	crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
	op_key, op->call_id, op_id, g_hash_table_size(lrm_state->pending_ops));

	} else if(op->interval != 0 && op->op_status == PCMK_LRM_OP_CANCELLED) {
	removed = TRUE;
	g_hash_table_remove(lrm_state->pending_ops, op_id);
	}

	switch (op->op_status) {
	case PCMK_LRM_OP_CANCELLED:
	crm_info("Result of %s operation for %s on %s: %s "
	CRM_XS " call=%d key=%s confirmed=%s",
	op->op_type, op->rsc_id, lrm_state->node_name,
	services_lrm_status_str(op->op_status),
	op->call_id, op_key, (removed? "true" : "false"));
	break;

	case PCMK_LRM_OP_DONE:
	do_crm_log(op->interval?LOG_INFO:LOG_NOTICE,
	"Result of %s operation for %s on %s: %s "
	CRM_XS " call=%d key=%s confirmed=%s rc=%d cib-update=%d",
	op->op_type, op->rsc_id, lrm_state->node_name,
	services_ocf_exitcode_str(op->rc),
	op->call_id, op_key, (removed? "true" : "false"),
	op->rc, update_id);
	break;

	case PCMK_LRM_OP_TIMEOUT:
	crm_err("Result of %s operation for %s on %s: %s "
	CRM_XS " call=%d key=%s timeout=%dms",
	op->op_type, op->rsc_id, lrm_state->node_name,
	services_lrm_status_str(op->op_status),
	op->call_id, op_key, op->timeout);
	break;

	default:
	crm_err("Result of %s operation for %s on %s: %s "
	CRM_XS " call=%d key=%s confirmed=%s status=%d cib-update=%d",
	op->op_type, op->rsc_id, lrm_state->node_name,
	services_lrm_status_str(op->op_status), op->call_id, op_key,
	(removed? "true" : "false"), op->op_status, update_id);
	}

	if (op->output) {
	char *prefix =
	crm_strdup_printf("%s-%s_%s_%d:%d", lrm_state->node_name, op->rsc_id, op->op_type, op->interval, op->call_id);

	if (op->rc) {
	crm_log_output(LOG_NOTICE, prefix, op->output);
	} else {
	crm_log_output(LOG_DEBUG, prefix, op->output);
	}
	free(prefix);
	}

	crmd_notify_resource_op(lrm_state->node_name, op);

	if (op->rsc_deleted) {
	crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
	delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
	}

	/* If a shutdown was escalated while operations were pending,
	* then the FSA will be stalled right now... allow it to continue
	*/
	mainloop_set_trigger(fsa_source);
	update_history_cache(lrm_state, rsc, op);

	lrmd_free_rsc_info(rsc);
	free(op_key);
	free(op_id);

	return TRUE;
	}
	diff --git a/fencing/admin.c b/fencing/admin.c
	index 2128ffdf63..cb647a7536 100644
	--- a/fencing/admin.c
	+++ b/fencing/admin.c
	@@ -1,585 +1,589 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <string.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>

	#include <crm/stonith-ng.h>
	#include <crm/cib.h>
	#include <crm/pengine/status.h>

	#include <crm/common/xml.h>


	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},
	{"quiet", 0, 0, 'q', "\tPrint only essential output"},

	{"-spacer-", 0, 0, '-', "Commands:"},
	{"list", 1, 0, 'l', "List devices that can terminate the specified host"},
	{"list-registered", 0, 0, 'L', "List all registered devices"},
	{"list-installed", 0, 0, 'I', "List all installed devices"},

	{"-spacer-", 0, 0, '-', ""},
	{"metadata", 0, 0, 'M', "Check the device's metadata"},
	{"query", 1, 0, 'Q', "Check the device's status"},
	{"fence", 1, 0, 'F', "Fence the named host"},
	{"unfence", 1, 0, 'U', "Unfence the named host"},
	{"reboot", 1, 0, 'B', "Reboot the named host"},
	{"confirm", 1, 0, 'C', "Confirm the named host is now safely down"},
	{"history", 1, 0, 'H', "Retrieve last fencing operation"},
	{"last", 1, 0, 'h', "Indicate when the named node was last fenced. Optional: --as-node-id"},

	{"-spacer-", 0, 0, '-', ""},
	{"register", 1, 0, 'R', "Register the named stonith device. Requires: --agent, optional: --option"},
	{"deregister", 1, 0, 'D', "De-register the named stonith device"},

	{"register-level", 1, 0, 'r',
	"Register a stonith level for the named target, specified as one of:\n\t"
	"NAME, @PATTERN, or ATTR=VALUE. Requires: --index, one or more --device entries"},
	{"deregister-level", 1, 0, 'd', "De-register a stonith level for the named target\n\t"
	"Target is specified as for --register-level. Requires: --index"},

	{"-spacer-", 0, 0, '-', ""},
	{"-spacer-", 0, 0, '-', "Options and modifiers:"},
	{"agent", 1, 0, 'a', "The agent (eg. fence_xvm) to instantiate when calling with --register"},
	{"env-option", 1, 0, 'e'},
	{"option", 1, 0, 'o'},
	{"tag", 1, 0, 'T', "Identify fencing operations with the specified tag.\n\t"
	"Useful when there are multiple entities that might be invoking stonith_admin(8)"},

	{"device", 1, 0, 'v', "A device to associate with a given host and stonith level"},
	{"index", 1, 0, 'i', "The stonith level (1-9)"},

	{"timeout", 1, 0, 't', "Operation timeout in seconds"},
	{"as-node-id", 0, 0, 'n', "(Advanced) The supplied node is the corosync nodeid (Only for use with --last)" },
	{"tolerance", 1, 0, 0, "(Advanced) Do nothing if an equivalent --fence request succeeded less than N seconds earlier" },

	{"list-all", 0, 0, 'L', "legacy alias for --list-registered"},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int st_opts = st_opt_sync_call \| st_opt_allow_suicide;

	GMainLoop *mainloop = NULL;
	struct {
	stonith_t *st;
	const char *target;
	const char *action;
	char *name;
	int timeout;
	int tolerance;
	int rc;
	} async_fence_data;

	static int
	try_mainloop_connect(void)
	{
	stonith_t *st = async_fence_data.st;
	int tries = 10;
	int i = 0;
	int rc = 0;

	for (i = 0; i < tries; i++) {
	crm_debug("Connecting as %s", async_fence_data.name);
	rc = st->cmds->connect(st, async_fence_data.name, NULL);

	if (!rc) {
	crm_debug("stonith client connection established");
	return 0;
	} else {
	crm_debug("stonith client connection failed");
	}
	sleep(1);
	}

	crm_err("Couldn not connect to stonithd. \n");
	return -1;
	}

	static void
	notify_callback(stonith_t * st, stonith_event_t * e)
	{
	if (e->result != pcmk_ok) {
	return;
	}

	if (safe_str_eq(async_fence_data.target, e->target) &&
	safe_str_eq(async_fence_data.action, e->action)) {

	async_fence_data.rc = e->result;
	g_main_loop_quit(mainloop);
	}
	}

	static void
	fence_callback(stonith_t * stonith, stonith_callback_data_t * data)
	{
	async_fence_data.rc = data->rc;

	g_main_loop_quit(mainloop);
	}

	static gboolean
	async_fence_helper(gpointer user_data)
	{
	stonith_t *st = async_fence_data.st;
	int call_id = 0;

	if (try_mainloop_connect()) {
	g_main_loop_quit(mainloop);
	return TRUE;
	}

	st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, notify_callback);

	call_id = st->cmds->fence(st,
	st_opt_allow_suicide,
	async_fence_data.target,
	async_fence_data.action,
	async_fence_data.timeout, async_fence_data.tolerance);

	if (call_id < 0) {
	g_main_loop_quit(mainloop);
	return TRUE;
	}

	st->cmds->register_callback(st,
	call_id,
	async_fence_data.timeout,
	st_opt_timeout_updates, NULL, "callback", fence_callback);

	return TRUE;
	}

	static int
	mainloop_fencing(stonith_t * st, const char target, const char action, int timeout, int tolerance)
	{
	crm_trigger_t *trig;

	async_fence_data.st = st;
	async_fence_data.target = target;
	async_fence_data.action = action;
	async_fence_data.timeout = timeout;
	async_fence_data.tolerance = tolerance;
	async_fence_data.rc = -1;

	trig = mainloop_add_trigger(G_PRIORITY_HIGH, async_fence_helper, NULL);
	mainloop_set_trigger(trig);

	mainloop = g_main_new(FALSE);
	g_main_run(mainloop);

	return async_fence_data.rc;
	}

	static int
	handle_level(stonith_t st, char target, int fence_level,
	stonith_key_value_t *devices, bool added)
	{
	char *node = NULL;
	char *pattern = NULL;
	char *name = NULL;
	char *value = strchr(target, '=');

	/* Determine if targeting by attribute, node name pattern or node name */
	if (value != NULL) {
	name = target;
	*value++ = '\0';
	} else if (*target == '@') {
	pattern = target + 1;
	} else {
	node = target;
	}

	/* Register or unregister level as appropriate */
	if (added) {
	return st->cmds->register_level_full(st, st_opts, node, pattern,
	name, value, fence_level,
	devices);
	}
	return st->cmds->remove_level_full(st, st_opts, node, pattern,
	name, value, fence_level);
	}

	int
	main(int argc, char **argv)
	{
	int flag;
	int rc = 0;
	int quiet = 0;
	int verbose = 0;
	int argerr = 0;
	int timeout = 120;
	int option_index = 0;
	int fence_level = 0;
	int no_connect = 0;
	int tolerance = 0;
	int as_nodeid = FALSE;

	char *name = NULL;
	char *value = NULL;
	char *target = NULL;
	const char *agent = NULL;
	const char *device = NULL;
	const char *longname = NULL;

	char action = 0;
	stonith_t *st = NULL;
	stonith_key_value_t *params = NULL;
	stonith_key_value_t *devices = NULL;
	stonith_key_value_t *dIter = NULL;

	crm_log_cli_init("stonith_admin");
	crm_set_options(NULL, "mode [options]", long_options,
	"Provides access to the stonith-ng API.\n"
	"\nAllows the administrator to add/remove/list devices, check device and host status and fence hosts\n");

	async_fence_data.name = strdup(crm_system_name);

	while (1) {
	flag = crm_get_option_long(argc, argv, &option_index, &longname);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	verbose = 1;
	crm_bump_log_level(argc, argv);
	break;
	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'I':
	no_connect = 1;
	/* fall through */
	case 'L':
	action = flag;
	break;
	case 'q':
	quiet = 1;
	break;
	case 'Q':
	case 'R':
	case 'D':
	action = flag;
	device = optarg;
	break;
	case 'T':
	free(async_fence_data.name);
	async_fence_data.name = crm_strdup_printf("%s.%s", crm_system_name, optarg);
	break;
	case 'a':
	agent = optarg;
	break;
	case 'l':
	target = optarg;
	action = 'L';
	break;
	case 'M':
	no_connect = 1;
	action = flag;
	break;
	case 't':
	timeout = crm_atoi(optarg, NULL);
	break;
	case 'B':
	case 'F':
	case 'U':
	/* using mainloop here */
	no_connect = 1;
	/* fall through */
	case 'C':
	/* Always log the input arguments */
	crm_log_args(argc, argv);
	target = optarg;
	action = flag;
	break;
	case 'n':
	as_nodeid = TRUE;
	break;
	case 'h':
	case 'H':
	case 'r':
	case 'd':
	target = optarg;
	action = flag;
	break;
	case 'i':
	fence_level = crm_atoi(optarg, NULL);
	break;
	case 'v':
	devices = stonith_key_value_add(devices, NULL, optarg);
	break;
	case 'o':
	crm_info("Scanning: -o %s", optarg);
	rc = sscanf(optarg, "%m[^=]=%m[^=]", &name, &value);
	if (rc != 2) {
	crm_err("Invalid option: -o %s", optarg);
	++argerr;
	} else {
	crm_info("Got: '%s'='%s'", name, value);
	params = stonith_key_value_add(params, name, value);
	}
	free(value); value = NULL;
	free(name); name = NULL;
	break;
	case 'e':
	{
	char *key = crm_concat("OCF_RESKEY", optarg, '_');
	const char *env = getenv(key);

	if (env == NULL) {
	crm_err("Invalid option: -e %s", optarg);
	++argerr;
	} else {
	crm_info("Got: '%s'='%s'", optarg, env);
	params = stonith_key_value_add(params, optarg, env);
	}
	}
	break;
	case 0:
	if (safe_str_eq("tolerance", longname)) {
	tolerance = crm_get_msec(optarg) / 1000; /* Send in seconds */
	}
	break;
	default:
	++argerr;
	break;
	}
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	crm_debug("Create");
	st = stonith_api_new();
	crm_debug("Created");

	if (!no_connect) {
	crm_debug("Connecting as %s", async_fence_data.name);
	rc = st->cmds->connect(st, async_fence_data.name, NULL);

	crm_debug("Connect: %d", rc);

	if (rc < 0) {
	goto done;
	}
	}

	switch (action) {
	case 'I':
	rc = st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout);
	for (dIter = devices; dIter; dIter = dIter->next) {
	fprintf(stdout, " %s\n", dIter->value);
	}
	if (rc == 0) {
	fprintf(stderr, "No devices found\n");

	} else if (rc > 0) {
	fprintf(stderr, "%d devices found\n", rc);
	rc = 0;
	}
	stonith_key_value_freeall(devices, 1, 1);
	break;
	case 'L':
	rc = st->cmds->query(st, st_opts, target, &devices, timeout);
	for (dIter = devices; dIter; dIter = dIter->next) {
	fprintf(stdout, " %s\n", dIter->value);
	}
	if (rc == 0) {
	fprintf(stderr, "No devices found\n");
	} else if (rc > 0) {
	fprintf(stderr, "%d devices found\n", rc);
	rc = 0;
	}
	stonith_key_value_freeall(devices, 1, 1);
	break;
	case 'Q':
	rc = st->cmds->monitor(st, st_opts, device, timeout);
	if (rc < 0) {
	rc = st->cmds->list(st, st_opts, device, NULL, timeout);
	}
	break;
	case 'R':
	rc = st->cmds->register_device(st, st_opts, device, "stonith-ng", agent, params);
	break;
	case 'D':
	rc = st->cmds->remove_device(st, st_opts, device);
	break;
	case 'd':
	case 'r':
	rc = handle_level(st, target, fence_level, devices, action == 'r');
	break;
	case 'M':
	if (agent == NULL) {
	printf("Please specify an agent to query using -a,--agent [value]\n");
	return -1;
	} else {
	char *buffer = NULL;

	rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, timeout);
	if (rc == pcmk_ok) {
	printf("%s\n", buffer);
	}
	free(buffer);
	}
	break;
	case 'C':
	rc = st->cmds->confirm(st, st_opts, target);
	break;
	case 'B':
	rc = mainloop_fencing(st, target, "reboot", timeout, tolerance);
	break;
	case 'F':
	rc = mainloop_fencing(st, target, "off", timeout, tolerance);
	break;
	case 'U':
	rc = mainloop_fencing(st, target, "on", timeout, tolerance);
	break;
	case 'h':
	{
	time_t when = 0;

	if(as_nodeid) {
	uint32_t nodeid = atol(target);
	when = stonith_api_time(nodeid, NULL, FALSE);
	} else {
	when = stonith_api_time(0, target, FALSE);
	}
	if(when) {
	printf("Node %s last kicked at: %s\n", target, ctime(&when));
	} else {
	printf("Node %s has never been kicked\n", target);
	}
	}
	break;
	case 'H':
	{
	stonith_history_t history, hp, *latest = NULL;

	rc = st->cmds->history(st, st_opts, target, &history, timeout);
	for (hp = history; hp; hp = hp->next) {
	char *action_s = NULL;
	time_t complete = hp->completed;

	if (hp->state == st_done) {
	latest = hp;
	}

	if (quiet \|\| !verbose) {
	continue;
	} else if (hp->action == NULL) {
	action_s = strdup("unknown");
	} else if (hp->action[0] != 'r') {
	action_s = crm_concat("turn", hp->action, ' ');
	} else {
	action_s = strdup(hp->action);
	}

	if (hp->state == st_failed) {
	printf("%s failed to %s node %s on behalf of %s from %s at %s\n",
	hp->delegate ? hp->delegate : "We", action_s, hp->target,
	hp->client, hp->origin, ctime(&complete));

	} else if (hp->state == st_done && hp->delegate) {
	printf("%s was able to %s node %s on behalf of %s from %s at %s\n",
	hp->delegate, action_s, hp->target,
	hp->client, hp->origin, ctime(&complete));

	} else if (hp->state == st_done) {
	printf("We were able to %s node %s on behalf of %s from %s at %s\n",
	action_s, hp->target, hp->client, hp->origin, ctime(&complete));
	} else {
	+ /* ocf:pacemaker:controld depends on "wishes to" being
	+ * in this output, when used with older versions of DLM
	+ * that don't report stateful_merge_wait
	+ */
	printf("%s at %s wishes to %s node %s - %d %d\n",
	hp->client, hp->origin, action_s, hp->target, hp->state, hp->completed);
	}

	free(action_s);
	}

	if (latest) {
	if (quiet) {
	printf("%d\n", latest->completed);
	} else {
	char *action_s = NULL;
	time_t complete = latest->completed;

	if (latest->action == NULL) {
	action_s = strdup("unknown");
	} else if (latest->action[0] != 'r') {
	action_s = crm_concat("turn", latest->action, ' ');
	} else {
	action_s = strdup(latest->action);
	}

	printf("%s was able to %s node %s on behalf of %s from %s at %s\n",
	latest->delegate ? latest->delegate : "We", action_s, latest->target,
	latest->client, latest->origin, ctime(&complete));

	free(action_s);
	}
	}
	break;
	} /* closing bracket for -H case */
	} /* closing bracket for switch case */

	done:
	free(async_fence_data.name);
	crm_info("Command returned: %s (%d)", pcmk_strerror(rc), rc);
	if (rc < 0) {
	printf("Command failed: %s\n", pcmk_strerror(rc));
	}

	stonith_key_value_freeall(params, 1, 1);
	st->cmds->disconnect(st);
	crm_debug("Disconnect: %d", rc);

	crm_debug("Destroy");
	stonith_api_delete(st);

	return rc;
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index f392fcdd34..de9698e438 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,3427 +1,3427 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <unpack.h>

	CRM_TRACE_INIT_DATA(pe_status);

	#define set_config_flag(data_set, option, flag) do { \
	const char *tmp = pe_pref(data_set->config_hash, option); \
	if(tmp) { \
	if(crm_is_true(tmp)) { \
	set_bit(data_set->flags, flag); \
	} else { \
	clear_bit(data_set->flags, flag); \
	} \
	} \
	} while(0)

	gboolean unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last_failure,
	enum action_fail_response failed, pe_working_set_t data_set);
	static gboolean determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node);

	static gboolean
	is_dangling_container_remote_node(node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (is_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	is_set(node->details->remote_rsc->flags, pe_rsc_orphan_container_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	void
	pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
	{
	CRM_CHECK(node, return);

	/* fence remote nodes living in a container by marking the container as failed. */
	if (is_container_remote_node(node)) {
	resource_t *rsc = node->details->remote_rsc->container;
	if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
	crm_warn("Remote node %s will be fenced by recovering container resource %s",
	node->details->uname, rsc->id, reason);
	/* node->details->unclean = TRUE; */
	node->details->remote_requires_reset = TRUE;
	set_bit(rsc->flags, pe_rsc_failed);
	}
	} else if (is_dangling_container_remote_node(node)) {
	crm_info("Fencing remote node %s has already occurred, container no longer exists. cleaning up dangling connection resource: %s",
	node->details->uname, reason);
	set_bit(node->details->remote_rsc->flags, pe_rsc_failed);

	} else if (is_baremetal_remote_node(node)) {
	if(pe_can_fence(data_set, node)) {
	crm_warn("Node %s will be fenced %s", node->details->uname, reason);
	} else {
	crm_warn("Node %s is unclean %s", node->details->uname, reason);
	}
	node->details->unclean = TRUE;
	node->details->remote_requires_reset = TRUE;

	} else if (node->details->unclean == FALSE) {
	if(pe_can_fence(data_set, node)) {
	crm_warn("Node %s will be fenced %s", node->details->uname, reason);
	} else {
	crm_warn("Node %s is unclean %s", node->details->uname, reason);
	}
	node->details->unclean = TRUE;
	} else {
	crm_trace("Huh? %s %s", node->details->uname, reason);
	}
	}

	gboolean
	unpack_config(xmlNode * config, pe_working_set_t * data_set)
	{
	const char *value = NULL;
	GHashTable *config_hash =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	xmlXPathObjectPtr xpathObj = NULL;

	if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
	xpathObj = xpath_search(data_set->input, "//nvpair[@name='provides' and @value='unfencing']");
	if(xpathObj && numXpathResults(xpathObj) > 0) {
	set_bit(data_set->flags, pe_flag_enable_unfencing);
	}
	freeXpathObject(xpathObj);
	}

	if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
	xpathObj = xpath_search(data_set->input, "//nvpair[@name='requires' and @value='unfencing']");
	if(xpathObj && numXpathResults(xpathObj) > 0) {
	set_bit(data_set->flags, pe_flag_enable_unfencing);
	}
	freeXpathObject(xpathObj);
	}


	#ifdef REDHAT_COMPAT_6
	if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
	xpathObj = xpath_search(data_set->input, "//primitive[@type='fence_scsi']");
	if(xpathObj && numXpathResults(xpathObj) > 0) {
	set_bit(data_set->flags, pe_flag_enable_unfencing);
	}
	freeXpathObject(xpathObj);
	}
	#endif

	data_set->config_hash = config_hash;

	unpack_instance_attributes(data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
	CIB_OPTIONS_FIRST, FALSE, data_set->now);

	verify_pe_options(data_set->config_hash);

	set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
	if(is_not_set(data_set->flags, pe_flag_startup_probes)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_notice("Relying on watchdog integration for fencing");
	set_bit(data_set->flags, pe_flag_have_stonith_resource);
	}

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	data_set->stonith_timeout = crm_get_msec(value);
	crm_debug("STONITH timeout: %d", data_set->stonith_timeout);

	set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
	crm_debug("STONITH of failed nodes is %s",
	is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");

	data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
	crm_trace("STONITH will %s nodes", data_set->stonith_action);

	set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
	crm_debug("Concurrent fencing is %s",
	is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled");

	set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
	crm_debug("Stop all active resources: %s",
	is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");

	set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
	if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(data_set->config_hash, "default-resource-stickiness");
	data_set->default_resource_stickiness = char2score(value);
	crm_debug("Default stickiness: %d", data_set->default_resource_stickiness);

	value = pe_pref(data_set->config_hash, "no-quorum-policy");

	if (safe_str_eq(value, "ignore")) {
	data_set->no_quorum_policy = no_quorum_ignore;

	} else if (safe_str_eq(value, "freeze")) {
	data_set->no_quorum_policy = no_quorum_freeze;

	} else if (safe_str_eq(value, "suicide")) {
	gboolean do_panic = FALSE;

	crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC, &do_panic);

	if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	crm_config_err
	("Setting no-quorum-policy=suicide makes no sense if stonith-enabled=false");
	}

	if (do_panic && is_set(data_set->flags, pe_flag_stonith_enabled)) {
	data_set->no_quorum_policy = no_quorum_suicide;

	} else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE && do_panic == FALSE) {
	crm_notice("Resetting no-quorum-policy to 'stop': The cluster has never had quorum");
	data_set->no_quorum_policy = no_quorum_stop;
	}

	} else {
	data_set->no_quorum_policy = no_quorum_stop;
	}

	switch (data_set->no_quorum_policy) {
	case no_quorum_freeze:
	crm_debug("On loss of CCM Quorum: Freeze resources");
	break;
	case no_quorum_stop:
	crm_debug("On loss of CCM Quorum: Stop ALL resources");
	break;
	case no_quorum_suicide:
	crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
	break;
	case no_quorum_ignore:
	crm_notice("On loss of CCM Quorum: Ignore");
	break;
	}

	set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
	crm_trace("Orphan resources are %s",
	is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");

	set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
	crm_trace("Orphan resource actions are %s",
	is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");

	set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
	crm_trace("Stopped resources are removed from the status section: %s",
	is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");

	set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
	crm_trace("Maintenance mode: %s",
	is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");

	if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
	clear_bit(data_set->flags, pe_flag_is_managed_default);
	} else {
	set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
	}
	crm_trace("By default resources are %smanaged",
	is_set(data_set->flags, pe_flag_is_managed_default) ? "" : "not ");

	set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
	crm_trace("Start failures are %s",
	is_set(data_set->flags,
	pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");

	node_score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
	node_score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
	node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));

	crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
	pe_pref(data_set->config_hash, "node-health-red"),
	pe_pref(data_set->config_hash, "node-health-yellow"),
	pe_pref(data_set->config_hash, "node-health-green"));

	data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
	crm_trace("Placement strategy: %s", data_set->placement_strategy);

	return TRUE;
	}

	static void
	destroy_digest_cache(gpointer ptr)
	{
	op_digest_cache_t *data = ptr;

	free_xml(data->params_all);
	free_xml(data->params_secure);
	free_xml(data->params_restart);

	free(data->digest_all_calc);
	free(data->digest_restart_calc);
	free(data->digest_secure_calc);

	free(data);
	}

	static node_t *
	create_node(const char id, const char uname, const char type, const char score, pe_working_set_t * data_set)
	{
	node_t *new_node = NULL;

	if (pe_find_node(data_set->nodes, uname) != NULL) {
	crm_config_warn("Detected multiple node entries with uname=%s"
	" - this is rarely intended", uname);
	}

	new_node = calloc(1, sizeof(node_t));
	if (new_node == NULL) {
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->fixed = FALSE;
	new_node->details = calloc(1, sizeof(struct node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->type = node_ping;

	if (safe_str_eq(type, "remote")) {
	new_node->details->type = node_remote;
	set_bit(data_set->flags, pe_flag_have_remote_nodes);
	} else if (type == NULL \|\| safe_str_eq(type, "member")
	\|\| safe_str_eq(type, NORMALNODE)) {
	new_node->details->type = node_member;
	}

	new_node->details->attrs = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str,
	g_hash_destroy_str);

	if (is_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup("#kind"), strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup("#kind"), strdup("cluster"));
	}

	new_node->details->utilization =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	g_hash_destroy_str);

	new_node->details->digest_cache =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	destroy_digest_cache);

	data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, GHashTable **rsc_name_check)
	{
	xmlNode *xml_rsc = NULL;
	xmlNode *xml_tmp = NULL;
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	char *tmp_id = NULL;

	for (attr_set = __xml_first_child(xml_obj); attr_set != NULL; attr_set = __xml_next_element(attr_set)) {
	if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) {
	continue;
	}

	for (attr = __xml_first_child(attr_set); attr != NULL; attr = __xml_next_element(attr)) {
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);

	if (safe_str_eq(name, XML_RSC_ATTR_REMOTE_NODE)) {
	remote_name = value;
	} else if (safe_str_eq(name, "remote-addr")) {
	remote_server = value;
	} else if (safe_str_eq(name, "remote-port")) {
	remote_port = value;
	} else if (safe_str_eq(name, "remote-connect-timeout")) {
	connect_timeout = value;
	} else if (safe_str_eq(name, "remote-allow-migrate")) {
	remote_allow_migrate=value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (*rsc_name_check == NULL) {
	*rsc_name_check = g_hash_table_new(crm_str_hash, g_str_equal);
	for (xml_rsc = __xml_first_child(parent); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) {
	const char *id = ID(xml_rsc);

	/* avoiding heap allocation here because we know the duration of this hashtable allows us to */
	g_hash_table_insert(rsc_name_check, (char ) id, (char *) id);
	}
	}

	if (g_hash_table_lookup(*rsc_name_check, remote_name)) {

	crm_err("Naming conflict with remote-node=%s. remote-nodes can not have the same name as a resource.",
	remote_name);
	return NULL;
	}

	xml_rsc = create_xml_node(parent, XML_CIB_TAG_RESOURCE);

	crm_xml_add(xml_rsc, XML_ATTR_ID, remote_name);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, "ocf");
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, "pacemaker");
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, "remote");

	xml_tmp = create_xml_node(xml_rsc, XML_TAG_META_SETS);
	tmp_id = crm_concat(remote_name, XML_TAG_META_SETS, '_');
	crm_xml_add(xml_tmp, XML_ATTR_ID, tmp_id);
	free(tmp_id);

	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "meta-attributes-container", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, XML_RSC_ATTR_CONTAINER);
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, container_id);
	free(tmp_id);

	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "meta-attributes-internal", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, XML_RSC_ATTR_INTERNAL_RSC);
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, "true");
	free(tmp_id);

	if (remote_allow_migrate) {
	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "meta-attributes-container", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, XML_OP_ATTR_ALLOW_MIGRATE);
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, remote_allow_migrate);
	free(tmp_id);
	}

	xml_tmp = create_xml_node(xml_rsc, "operations");
	attr = create_xml_node(xml_tmp, XML_ATTR_OP);
	tmp_id = crm_concat(remote_name, "monitor-interval-30s", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_ATTR_TIMEOUT, "30s");
	crm_xml_add(attr, XML_LRM_ATTR_INTERVAL, "30s");
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "monitor");
	free(tmp_id);

	if (connect_timeout) {
	attr = create_xml_node(xml_tmp, XML_ATTR_OP);
	tmp_id = crm_concat(remote_name, "start-interval-0", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_ATTR_TIMEOUT, connect_timeout);
	crm_xml_add(attr, XML_LRM_ATTR_INTERVAL, "0");
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "start");
	free(tmp_id);
	}

	if (remote_port \|\| remote_server) {
	xml_tmp = create_xml_node(xml_rsc, XML_TAG_ATTR_SETS);
	tmp_id = crm_concat(remote_name, XML_TAG_ATTR_SETS, '_');
	crm_xml_add(xml_tmp, XML_ATTR_ID, tmp_id);
	free(tmp_id);

	if (remote_server) {
	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "instance-attributes-addr", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "addr");
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, remote_server);
	free(tmp_id);
	}
	if (remote_port) {
	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "instance-attributes-port", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "port");
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, remote_port);
	free(tmp_id);
	}
	}

	return remote_name;
	}

	static void
	handle_startup_fencing(pe_working_set_t data_set, node_t new_node)
	{
	static const char *blind_faith = NULL;
	static gboolean unseen_are_unclean = TRUE;

	if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
	/* ignore fencing remote-nodes that don't have a conneciton resource associated
	* with them. This happens when remote-node entries get left in the nodes section
	* after the connection resource is removed */
	return;
	}

	blind_faith = pe_pref(data_set->config_hash, "startup-fencing");

	if (crm_is_true(blind_faith) == FALSE) {
	unseen_are_unclean = FALSE;
	crm_warn("Blind faith: not fencing unseen nodes");
	}

	if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE
	\|\| unseen_are_unclean == FALSE) {
	/* blind faith... */
	new_node->details->unclean = FALSE;

	} else {
	/* all nodes are unclean until we've seen their
	* status entry
	*/
	new_node->details->unclean = TRUE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = __xml_first_child(xml_nodes); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	crm_config_err("Must specify id tag in <node>");
	continue;
	}
	new_node = create_node(id, uname, type, score, data_set);

	if (new_node == NULL) {
	return FALSE;
	}

	/* if(data_set->have_quorum == FALSE */
	/* && data_set->no_quorum_policy == no_quorum_stop) { */
	/* /\* start shutting resources down \/ /
	/* new_node->weight = -INFINITY; */
	/* } */

	handle_startup_fencing(data_set, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, data_set);
	unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_UTILIZATION, NULL,
	new_node->details->utilization, NULL, FALSE, data_set->now);

	crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
	}
	}

	if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
	crm_info("Creating a fake local node");
	create_node(data_set->localhost, data_set->localhost, NULL, 0, data_set);
	}

	return TRUE;
	}

	static void
	setup_container(resource_t * rsc, pe_working_set_t * data_set)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	setup_container(child_rsc, data_set);
	}
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
	if (container_id && safe_str_neq(container_id, rsc->id)) {
	resource_t *container = pe_find_resource(data_set->resources, container_id);

	if (container) {
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
	} else {
	pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	GHashTable *rsc_name_check = NULL;

	/* generate remote nodes from resource config before unpacking resources */
	for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	const char *new_node_id = NULL;

	/* first check if this is a bare metal remote node. Bare metal remote nodes
	* are defined as a resource primitive only. */
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found baremetal remote node %s in container resource %s", new_node_id, ID(xml_obj));
	create_node(new_node_id, new_node_id, "remote", NULL, data_set);
	}
	continue;
	}

	/* Now check for guest remote nodes.
	* guest remote nodes are defined within a resource primitive.
	* Example1: a vm resource might be configured as a remote node.
	* Example2: a vm resource might be configured within a group to be a remote node.
	* Note: right now we only support guest remote nodes in as a standalone primitive
	* or a primitive within a group. No cloned primitives can be a guest remote node
	* right now */
	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, TRUE)) {
	/* expands a metadata defined remote resource into the xml config
	* as an actual rsc primitive to be unpacked later. */
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, &rsc_name_check);

	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest remote node %s in container resource %s", new_node_id, ID(xml_obj));
	create_node(new_node_id, new_node_id, "remote", NULL, data_set);
	}
	continue;

	} else if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) {
	xmlNode *xml_obj2 = NULL;
	/* search through a group to see if any of the primitive contain a remote node. */
	for (xml_obj2 = __xml_first_child(xml_obj); xml_obj2 != NULL; xml_obj2 = __xml_next_element(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, &rsc_name_check);

	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest remote node %s in container resource %s which is in group %s", new_node_id, ID(xml_obj2), ID(xml_obj));
	create_node(new_node_id, new_node_id, "remote", NULL, data_set);
	}
	}
	}
	}
	if (rsc_name_check) {
	g_hash_table_destroy(rsc_name_check);
	}

	return TRUE;
	}


	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the PE calculations.
	*/
	static void
	link_rsc2remotenode(pe_working_set_t data_set, resource_t new_rsc)
	{
	node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (is_set(data_set->flags, pe_flag_quick_location)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	print_resource(LOG_DEBUG_3, "Linking remote-node connection resource, ", new_rsc, FALSE);

	remote_node = pe_find_node(data_set->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return;);

	remote_node->details->remote_rsc = new_rsc;
	/* If this is a baremetal remote-node (no container resource
	* associated with it) then we need to handle startup fencing the same way
	* as cluster nodes. */
	if (new_rsc->container == NULL) {
	handle_startup_fencing(data_set, remote_node);
	} else {
	/* At this point we know if the remote node is a container or baremetal
	* remote node, update the #kind attribute if a container is involved */
	g_hash_table_replace(remote_node->details->attrs, strdup("#kind"), strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	gboolean
	unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	GListPtr gIter = NULL;

	data_set->template_rsc_sets =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	destroy_tag);

	for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	resource_t *new_rsc = NULL;

	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
	const char *template_id = ID(xml_obj);

	if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
	template_id, NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
	}
	continue;
	}

	crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
	if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
	data_set->resources = g_list_append(data_set->resources, new_rsc);

	if (xml_contains_remote_node(xml_obj)) {
	new_rsc->is_remote_node = TRUE;
	}
	print_resource(LOG_DEBUG_3, "Added ", new_rsc, FALSE);

	} else {
	crm_config_err("Failed unpacking %s %s",
	crm_element_name(xml_obj), crm_element_value(xml_obj, XML_ATTR_ID));
	if (new_rsc != NULL && new_rsc->fns != NULL) {
	new_rsc->fns->free(new_rsc);
	}
	}
	}

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	setup_container(rsc, data_set);
	link_rsc2remotenode(data_set, rsc);
	}

	data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
	if (is_set(data_set->flags, pe_flag_quick_location)) {
	/* Ignore */

	} else if (is_set(data_set->flags, pe_flag_stonith_enabled)
	&& is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {

	crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
	crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
	crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
	{
	xmlNode *xml_tag = NULL;

	data_set->tags =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_tag);

	for (xml_tag = __xml_first_child(xml_tags); xml_tag != NULL; xml_tag = __xml_next_element(xml_tag)) {
	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (crm_str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, TRUE) == FALSE) {
	continue;
	}

	if (tag_id == NULL) {
	crm_config_err("Failed unpacking %s: %s should be specified",
	crm_element_name(xml_tag), XML_ATTR_ID);
	continue;
	}

	for (xml_obj_ref = __xml_first_child(xml_tag); xml_obj_ref != NULL; xml_obj_ref = __xml_next_element(xml_obj_ref)) {
	const char *obj_ref = ID(xml_obj_ref);

	if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) {
	continue;
	}

	if (obj_ref == NULL) {
	crm_config_err("Failed unpacking %s for tag %s: %s should be specified",
	crm_element_name(xml_obj_ref), tag_id, XML_ATTR_ID);
	continue;
	}

	if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (ticket_id == NULL \|\| strlen(ticket_id) == 0) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, data_set);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = crm_element_value(xml_ticket, prop_name);

	if (crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	ticket->last_granted = crm_parse_int(last_granted, 0);
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = __xml_first_child(xml_tickets); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
	continue;
	}
	unpack_ticket_state(xml_obj, data_set);
	}

	return TRUE;
	}

	/* Compatibility with the deprecated ticket state section:
	* "/cib/status/tickets/instance_attributes" */
	static void
	get_ticket_state_legacy(gpointer key, gpointer value, gpointer user_data)
	{
	const char *long_key = key;
	char *state_key = NULL;

	const char *granted_prefix = "granted-ticket-";
	const char *last_granted_prefix = "last-granted-";
	static int granted_prefix_strlen = 0;
	static int last_granted_prefix_strlen = 0;

	const char *ticket_id = NULL;
	const char *is_granted = NULL;
	const char *last_granted = NULL;
	const char *sep = NULL;

	ticket_t *ticket = NULL;
	pe_working_set_t *data_set = user_data;

	if (granted_prefix_strlen == 0) {
	granted_prefix_strlen = strlen(granted_prefix);
	}

	if (last_granted_prefix_strlen == 0) {
	last_granted_prefix_strlen = strlen(last_granted_prefix);
	}

	if (strstr(long_key, granted_prefix) == long_key) {
	ticket_id = long_key + granted_prefix_strlen;
	if (strlen(ticket_id)) {
	state_key = strdup("granted");
	is_granted = value;
	}
	} else if (strstr(long_key, last_granted_prefix) == long_key) {
	ticket_id = long_key + last_granted_prefix_strlen;
	if (strlen(ticket_id)) {
	state_key = strdup("last-granted");
	last_granted = value;
	}
	} else if ((sep = strrchr(long_key, '-'))) {
	ticket_id = sep + 1;
	state_key = strndup(long_key, strlen(long_key) - strlen(sep));
	}

	if (ticket_id == NULL \|\| strlen(ticket_id) == 0) {
	free(state_key);
	return;
	}

	if (state_key == NULL \|\| strlen(state_key) == 0) {
	free(state_key);
	return;
	}

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, data_set);
	if (ticket == NULL) {
	free(state_key);
	return;
	}
	}

	g_hash_table_replace(ticket->state, state_key, strdup(value));

	if (is_granted) {
	if (crm_is_true(is_granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	} else if (last_granted) {
	ticket->last_granted = crm_parse_int(last_granted, 0);
	}
	}

	/* remove nodes that are down, stopping */
	/* create +ve rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode * status, pe_working_set_t * data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;

	xmlNode *state = NULL;
	xmlNode *lrm_rsc = NULL;
	node_t *this_node = NULL;

	crm_trace("Beginning unpack");

	if (data_set->tickets == NULL) {
	data_set->tickets =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket);
	}

	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
	xmlNode *xml_tickets = state;
	GHashTable *state_hash = NULL;

	/* Compatibility with the deprecated ticket state section:
	* Unpack the attributes in the deprecated "/cib/status/tickets/instance_attributes" if it exists. */
	state_hash =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	g_hash_destroy_str);

	unpack_instance_attributes(data_set->input, xml_tickets, XML_TAG_ATTR_SETS, NULL,
	state_hash, NULL, TRUE, data_set->now);

	g_hash_table_foreach(state_hash, get_ticket_state_legacy, data_set);

	if (state_hash) {
	g_hash_table_destroy(state_hash);
	}

	/* Unpack the new "/cib/status/tickets/ticket_state"s */
	unpack_tickets_state(xml_tickets, data_set);
	}

	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
	xmlNode *attrs = NULL;
	const char *resource_discovery_enabled = NULL;

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if (uname == NULL) {
	/* error */
	continue;

	} else if (this_node == NULL) {
	crm_config_warn("Node %s in status section no longer exists", uname);
	continue;

	} else if (is_remote_node(this_node)) {
	/* online state for remote nodes is determined by the
	* rsc state after all the unpacking is done. we do however
	* need to mark whether or not the node has been fenced as this plays
	* a role during unpacking cluster node resource state */
	this_node->details->remote_was_fenced =
	crm_atoi(crm_element_value(state, XML_NODE_IS_FENCED), "0");
	continue;
	}

	crm_trace("Processing node id=%s, uname=%s", id, uname);

	/* Mark the node as provisionally clean
	* - at least we have seen it in the current cluster's lifetime
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, data_set);

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
	crm_info("Node %s is in standby-mode", this_node->details->uname);
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance"))) {
	crm_info("Node %s is in maintenance-mode", this_node->details->uname);
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
	XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
	}

	crm_trace("determining node state");
	determine_online_status(state, this_node, data_set);

	if (this_node->details->online && data_set->no_quorum_policy == no_quorum_suicide) {
	/* Everything else should flow from this automatically
	* At least until the PE becomes able to migrate off healthy resources
	*/
	pe_fence_node(data_set, this_node, "because the cluster does not have quorum");
	}
	}
	}

	/* Now that we know all node states, we can safely handle migration ops */
	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
	continue;
	}

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if (this_node == NULL) {
	crm_info("Node %s is unknown", id);
	continue;

	} else if (is_remote_node(this_node)) {

	/* online status of remote node can not be determined until all other
	* resource status is unpacked. */
	continue;
	} else if (this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_trace("Processing lrm resource entries on healthy node: %s",
	this_node->details->uname);
	lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
	unpack_lrm_resources(this_node, lrm_rsc, data_set);
	}
	}

	/* now that the rest of the cluster's status is determined
	* calculate remote-nodes */
	unpack_remote_status(status, data_set);

	return TRUE;
	}

	gboolean
	unpack_remote_status(xmlNode * status, pe_working_set_t * data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;
	const char *shutdown = NULL;

	GListPtr gIter = NULL;

	xmlNode *state = NULL;
	xmlNode *lrm_rsc = NULL;
	node_t *this_node = NULL;

	if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
	crm_trace("no remote nodes to unpack");
	return TRUE;
	}

	/* get online status */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	this_node = gIter->data;

	if ((this_node == NULL) \|\| (is_remote_node(this_node) == FALSE)) {
	continue;
	}
	determine_remote_online_status(data_set, this_node);
	}

	/* process attributes */
	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	const char *resource_discovery_enabled = NULL;
	xmlNode *attrs = NULL;
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
	continue;
	}

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if ((this_node == NULL) \|\| (is_remote_node(this_node) == FALSE)) {
	continue;
	}
	crm_trace("Processing remote node id=%s, uname=%s", id, uname);

	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, data_set);

	shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);
	if (shutdown != NULL && safe_str_neq("0", shutdown)) {
	resource_t *rsc = this_node->details->remote_rsc;

	crm_info("Node %s is shutting down", this_node->details->uname);
	this_node->details->shutdown = TRUE;
	rsc->next_role = RSC_ROLE_STOPPED;
	}

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
	crm_info("Node %s is in standby-mode", this_node->details->uname);
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance"))) {
	crm_info("Node %s is in maintenance-mode", this_node->details->uname);
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.",
	XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
	} else {
	/* if we're here, this is either a baremetal node and fencing is enabled,
	* or this is a container node which we don't care if fencing is enabled
	* or not on. container nodes are 'fenced' by recovering the container resource
	* regardless of whether fencing is enabled. */
	crm_info("Node %s has resource discovery disabled", this_node->details->uname);
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/* process node rsc status */
	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
	continue;
	}

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if ((this_node == NULL) \|\| (is_remote_node(this_node) == FALSE)) {
	continue;
	}
	crm_trace("Processing lrm resource entries on healthy remote node: %s",
	this_node->details->uname);
	lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
	unpack_lrm_resources(this_node, lrm_rsc, data_set);
	}

	return TRUE;
	}

	static gboolean
	determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
	node_t * this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);

	if (!crm_is_true(in_cluster)) {
	crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));

	} else if (safe_str_eq(is_peer, ONLINESTATUS)) {
	if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
	online = TRUE;
	} else {
	crm_debug("Node is not ready to run resources: %s", join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("CRMd is down: in_cluster=%s", crm_str(in_cluster));
	crm_trace("\tis_peer=%s, join=%s, expected=%s",
	crm_str(is_peer), crm_str(join), crm_str(exp_state));

	} else {
	/* mark it unclean */
	pe_fence_node(data_set, this_node, "because node is unexpectedly down");
	crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
	crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
	}
	return online;
	}

	static gboolean
	determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
	node_t * this_node)
	{
	gboolean online = FALSE;
	gboolean do_terminate = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
	const char *terminate = g_hash_table_lookup(this_node->details->attrs, "terminate");

	/*
	- XML_NODE_IN_CLUSTER ::= true\|false
	- XML_NODE_IS_PEER ::= true\|false\|online\|offline
	- XML_NODE_JOIN_STATE ::= member\|down\|pending\|banned
	- XML_NODE_EXPECTED ::= member\|down
	*/

	if (crm_is_true(terminate)) {
	do_terminate = TRUE;

	} else if (terminate != NULL && strlen(terminate) > 0) {
	/* could be a time() value */
	char t = terminate[0];

	if (t != '0' && isdigit(t)) {
	do_terminate = TRUE;
	}
	}

	crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
	this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
	crm_str(join), crm_str(exp_state), do_terminate);

	online = crm_is_true(in_cluster);
	if (safe_str_eq(is_peer, ONLINESTATUS)) {
	is_peer = XML_BOOLEAN_YES;
	}
	if (exp_state == NULL) {
	exp_state = CRMD_JOINSTATE_DOWN;
	}

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", this_node->details->uname);

	/* Slightly different criteria since we can't shut down a dead peer */
	online = crm_is_true(is_peer);

	} else if (in_cluster == NULL) {
	pe_fence_node(data_set, this_node, "because the peer has not been seen by the cluster");

	} else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
	pe_fence_node(data_set, this_node, "because it failed the pacemaker membership criteria");

	} else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {

	if (crm_is_true(in_cluster) \|\| crm_is_true(is_peer)) {
	crm_info("- Node %s is not ready to run resources", this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up", this_node->details->uname);
	}

	} else if (do_terminate && safe_str_eq(join, CRMD_JOINSTATE_DOWN)
	&& crm_is_true(in_cluster) == FALSE && crm_is_true(is_peer) == FALSE) {
	crm_info("Node %s was just shot", this_node->details->uname);
	online = FALSE;

	} else if (crm_is_true(in_cluster) == FALSE) {
	pe_fence_node(data_set, this_node, "because the node is no longer part of the cluster");

	} else if (crm_is_true(is_peer) == FALSE) {
	pe_fence_node(data_set, this_node, "because our peer process is no longer available");

	/* Everything is running at this point, now check join state */
	} else if (do_terminate) {
	pe_fence_node(data_set, this_node, "because termination was requested");

	} else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
	crm_info("Node %s is active", this_node->details->uname);

	} else if (safe_str_eq(join, CRMD_JOINSTATE_PENDING)
	\|\| safe_str_eq(join, CRMD_JOINSTATE_DOWN)) {
	crm_info("Node %s is not ready to run resources", this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(data_set, this_node, "because the peer was in an unknown state");
	crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
	this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
	crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
	}

	return online;
	}

	static gboolean
	determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
	{
	resource_t *rsc = this_node->details->remote_rsc;
	resource_t *container = NULL;

	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	CRM_ASSERT(rsc != NULL);

	/* If the resource is currently started, mark it online. */
	if (rsc->role == RSC_ROLE_STARTED) {
	crm_trace("Remote node %s is set to ONLINE. role == started", this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
	crm_trace("Remote node %s shutdown. transition from start to stop role", this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && is_set(container->flags, pe_rsc_failed)) {
	crm_trace("Remote node %s is set to UNCLEAN. rsc failed.", this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if(is_set(rsc->flags, pe_rsc_failed)) {
	crm_trace("Remote node %s is set to OFFLINE. rsc failed.", this_node->details->id);
	this_node->details->online = FALSE;

	} else if (rsc->role == RSC_ROLE_STOPPED
	\|\| (container && container->role == RSC_ROLE_STOPPED)) {

	crm_trace("Remote node %s is set to OFFLINE. node is stopped.", this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	return this_node->details->online;
	}

	gboolean
	determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set)
	{
	gboolean online = FALSE;
	const char *shutdown = NULL;
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);

	if (this_node == NULL) {
	crm_config_err("No node to check");
	return online;
	}

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;
	shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);

	if (shutdown != NULL && safe_str_neq("0", shutdown)) {
	this_node->details->shutdown = TRUE;

	} else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	online = determine_online_status_no_fencing(data_set, node_state, this_node);

	} else {
	online = determine_online_status_fencing(data_set, node_state, this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* dont run resources here */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("Node %s is not a pacemaker node", this_node->details->uname);

	} else if (this_node->details->unclean) {
	pe_proc_warn("Node %s is unclean", this_node->details->uname);

	} else if (this_node->details->online) {
	crm_info("Node %s is %s", this_node->details->uname,
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("Node %s is offline", this_node->details->uname);
	}

	return online;
	}

	char *
	clone_strip(const char *last_rsc_id)
	{
	int lpc = 0;
	char *zero = NULL;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	lpc = strlen(last_rsc_id);
	while (--lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	crm_err("Empty string: %s", last_rsc_id);
	return NULL;
	break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	zero = calloc(1, lpc + 1);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = 0;
	return zero;
	default:
	goto done;
	}
	}
	done:
	zero = strdup(last_rsc_id);
	return zero;
	}

	char *
	clone_zero(const char *last_rsc_id)
	{
	int lpc = 0;
	char *zero = NULL;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if (last_rsc_id != NULL) {
	lpc = strlen(last_rsc_id);
	}

	while (--lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	return NULL;
	break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	zero = calloc(1, lpc + 3);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = ':';
	zero[lpc + 1] = '0';
	zero[lpc + 2] = 0;
	return zero;
	default:
	goto done;
	}
	}
	done:
	lpc = strlen(last_rsc_id);
	zero = calloc(1, lpc + 3);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = ':';
	zero[lpc + 1] = '0';
	zero[lpc + 2] = 0;
	crm_trace("%s -> %s", last_rsc_id, zero);
	return zero;
	}

	static resource_t *
	create_fake_resource(const char rsc_id, xmlNode rsc_entry, pe_working_set_t * data_set)
	{
	resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	rsc->is_remote_node = TRUE;
	node = pe_find_node(data_set->nodes, rsc_id);
	if (node == NULL) {
	node = create_node(rsc_id, rsc_id, "remote", NULL, data_set);
	}
	link_rsc2remotenode(data_set, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	set_bit(rsc->flags, pe_rsc_orphan_container_filler);
	}
	set_bit(rsc->flags, pe_rsc_orphan);
	data_set->resources = g_list_append(data_set->resources, rsc);
	return rsc;
	}

	extern resource_t create_child_clone(resource_t rsc, int sub_id, pe_working_set_t * data_set);

	static resource_t *
	find_anonymous_clone(pe_working_set_t * data_set, node_t * node, resource_t * parent,
	const char *rsc_id)
	{
	GListPtr rIter = NULL;
	resource_t *rsc = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(parent->variant == pe_clone \|\| parent->variant == pe_master);
	CRM_ASSERT(is_not_set(parent->flags, pe_rsc_unique));

	/* Find an instance active (or partially active for grouped clones) on the specified node */
	pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GListPtr nIter = NULL;
	GListPtr locations = NULL;
	resource_t *child = rIter->data;

	child->fns->location(child, &locations, TRUE);
	if (locations == NULL) {
	pe_rsc_trace(child, "Resource %s, skip inactive", child->id);
	continue;
	}

	for (nIter = locations; nIter && rsc == NULL; nIter = nIter->next) {
	node_t *childnode = nIter->data;

	if (childnode->details == node->details) {
	/* ->find_rsc() because we might be a cloned group */
	rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
	if(rsc) {
	pe_rsc_trace(rsc, "Resource %s, active", rsc->id);
	}
	}

	/* Keep this block, it means we'll do the right thing if
	* anyone toggles the unique flag to 'off'
	*/
	if (rsc && rsc->running_on) {
	crm_notice("/Anonymous/ clone %s is already running on %s",
	parent->id, node->details->uname);
	skip_inactive = TRUE;
	rsc = NULL;
	}
	}

	g_list_free(locations);
	}

	/* Find an inactive instance */
	if (skip_inactive == FALSE) {
	pe_rsc_trace(parent, "Looking for %s anywhere", rsc_id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GListPtr locations = NULL;
	resource_t *child = rIter->data;

	if (is_set(child->flags, pe_rsc_block)) {
	pe_rsc_trace(child, "Skip: blocked in stopped state");
	continue;
	}

	child->fns->location(child, &locations, TRUE);
	if (locations == NULL) {
	/* ->find_rsc() because we might be a cloned group */
	rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
	pe_rsc_trace(parent, "Resource %s, empty slot", rsc->id);
	}
	g_list_free(locations);
	}
	}

	if (rsc == NULL) {
	/* Create an extra orphan */
	resource_t *top = create_child_clone(parent, -1, data_set);

	/* ->find_rsc() because we might be a cloned group */
	rsc = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);
	CRM_ASSERT(rsc != NULL);

	pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s", top->id, parent->id, rsc_id,
	node->details->uname);
	}

	if (safe_str_neq(rsc_id, rsc->id)) {
	pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, node->details->uname, rsc->id,
	is_set(rsc->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
	}

	return rsc;
	}

	static resource_t *
	unpack_find_resource(pe_working_set_t * data_set, node_t * node, const char *rsc_id,
	xmlNode * rsc_entry)
	{
	resource_t *rsc = NULL;
	resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(data_set->resources, rsc_id);

	/* no match */
	if (rsc == NULL) {
	/* Even when clone-max=0, we still create a single :0 orphan to match against */
	char *tmp = clone_zero(rsc_id);
	resource_t *clone0 = pe_find_resource(data_set->resources, tmp);

	if (clone0 && is_not_set(clone0->flags, pe_rsc_unique)) {
	rsc = clone0;
	} else {
	crm_trace("%s is not known as %s either", rsc_id, tmp);
	}

	parent = uber_parent(clone0);
	free(tmp);

	crm_trace("%s not found: %s", rsc_id, parent ? parent->id : "orphan");

	} else if (rsc->variant > pe_native) {
	- crm_trace("%s is no longer a primitve resource, the lrm_resource entry is obsolete",
	+ crm_trace("%s is no longer a primitive resource, the lrm_resource entry is obsolete",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (parent && parent->variant > pe_group) {
	if (is_not_set(parent->flags, pe_rsc_unique)) {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(data_set, node, parent, base);
	CRM_ASSERT(rsc != NULL);
	free(base);
	}

	if (rsc && safe_str_neq(rsc_id, rsc->id)) {
	free(rsc->clone_name);
	rsc->clone_name = strdup(rsc_id);
	}
	}

	return rsc;
	}

	static resource_t *
	process_orphan_resource(xmlNode * rsc_entry, node_t * node, pe_working_set_t * data_set)
	{
	resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
	rsc = create_fake_resource(rsc_id, rsc_entry, data_set);

	if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
	clear_bit(rsc->flags, pe_rsc_managed);

	} else {
	print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);

	CRM_CHECK(rsc != NULL, return NULL);
	resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
	}
	return rsc;
	}

	static void
	process_rsc_state(resource_t * rsc, node_t * node,
	enum action_fail_response on_fail,
	xmlNode * migrate_op, pe_working_set_t * data_set)
	{
	node_t *tmpnode = NULL;
	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));

	/* process current state */
	if (rsc->role != RSC_ROLE_UNKNOWN) {
	resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	node_t *n = node_copy(node);

	pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
	n->details->uname);
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (is_set(iter->flags, pe_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	if (rsc->role > RSC_ROLE_STOPPED
	&& node->details->online == FALSE && is_set(rsc->flags, pe_rsc_managed)) {

	char *reason = NULL;
	gboolean should_fence = FALSE;

	/* if this is a remote_node living in a container, fence the container
	* by recovering it. Mark the resource as unmanaged. Once the container
	* and remote connenction are re-established, the status section will
	* get reset in the crmd freeing up this resource to run again once we
	* are sure we know the resources state. */
	if (is_container_remote_node(node)) {
	set_bit(rsc->flags, pe_rsc_failed);

	should_fence = TRUE;
	} else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
	if (is_baremetal_remote_node(node) && node->details->remote_rsc && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
	/* setting unseen = true means that fencing of the remote node will
	* only occur if the connection resource is not going to start somewhere.
	* This allows connection resources on a failed cluster-node to move to
	* another node without requiring the baremetal remote nodes to be fenced
	* as well. */
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("because %s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("because %s is thought to be active there", rsc->id);
	}
	pe_fence_node(data_set, node, reason);
	}
	free(reason);
	}

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = action_fail_ignore;
	}

	switch (on_fail) {
	case action_fail_ignore:
	/* nothing to do */
	break;

	case action_fail_fence:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	pe_fence_node(data_set, node, "because of resource failure(s)");
	break;

	case action_fail_standby:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case action_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	clear_bit(rsc->flags, pe_rsc_managed);
	set_bit(rsc->flags, pe_rsc_block);
	break;

	case action_fail_migrate:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
	break;

	case action_fail_stop:
	rsc->next_role = RSC_ROLE_STOPPED;
	break;

	case action_fail_recover:
	if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	set_bit(rsc->flags, pe_rsc_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case action_fail_restart_container:
	set_bit(rsc->flags, pe_rsc_failed);

	if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	stop_action(rsc, node, FALSE);
	}
	break;
	case action_fail_reset_remote:
	set_bit(rsc->flags, pe_rsc_failed);
	if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(data_set->nodes, rsc->id);
	}
	if (tmpnode &&
	is_baremetal_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* connection resource to baremetal resource failed in a way that
	* should result in fencing the remote-node. */
	pe_fence_node(data_set, tmpnode, "because of connection failure(s)");
	}
	}

	/* require the stop action regardless if fencing is occuring or not. */
	if (rsc->role > RSC_ROLE_STOPPED) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_interval) {
	rsc->next_role = RSC_ROLE_STOPPED;
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(data_set->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	if (is_set(rsc->flags, pe_rsc_orphan)) {
	if (is_set(rsc->flags, pe_rsc_managed)) {
	crm_config_warn("Detected active orphan %s running on %s",
	rsc->id, node->details->uname);
	} else {
	crm_config_warn("Cluster configured not to stop active orphans."
	" %s must be stopped manually on %s",
	rsc->id, node->details->uname);
	}
	}

	native_add_running(rsc, node, data_set);
	if (on_fail != action_fail_ignore) {
	set_bit(rsc->flags, pe_rsc_failed);
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	char *key = stop_key(rsc);
	GListPtr possible_matches = find_actions(rsc->actions, key, node);
	GListPtr gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	action_t stop = (action_t ) gIter->data;

	stop->flags \|= pe_action_optional;
	}

	g_list_free(possible_matches);
	free(key);
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(node_t * node, resource_t * rsc,
	int start_index, int stop_index,
	GListPtr sorted_op_list, pe_working_set_t * data_set)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GListPtr gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	int interval = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);
	const char *interval_s = NULL;

	counter++;

	if (node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
	continue;

	} else if (counter < start_index) {
	pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
	continue;
	}

	interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");
	if (interval == 0) {
	pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (safe_str_eq(status, "-1")) {
	pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = generate_op_key(rsc->id, task, interval);
	pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
	custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
	}
	}

	void
	calculate_active_ops(GListPtr sorted_op_list, int start_index, int stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_master_start = -1;
	const char *task = NULL;
	const char *status = NULL;
	GListPtr gIter = sorted_op_list;

	*stop_index = -1;
	*start_index = -1;

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (safe_str_eq(task, CRMD_ACTION_STOP)
	&& safe_str_eq(status, "0")) {
	*stop_index = counter;

	} else if (safe_str_eq(task, CRMD_ACTION_START) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (safe_str_eq(rc, "0") \|\| safe_str_eq(rc, "8")) {
	implied_monitor_start = counter;
	}
	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) \|\| safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	implied_master_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_master_start != -1) {
	*start_index = implied_master_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	static resource_t *
	unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;

	const char *task = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	resource_t *rsc = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	xmlNode *migrate_op = NULL;
	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = FALSE;
	enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;

	crm_trace("[%s] Processing %s on %s",
	crm_element_name(rsc_entry), rsc_id, node->details->uname);

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
	if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	/* find the resource */
	rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
	if (rsc == NULL) {
	rsc = process_orphan_resource(rsc_entry, node, data_set);
	}
	CRM_ASSERT(rsc != NULL);

	/* process operations */
	saved_role = rsc->role;
	on_fail = action_fail_ignore;
	rsc->role = RSC_ROLE_UNKNOWN;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	migrate_op = rsc_op;
	}

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail, data_set);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail, migrate_op, data_set);

	if (get_target_role(rsc, &req_role)) {
	if (rsc->next_role == RSC_ROLE_UNKNOWN \|\| req_role < rsc->next_role) {
	pe_rsc_debug(rsc, "%s: Overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role), role2text(req_role));
	rsc->next_role = req_role;

	} else if (req_role > rsc->next_role) {
	pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role), role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
	{
	xmlNode *rsc_entry = NULL;
	for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
	rsc_entry = __xml_next_element(rsc_entry)) {

	resource_t *rsc;
	resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (safe_str_neq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(data_set->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(data_set->resources, rsc_id);
	if (rsc == NULL \|\|
	is_set(rsc->flags, pe_rsc_orphan_container_filler) == FALSE \|\|
	rsc->container != NULL) {
	continue;
	}

	pe_rsc_trace(rsc, "Mapped orphaned rsc %s's container to %s", rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	gboolean
	unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
	{
	xmlNode *rsc_entry = NULL;
	gboolean found_orphaned_container_filler = FALSE;
	GListPtr unexpected_containers = NULL;
	GListPtr gIter = NULL;
	resource_t *remote = NULL;

	CRM_CHECK(node != NULL, return FALSE);

	crm_trace("Unpacking resources on %s", node->details->uname);

	for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
	rsc_entry = __xml_next_element(rsc_entry)) {

	if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
	resource_t *rsc;
	rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set);
	if (!rsc) {
	continue;
	}
	if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
	found_orphaned_container_filler = TRUE;
	}
	if (is_set(rsc->flags, pe_rsc_unexpectedly_running)) {
	remote = rsc_contains_remote_node(data_set, rsc);
	if (remote) {
	unexpected_containers = g_list_append(unexpected_containers, remote);
	}
	}
	}
	}

	/* If a container resource is unexpectedly up... and the remote-node
	* connection resource for that container is not up, the entire container
	* must be recovered. */
	for (gIter = unexpected_containers; gIter != NULL; gIter = gIter->next) {
	remote = (resource_t *) gIter->data;
	if (remote->role != RSC_ROLE_STARTED) {
	crm_warn("Recovering container resource %s. Resource is unexpectedly running and involves a remote-node.", remote->container->id);
	set_bit(remote->container->flags, pe_rsc_failed);
	}
	}

	/* now that all the resource state has been unpacked for this node
	* we have to go back and map any orphaned container fillers to their
	* container resource */
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(lrm_rsc_list, data_set);
	}
	g_list_free(unexpected_containers);
	return TRUE;
	}

	static void
	set_active(resource_t * rsc)
	{
	resource_t *top = uber_parent(rsc);

	if (top && top->variant == pe_master) {
	rsc->role = RSC_ROLE_SLAVE;
	} else {
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define STATUS_PATH_MAX 1024
	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	pe_working_set_t * data_set)
	{
	int offset = 0;
	char xpath[STATUS_PATH_MAX];

	offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
	resource);

	/* Need to check against transition_magic too? */
	if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
	source);
	} else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
	source);
	} else {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
	}

	CRM_LOG_ASSERT(offset > 0);
	return get_xpath_object(xpath, data_set->input, LOG_DEBUG);
	}

	static void
	unpack_rsc_migration(resource_t rsc, node_t node, xmlNode xml_op, pe_working_set_t data_set)
	{

	/*
	* The normal sequence is (now): migrate_to(Src) -> migrate_from(Tgt) -> stop(Src)
	*
	* So if a migrate_to is followed by a stop, then we dont need to care what
	* happended on the target node
	*
	* Without the stop, we need to look for a successful migrate_from.
	* This would also imply we're no longer running on the source
	*
	* Without the stop, and without a migrate_from op we make sure the resource
	* gets stopped on both source and target (assuming the target is up)
	*
	*/
	int stop_id = 0;
	int task_id = 0;
	xmlNode *stop_op =
	find_lrm_op(rsc->id, CRMD_ACTION_STOP, node->details->id, NULL, data_set);

	if (stop_op) {
	crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
	}

	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);

	if (stop_op == NULL \|\| stop_id < task_id) {
	int from_rc = 0, from_status = 0;
	const char *migrate_source =
	crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *migrate_target =
	crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	node_t *target = pe_find_node(data_set->nodes, migrate_target);
	node_t *source = pe_find_node(data_set->nodes, migrate_source);
	xmlNode *migrate_from =
	find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
	data_set);

	rsc->role = RSC_ROLE_STARTED; /* can be master? */
	if (migrate_from) {
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
	pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
	ID(migrate_from), migrate_target, from_status, from_rc);
	}

	if (migrate_from && from_rc == PCMK_OCF_OK
	&& from_status == PCMK_LRM_OP_DONE) {
	pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
	migrate_source);

	/* all good
	* just need to arrange for the stop action to get sent
	* but _without_ affecting the target somehow
	*/
	rsc->role = RSC_ROLE_STOPPED;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);

	} else if (migrate_from) { /* Failed */
	if (target && target->details->online) {
	pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
	target->details->online);
	native_add_running(rsc, target, data_set);
	}

	} else { /* Pending or complete but erased */
	if (target && target->details->online) {
	pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
	target->details->online);

	native_add_running(rsc, target, data_set);
	if (source && source->details->online) {
	/* If we make it here we have a partial migration. The migrate_to
	* has completed but the migrate_from on the target has not. Hold on
	* to the target and source on the resource. Later on if we detect that
	* the resource is still going to run on that target, we may continue
	* the migration */
	rsc->partial_migration_target = target;
	rsc->partial_migration_source = source;
	}
	} else {
	/* Consider it failed here - forces a restart, prevents migration */
	set_bit(rsc->flags, pe_rsc_failed);
	clear_bit(rsc->flags, pe_rsc_allow_migrate);
	}
	}
	}
	}

	static void
	unpack_rsc_migration_failure(resource_t rsc, node_t node, xmlNode xml_op, pe_working_set_t data_set)
	{
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);

	CRM_ASSERT(rsc);
	if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	int stop_id = 0;
	int migrate_id = 0;
	const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	xmlNode *stop_op =
	find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_source, NULL, data_set);
	xmlNode *migrate_op =
	find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE, migrate_source, migrate_target,
	data_set);

	if (stop_op) {
	crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
	}
	if (migrate_op) {
	crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
	}

	/* Get our state right */
	rsc->role = RSC_ROLE_STARTED; /* can be master? */

	if (stop_op == NULL \|\| stop_id < migrate_id) {
	node_t *source = pe_find_node(data_set->nodes, migrate_source);

	if (source && source->details->online) {
	native_add_running(rsc, source, data_set);
	}
	}

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
	int stop_id = 0;
	int migrate_id = 0;
	const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	xmlNode *stop_op =
	find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_target, NULL, data_set);
	xmlNode *migrate_op =
	find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
	data_set);

	if (stop_op) {
	crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
	}
	if (migrate_op) {
	crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
	}

	/* Get our state right */
	rsc->role = RSC_ROLE_STARTED; /* can be master? */

	if (stop_op == NULL \|\| stop_id < migrate_id) {
	node_t *target = pe_find_node(data_set->nodes, migrate_target);

	pe_rsc_trace(rsc, "Stop: %p %d, Migrated: %p %d", stop_op, stop_id, migrate_op,
	migrate_id);
	if (target && target->details->online) {
	native_add_running(rsc, target, data_set);
	}

	} else if (migrate_op == NULL) {
	/* Make sure it gets cleaned up, the stop may pre-date the migrate_from */
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
	}
	}
	}

	static void
	record_failed_op(xmlNode op, node_t node, pe_working_set_t * data_set)
	{
	xmlNode *xIter = NULL;
	const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);

	if ((node->details->shutdown) && (node->details->online == FALSE)) {
	return;
	}

	for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
	const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
	const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);

	if(safe_str_eq(op_key, key) && safe_str_eq(uname, node->details->uname)) {
	crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
	return;
	}
	}

	crm_trace("Adding entry %s on %s", op_key, node->details->uname);
	crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
	add_node_copy(data_set->failed, op);
	}

	static const char get_op_key(xmlNode xml_op)
	{
	const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	if(key == NULL) {
	key = ID(xml_op);
	}
	return key;
	}

	static void
	unpack_rsc_op_failure(resource_t * rsc, node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
	enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	int interval = 0;
	bool is_probe = FALSE;
	action_t *action = NULL;

	const char *key = get_op_key(xml_op);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);

	CRM_ASSERT(rsc);

	*last_failure = xml_op;

	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
	if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	is_probe = TRUE;
	pe_rsc_trace(rsc, "is a probe: %s", key);
	}

	if (rc != PCMK_OCF_NOT_INSTALLED \|\| is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	crm_warn("Processing failed op %s for %s on %s: %s (%d)",
	task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
	rc);

	record_failed_op(xml_op, node, data_set);

	} else {
	crm_trace("Processing failed op %s for %s on %s: %s (%d)",
	task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
	rc);
	}

	action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
	if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) \|\|
	(action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) \|\|
	(action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) \|\|
	(*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
	pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
	fail2text(action->on_fail), action->uuid, key);
	*on_fail = action->on_fail;
	}

	if (safe_str_eq(task, CRMD_ACTION_STOP)) {
	resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	unpack_rsc_migration_failure(rsc, node, xml_op, data_set);

	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	/*
	* staying in role=master ends up putting the PE/TE into a loop
	* setting role=slave is not dangerous because no master will be
	* promoted until the failed resource has been fully stopped
	*/
	rsc->next_role = RSC_ROLE_STOPPED;
	if (action->on_fail == action_fail_block) {
	rsc->role = RSC_ROLE_MASTER;

	} else {
	crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
	rsc->role = RSC_ROLE_SLAVE;
	}

	} else if (compare_version("2.0", op_version) > 0 && safe_str_eq(task, CRMD_ACTION_START)) {
	crm_warn("Compatibility handling for failed op %s on %s", key, node->details->uname);
	resource_location(rsc, node, -INFINITY, "__legacy_start__", data_set);
	}

	if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
	/* leave stopped */
	pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
	rsc->role = RSC_ROLE_STOPPED;

	} else if (rsc->role < RSC_ROLE_STARTED) {
	pe_rsc_trace(rsc, "Setting %s active", rsc->id);
	set_active(rsc);
	}

	pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
	rsc->id, role2text(rsc->role),
	node->details->unclean ? "true" : "false",
	fail2text(action->on_fail), role2text(action->fail_role));

	if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
	rsc->next_role = action->fail_role;
	}

	if (action->fail_role == RSC_ROLE_STOPPED) {
	int score = -INFINITY;

	resource_t *fail_rsc = rsc;

	if (fail_rsc->parent) {
	resource_t *parent = uber_parent(fail_rsc);

	if ((parent->variant == pe_clone \|\| parent->variant == pe_master)
	&& is_not_set(parent->flags, pe_rsc_unique)) {
	/* for clone and master resources, if a child fails on an operation
	* with on-fail = stop, all the resources fail. Do this by preventing
	* the parent from coming up again. */
	fail_rsc = parent;
	}
	}
	crm_warn("Making sure %s doesn't come up again", fail_rsc->id);
	/* make sure it doesn't come up again */
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	fail_rsc->allowed_nodes = node_hash_from_list(data_set->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	pe_free_action(action);
	}

	static int
	determine_op_status(
	resource_t rsc, int rc, int target_rc, node_t node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	int interval = 0;
	int result = PCMK_LRM_OP_DONE;

	const char *key = get_op_key(xml_op);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);

	bool is_probe = FALSE;

	CRM_ASSERT(rsc);
	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
	if (interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	is_probe = TRUE;
	}

	if (target_rc >= 0 && target_rc != rc) {
	result = PCMK_LRM_OP_ERROR;
	pe_rsc_debug(rsc, "%s on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
	key, node->details->uname,
	services_ocf_exitcode_str(rc), rc,
	services_ocf_exitcode_str(target_rc), target_rc);
	}

	/* we could clean this up significantly except for old LRMs and CRMs that
	* didn't include target_rc and liked to remap status
	*/
	switch (rc) {
	case PCMK_OCF_OK:
	if (is_probe && target_rc == 7) {
	result = PCMK_LRM_OP_DONE;
	set_bit(rsc->flags, pe_rsc_unexpectedly_running);
	pe_rsc_info(rsc, "Operation %s found resource %s active on %s",
	task, rsc->id, node->details->uname);

	/* legacy code for pre-0.6.5 operations */
	} else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) {
	/* catch status ops that return 0 instead of 8 while they
	* are supposed to be in master mode
	*/
	result = PCMK_LRM_OP_ERROR;
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe \|\| target_rc == rc \|\| is_not_set(rsc->flags, pe_rsc_managed)) {
	result = PCMK_LRM_OP_DONE;
	rsc->role = RSC_ROLE_STOPPED;

	/* clear any previous failure actions */
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;

	} else if (safe_str_neq(task, CRMD_ACTION_STOP)) {
	result = PCMK_LRM_OP_ERROR;
	}
	break;

	case PCMK_OCF_RUNNING_MASTER:
	if (is_probe) {
	result = PCMK_LRM_OP_DONE;
	pe_rsc_info(rsc, "Operation %s found resource %s active in master mode on %s",
	task, rsc->id, node->details->uname);

	} else if (target_rc == rc) {
	/* nothing to do */

	} else if (target_rc >= 0) {
	result = PCMK_LRM_OP_ERROR;

	/* legacy code for pre-0.6.5 operations */
	} else if (safe_str_neq(task, CRMD_ACTION_STATUS)
	\|\| rsc->role != RSC_ROLE_MASTER) {
	result = PCMK_LRM_OP_ERROR;
	if (rsc->role != RSC_ROLE_MASTER) {
	crm_err("%s reported %s in master mode on %s",
	key, rsc->id, node->details->uname);
	}
	}
	rsc->role = RSC_ROLE_MASTER;
	break;

	case PCMK_OCF_DEGRADED_MASTER:
	case PCMK_OCF_FAILED_MASTER:
	rsc->role = RSC_ROLE_MASTER;
	result = PCMK_LRM_OP_ERROR;
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	result = PCMK_LRM_OP_ERROR_FATAL;
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	if (rc == PCMK_OCF_UNIMPLEMENT_FEATURE && interval > 0) {
	result = PCMK_LRM_OP_NOTSUPPORTED;
	break;

	} else if(pe_can_fence(data_set, node) == FALSE
	&& safe_str_eq(task, CRMD_ACTION_STOP)) {
	/* If a stop fails and we can't fence, there's nothing else we can do */
	pe_proc_err("No further recovery can be attempted for %s: %s action failed with '%s' (%d)",
	rsc->id, task, services_ocf_exitcode_str(rc), rc);
	clear_bit(rsc->flags, pe_rsc_managed);
	set_bit(rsc->flags, pe_rsc_block);
	}
	result = PCMK_LRM_OP_ERROR_HARD;
	break;

	default:
	if (result == PCMK_LRM_OP_DONE) {
	crm_info("Treating %s (rc=%d) on %s as an ERROR",
	key, rc, node->details->uname);
	result = PCMK_LRM_OP_ERROR;
	}
	}

	return result;
	}

	static bool check_operation_expiry(resource_t rsc, node_t node, int rc, xmlNode xml_op, pe_working_set_t data_set)
	{
	bool expired = FALSE;
	time_t last_failure = 0;
	int clear_failcount = 0;
	int interval = 0;
	int failure_timeout = rsc->failure_timeout;
	const char *key = get_op_key(xml_op);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);

	/* clearing recurring monitor operation failures automatically
	* needs to be carefully considered */
	if (safe_str_eq(crm_element_value(xml_op, XML_LRM_ATTR_TASK), "monitor") &&
	safe_str_neq(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL), "0")) {

	/* TODO, in the future we should consider not clearing recurring monitor
	* op failures unless the last action for a resource was a "stop" action.
	* otherwise it is possible that clearing the monitor failure will result
	* in the resource being in an undeterministic state.
	*
	* For now we handle this potential undeterministic condition for remote
	* node connection resources by not clearing a recurring monitor op failure
	* until after the node has been fenced. */

	if (is_set(data_set->flags, pe_flag_stonith_enabled) &&
	(rsc->remote_reconnect_interval)) {

	node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
	if (remote_node && remote_node->details->remote_was_fenced == 0) {
	if (strstr(ID(xml_op), "last_failure")) {
	crm_info("Waiting to clear monitor failure for remote node %s until fencing has occurred", rsc->id);
	}
	/* disabling failure timeout for this operation because we believe
	* fencing of the remote node should occur first. */
	failure_timeout = 0;
	}
	}
	}

	if (failure_timeout > 0) {
	int last_run = 0;

	if (crm_element_value_int(xml_op, XML_RSC_OP_LAST_CHANGE, &last_run) == 0) {
	time_t now = get_effective_time(data_set);

	if (now > (last_run + failure_timeout)) {
	expired = TRUE;
	}
	}
	}

	if (expired) {
	if (failure_timeout > 0) {
	int fc = get_failcount_full(node, rsc, &last_failure, FALSE, xml_op, data_set);
	if(fc) {
	if (get_failcount_full(node, rsc, &last_failure, TRUE, xml_op, data_set) == 0) {
	clear_failcount = 1;
	crm_notice("Clearing expired failcount for %s on %s", rsc->id, node->details->uname);

	} else {
	expired = FALSE;
	}
	} else if (rsc->remote_reconnect_interval && strstr(ID(xml_op), "last_failure")) {
	/* always clear last failure when reconnect interval is set */
	clear_failcount = 1;
	}
	}

	} else if (strstr(ID(xml_op), "last_failure") &&
	((strcmp(task, "start") == 0) \|\| (strcmp(task, "monitor") == 0))) {

	op_digest_cache_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);

	if (digest_data->rc == RSC_DIGEST_UNKNOWN) {
	crm_trace("rsc op %s/%s on node %s does not have a op digest to compare against", rsc->id,
	key, node->details->id);
	} else if (digest_data->rc != RSC_DIGEST_MATCH) {
	clear_failcount = 1;
	crm_info
	("Clearing failcount for %s on %s, %s failed and now resource parameters have changed.",
	task, rsc->id, node->details->uname);
	}
	}

	if (clear_failcount) {
	action_t *clear_op = NULL;

	clear_op = custom_action(rsc, crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_'),
	CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
	add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	}

	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
	if(expired && interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	switch(rc) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_MASTER:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_MASTER:
	/* Don't expire probes that return these values */
	expired = FALSE;
	break;
	}
	}

	return expired;
	}

	int get_target_rc(xmlNode *xml_op)
	{
	int dummy = 0;
	int target_rc = 0;
	char *dummy_string = NULL;
	const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
	if (key == NULL) {
	return -1;
	}

	decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
	free(dummy_string);

	return target_rc;
	}

	static enum action_fail_response
	get_action_on_fail(resource_t rsc, const char key, const char task, pe_working_set_t data_set)
	{
	int result = action_fail_recover;
	action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);

	result = action->on_fail;
	pe_free_action(action);

	return result;
	}

	static void
	update_resource_state(resource_t * rsc, node_t * node, xmlNode * xml_op, const char * task, int rc,
	xmlNode * last_failure, enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	gboolean clear_past_failure = FALSE;

	CRM_ASSERT(rsc);
	CRM_ASSERT(xml_op);

	if (rc == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = TRUE;

	} else if (rc == PCMK_OCF_NOT_INSTALLED) {
	rsc->role = RSC_ROLE_STOPPED;

	} else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
	if (last_failure) {
	const char *op_key = get_op_key(xml_op);
	const char *last_failure_key = get_op_key(last_failure);

	if (safe_str_eq(op_key, last_failure_key)) {
	clear_past_failure = TRUE;
	}
	}

	if (rsc->role < RSC_ROLE_STARTED) {
	set_active(rsc);
	}

	} else if (safe_str_eq(task, CRMD_ACTION_START)) {
	rsc->role = RSC_ROLE_STARTED;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
	rsc->role = RSC_ROLE_STOPPED;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	/* Demote from Master does not clear an error */
	rsc->role = RSC_ROLE_SLAVE;

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	rsc->role = RSC_ROLE_STARTED;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
	unpack_rsc_migration(rsc, node, xml_op, data_set);

	} else if (rsc->role < RSC_ROLE_STARTED) {
	pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
	set_active(rsc);
	}

	/* clear any previous failure actions */
	if (clear_past_failure) {
	switch (*on_fail) {
	case action_fail_stop:
	case action_fail_fence:
	case action_fail_migrate:
	case action_fail_standby:
	pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
	rsc->id, fail2text(*on_fail));
	break;

	case action_fail_block:
	case action_fail_ignore:
	case action_fail_recover:
	case action_fail_restart_container:
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;
	break;
	case action_fail_reset_remote:
	if (rsc->remote_reconnect_interval == 0) {
	/* when reconnect delay is not in use, the connection is allowed
	* to start again after the remote node is fenced and completely
	* stopped. Otherwise, with reconnect delay we wait for the failure
	* to be cleared entirely before reconnected can be attempted. */
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;
	}
	break;
	}
	}
	}

	gboolean
	unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last_failure,
	enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	int task_id = 0;

	const char *key = NULL;
	const char *task = NULL;
	const char *task_key = NULL;

	int rc = 0;
	int status = PCMK_LRM_OP_PENDING-1;
	int target_rc = get_target_rc(xml_op);
	int interval = 0;

	gboolean expired = FALSE;
	resource_t *parent = rsc;
	enum action_fail_response failure_strategy = action_fail_recover;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(node != NULL, return FALSE);
	CRM_CHECK(xml_op != NULL, return FALSE);

	task_key = get_op_key(xml_op);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
	crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);

	CRM_CHECK(task != NULL, return FALSE);
	CRM_CHECK(status <= PCMK_LRM_OP_NOT_INSTALLED, return FALSE);
	CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return FALSE);

	if (safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
	/* safe to ignore these */
	return TRUE;
	}

	if (is_not_set(rsc->flags, pe_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
	task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));

	if (node->details->unclean) {
	pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
	" Further action depends on the value of the stop's on-fail attribue",
	node->details->uname, rsc->id);
	}

	if (status == PCMK_LRM_OP_ERROR) {
	/* Older versions set this if rc != 0 but its up to us to decide */
	status = PCMK_LRM_OP_DONE;
	}

	if(status != PCMK_LRM_OP_NOT_INSTALLED) {
	expired = check_operation_expiry(rsc, node, rc, xml_op, data_set);
	}

	/* Degraded results are informational only, re-map them to their error-free equivalents */
	if (rc == PCMK_OCF_DEGRADED && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	rc = PCMK_OCF_OK;

	/* Add them to the failed list to highlight them for the user */
	if ((node->details->shutdown == FALSE) \|\| (node->details->online == TRUE)) {
	crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED, PCMK_OCF_OK);
	record_failed_op(xml_op, node, data_set);
	}

	} else if (rc == PCMK_OCF_DEGRADED_MASTER && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	rc = PCMK_OCF_RUNNING_MASTER;

	/* Add them to the failed list to highlight them for the user */
	if ((node->details->shutdown == FALSE) \|\| (node->details->online == TRUE)) {
	crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED_MASTER, PCMK_OCF_RUNNING_MASTER);
	record_failed_op(xml_op, node, data_set);
	}
	}

	if (expired && target_rc != rc) {
	const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);

	pe_rsc_debug(rsc, "Expired operation '%s' on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
	key, node->details->uname,
	services_ocf_exitcode_str(rc), rc,
	services_ocf_exitcode_str(target_rc), target_rc);

	if(interval == 0) {
	crm_notice("Ignoring expired calculated failure %s (rc=%d, magic=%s) on %s",
	task_key, rc, magic, node->details->uname);
	goto done;

	} else if(node->details->online && node->details->unclean == FALSE) {
	crm_notice("Re-initiated expired calculated failure %s (rc=%d, magic=%s) on %s",
	task_key, rc, magic, node->details->uname);
	/* This is SO horrible, but we don't have access to CancelXmlOp() yet */
	crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
	goto done;
	}
	}

	if(status == PCMK_LRM_OP_DONE \|\| status == PCMK_LRM_OP_ERROR) {
	status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
	}

	pe_rsc_trace(rsc, "Handling status: %d", status);
	switch (status) {
	case PCMK_LRM_OP_CANCELLED:
	/* do nothing?? */
	pe_err("Don't know what to do for cancelled ops yet");
	break;

	case PCMK_LRM_OP_PENDING:
	if (safe_str_eq(task, CRMD_ACTION_START)) {
	set_bit(rsc->flags, pe_rsc_start_pending);
	set_active(rsc);

	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
	/* If a pending migrate_to action is out on a unclean node,
	* we have to force the stop action on the target. */
	const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
	node_t *target = pe_find_node(data_set->nodes, migrate_target);
	if (target) {
	stop_action(rsc, target, FALSE);
	}
	}

	if (rsc->pending_task == NULL) {
	if (safe_str_eq(task, CRMD_ACTION_STATUS) && interval == 0) {
	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* native.c:native_pending_task().
	*/
	/rsc->pending_task = strdup("probe");/

	} else {
	rsc->pending_task = strdup(task);
	}
	}
	break;

	case PCMK_LRM_OP_DONE:
	pe_rsc_trace(rsc, "%s/%s completed on %s", rsc->id, task, node->details->uname);
	update_resource_state(rsc, node, xml_op, task, rc, *last_failure, on_fail, data_set);
	break;

	case PCMK_LRM_OP_NOT_INSTALLED:
	failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
	if (failure_strategy == action_fail_ignore) {
	crm_warn("Cannot ignore failed %s (status=%d, rc=%d) on %s: "
	"Resource agent doesn't exist",
	task_key, status, rc, node->details->uname);
	/* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
	*on_fail = action_fail_migrate;
	}
	resource_location(parent, node, -INFINITY, "hard-error", data_set);
	unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
	break;

	case PCMK_LRM_OP_ERROR:
	case PCMK_LRM_OP_ERROR_HARD:
	case PCMK_LRM_OP_ERROR_FATAL:
	case PCMK_LRM_OP_TIMEOUT:
	case PCMK_LRM_OP_NOTSUPPORTED:

	failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
	if ((failure_strategy == action_fail_ignore)
	\|\| (failure_strategy == action_fail_restart_container
	&& safe_str_eq(task, CRMD_ACTION_STOP))) {

	crm_warn("Pretending the failure of %s (rc=%d) on %s succeeded",
	task_key, rc, node->details->uname);

	update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure, on_fail, data_set);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	set_bit(rsc->flags, pe_rsc_failure_ignored);

	record_failed_op(xml_op, node, data_set);

	if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);

	if(status == PCMK_LRM_OP_ERROR_HARD) {
	do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
	"Preventing %s from re-starting on %s: operation %s failed '%s' (%d)",
	parent->id, node->details->uname,
	task, services_ocf_exitcode_str(rc), rc);

	resource_location(parent, node, -INFINITY, "hard-error", data_set);

	} else if(status == PCMK_LRM_OP_ERROR_FATAL) {
	crm_err("Preventing %s from re-starting anywhere: operation %s failed '%s' (%d)",
	parent->id, task, services_ocf_exitcode_str(rc), rc);

	resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
	}
	}
	break;
	}

	done:
	pe_rsc_trace(rsc, "Resource %s after %s: role=%s", rsc->id, task, role2text(rsc->role));
	return TRUE;
	}

	gboolean
	add_node_attrs(xmlNode * xml_obj, node_t * node, gboolean overwrite, pe_working_set_t * data_set)
	{
	const char *cluster_name = NULL;

	g_hash_table_insert(node->details->attrs,
	strdup("#uname"), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup("#" XML_ATTR_ID), strdup(node->details->id));
	if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
	data_set->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup("#" XML_ATTR_DC), strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup("#" XML_ATTR_DC), strdup(XML_BOOLEAN_FALSE));
	}

	cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup("#cluster-name"), strdup(cluster_name));
	}

	unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
	node->details->attrs, NULL, overwrite, data_set->now);

	if (g_hash_table_lookup(node->details->attrs, "#site-name") == NULL) {
	const char *site_name = g_hash_table_lookup(node->details->attrs, "site-name");

	if (site_name) {
	/* Prefix '#' to the key */
	g_hash_table_insert(node->details->attrs, strdup("#site-name"), strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs, strdup("#site-name"), strdup(cluster_name));
	}
	}
	return TRUE;
	}

	static GListPtr
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GListPtr gIter = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
	if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GListPtr
	find_operations(const char rsc, const char node, gboolean active_filter,
	pe_working_set_t * data_set)
	{
	GListPtr output = NULL;
	GListPtr intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);

	node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = __xml_first_child(status); node_state != NULL;
	node_state = __xml_next_element(node_state)) {

	if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);

	if (node != NULL && safe_str_neq(uname, node)) {
	continue;
	}

	this_node = pe_find_node(data_set->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (is_remote_node(this_node)) {
	determine_remote_online_status(data_set, this_node);

	} else {
	determine_online_status(node_state, this_node, data_set);
	}

	if (this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = __xml_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = __xml_next_element(lrm_rsc)) {
	if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {

	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);

	if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/pengine/native.c b/pengine/native.c
	index 095ad0767e..98ecd64146 100644
	--- a/pengine/native.c
	+++ b/pengine/native.c
	@@ -1,3318 +1,3318 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <pengine.h>
	#include <crm/pengine/rules.h>
	#include <crm/msg_xml.h>
	#include <allocate.h>
	#include <utils.h>
	#include <crm/services.h>

	/* #define DELETE_THEN_REFRESH 1 // The crmd will remove the resource from the CIB itself, making this redundant */
	#define INFINITY_HACK (INFINITY * -100)

	#define VARIANT_NATIVE 1
	#include <lib/pengine/variant.h>

	gboolean update_action(action_t * then);
	void native_rsc_colocation_rh_must(resource_t * rsc_lh, gboolean update_lh,
	resource_t * rsc_rh, gboolean update_rh);

	void native_rsc_colocation_rh_mustnot(resource_t * rsc_lh, gboolean update_lh,
	resource_t * rsc_rh, gboolean update_rh);

	void Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set);
	void RecurringOp(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set);
	void Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node,
	pe_working_set_t * data_set);
	void RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set);
	void pe_post_notify(resource_t * rsc, node_t * node, action_t * op,
	notify_data_t * n_data, pe_working_set_t * data_set);

	void ReloadRsc(resource_t * rsc, node_t node, pe_working_set_t data_set);
	gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set);
	gboolean StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean PromoteRsc(resource_t * rsc, node_t * next, gboolean optional,
	pe_working_set_t * data_set);
	gboolean RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
	gboolean NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);

	/* INDENT-OFF */
	enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* Current State */
	/* Next State: Unknown Stopped Started Slave Master */
	/* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
	/* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, },
	/* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Slave */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	/* Master */ { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_SLAVE, RSC_ROLE_MASTER, },
	};

	gboolean (rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(resource_t,node_t,gboolean,pe_working_set_t) = {
	/* Current State */
	/* Next State: Unknown Stopped Started Slave Master */
	/* Unknown */ { RoleError, StopRsc, RoleError, RoleError, RoleError, },
	/* Stopped */ { RoleError, NullOp, StartRsc, StartRsc, RoleError, },
	/* Started */ { RoleError, StopRsc, NullOp, NullOp, PromoteRsc, },
	/* Slave */ { RoleError, StopRsc, StopRsc, NullOp, PromoteRsc, },
	/* Master */ { RoleError, DemoteRsc, DemoteRsc, DemoteRsc, NullOp, },
	};
	/* INDENT-ON */

	static action_t * get_first_named_action(resource_t * rsc, const char action, gboolean only_valid, node_t current);

	static gboolean
	native_choose_node(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
	{
	/*
	1. Sort by weight
	2. color.chosen_node = the node (of those with the highest wieght)
	with the fewest resources
	3. remove color.chosen_node from all other colors
	*/
	GListPtr nodes = NULL;
	node_t *chosen = NULL;

	int lpc = 0;
	int multiple = 0;
	int length = 0;
	gboolean result = FALSE;

	process_utilization(rsc, &prefer, data_set);

	length = g_hash_table_size(rsc->allowed_nodes);

	if (is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to ? TRUE : FALSE;
	}

	if (prefer) {
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
	if (chosen && chosen->weight >= 0 && can_run_resources(chosen)) {
	pe_rsc_trace(rsc,
	"Using preferred node %s for %s instead of choosing from %d candidates",
	chosen->details->uname, rsc->id, length);
	} else if (chosen && chosen->weight < 0) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable", chosen->details->uname,
	rsc->id);
	chosen = NULL;
	} else if (chosen && can_run_resources(chosen)) {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable", chosen->details->uname,
	rsc->id);
	chosen = NULL;
	} else {
	pe_rsc_trace(rsc, "Preferred node %s for %s was unknown", prefer->details->uname,
	rsc->id);
	}
	}

	if (chosen == NULL && rsc->allowed_nodes) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = g_list_sort_with_data(nodes, sort_node_weight, g_list_nth_data(rsc->running_on, 0));

	chosen = g_list_nth_data(nodes, 0);
	pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates",
	chosen ? chosen->details->uname : "<none>", rsc->id, length);

	if (chosen && chosen->weight > 0 && can_run_resources(chosen)) {
	node_t *running = g_list_nth_data(rsc->running_on, 0);

	if (running && can_run_resources(running) == FALSE) {
	pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
	rsc->id, running->details->uname);
	running = NULL;
	}

	for (lpc = 1; lpc < length && running; lpc++) {
	node_t *tmp = g_list_nth_data(nodes, lpc);

	if (tmp->weight == chosen->weight) {
	multiple++;
	if (tmp->details == running->details) {
	/* prefer the existing node if scores are equal */
	chosen = tmp;
	}
	}
	}
	}
	}

	if (multiple > 1) {
	int log_level = LOG_INFO;
	static char score[33];

	score2char_stack(chosen->weight, score, sizeof(score));

	if (chosen->weight >= INFINITY) {
	log_level = LOG_WARNING;
	}

	do_crm_log(log_level, "%d nodes with equal score (%s) for"
	" running %s resources. Chose %s.",
	multiple, score, rsc->id, chosen->details->uname);
	}

	result = native_assign_node(rsc, nodes, chosen, FALSE);
	g_list_free(nodes);
	return result;
	}

	static int
	node_list_attr_score(GHashTable * list, const char attr, const char value)
	{
	GHashTableIter iter;
	node_t *node = NULL;
	int best_score = -INFINITY;
	const char *best_node = NULL;

	if (attr == NULL) {
	attr = "#" XML_ATTR_UNAME;
	}

	g_hash_table_iter_init(&iter, list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	int weight = node->weight;

	if (can_run_resources(node) == FALSE) {
	weight = -INFINITY;
	}
	if (weight > best_score \|\| best_node == NULL) {
	const char *tmp = g_hash_table_lookup(node->details->attrs, attr);

	if (safe_str_eq(value, tmp)) {
	best_score = weight;
	best_node = node->details->uname;
	}
	}
	}

	if (safe_str_neq(attr, "#" XML_ATTR_UNAME)) {
	crm_info("Best score for %s=%s was %s with %d",
	attr, value, best_node ? best_node : "<none>", best_score);
	}

	return best_score;
	}

	static void
	node_hash_update(GHashTable * list1, GHashTable * list2, const char *attr, float factor,
	gboolean only_positive)
	{
	int score = 0;
	int new_score = 0;
	GHashTableIter iter;
	node_t *node = NULL;

	if (attr == NULL) {
	attr = "#" XML_ATTR_UNAME;
	}

	g_hash_table_iter_init(&iter, list1);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	CRM_LOG_ASSERT(node != NULL);
	if(node == NULL) { continue; };

	score = node_list_attr_score(list2, attr, g_hash_table_lookup(node->details->attrs, attr));
	new_score = merge_weights(factor * score, node->weight);

	if (factor < 0 && score < 0) {
	/* Negative preference for a node with a negative score
	* should not become a positive preference
	*
	* TODO - Decide if we want to filter only if weight == -INFINITY
	*
	*/
	crm_trace("%s: Filtering %d + %f%d (factor score)",
	node->details->uname, node->weight, factor, score);

	} else if (node->weight == INFINITY_HACK) {
	crm_trace("%s: Filtering %d + %f*%d (node < 0)",
	node->details->uname, node->weight, factor, score);

	} else if (only_positive && new_score < 0 && node->weight > 0) {
	node->weight = INFINITY_HACK;
	crm_trace("%s: Filtering %d + %f*%d (score > 0)",
	node->details->uname, node->weight, factor, score);

	} else if (only_positive && new_score < 0 && node->weight == 0) {
	crm_trace("%s: Filtering %d + %f*%d (score == 0)",
	node->details->uname, node->weight, factor, score);

	} else {
	crm_trace("%s: %d + %f*%d", node->details->uname, node->weight, factor, score);
	node->weight = new_score;
	}
	}
	}

	GHashTable *
	node_hash_dup(GHashTable * hash)
	{
	/* Hack! */
	GListPtr list = g_hash_table_get_values(hash);
	GHashTable *result = node_hash_from_list(list);

	g_list_free(list);
	return result;
	}

	GHashTable *
	native_merge_weights(resource_t * rsc, const char rhs, GHashTable nodes, const char *attr,
	float factor, enum pe_weights flags)
	{
	return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags);
	}

	GHashTable *
	rsc_merge_weights(resource_t * rsc, const char rhs, GHashTable nodes, const char *attr,
	float factor, enum pe_weights flags)
	{
	GHashTable *work = NULL;
	int multiplier = 1;

	if (factor < 0) {
	multiplier = -1;
	}

	if (is_set(rsc->flags, pe_rsc_merging)) {
	pe_rsc_info(rsc, "%s: Breaking dependency loop at %s", rhs, rsc->id);
	return nodes;
	}

	set_bit(rsc->flags, pe_rsc_merging);

	if (is_set(flags, pe_weights_init)) {
	if (rsc->variant == pe_group && rsc->children) {
	GListPtr last = rsc->children;

	while (last->next != NULL) {
	last = last->next;
	}

	pe_rsc_trace(rsc, "Merging %s as a group %p %p", rsc->id, rsc->children, last);
	work = rsc_merge_weights(last->data, rhs, NULL, attr, factor, flags);

	} else {
	work = node_hash_dup(rsc->allowed_nodes);
	}
	clear_bit(flags, pe_weights_init);

	} else if (rsc->variant == pe_group && rsc->children) {
	GListPtr iter = rsc->children;

	pe_rsc_trace(rsc, "%s: Combining scores from %d children of %s", rhs, g_list_length(iter), rsc->id);
	work = node_hash_dup(nodes);
	for(iter = rsc->children; iter->next != NULL; iter = iter->next) {
	work = rsc_merge_weights(iter->data, rhs, work, attr, factor, flags);
	}

	} else {
	pe_rsc_trace(rsc, "%s: Combining scores from %s", rhs, rsc->id);
	work = node_hash_dup(nodes);
	node_hash_update(work, rsc->allowed_nodes, attr, factor,
	is_set(flags, pe_weights_positive));
	}

	if (is_set(flags, pe_weights_rollback) && can_run_any(work) == FALSE) {
	pe_rsc_info(rsc, "%s: Rolling back scores from %s", rhs, rsc->id);
	g_hash_table_destroy(work);
	clear_bit(rsc->flags, pe_rsc_merging);
	return nodes;
	}

	if (can_run_any(work)) {
	GListPtr gIter = NULL;

	if (is_set(flags, pe_weights_forward)) {
	gIter = rsc->rsc_cons;
	crm_trace("Checking %d additional colocation constraints", g_list_length(gIter));

	} else if(rsc->variant == pe_group && rsc->children) {
	GListPtr last = rsc->children;

	while (last->next != NULL) {
	last = last->next;
	}

	gIter = ((resource_t*)last->data)->rsc_cons_lhs;
	crm_trace("Checking %d additional optional group colocation constraints from %s",
	g_list_length(gIter), ((resource_t*)last->data)->id);

	} else {
	gIter = rsc->rsc_cons_lhs;
	crm_trace("Checking %d additional optional colocation constraints %s", g_list_length(gIter), rsc->id);
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t *other = NULL;
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	if (is_set(flags, pe_weights_forward)) {
	other = constraint->rsc_rh;
	} else {
	other = constraint->rsc_lh;
	}

	pe_rsc_trace(rsc, "Applying %s (%s)", constraint->id, other->id);
	work = rsc_merge_weights(other, rhs, work, constraint->node_attribute,
	multiplier * (float)constraint->score / INFINITY, flags\|pe_weights_rollback);
	dump_node_scores(LOG_TRACE, NULL, rhs, work);
	}

	}

	if (is_set(flags, pe_weights_positive)) {
	node_t *node = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->weight == INFINITY_HACK) {
	node->weight = 1;
	}
	}
	}

	if (nodes) {
	g_hash_table_destroy(nodes);
	}

	clear_bit(rsc->flags, pe_rsc_merging);
	return work;
	}

	node_t *
	native_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	int alloc_details = scores_log_level + 1;

	if (rsc->parent && is_not_set(rsc->parent->flags, pe_rsc_allocating)) {
	/* never allocate children on their own */
	pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id,
	rsc->parent->id);
	rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
	}

	if (is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->allocated_to;
	}

	if (is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	set_bit(rsc->flags, pe_rsc_allocating);
	print_resource(alloc_details, "Allocating: ", rsc, FALSE);
	dump_node_scores(alloc_details, rsc, "Pre-allloc", rsc->allowed_nodes);

	for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	GHashTable *archive = NULL;
	resource_t *rsc_rh = constraint->rsc_rh;

	pe_rsc_trace(rsc, "%s: Pre-Processing %s (%s, %d, %s)",
	rsc->id, constraint->id, rsc_rh->id,
	constraint->score, role2text(constraint->role_lh));
	if (constraint->role_lh >= RSC_ROLE_MASTER
	\|\| (constraint->score < 0 && constraint->score > -INFINITY)) {
	archive = node_hash_dup(rsc->allowed_nodes);
	}
	rsc_rh->cmds->allocate(rsc_rh, NULL, data_set);
	rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint);
	if (archive && can_run_any(rsc->allowed_nodes) == FALSE) {
	pe_rsc_info(rsc, "%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive) {
	g_hash_table_destroy(archive);
	}
	}

	dump_node_scores(alloc_details, rsc, "Post-coloc", rsc->allowed_nodes);

	for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	rsc->allowed_nodes =
	constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute,
	(float)constraint->score / INFINITY,
	pe_weights_rollback);
	}

	print_resource(LOG_DEBUG_2, "Allocating: ", rsc, FALSE);
	if (rsc->next_role == RSC_ROLE_STOPPED) {
	pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id);
	/* make sure it doesn't come up again */
	resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);

	} else if(rsc->next_role > rsc->role
	&& is_set(data_set->flags, pe_flag_have_quorum) == FALSE
	&& data_set->no_quorum_policy == no_quorum_freeze) {
	crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	rsc->next_role = rsc->role;
	}

	dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__,
	rsc->allowed_nodes);
	if (is_set(data_set->flags, pe_flag_stonith_enabled)
	&& is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
	clear_bit(rsc->flags, pe_rsc_managed);
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	const char *reason = NULL;
	node_t *assign_to = NULL;

	rsc->next_role = rsc->role;
	if (rsc->running_on == NULL) {
	reason = "inactive";
	} else if (rsc->role == RSC_ROLE_MASTER) {
	assign_to = rsc->running_on->data;
	reason = "master";
	} else if (is_set(rsc->flags, pe_rsc_failed)) {
	assign_to = rsc->running_on->data;
	reason = "failed";
	} else {
	assign_to = rsc->running_on->data;
	reason = "active";
	}
	pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id,
	assign_to ? assign_to->details->uname : "'nowhere'", reason);
	native_assign_node(rsc, NULL, assign_to, TRUE);

	} else if (is_set(data_set->flags, pe_flag_stop_everything)) {
	pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id);
	native_assign_node(rsc, NULL, NULL, TRUE);

	} else if (is_set(rsc->flags, pe_rsc_provisional)
	&& native_choose_node(rsc, prefer, data_set)) {
	pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);

	} else if (rsc->allocated_to == NULL) {
	if (is_not_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if (rsc->running_on != NULL) {
	pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}

	} else {
	pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id,
	rsc->allocated_to->details->uname);
	}

	clear_bit(rsc->flags, pe_rsc_allocating);
	print_resource(LOG_DEBUG_3, "Allocated ", rsc, TRUE);

	if (rsc->is_remote_node) {
	node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);

	CRM_ASSERT(remote_node != NULL);
	if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) {
	crm_trace("Setting remote node %s to ONLINE", remote_node->details->id);
	remote_node->details->online = TRUE;
	/* We shouldn't consider an unseen remote-node unclean if we are going
	* to try and connect to it. Otherwise we get an unnecessary fence */
	if (remote_node->details->unseen == TRUE) {
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Setting remote node %s to SHUTDOWN. next role = %s, allocated=%s",
	remote_node->details->id, role2text(rsc->next_role), rsc->allocated_to ? "true" : "false");
	remote_node->details->shutdown = TRUE;
	}
	}

	return rsc->allocated_to;
	}

	static gboolean
	is_op_dup(resource_t * rsc, const char name, const char interval)
	{
	gboolean dup = FALSE;
	const char *id = NULL;
	const char *value = NULL;
	xmlNode *operation = NULL;

	CRM_ASSERT(rsc);
	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {
	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	value = crm_element_value(operation, "name");
	if (safe_str_neq(value, name)) {
	continue;
	}

	value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (value == NULL) {
	value = "0";
	}

	if (safe_str_neq(value, interval)) {
	continue;
	}

	if (id == NULL) {
	id = ID(operation);

	} else {
	crm_config_err("Operation %s is a duplicate of %s", ID(operation), id);
	crm_config_err
	("Do not use the same (name, interval) combination more than once per resource");
	dup = TRUE;
	}
	}
	}

	return dup;
	}

	void
	RecurringOp(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *interval = NULL;
	const char *node_uname = NULL;

	unsigned long long interval_ms = 0;
	action_t *mon = NULL;
	gboolean is_optional = TRUE;
	GListPtr possible_matches = NULL;

	/* Only process for the operations without role="Stopped" */
	value = crm_element_value(operation, "role");
	if (value && text2role(value) == RSC_ROLE_STOPPED) {
	return;
	}

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s",
	ID(operation), rsc->id, role2text(rsc->next_role),
	node ? node->details->uname : "n/a");

	if (node != NULL) {
	node_uname = node->details->uname;
	}

	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_get_interval(interval);

	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval)) {
	return;
	}

	if (safe_str_eq(name, RSC_STOP)
	\|\| safe_str_eq(name, RSC_START)
	\|\| safe_str_eq(name, RSC_DEMOTE)
	\|\| safe_str_eq(name, RSC_PROMOTE)
	) {
	crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name);
	return;
	}

	key = generate_op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	/* disabled */
	free(key);
	return;
	}

	if (start != NULL) {
	pe_rsc_trace(rsc, "Marking %s %s due to %s",
	key, is_set(start->flags, pe_action_optional) ? "optional" : "mandatory",
	start->uuid);
	is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
	} else {
	pe_rsc_trace(rsc, "Marking %s optional", key);
	is_optional = TRUE;
	}

	/* start a monitor for an already active resource */
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches == NULL) {
	is_optional = FALSE;
	pe_rsc_trace(rsc, "Marking %s mandatory: not active", key);

	} else {
	GListPtr gIter = NULL;

	for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) {
	action_t op = (action_t ) gIter->data;

	if (is_set(op->flags, pe_action_reschedule)) {
	is_optional = FALSE;
	break;
	}
	}
	g_list_free(possible_matches);
	}

	if ((rsc->next_role == RSC_ROLE_MASTER && value == NULL)
	\|\| (value != NULL && text2role(value) != rsc->next_role)) {
	int log_level = LOG_DEBUG_2;
	const char *result = "Ignoring";

	if (is_optional) {
	char *local_key = strdup(key);

	log_level = LOG_INFO;
	result = "Cancelling";
	/* its running : cancel it */

	mon = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set);

	free(mon->task);
	free(mon->cancel_task);
	mon->task = strdup(RSC_CANCEL);
	mon->cancel_task = strdup(name);
	add_hash_param(mon->meta, XML_LRM_ATTR_INTERVAL, interval);
	add_hash_param(mon->meta, XML_LRM_ATTR_TASK, name);

	local_key = NULL;

	switch (rsc->role) {
	case RSC_ROLE_SLAVE:
	case RSC_ROLE_STARTED:
	if (rsc->next_role == RSC_ROLE_MASTER) {
	local_key = promote_key(rsc);

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	local_key = stop_key(rsc);
	}

	break;
	case RSC_ROLE_MASTER:
	local_key = demote_key(rsc);
	break;
	default:
	break;
	}

	if (local_key) {
	custom_action_order(rsc, NULL, mon, rsc, local_key, NULL,
	pe_order_runnable_left, data_set);
	}

	mon = NULL;
	}

	do_crm_log(log_level, "%s action %s (%s vs. %s)",
	result, key, value ? value : role2text(RSC_ROLE_SLAVE),
	role2text(rsc->next_role));

	free(key);
	key = NULL;
	return;
	}

	mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
	key = mon->uuid;
	if (is_optional) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(node_uname), mon->uuid);
	}

	if (start == NULL \|\| is_set(start->flags, pe_action_runnable) == FALSE) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : start un-runnable)", crm_str(node_uname),
	mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear);

	} else if (node == NULL \|\| node->details->online == FALSE \|\| node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)", crm_str(node_uname),
	mon->uuid);
	update_action_flags(mon, pe_action_runnable \| pe_action_clear);

	} else if (is_set(mon->flags, pe_action_optional) == FALSE) {
	pe_rsc_info(rsc, " Start recurring %s (%llus) for %s on %s", mon->task, interval_ms / 1000,
	rsc->id, crm_str(node_uname));
	}

	if (rsc->next_role == RSC_ROLE_MASTER) {
	char *running_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);

	add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master);
	free(running_master);
	}

	if (node == NULL \|\| is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, start_key(rsc), NULL,
	NULL, strdup(key), mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	custom_action_order(rsc, reload_key(rsc), NULL,
	NULL, strdup(key), mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	if (rsc->next_role == RSC_ROLE_MASTER) {
	custom_action_order(rsc, promote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);

	} else if (rsc->role == RSC_ROLE_MASTER) {
	custom_action_order(rsc, demote_key(rsc), NULL,
	rsc, NULL, mon,
	pe_order_optional \| pe_order_runnable_left, data_set);
	}
	}
	}

	void
	Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set)
	{
	if (is_not_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {
	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	RecurringOp(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	void
	RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node,
	xmlNode * operation, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *interval = NULL;
	const char *node_uname = NULL;

	unsigned long long interval_ms = 0;
	GListPtr possible_matches = NULL;
	GListPtr gIter = NULL;

	/* TODO: Support of non-unique clone */
	if (is_set(rsc->flags, pe_rsc_unique) == FALSE) {
	return;
	}

	/* Only process for the operations with role="Stopped" */
	role = crm_element_value(operation, "role");
	if (role == NULL \|\| text2role(role) != RSC_ROLE_STOPPED) {
	return;
	}

	pe_rsc_trace(rsc,
	"Creating recurring actions %s for %s in role %s on nodes where it'll not be running",
	ID(operation), rsc->id, role2text(rsc->next_role));

	if (node != NULL) {
	node_uname = node->details->uname;
	}

	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	interval_ms = crm_get_interval(interval);

	if (interval_ms == 0) {
	return;
	}

	name = crm_element_value(operation, "name");
	if (is_op_dup(rsc, name, interval)) {
	return;
	}

	if (safe_str_eq(name, RSC_STOP)
	\|\| safe_str_eq(name, RSC_START)
	\|\| safe_str_eq(name, RSC_DEMOTE)
	\|\| safe_str_eq(name, RSC_PROMOTE)
	) {
	crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name);
	return;
	}

	key = generate_op_key(rsc->id, name, interval_ms);
	if (find_rsc_op_entry(rsc, key) == NULL) {
	/* disabled */
	free(key);
	return;
	}

	/* if the monitor exists on the node where the resource will be running, cancel it */
	if (node != NULL) {
	possible_matches = find_actions_exact(rsc->actions, key, node);
	if (possible_matches) {
	action_t *cancel_op = NULL;
	char *local_key = strdup(key);

	g_list_free(possible_matches);

	cancel_op = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set);

	free(cancel_op->task);
	free(cancel_op->cancel_task);
	cancel_op->task = strdup(RSC_CANCEL);
	cancel_op->cancel_task = strdup(name);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL, interval);
	add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, name);

	local_key = NULL;

	if (rsc->next_role == RSC_ROLE_STARTED \|\| rsc->next_role == RSC_ROLE_SLAVE) {
	/* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
	/* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
	custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL,
	pe_order_runnable_left, data_set);
	}

	pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s",
	key, role, role2text(rsc->next_role), crm_str(node_uname));
	}
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t stop_node = (node_t ) gIter->data;
	const char *stop_node_uname = stop_node->details->uname;
	gboolean is_optional = TRUE;
	gboolean probe_is_optional = TRUE;
	gboolean stop_is_optional = TRUE;
	action_t *stopped_mon = NULL;
	char *rc_inactive = NULL;
	GListPtr probe_complete_ops = NULL;
	GListPtr stop_ops = NULL;
	GListPtr local_gIter = NULL;
	char *stop_op_key = NULL;

	if (node_uname && safe_str_eq(stop_node_uname, node_uname)) {
	continue;
	}

	pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s",
	ID(operation), rsc->id, crm_str(stop_node_uname));

	/* start a monitor for an already stopped resource */
	possible_matches = find_actions_exact(rsc->actions, key, stop_node);
	if (possible_matches == NULL) {
	pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key,
	crm_str(stop_node_uname));
	is_optional = FALSE;
	} else {
	pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key,
	crm_str(stop_node_uname));
	is_optional = TRUE;
	g_list_free(possible_matches);
	}

	stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set);

	rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
	add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
	free(rc_inactive);

	if (is_set(rsc->flags, pe_rsc_managed)) {
	char *probe_key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0);
	GListPtr probes = find_actions(rsc->actions, probe_key, stop_node);
	GListPtr pIter = NULL;

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	action_t probe = (action_t ) pIter->data;

	order_actions(probe, stopped_mon, pe_order_runnable_left);
	crm_trace("%s then %s on %s\n", probe->uuid, stopped_mon->uuid, stop_node->details->uname);
	}

	g_list_free(probes);
	free(probe_key);
	}

	if (probe_complete_ops) {
	g_list_free(probe_complete_ops);
	}

	stop_op_key = stop_key(rsc);
	stop_ops = find_actions_exact(rsc->actions, stop_op_key, stop_node);

	for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
	action_t stop = (action_t ) local_gIter->data;

	if (is_set(stop->flags, pe_action_optional) == FALSE) {
	stop_is_optional = FALSE;
	}

	if (is_set(stop->flags, pe_action_runnable) == FALSE) {
	crm_debug("%s\t %s (cancelled : stop un-runnable)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear);
	}

	if (is_set(rsc->flags, pe_rsc_managed)) {
	custom_action_order(rsc, strdup(stop_op_key), stop,
	NULL, strdup(key), stopped_mon,
	pe_order_implies_then \| pe_order_runnable_left, data_set);
	}

	}

	if (stop_ops) {
	g_list_free(stop_ops);
	}
	free(stop_op_key);

	if (is_optional == FALSE && probe_is_optional && stop_is_optional
	&& is_set(rsc->flags, pe_rsc_managed) == FALSE) {
	pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged",
	key, crm_str(stop_node_uname));
	update_action_flags(stopped_mon, pe_action_optional);
	}

	if (is_set(stopped_mon->flags, pe_action_optional)) {
	pe_rsc_trace(rsc, "%s\t %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
	}

	if (stop_node->details->online == FALSE \|\| stop_node->details->unclean) {
	pe_rsc_debug(rsc, "%s\t %s (cancelled : no node available)",
	crm_str(stop_node_uname), stopped_mon->uuid);
	update_action_flags(stopped_mon, pe_action_runnable \| pe_action_clear);
	}

	if (is_set(stopped_mon->flags, pe_action_runnable)
	&& is_set(stopped_mon->flags, pe_action_optional) == FALSE) {
	crm_notice(" Start recurring %s (%llus) for %s on %s", stopped_mon->task,
	interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
	}
	}

	free(key);
	}

	void
	Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set)
	{
	if (is_not_set(rsc->flags, pe_rsc_maintenance) &&
	(node == NULL \|\| node->details->maintenance == FALSE)) {
	xmlNode *operation = NULL;

	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {
	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	RecurringOp_Stopped(rsc, start, node, operation, data_set);
	}
	}
	}
	}

	static void
	handle_migration_actions(resource_t * rsc, node_t current, node_t chosen, pe_working_set_t * data_set)
	{
	action_t *migrate_to = NULL;
	action_t *migrate_from = NULL;
	action_t *start = NULL;
	action_t *stop = NULL;
	gboolean partial = rsc->partial_migration_target ? TRUE : FALSE;

	pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s",
	rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE");
	start = start_action(rsc, chosen, TRUE);
	stop = stop_action(rsc, current, TRUE);

	if (partial == FALSE) {
	migrate_to = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), RSC_MIGRATE, current, TRUE, TRUE, data_set);
	}

	migrate_from = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), RSC_MIGRATED, chosen, TRUE, TRUE, data_set);

	if ((migrate_to && migrate_from) \|\| (migrate_from && partial)) {

	set_bit(start->flags, pe_action_migrate_runnable);
	set_bit(stop->flags, pe_action_migrate_runnable);

	update_action_flags(start, pe_action_pseudo); /* easier than trying to delete it from the graph */

	/* order probes before migrations */
	if (partial) {
	set_bit(migrate_from->flags, pe_action_migrate_runnable);
	migrate_from->needs = start->needs;

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, pe_order_optional, data_set);

	} else {
	set_bit(migrate_from->flags, pe_action_migrate_runnable);
	set_bit(migrate_to->flags, pe_action_migrate_runnable);
	migrate_to->needs = start->needs;

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_STATUS, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL, pe_order_optional, data_set);
	custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, pe_order_optional \| pe_order_implies_first_migratable, data_set);
	}

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, pe_order_optional \| pe_order_implies_first_migratable, data_set);
	custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional \| pe_order_implies_first_migratable \| pe_order_pseudo_left, data_set);

	}

	if (migrate_to) {
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);

	/* pcmk remote connections don't require pending to be recorded in cib.
	* We can optimize cib writes by only setting PENDING for non pcmk remote
	* connection resources */
	if (rsc->is_remote_node == FALSE) {
	/* migrate_to takes place on the source node, but can
	* have an effect on the target node depending on how
	* the agent is written. Because of this, we have to maintain
	* a record that the migrate_to occurred incase the source node
	* loses membership while the migrate_to action is still in-flight. */
	add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
	}
	}

	if (migrate_from) {
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
	add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
	}
	}

	void
	native_create_actions(resource_t * rsc, pe_working_set_t * data_set)
	{
	action_t *start = NULL;
	node_t *chosen = NULL;
	node_t *current = NULL;
	gboolean need_stop = FALSE;
	gboolean is_moving = FALSE;
	gboolean allow_migrate = is_set(rsc->flags, pe_rsc_allow_migrate) ? TRUE : FALSE;

	GListPtr gIter = NULL;
	int num_active_nodes = 0;
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;

	CRM_ASSERT(rsc);
	chosen = rsc->allocated_to;
	if (chosen != NULL && rsc->next_role == RSC_ROLE_UNKNOWN) {
	rsc->next_role = RSC_ROLE_STARTED;
	pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));

	} else if (rsc->next_role == RSC_ROLE_UNKNOWN) {
	rsc->next_role = RSC_ROLE_STOPPED;
	pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));
	}

	pe_rsc_trace(rsc, "Processing state transition for %s %p: %s->%s", rsc->id, rsc,
	role2text(rsc->role), role2text(rsc->next_role));

	if (rsc->running_on) {
	current = rsc->running_on->data;
	}

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t n = (node_t ) gIter->data;
	if (rsc->partial_migration_source &&
	(n->details == rsc->partial_migration_source->details)) {
	current = rsc->partial_migration_source;
	}
	num_active_nodes++;
	}

	for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;

	action_t *stop = stop_action(rsc, current, FALSE);

	set_bit(stop->flags, pe_action_dangle);
	pe_rsc_trace(rsc, "Forcing a cleanup of %s on %s", rsc->id, current->details->uname);

	if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, FALSE, data_set);
	}
	}

	if (num_active_nodes > 1) {

	if (num_active_nodes == 2
	&& chosen
	&& rsc->partial_migration_target
	&& rsc->partial_migration_source
	&& (current->details == rsc->partial_migration_source->details)
	&& (chosen->details == rsc->partial_migration_target->details)) {
	/* Here the chosen node is still the migration target from a partial
	* migration. Attempt to continue the migration instead of recovering
	* by stopping the resource everywhere and starting it on a single node. */
	pe_rsc_trace(rsc,
	"Will attempt to continue with a partial migration to target %s from %s",
	rsc->partial_migration_target->details->id,
	rsc->partial_migration_source->details->id);
	} else {
	const char *type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if(rsc->partial_migration_target && rsc->partial_migration_source) {
	crm_notice("Resource %s can no longer migrate to %s. Stopping on %s too", rsc->id,
	rsc->partial_migration_target->details->uname,
	rsc->partial_migration_source->details->uname);

	} else {
	pe_proc_err("Resource %s (%s::%s) is active on %d nodes %s",
	rsc->id, class, type, num_active_nodes, recovery2text(rsc->recovery_type));
	crm_warn("See %s for more information.",
	"http://clusterlabs.org/wiki/FAQ#Resource_is_Too_Active");
	}

	if (rsc->recovery_type == recovery_stop_start) {
	need_stop = TRUE;
	}

	/* If by chance a partial migration is in process,
	* but the migration target is not chosen still, clear all
	* partial migration data. */
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = FALSE;
	}
	}

	if (is_set(rsc->flags, pe_rsc_start_pending)) {
	start = start_action(rsc, chosen, TRUE);
	set_bit(start->flags, pe_action_print_always);
	}

	if (current && chosen && current->details != chosen->details) {
	pe_rsc_trace(rsc, "Moving %s", rsc->id);
	is_moving = TRUE;
	need_stop = TRUE;

	} else if (is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_trace(rsc, "Recovering %s", rsc->id);
	need_stop = TRUE;

	} else if (is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "Block %s", rsc->id);
	need_stop = TRUE;

	} else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
	/* Recovery of a promoted resource */
	start = start_action(rsc, chosen, TRUE);
	if (is_set(start->flags, pe_action_optional) == FALSE) {
	pe_rsc_trace(rsc, "Forced start %s", rsc->id);
	need_stop = TRUE;
	}
	}

	pe_rsc_trace(rsc, "Creating actions for %s: %s->%s", rsc->id,
	role2text(rsc->role), role2text(rsc->next_role));

	role = rsc->role;
	/* Potentiall optional steps on brining the resource down and back up to the same level */
	while (role != RSC_ROLE_STOPPED) {
	next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
	pe_rsc_trace(rsc, "Down: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
	rsc->id, need_stop ? " required" : "");
	if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}


	while (rsc->role <= rsc->next_role && role != rsc->role && is_not_set(rsc->flags, pe_rsc_block)) {
	next_role = rsc_state_matrix[role][rsc->role];
	pe_rsc_trace(rsc, "Up: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
	rsc->id, need_stop ? " required" : "");
	if (rsc_action_matrix[role][next_role] (rsc, chosen, !need_stop, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}
	role = rsc->role;

	/* Required steps from this role to the next */
	while (role != rsc->next_role) {
	next_role = rsc_state_matrix[role][rsc->next_role];
	pe_rsc_trace(rsc, "Role: Executing: %s->%s = (%s)", role2text(role), role2text(next_role), rsc->id);
	if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
	break;
	}
	role = next_role;
	}

	if(is_set(rsc->flags, pe_rsc_block)) {
	pe_rsc_trace(rsc, "No monitor additional ops for blocked resource");

	} else if (rsc->next_role != RSC_ROLE_STOPPED \|\| is_set(rsc->flags, pe_rsc_managed) == FALSE) {
	pe_rsc_trace(rsc, "Monitor ops for active resource");
	start = start_action(rsc, chosen, TRUE);
	Recurring(rsc, start, chosen, data_set);
	Recurring_Stopped(rsc, start, chosen, data_set);
	} else {
	pe_rsc_trace(rsc, "Monitor ops for in-active resource");
	Recurring_Stopped(rsc, NULL, NULL, data_set);
	}

	/* if we are stuck in a partial migration, where the target
	* of the partial migration no longer matches the chosen target.
	* A full stop/start is required */
	if (rsc->partial_migration_target && (chosen == NULL \|\| rsc->partial_migration_target->details != chosen->details)) {
	pe_rsc_trace(rsc, "Not allowing partial migration to continue. %s", rsc->id);
	allow_migrate = FALSE;

	} else if (is_moving == FALSE \|\|
	is_not_set(rsc->flags, pe_rsc_managed) \|\|
	is_set(rsc->flags, pe_rsc_failed) \|\|
	is_set(rsc->flags, pe_rsc_start_pending) \|\|
	(current->details->unclean == TRUE) \|\|
	rsc->next_role < RSC_ROLE_STARTED) {

	allow_migrate = FALSE;
	}

	if (allow_migrate) {
	handle_migration_actions(rsc, current, chosen, data_set);
	}
	}

	static void
	rsc_avoids_remote_nodes(resource_t *rsc)
	{
	GHashTableIter iter;
	node_t *node = NULL;
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->details->remote_rsc) {
	node->weight = -INFINITY;
	}
	}
	}

	void
	native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
	{
	/* This function is on the critical path and worth optimizing as much as possible */

	resource_t *top = uber_parent(rsc);
	int type = pe_order_optional \| pe_order_implies_then \| pe_order_restart;
	gboolean is_stonith = is_set(rsc->flags, pe_rsc_fence_device);

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, type, data_set);

	if (top->variant == pe_master \|\| rsc->role > RSC_ROLE_SLAVE) {
	custom_action_order(rsc, generate_op_key(rsc->id, RSC_DEMOTE, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_implies_first_master, data_set);

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_START, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_PROMOTE, 0), NULL,
	pe_order_runnable_left, data_set);
	}

	if (is_stonith == FALSE
	&& is_set(data_set->flags, pe_flag_enable_unfencing)
	&& is_set(rsc->flags, pe_rsc_needs_unfencing)
	&& is_not_set(rsc->flags, pe_rsc_have_unfencing)) {
	/* Check if the node needs to be unfenced first */
	node_t *node = NULL;
	GHashTableIter iter;

	if(rsc != top) {
	/* Only create these constraints once, rsc is almost certainly cloned */
	clear_bit_recursive(top, pe_rsc_have_unfencing);
	}

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	action_t *unfence = pe_fence_op(node, "on", TRUE, data_set);

	custom_action_order(top, generate_op_key(top->id, top == rsc?RSC_STOP:RSC_STOPPED, 0), NULL,
	NULL, strdup(unfence->uuid), unfence,
	pe_order_optional, data_set);

	crm_debug("Stopping %s prior to unfencing %s", top->id, unfence->uuid);

	custom_action_order(NULL, strdup(unfence->uuid), unfence,
	top, generate_op_key(top->id, RSC_START, 0), NULL,
	pe_order_implies_then_on_node, data_set);
	}
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);
	return;
	}

	{
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);

	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, strdup(all_stopped->task), all_stopped,
	pe_order_implies_then \| pe_order_runnable_left, data_set);
	}

	if (g_hash_table_size(rsc->utilization) > 0
	&& safe_str_neq(data_set->placement_strategy, "default")) {
	GHashTableIter iter;
	node_t *next = NULL;
	GListPtr gIter = NULL;

	pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s",
	rsc->id, data_set->placement_strategy);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;

	char *load_stopped_task = crm_concat(LOAD_STOPPED, current->details->uname, '_');
	action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = node_copy(current);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear);
	}

	custom_action_order(rsc, stop_key(rsc), NULL,
	NULL, load_stopped_task, load_stopped, pe_order_load, data_set);
	}

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&next)) {
	char *load_stopped_task = crm_concat(LOAD_STOPPED, next->details->uname, '_');
	action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);

	if (load_stopped->node == NULL) {
	load_stopped->node = node_copy(next);
	update_action_flags(load_stopped, pe_action_optional \| pe_action_clear);
	}

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, start_key(rsc), NULL, pe_order_load, data_set);

	custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
	rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL,
	pe_order_load, data_set);

	free(load_stopped_task);
	}
	}

	if (rsc->container) {
	resource_t *remote_rsc = NULL;

	/* find out if the container is associated with remote node connection resource */
	if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else if (rsc->is_remote_node == FALSE) {
	remote_rsc = rsc_contains_remote_node(data_set, rsc->container);
	}

	/* if the container is a remote-node, force the resource within the container
	* instead of colocating the resource with the container. */
	if (remote_rsc) {
	GHashTableIter iter;
	node_t *node = NULL;
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}
	} else {

	crm_trace("Generating order and colocation rules for rsc %s with container %s", rsc->id, rsc->container->id);
	custom_action_order(rsc->container, generate_op_key(rsc->container->id, RSC_START, 0), NULL,
	rsc, generate_op_key(rsc->id, RSC_START, 0), NULL,
	pe_order_implies_then \| pe_order_runnable_left, data_set);

	custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
	rsc->container, generate_op_key(rsc->container->id, RSC_STOP, 0), NULL,
	pe_order_implies_first, data_set);

	- rsc_colocation_new("resource-with-containter", NULL, INFINITY, rsc, rsc->container, NULL,
	+ rsc_colocation_new("resource-with-container", NULL, INFINITY, rsc, rsc->container, NULL,
	NULL, data_set);
	}
	}

	if (rsc->is_remote_node \|\| is_stonith) {
	/* don't allow remote nodes to run stonith devices
	* or remote connection resources.*/
	rsc_avoids_remote_nodes(rsc);
	}

	/* If this rsc is a remote connection resource associated
	* with a container ( which will most likely be a virtual guest )
	* do not allow the container to live on any remote-nodes.
	* remote-nodes managing nested remote-nodes should not be allowed. */
	if (rsc->is_remote_node && rsc->container) {
	rsc_avoids_remote_nodes(rsc->container);
	}
	}

	void
	native_rsc_colocation_lh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", constraint->id);
	return;

	} else if (constraint->rsc_rh == NULL) {
	pe_err("rsc_rh was NULL for %s", constraint->id);
	return;
	}

	pe_rsc_trace(rsc_lh, "Processing colocation constraint between %s and %s", rsc_lh->id,
	rsc_rh->id);

	rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
	}

	enum filter_colocation_res
	filter_colocation_constraint(resource_t * rsc_lh, resource_t * rsc_rh,
	rsc_colocation_t * constraint, gboolean preview)
	{
	if (constraint->score == 0) {
	return influence_nothing;
	}

	/* rh side must be allocated before we can process constraint */
	if (preview == FALSE && is_set(rsc_rh->flags, pe_rsc_provisional)) {
	return influence_nothing;
	}

	if ((constraint->role_lh >= RSC_ROLE_SLAVE) &&
	rsc_lh->parent &&
	rsc_lh->parent->variant == pe_master && is_not_set(rsc_lh->flags, pe_rsc_provisional)) {

	/* LH and RH resources have already been allocated, place the correct
	* priority oh LH rsc for the given multistate resource role */
	return influence_rsc_priority;
	}

	if (preview == FALSE && is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
	/* error check */
	struct node_shared_s *details_lh;
	struct node_shared_s *details_rh;

	if ((constraint->score > -INFINITY) && (constraint->score < INFINITY)) {
	return influence_nothing;
	}

	details_rh = rsc_rh->allocated_to ? rsc_rh->allocated_to->details : NULL;
	details_lh = rsc_lh->allocated_to ? rsc_lh->allocated_to->details : NULL;

	if (constraint->score == INFINITY && details_lh != details_rh) {
	crm_err("%s and %s are both allocated"
	" but to different nodes: %s vs. %s",
	rsc_lh->id, rsc_rh->id,
	details_lh ? details_lh->uname : "n/a", details_rh ? details_rh->uname : "n/a");

	} else if (constraint->score == -INFINITY && details_lh == details_rh) {
	crm_err("%s and %s are both allocated"
	" but to the SAME node: %s",
	rsc_lh->id, rsc_rh->id, details_rh ? details_rh->uname : "n/a");
	}

	return influence_nothing;
	}

	if (constraint->score > 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) {
	crm_trace("LH: Skipping constraint: \"%s\" state filter nextrole is %s",
	role2text(constraint->role_lh), role2text(rsc_lh->next_role));
	return influence_nothing;
	}

	if (constraint->score > 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) {
	crm_trace("RH: Skipping constraint: \"%s\" state filter", role2text(constraint->role_rh));
	return FALSE;
	}

	if (constraint->score < 0
	&& constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) {
	crm_trace("LH: Skipping -ve constraint: \"%s\" state filter",
	role2text(constraint->role_lh));
	return influence_nothing;
	}

	if (constraint->score < 0
	&& constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) {
	crm_trace("RH: Skipping -ve constraint: \"%s\" state filter",
	role2text(constraint->role_rh));
	return influence_nothing;
	}

	return influence_rsc_location;
	}

	static void
	influence_priority(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	const char *rh_value = NULL;
	const char *lh_value = NULL;
	const char *attribute = "#id";
	int score_multiplier = 1;

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	if (!rsc_rh->allocated_to \|\| !rsc_lh->allocated_to) {
	return;
	}

	lh_value = g_hash_table_lookup(rsc_lh->allocated_to->details->attrs, attribute);
	rh_value = g_hash_table_lookup(rsc_rh->allocated_to->details->attrs, attribute);

	if (!safe_str_eq(lh_value, rh_value)) {
	if(constraint->score == INFINITY && constraint->role_lh == RSC_ROLE_MASTER) {
	rsc_lh->priority = -INFINITY;
	}
	return;
	}

	if (constraint->role_rh && (constraint->role_rh != rsc_rh->next_role)) {
	return;
	}

	if (constraint->role_lh == RSC_ROLE_SLAVE) {
	score_multiplier = -1;
	}

	rsc_lh->priority = merge_weights(score_multiplier * constraint->score, rsc_lh->priority);
	}

	static void
	colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	const char *tmp = NULL;
	const char *value = NULL;
	const char *attribute = "#id";

	GHashTable *work = NULL;
	gboolean do_check = FALSE;

	GHashTableIter iter;
	node_t *node = NULL;

	if (constraint->node_attribute != NULL) {
	attribute = constraint->node_attribute;
	}

	if (rsc_rh->allocated_to) {
	value = g_hash_table_lookup(rsc_rh->allocated_to->details->attrs, attribute);
	do_check = TRUE;

	} else if (constraint->score < 0) {
	/* nothing to do:
	* anti-colocation with something thats not running
	*/
	return;
	}

	work = node_hash_dup(rsc_lh->allowed_nodes);

	g_hash_table_iter_init(&iter, work);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	tmp = g_hash_table_lookup(node->details->attrs, attribute);
	if (do_check && safe_str_eq(tmp, value)) {
	if (constraint->score < INFINITY) {
	pe_rsc_trace(rsc_lh, "%s: %s.%s += %d", constraint->id, rsc_lh->id,
	node->details->uname, constraint->score);
	node->weight = merge_weights(constraint->score, node->weight);
	}

	} else if (do_check == FALSE \|\| constraint->score >= INFINITY) {
	pe_rsc_trace(rsc_lh, "%s: %s.%s -= %d (%s)", constraint->id, rsc_lh->id,
	node->details->uname, constraint->score,
	do_check ? "failed" : "unallocated");
	node->weight = merge_weights(-constraint->score, node->weight);
	}
	}

	if (can_run_any(work)
	\|\| constraint->score <= -INFINITY \|\| constraint->score >= INFINITY) {
	g_hash_table_destroy(rsc_lh->allowed_nodes);
	rsc_lh->allowed_nodes = work;
	work = NULL;

	} else {
	static char score[33];

	score2char_stack(constraint->score, score, sizeof(score));

	pe_rsc_info(rsc_lh, "%s: Rolling back scores from %s (%d, %s)",
	rsc_lh->id, rsc_rh->id, do_check, score);
	}

	if (work) {
	g_hash_table_destroy(work);
	}
	}

	void
	native_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	enum filter_colocation_res filter_results;

	CRM_ASSERT(rsc_lh);
	CRM_ASSERT(rsc_rh);
	filter_results = filter_colocation_constraint(rsc_lh, rsc_rh, constraint, FALSE);
	pe_rsc_trace(rsc_lh, "%sColocating %s with %s (%s, weight=%d, filter=%d)",
	constraint->score >= 0 ? "" : "Anti-",
	rsc_lh->id, rsc_rh->id, constraint->id, constraint->score, filter_results);

	switch (filter_results) {
	case influence_rsc_priority:
	influence_priority(rsc_lh, rsc_rh, constraint);
	break;
	case influence_rsc_location:
	pe_rsc_trace(rsc_lh, "%sColocating %s with %s (%s, weight=%d)",
	constraint->score >= 0 ? "" : "Anti-",
	rsc_lh->id, rsc_rh->id, constraint->id, constraint->score);
	colocation_match(rsc_lh, rsc_rh, constraint);
	break;
	case influence_nothing:
	default:
	return;
	}
	}

	static gboolean
	filter_rsc_ticket(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket)
	{
	if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) {
	pe_rsc_trace(rsc_lh, "LH: Skipping constraint: \"%s\" state filter",
	role2text(rsc_ticket->role_lh));
	return FALSE;
	}

	return TRUE;
	}

	void
	rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set)
	{
	if (rsc_ticket == NULL) {
	pe_err("rsc_ticket was NULL");
	return;
	}

	if (rsc_lh == NULL) {
	pe_err("rsc_lh was NULL for %s", rsc_ticket->id);
	return;
	}

	if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) {
	return;
	}

	if (rsc_lh->children) {
	GListPtr gIter = rsc_lh->children;

	pe_rsc_trace(rsc_lh, "Processing ticket dependencies from %s", rsc_lh->id);

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	rsc_ticket_constraint(child_rsc, rsc_ticket, data_set);
	}
	return;
	}

	pe_rsc_trace(rsc_lh, "%s: Processing ticket dependency on %s (%s, %s)",
	rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id,
	role2text(rsc_ticket->role_lh));

	if (rsc_ticket->ticket->granted == FALSE && g_list_length(rsc_lh->running_on) > 0) {
	GListPtr gIter = NULL;

	switch (rsc_ticket->loss_policy) {
	case loss_ticket_stop:
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	break;

	case loss_ticket_demote:
	/Promotion score will be set to -INFINITY in master_promotion_order() /
	if (rsc_ticket->role_lh != RSC_ROLE_MASTER) {
	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
	}
	break;

	case loss_ticket_fence:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}

	resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);

	for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	pe_fence_node(data_set, node, "because deadman ticket was lost");
	}
	break;

	case loss_ticket_freeze:
	if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
	return;
	}
	if (g_list_length(rsc_lh->running_on) > 0) {
	clear_bit(rsc_lh->flags, pe_rsc_managed);
	set_bit(rsc_lh->flags, pe_rsc_block);
	}
	break;
	}

	} else if (rsc_ticket->ticket->granted == FALSE) {

	if (rsc_ticket->role_lh != RSC_ROLE_MASTER \|\| rsc_ticket->loss_policy == loss_ticket_stop) {
	resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set);
	}

	} else if (rsc_ticket->ticket->standby) {

	if (rsc_ticket->role_lh != RSC_ROLE_MASTER \|\| rsc_ticket->loss_policy == loss_ticket_stop) {
	resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set);
	}
	}
	}

	enum pe_action_flags
	native_action_flags(action_t * action, node_t * node)
	{
	return action->flags;
	}

	enum pe_graph_flags
	native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
	enum pe_action_flags filter, enum pe_ordering type)
	{
	/* flags == get_action_flags(first, then_node) called from update_action() */
	enum pe_graph_flags changed = pe_graph_none;
	enum pe_action_flags then_flags = then->flags;
	enum pe_action_flags first_flags = first->flags;

	crm_trace( "Testing %s on %s (0x%.6x) with %s 0x%.6x",
	first->uuid, first->node ? first->node->details->uname : "[none]",
	first->flags, then->uuid, then->flags);

	if (type & pe_order_asymmetrical) {
	resource_t *then_rsc = then->rsc;
	enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;

	if (!then_rsc) {
	/* ignore */
	} else if ((then_rsc_role == RSC_ROLE_STOPPED) && safe_str_eq(then->task, RSC_STOP)) {
	/* ignore... if 'then' is supposed to be stopped after 'first', but
	* then is already stopped, there is nothing to be done when non-symmetrical. */
	} else if ((then_rsc_role >= RSC_ROLE_STARTED)
	&& safe_str_eq(then->task, RSC_START)
	&& then->node
	&& then_rsc->running_on
	&& g_list_length(then_rsc->running_on) == 1
	&& then->node->details == ((node_t *) then_rsc->running_on->data)->details) {
	/* ignore... if 'then' is supposed to be started after 'first', but
	* then is already started, there is nothing to be done when non-symmetrical. */
	} else if (!(first->flags & pe_action_runnable)) {
	/* prevent 'then' action from happening if 'first' is not runnable and
	* 'then' has not yet occurred. */
	pe_clear_action_bit(then, pe_action_runnable);
	pe_clear_action_bit(then, pe_action_optional);
	pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid);
	} else {
	/* ignore... then is allowed to start/stop if it wants to. */
	}
	}

	if (type & pe_order_implies_first) {
	if ((filter & pe_action_optional) && (flags & pe_action_optional) == 0) {
	pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);

	pe_clear_action_bit(first, pe_action_optional);
	}

	if (is_set(flags, pe_action_migrate_runnable) &&
	is_set(then->flags, pe_action_migrate_runnable) == FALSE &&
	is_set(then->flags, pe_action_optional) == FALSE) {

	pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s",
	first->uuid, then->uuid);
	pe_clear_action_bit(first, pe_action_migrate_runnable);
	}
	}

	if (type & pe_order_implies_first_master) {
	if ((filter & pe_action_optional) &&
	((then->flags & pe_action_optional) == FALSE) &&
	then->rsc && (then->rsc->role == RSC_ROLE_MASTER)) {
	pe_clear_action_bit(first, pe_action_optional);

	if (is_set(first->flags, pe_action_migrate_runnable) &&
	is_set(then->flags, pe_action_migrate_runnable) == FALSE) {

	pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid);
	pe_clear_action_bit(first, pe_action_migrate_runnable);
	}
	pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
	}
	}

	if ((type & pe_order_implies_first_migratable)
	&& is_set(filter, pe_action_optional)) {

	if (((then->flags & pe_action_migrate_runnable) == FALSE) \|\|
	((then->flags & pe_action_runnable) == FALSE)) {

	pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid);
	pe_clear_action_bit(first, pe_action_runnable);
	}

	if ((then->flags & pe_action_optional) == 0) {

	pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid);
	pe_clear_action_bit(first, pe_action_optional);
	}
	}

	if ((type & pe_order_pseudo_left)
	&& is_set(filter, pe_action_optional)) {

	if ((first->flags & pe_action_runnable) == FALSE) {
	pe_clear_action_bit(then, pe_action_migrate_runnable);
	pe_clear_action_bit(then, pe_action_pseudo);
	pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid);
	}

	}

	if (is_set(type, pe_order_runnable_left)
	&& is_set(filter, pe_action_runnable)
	&& is_set(then->flags, pe_action_runnable)
	&& is_set(flags, pe_action_runnable) == FALSE) {
	pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid);
	pe_clear_action_bit(then, pe_action_runnable);
	pe_clear_action_bit(then, pe_action_migrate_runnable);
	}

	if (is_set(type, pe_order_implies_then)
	&& is_set(filter, pe_action_optional)
	&& is_set(then->flags, pe_action_optional)
	&& is_set(flags, pe_action_optional) == FALSE) {

	/* in this case, treat migrate_runnable as if first is optional */
	if (is_set(first->flags, pe_action_migrate_runnable) == FALSE) {
	pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid);
	pe_clear_action_bit(then, pe_action_optional);
	}
	}

	if (is_set(type, pe_order_restart)) {
	const char *reason = NULL;

	CRM_ASSERT(first->rsc && first->rsc->variant == pe_native);
	CRM_ASSERT(then->rsc && then->rsc->variant == pe_native);

	if ((filter & pe_action_runnable)
	&& (then->flags & pe_action_runnable) == 0
	&& (then->rsc->flags & pe_rsc_managed)) {
	reason = "shutdown";
	}

	if ((filter & pe_action_optional) && (then->flags & pe_action_optional) == 0) {
	reason = "recover";
	}

	if (reason && is_set(first->flags, pe_action_optional)) {
	if (is_set(first->flags, pe_action_runnable)
	\|\| is_not_set(then->flags, pe_action_optional)) {
	pe_rsc_trace(first->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid);
	pe_clear_action_bit(first, pe_action_optional);
	}
	}

	if (reason && is_not_set(first->flags, pe_action_optional)
	&& is_not_set(first->flags, pe_action_runnable)) {
	pe_rsc_trace(then->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid);
	pe_clear_action_bit(then, pe_action_runnable);
	}

	if (reason &&
	is_not_set(first->flags, pe_action_optional) &&
	is_set(first->flags, pe_action_migrate_runnable) &&
	is_not_set(then->flags, pe_action_migrate_runnable)) {

	pe_clear_action_bit(first, pe_action_migrate_runnable);
	}

	}

	if (then_flags != then->flags) {
	changed \|= pe_graph_updated_then;
	pe_rsc_trace(then->rsc,
	"Then: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	then->uuid, then->node ? then->node->details->uname : "[none]", then->flags,
	then_flags, first->uuid, first->flags);

	if(then->rsc && then->rsc->parent) {
	/* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */
	update_action(then);
	}
	}

	if (first_flags != first->flags) {
	changed \|= pe_graph_updated_first;
	pe_rsc_trace(first->rsc,
	"First: Flags for %s on %s are now 0x%.6x (was 0x%.6x) because of %s 0x%.6x",
	first->uuid, first->node ? first->node->details->uname : "[none]",
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	void
	native_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
	{
	GListPtr gIter = NULL;
	GHashTableIter iter;
	node_t *node = NULL;

	if (constraint == NULL) {
	pe_err("Constraint is NULL");
	return;

	} else if (rsc == NULL) {
	pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id);
	return;
	}

	pe_rsc_trace(rsc, "Applying %s (%s) to %s", constraint->id,
	role2text(constraint->role_filter), rsc->id);

	/* take "lifetime" into account */
	if (constraint->role_filter > RSC_ROLE_UNKNOWN && constraint->role_filter != rsc->next_role) {
	pe_rsc_debug(rsc, "Constraint (%s) is not active (role : %s vs. %s)",
	constraint->id, role2text(constraint->role_filter), role2text(rsc->next_role));
	return;

	} else if (is_active(constraint) == FALSE) {
	pe_rsc_trace(rsc, "Constraint (%s) is not active", constraint->id);
	return;
	}

	if (constraint->node_list_rh == NULL) {
	pe_rsc_trace(rsc, "RHS of constraint %s is NULL", constraint->id);
	return;
	}

	for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	node_t *other_node = NULL;

	other_node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (other_node != NULL) {
	pe_rsc_trace(rsc, "%s + %s: %d + %d",
	node->details->uname,
	other_node->details->uname, node->weight, other_node->weight);
	other_node->weight = merge_weights(other_node->weight, node->weight);

	} else {
	other_node = node_copy(node);

	g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node);
	}

	if (other_node->rsc_discover_mode < constraint->discover_mode) {
	if (constraint->discover_mode == discover_exclusive) {
	rsc->exclusive_discover = TRUE;
	}
	/* exclusive > never > always... always is default */
	other_node->rsc_discover_mode = constraint->discover_mode;
	}
	}

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	pe_rsc_trace(rsc, "%s + %s : %d", rsc->id, node->details->uname, node->weight);
	}
	}

	void
	native_expand(resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
	graph_element_from_action(action, data_set);
	}

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}
	}

	#define log_change(fmt, args...) do { \
	if(terminal) { \
	printf(" * "fmt"\n", ##args); \
	} else { \
	crm_notice(fmt, ##args); \
	} \
	} while(0)

	#define STOP_SANITY_ASSERT(lineno) do { \
	if(current && current->details->unclean) { \
	/* It will be a pseudo op */ \
	} else if(stop == NULL) { \
	crm_err("%s:%d: No stop action exists for %s", __FUNCTION__, lineno, rsc->id); \
	CRM_ASSERT(stop != NULL); \
	} else if(is_set(stop->flags, pe_action_optional)) { \
	crm_err("%s:%d: Action %s is still optional", __FUNCTION__, lineno, stop->uuid); \
	CRM_ASSERT(is_not_set(stop->flags, pe_action_optional)); \
	} \
	} while(0)

	void
	LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
	{
	node_t *next = NULL;
	node_t *current = NULL;

	action_t *stop = NULL;
	action_t *start = NULL;
	action_t *demote = NULL;
	action_t *promote = NULL;

	char *key = NULL;
	gboolean moving = FALSE;
	GListPtr possible_matches = NULL;

	if (rsc->children) {
	GListPtr gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	LogActions(child_rsc, data_set, terminal);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	if (g_list_length(rsc->running_on) > 1 && rsc->partial_migration_source) {
	current = rsc->partial_migration_source;
	} else {
	current = rsc->running_on->data;
	}

	if (rsc->role == RSC_ROLE_STOPPED) {
	/*
	* This can occur when resources are being recovered
	* We fiddle with the current role in native_create_actions()
	*/
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	if (current == NULL && is_set(rsc->flags, pe_rsc_orphan)) {
	/* Don't log stopped orphans */
	return;
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)
	\|\| (current == NULL && next == NULL)) {
	pe_rsc_info(rsc, "Leave %s\t(%s%s)",
	rsc->id, role2text(rsc->role), is_not_set(rsc->flags,
	pe_rsc_managed) ? " unmanaged" : "");
	return;
	}

	if (current != NULL && next != NULL && safe_str_neq(current->details->id, next->details->id)) {
	moving = TRUE;
	}

	key = start_key(rsc);
	possible_matches = find_actions(rsc->actions, key, next);
	free(key);
	if (possible_matches) {
	start = possible_matches->data;
	g_list_free(possible_matches);
	}

	key = stop_key(rsc);
	if(start == NULL \|\| is_set(start->flags, pe_action_runnable) == FALSE) {
	possible_matches = find_actions(rsc->actions, key, NULL);
	} else {
	possible_matches = find_actions(rsc->actions, key, current);
	}
	free(key);
	if (possible_matches) {
	stop = possible_matches->data;
	g_list_free(possible_matches);
	}

	key = promote_key(rsc);
	possible_matches = find_actions(rsc->actions, key, next);
	free(key);
	if (possible_matches) {
	promote = possible_matches->data;
	g_list_free(possible_matches);
	}

	key = demote_key(rsc);
	possible_matches = find_actions(rsc->actions, key, next);
	free(key);
	if (possible_matches) {
	demote = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (rsc->role == rsc->next_role) {
	action_t *migrate_to = NULL;
	key = generate_op_key(rsc->id, RSC_MIGRATED, 0);
	possible_matches = find_actions(rsc->actions, key, next);
	free(key);

	if (possible_matches) {
	migrate_to = possible_matches->data;
	}

	CRM_CHECK(next != NULL,);
	if (next == NULL) {
	} else if (migrate_to && is_set(migrate_to->flags, pe_action_runnable) && current) {
	log_change("Migrate %s\t(%s %s -> %s)",
	rsc->id, role2text(rsc->role), current->details->uname,
	next->details->uname);

	} else if (is_set(rsc->flags, pe_rsc_reload)) {
	log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);

	} else if (start == NULL \|\| is_set(start->flags, pe_action_optional)) {
	pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role),
	next->details->uname);

	} else if (start && is_set(start->flags, pe_action_runnable) == FALSE) {
	log_change("Stop %s\t(%s %s%s)", rsc->id, role2text(rsc->role), current?current->details->uname:"N/A",
	stop && is_not_set(stop->flags, pe_action_runnable) ? " - blocked" : "");
	STOP_SANITY_ASSERT(__LINE__);

	} else if (moving && current) {
	log_change("%s %s\t(%s %s -> %s)",
	is_set(rsc->flags, pe_rsc_failed) ? "Recover" : "Move ",
	rsc->id, role2text(rsc->role),
	current->details->uname, next->details->uname);

	} else if (is_set(rsc->flags, pe_rsc_failed)) {
	log_change("Recover %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);
	STOP_SANITY_ASSERT(__LINE__);

	} else {
	log_change("Restart %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname);
	/* STOP_SANITY_ASSERT(__LINE__); False positive for migrate-fail-7 */
	}

	g_list_free(possible_matches);
	return;
	}

	if (rsc->role > RSC_ROLE_SLAVE && rsc->role > rsc->next_role) {
	CRM_CHECK(current != NULL,);
	if (current != NULL) {
	gboolean allowed = FALSE;

	if (demote != NULL && (demote->flags & pe_action_runnable)) {
	allowed = TRUE;
	}

	log_change("Demote %s\t(%s -> %s %s%s)",
	rsc->id,
	role2text(rsc->role),
	role2text(rsc->next_role),
	current->details->uname, allowed ? "" : " - blocked");

	if (stop != NULL && is_not_set(stop->flags, pe_action_optional)
	&& rsc->next_role > RSC_ROLE_STOPPED && moving == FALSE) {
	if (is_set(rsc->flags, pe_rsc_failed)) {
	log_change("Recover %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next->details->uname);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (is_set(rsc->flags, pe_rsc_reload)) {
	log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role),
	next->details->uname);

	} else {
	log_change("Restart %s\t(%s %s)",
	rsc->id, role2text(rsc->next_role), next->details->uname);
	STOP_SANITY_ASSERT(__LINE__);
	}
	}
	}

	} else if (rsc->next_role == RSC_ROLE_STOPPED) {
	GListPtr gIter = NULL;

	CRM_CHECK(current != NULL,);

	key = stop_key(rsc);
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	action_t *stop_op = NULL;
	gboolean allowed = FALSE;

	possible_matches = find_actions(rsc->actions, key, node);
	if (possible_matches) {
	stop_op = possible_matches->data;
	g_list_free(possible_matches);
	}

	if (stop_op && (stop_op->flags & pe_action_runnable)) {
	STOP_SANITY_ASSERT(__LINE__);
	allowed = TRUE;
	}

	log_change("Stop %s\t(%s%s)", rsc->id, node->details->uname,
	allowed ? "" : " - blocked");
	}

	free(key);
	}

	if (moving) {
	log_change("Move %s\t(%s %s -> %s)",
	rsc->id, role2text(rsc->next_role), current->details->uname,
	next->details->uname);
	STOP_SANITY_ASSERT(__LINE__);
	}

	if (rsc->role == RSC_ROLE_STOPPED) {
	gboolean allowed = FALSE;

	if (start && (start->flags & pe_action_runnable)) {
	allowed = TRUE;
	}

	CRM_CHECK(next != NULL,);
	if (next != NULL) {
	log_change("Start %s\t(%s%s)", rsc->id, next->details->uname,
	allowed ? "" : " - blocked");
	}
	if (allowed == FALSE) {
	return;
	}
	}

	if (rsc->next_role > RSC_ROLE_SLAVE && rsc->role < rsc->next_role) {
	gboolean allowed = FALSE;

	CRM_LOG_ASSERT(next);
	if (stop != NULL && is_not_set(stop->flags, pe_action_optional)
	&& rsc->role > RSC_ROLE_STOPPED) {
	if (is_set(rsc->flags, pe_rsc_failed)) {
	log_change("Recover %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next?next->details->uname:NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else if (is_set(rsc->flags, pe_rsc_reload)) {
	log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role),
	next?next->details->uname:NULL);
	STOP_SANITY_ASSERT(__LINE__);

	} else {
	log_change("Restart %s\t(%s %s)",
	rsc->id, role2text(rsc->role), next?next->details->uname:NULL);
	STOP_SANITY_ASSERT(__LINE__);
	}
	}

	if (promote && (promote->flags & pe_action_runnable)) {
	allowed = TRUE;
	}

	log_change("Promote %s\t(%s -> %s %s%s)",
	rsc->id,
	role2text(rsc->role),
	role2text(rsc->next_role),
	next?next->details->uname:NULL,
	allowed ? "" : " - blocked");
	}
	}

	gboolean
	StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;
	action_t *stop;

	if (rsc->partial_migration_target) {
	if (rsc->partial_migration_target->details == current->details) {
	pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname,
	next->details->uname, rsc->id);
	continue;
	} else {
	pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id);
	optional = FALSE;
	}
	}

	pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname);
	stop = stop_action(rsc, current, optional);

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	update_action_flags(stop, pe_action_runnable \| pe_action_clear);
	}

	if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
	DeleteRsc(rsc, current, optional, data_set);
	}
	}

	return TRUE;
	}

	gboolean
	StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	action_t *start = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s on %s %d", rsc->id, next ? next->details->uname : "N/A", optional);
	start = start_action(rsc, next, TRUE);
	if (is_set(start->flags, pe_action_runnable) && optional == FALSE) {
	update_action_flags(start, pe_action_optional \| pe_action_clear);
	}
	return TRUE;
	}

	gboolean
	PromoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	char *key = NULL;
	GListPtr gIter = NULL;
	gboolean runnable = TRUE;
	GListPtr action_list = NULL;

	CRM_ASSERT(rsc);
	CRM_CHECK(next != NULL, return FALSE);
	pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname);

	key = start_key(rsc);
	action_list = find_actions_exact(rsc->actions, key, next);
	free(key);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t start = (action_t ) gIter->data;

	if (is_set(start->flags, pe_action_runnable) == FALSE) {
	runnable = FALSE;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	promote_action(rsc, next, optional);
	return TRUE;
	}

	pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id);

	key = promote_key(rsc);
	action_list = find_actions_exact(rsc->actions, key, next);
	free(key);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t promote = (action_t ) gIter->data;

	update_action_flags(promote, pe_action_runnable \| pe_action_clear);
	}

	g_list_free(action_list);
	return TRUE;
	}

	gboolean
	DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);

	/* CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */
	for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	node_t current = (node_t ) gIter->data;

	pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A");
	demote_action(rsc, current, optional);
	}
	return TRUE;
	}

	gboolean
	RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A");
	CRM_CHECK(FALSE, return FALSE);
	return FALSE;
	}

	gboolean
	NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
	{
	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s", rsc->id);
	return FALSE;
	}

	gboolean
	DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set)
	{
	if (is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
	return FALSE;

	} else if (node == NULL) {
	pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id);
	return FALSE;

	} else if (node->details->unclean \|\| node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id,
	node->details->uname);
	return FALSE;
	}

	crm_notice("Removing %s from %s", rsc->id, node->details->uname);

	delete_action(rsc, node, optional);

	new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,
	optional ? pe_order_implies_then : pe_order_optional, data_set);

	return TRUE;
	}

	#include <../lib/pengine/unpack.h>
	#define set_char(x) last_rsc_id[lpc] = x; complete = TRUE;
	static char *
	increment_clone(char *last_rsc_id)
	{
	int lpc = 0;
	int len = 0;
	char *tmp = NULL;
	gboolean complete = FALSE;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if (last_rsc_id != NULL) {
	len = strlen(last_rsc_id);
	}

	lpc = len - 1;
	while (complete == FALSE && lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	lpc--;
	break;
	case '0':
	set_char('1');
	break;
	case '1':
	set_char('2');
	break;
	case '2':
	set_char('3');
	break;
	case '3':
	set_char('4');
	break;
	case '4':
	set_char('5');
	break;
	case '5':
	set_char('6');
	break;
	case '6':
	set_char('7');
	break;
	case '7':
	set_char('8');
	break;
	case '8':
	set_char('9');
	break;
	case '9':
	last_rsc_id[lpc] = '0';
	lpc--;
	break;
	case ':':
	tmp = last_rsc_id;
	last_rsc_id = calloc(1, len + 2);
	memcpy(last_rsc_id, tmp, len);
	last_rsc_id[++lpc] = '1';
	last_rsc_id[len] = '0';
	last_rsc_id[len + 1] = 0;
	complete = TRUE;
	free(tmp);
	break;
	default:
	crm_err("Unexpected char: %c (%d)", last_rsc_id[lpc], lpc);
	return NULL;
	break;
	}
	}
	return last_rsc_id;
	}

	static node_t *
	probe_grouped_clone(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
	{
	node_t *running = NULL;
	resource_t *top = uber_parent(rsc);

	if (running == NULL && is_set(top->flags, pe_rsc_unique) == FALSE) {
	/* Annoyingly we also need to check any other clone instances
	* Clumsy, but it will work.
	*
	* An alternative would be to update known_on for every peer
	* during process_rsc_state()
	*
	* This code desperately needs optimization
	* ptest -x with 100 nodes, 100 clones and clone-max=10:
	* No probes O(25s)
	* Detection without clone loop O(3m)
	* Detection with clone loop O(8m)

	ptest[32211]: 2010/02/18_14:27:55 CRIT: stage5: Probing for unknown resources
	ptest[32211]: 2010/02/18_14:33:39 CRIT: stage5: Done
	ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Updating action states
	ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Done

	*/
	char *clone_id = clone_zero(rsc->id);
	resource_t *peer = pe_find_resource(top->children, clone_id);

	while (peer && running == NULL) {
	running = pe_hash_table_lookup(peer->known_on, node->details->id);
	if (running != NULL) {
	/* we already know the status of the resource on this node */
	pe_rsc_trace(rsc, "Skipping active clone: %s", rsc->id);
	free(clone_id);
	return running;
	}
	clone_id = increment_clone(clone_id);
	peer = pe_find_resource(data_set->resources, clone_id);
	}

	free(clone_id);
	}
	return running;
	}

	gboolean
	native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
	gboolean force, pe_working_set_t * data_set)
	{
	enum pe_ordering flags = pe_order_optional;
	char *key = NULL;
	action_t *probe = NULL;
	node_t *running = NULL;
	node_t *allowed = NULL;
	resource_t *top = uber_parent(rsc);

	static const char *rc_master = NULL;
	static const char *rc_inactive = NULL;

	if (rc_inactive == NULL) {
	rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
	rc_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);
	}

	CRM_CHECK(node != NULL, return FALSE);
	if (force == FALSE && is_not_set(data_set->flags, pe_flag_startup_probes)) {
	pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id);
	return FALSE;
	} else if (force == FALSE && is_container_remote_node(node)) {
	pe_rsc_trace(rsc, "Skipping active resource detection for %s on container %s",
	rsc->id, node->details->id);
	return FALSE;
	}

	if (is_remote_node(node)) {
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (safe_str_eq(class, "stonith")) {
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, remote-nodes do not run stonith agents.", rsc->id, node->details->id);
	return FALSE;
	} else if (rsc_contains_remote_node(data_set, rsc)) {
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, remote-nodes can not run resources that contain connection resources.", rsc->id, node->details->id);
	return FALSE;
	} else if (rsc->is_remote_node) {
	pe_rsc_trace(rsc, "Skipping probe for %s on node %s, remote-nodes can not run connection resources", rsc->id, node->details->id);
	return FALSE;
	}
	}

	if (rsc->children) {
	GListPtr gIter = NULL;
	gboolean any_created = FALSE;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
	\|\| any_created;
	}

	return any_created;

	} else if (rsc->container) {
	pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id);
	return FALSE;
	}

	if (is_set(rsc->flags, pe_rsc_orphan)) {
	pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id);
	return FALSE;
	}

	running = g_hash_table_lookup(rsc->known_on, node->details->id);
	if (running == NULL && is_set(rsc->flags, pe_rsc_unique) == FALSE) {
	/* Anonymous clones */
	if (rsc->parent == top) {
	running = g_hash_table_lookup(rsc->parent->known_on, node->details->id);

	} else {
	/* Grouped anonymous clones need extra special handling */
	running = probe_grouped_clone(rsc, node, data_set);
	}
	}

	if (force == FALSE && running != NULL) {
	/* we already know the status of the resource on this node */
	pe_rsc_trace(rsc, "Skipping active: %s on %s", rsc->id, node->details->uname);
	return FALSE;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	if (allowed == NULL) {
	/* exclusive discover is enabled and this node is not in the allowed list. */
	return FALSE;
	} else if (allowed->rsc_discover_mode != discover_exclusive) {
	/* exclusive discover is enabled and this node is not marked
	* as a node this resource should be discovered on */
	return FALSE;
	}
	}
	if (allowed && allowed->rsc_discover_mode == discover_never) {
	/* this resource is marked as not needing to be discovered on this node */
	return FALSE;
	}


	key = generate_op_key(rsc->id, RSC_STATUS, 0);
	probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
	update_action_flags(probe, pe_action_optional \| pe_action_clear);

	/* If enabled, require unfencing before probing any fence devices
	* but ensure it happens after any resources that require
	* unfencing have been probed.
	*
	* Doing it the other way (requiring unfencing after probing
	* resources that need it) would result in the node being
	* unfenced, and all its resources being stopped, whenever a new
	* resource is added. Which would be highly suboptimal.
	*
	* So essentially, at the point the fencing device(s) have been
	* probed, we know the state of all resources that require
	* unfencing and that unfencing occurred.
	*/
	if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) {
	trigger_unfencing(NULL, node, "node discovery", probe, data_set);
	probe->priority = INFINITY; /* Ensure this runs if unfencing succeeds */

	} else if(is_set(rsc->flags, pe_rsc_needs_unfencing)) {
	action_t *unfence = pe_fence_op(node, "on", TRUE, data_set);

	order_actions(probe, unfence, pe_order_optional);
	}

	/*
	* We need to know if it's running_on (not just known_on) this node
	* to correctly determine the target rc.
	*/
	running = pe_find_node_id(rsc->running_on, node->details->id);
	if (running == NULL) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);

	} else if (rsc->role == RSC_ROLE_MASTER) {
	add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_master);
	}

	crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role),
	is_set(probe->flags, pe_action_runnable), rsc->running_on);

	if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) {
	top = rsc;

	} else if (top->variant < pe_clone) {
	top = rsc;

	} else {
	crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id);
	}

	if(is_not_set(probe->flags, pe_action_runnable) && rsc->running_on == NULL) {
	/* Prevent the start from occuring if rsc isn't active, but
	* don't cause it to stop if it was active already
	*/
	flags \|= pe_order_runnable_left;
	}

	custom_action_order(rsc, NULL, probe,
	top, generate_op_key(top->id, RSC_START, 0), NULL,
	flags, data_set);

	/* Before any reloads, if they exist */
	custom_action_order(rsc, NULL, probe,
	top, reload_key(rsc), NULL,
	pe_order_optional, data_set);

	if (node->details->shutdown == FALSE) {
	custom_action_order(rsc, NULL, probe,
	rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_optional, data_set);
	}

	if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* Normally rsc.start depends on probe complete which depends
	* on rsc.probe. But this can't be the case in this scenario as
	* it would create graph loops.
	*
	* So instead we explicitly order 'rsc.probe then rsc.start'
	*/

	} else {
	order_actions(probe, complete, pe_order_implies_then);
	}
	return TRUE;
	}

	static void
	native_start_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
	{
	node_t *target = stonith_op ? stonith_op->node : NULL;

	GListPtr gIter = NULL;
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
	action_t *stonith_done = get_pseudo_op(STONITH_DONE, data_set);

	for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if(action->needs == rsc_req_nothing) {
	} else if (action->needs == rsc_req_stonith) {
	order_actions(stonith_done, action, pe_order_optional);

	} else if (target != NULL
	&& safe_str_eq(action->task, RSC_START)
	&& NULL == pe_hash_table_lookup(rsc->known_on, target->details->id)) {
	/* if known == NULL, then we dont know if
	* the resource is active on the node
	* we're about to shoot
	*
	* in this case, regardless of action->needs,
	* the only safe option is to wait until
	* the node is shot before doing anything
	* to with the resource
	*
	* its analogous to waiting for all the probes
	* for rscX to complete before starting rscX
	*
	* the most likely explaination is that the
	* DC died and took its status with it
	*/

	pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
	target->details->uname);
	order_actions(all_stopped, action, pe_order_optional \| pe_order_runnable_left);
	}
	}
	}

	/* User data to pass to guest node iterator */
	struct action_list_s {
	GListPtr search_list; /* list of actions to search */
	GListPtr result_list; /* list of matching actions for this node */
	const char key; / action key to match */
	};

	/*!
	* \internal
	* \brief Prepend a node's actions matching a key to a list
	*
	* \param[in] node Guest node
	* \param[in/out] data User data
	*/
	static void prepend_node_actions(const node_t node, void data)
	{
	GListPtr actions;
	struct action_list_s info = (struct action_list_s ) data;

	actions = find_actions(info->search_list, info->key, node);
	info->result_list = g_list_concat(actions, info->result_list);
	}

	static GListPtr
	find_fence_target_node_actions(GListPtr search_list, const char key, node_t fence_target, pe_working_set_t *data_set)
	{
	struct action_list_s action_list;

	/* Actions on the target that match the key are implied by the fencing */
	action_list.search_list = search_list;
	action_list.result_list = find_actions(search_list, key, fence_target);
	action_list.key = key;

	/*
	* If the target is a host for any guest nodes, actions on those nodes
	* that match the key are also implied by the fencing.
	*/
	pe_foreach_guest_node(data_set, fence_target, prepend_node_actions, &action_list);

	return action_list.result_list;
	}

	static void
	native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
	{
	char *key = NULL;
	GListPtr gIter = NULL;
	GListPtr action_list = NULL;

	action_t *start = NULL;
	resource_t *top = uber_parent(rsc);

	key = start_key(rsc);
	action_list = find_actions(rsc->actions, key, NULL);
	if(action_list) {
	start = action_list->data;
	}

	g_list_free(action_list);
	free(key);

	key = stop_key(rsc);
	action_list = find_fence_target_node_actions(rsc->actions, key, stonith_op->node, data_set);
	free(key);

	/* add the stonith OP as a stop pre-req and the mark the stop
	* as a pseudo op - since its now redundant
	*/

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (action->node->details->online
	&& action->node->details->unclean == FALSE && is_set(rsc->flags, pe_rsc_failed)) {
	continue;
	}

	if (is_set(rsc->flags, pe_rsc_failed)) {
	crm_notice("Stop of failed resource %s is"
	" implicit after %s is fenced", rsc->id, action->node->details->uname);
	} else {
	crm_info("%s is implicit after %s is fenced",
	action->uuid, action->node->details->uname);
	}

	/* the stop would never complete and is
	* now implied by the stonith operation
	*/
	update_action_flags(action, pe_action_pseudo);
	update_action_flags(action, pe_action_runnable);
	update_action_flags(action, pe_action_implied_by_stonith);

	if(start == NULL \|\| start->needs > rsc_req_quorum) {
	enum pe_ordering flags = pe_order_optional;
	action_t *parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);

	if(stonith_op->node->details->remote_rsc) {
	flags \|= pe_order_preserve;
	}
	order_actions(stonith_op, action, flags);
	order_actions(stonith_op, parent_stop, flags);
	}

	if (is_set(rsc->flags, pe_rsc_notify)) {
	/* Create a second notification that will be delivered
	* immediately after the node is fenced
	*
	* Basic problem:
	* - C is a clone active on the node to be shot and stopping on another
	* - R is a resource that depends on C
	*
	* + C.stop depends on R.stop
	* + C.stopped depends on STONITH
	* + C.notify depends on C.stopped
	* + C.healthy depends on C.notify
	* + R.stop depends on C.healthy
	*
	* The extra notification here changes
	* + C.healthy depends on C.notify
	* into:
	* + C.healthy depends on C.notify'
	* + C.notify' depends on STONITH'
	* thus breaking the loop
	*/
	notify_data_t *n_data =
	create_notification_boundaries(rsc, RSC_STOP, NULL, stonith_op, data_set);
	crm_info("Creating secondary notification for %s", action->uuid);

	collect_notification_data(rsc, TRUE, FALSE, n_data);
	g_hash_table_insert(n_data->keys, strdup("notify_stop_resource"), strdup(rsc->id));
	g_hash_table_insert(n_data->keys, strdup("notify_stop_uname"),
	strdup(action->node->details->uname));
	create_notifications(uber_parent(rsc), n_data, data_set);
	free_notification_data(n_data);
	}

	/* From Bug #1601, successful fencing must be an input to a failed resources stop action.

	However given group(rA, rB) running on nodeX and B.stop has failed,
	A := stop healthy resource (rA.stop)
	B := stop failed resource (pseudo operation B.stop)
	C := stonith nodeX
	A requires B, B requires C, C requires A
	This loop would prevent the cluster from making progress.

	This block creates the "C requires A" dependency and therefore must (at least
	for now) be disabled.

	Instead, run the block above and treat all resources on nodeX as B would be
	(marked as a pseudo op depending on the STONITH).

	TODO: Break the "A requires B" dependency in update_action() and re-enable this block

	} else if(is_stonith == FALSE) {
	crm_info("Moving healthy resource %s"
	" off %s before fencing",
	rsc->id, node->details->uname);

	* stop healthy resources before the
	* stonith op
	*
	custom_action_order(
	rsc, stop_key(rsc), NULL,
	NULL,strdup(CRM_OP_FENCE),stonith_op,
	pe_order_optional, data_set);
	*/
	}

	g_list_free(action_list);

	key = demote_key(rsc);
	action_list = find_fence_target_node_actions(rsc->actions, key, stonith_op->node, data_set);
	free(key);

	for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (action->node->details->online == FALSE \|\| action->node->details->unclean == TRUE
	\|\| is_set(rsc->flags, pe_rsc_failed)) {
	if (is_set(rsc->flags, pe_rsc_failed)) {
	pe_rsc_info(rsc, "Demote of failed resource %s is"
	" implict after %s is fenced", rsc->id, action->node->details->uname);
	} else {
	pe_rsc_info(rsc, "%s is implicit after %s is fenced",
	action->uuid, action->node->details->uname);
	}
	/* the stop would never complete and is
	* now implied by the stonith operation
	*/
	crm_trace("here - 1");
	update_action_flags(action, pe_action_pseudo);
	update_action_flags(action, pe_action_runnable);

	if (start == NULL \|\| start->needs > rsc_req_quorum) {
	order_actions(stonith_op, action, pe_order_preserve\|pe_order_optional);
	}
	}
	}

	g_list_free(action_list);
	}

	void
	rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
	{
	if (rsc->children) {
	GListPtr gIter = NULL;

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	rsc_stonith_ordering(child_rsc, stonith_op, data_set);
	}
	return;
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);
	return;
	}

	/* Start constraints */
	native_start_constraints(rsc, stonith_op, data_set);

	/* Stop constraints */
	if (stonith_op) {
	native_stop_constraints(rsc, stonith_op, data_set);
	}
	}

	enum stack_activity {
	stack_stable = 0,
	stack_starting = 1,
	stack_stopping = 2,
	stack_middle = 4,
	};

	static action_t *
	get_first_named_action(resource_t * rsc, const char action, gboolean only_valid, node_t current)
	{
	action_t *a = NULL;
	GListPtr action_list = NULL;
	char *key = generate_op_key(rsc->id, action, 0);

	action_list = find_actions(rsc->actions, key, current);

	if (action_list == NULL \|\| action_list->data == NULL) {
	crm_trace("%s: no %s action", rsc->id, action);
	free(key);
	return NULL;
	}

	a = action_list->data;
	g_list_free(action_list);

	if (only_valid && is_set(a->flags, pe_action_pseudo)) {
	crm_trace("%s: pseudo", key);
	a = NULL;

	} else if (only_valid && is_not_set(a->flags, pe_action_runnable)) {
	crm_trace("%s: runnable", key);
	a = NULL;
	}

	free(key);
	return a;
	}

	void
	ReloadRsc(resource_t * rsc, node_t node, pe_working_set_t data_set)
	{
	GListPtr gIter = NULL;
	action_t *other = NULL;
	action_t *reload = NULL;

	if (rsc->children) {
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	ReloadRsc(child_rsc, node, data_set);
	}
	return;

	} else if (rsc->variant > pe_native) {
	/* Complex resource with no children */
	return;

	} else if (is_not_set(rsc->flags, pe_rsc_managed)) {
	pe_rsc_trace(rsc, "%s: unmanaged", rsc->id);
	return;

	} else if (is_set(rsc->flags, pe_rsc_failed) \|\| is_set(rsc->flags, pe_rsc_start_pending)) {
	pe_rsc_trace(rsc, "%s: general resource state: flags=0x%.16llx", rsc->id, rsc->flags);
	stop_action(rsc, node, FALSE); /* Force a full restart, overkill? */
	return;

	} else if (node == NULL) {
	pe_rsc_trace(rsc, "%s: not active", rsc->id);
	return;
	}

	pe_rsc_trace(rsc, "Processing %s", rsc->id);
	set_bit(rsc->flags, pe_rsc_reload);

	reload = custom_action(
	rsc, reload_key(rsc), CRMD_ACTION_RELOAD, node, FALSE, TRUE, data_set);

	/* stop = stop_action(rsc, node, optional); */
	other = get_first_named_action(rsc, RSC_STOP, TRUE, node);
	if (other != NULL) {
	order_actions(reload, other, pe_order_optional);
	}

	other = get_first_named_action(rsc, RSC_DEMOTE, TRUE, node);
	if (other != NULL) {
	order_actions(reload, other, pe_order_optional);
	}
	}

	void
	native_append_meta(resource_t * rsc, xmlNode * xml)
	{
	char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
	resource_t iso_parent, last_parent, *parent;

	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
	crm_xml_add(xml, name, value);
	free(name);
	}

	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
	if (value) {
	char *name = NULL;

	name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container) {
	crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id);
	}
	}

	last_parent = iso_parent = rsc;
	while (iso_parent != NULL) {
	char *name = NULL;
	char *iso = NULL;

	if (iso_parent->isolation_wrapper == NULL) {
	last_parent = iso_parent;
	iso_parent = iso_parent->parent;
	continue;
	}

	/* name of wrapper script this resource is routed through. */
	name = crm_meta_name(XML_RSC_ATTR_ISOLATION_WRAPPER);
	crm_xml_add(xml, name, iso_parent->isolation_wrapper);
	free(name);

	/* instance name for isolated environment */
	name = crm_meta_name(XML_RSC_ATTR_ISOLATION_INSTANCE);
	if (iso_parent->variant >= pe_clone) {
	/* if isolation is set at the clone/master level, we have to
	* give this resource the unique isolation instance associated
	* with the clone child (last_parent)*/

	/* Example: cloned group. group is container
	* clone myclone - iso_parent
	* group mygroup - last_parent (this is the iso environment)
	* rsc myrsc1 - rsc
	* rsc myrsc2
	* The group is what is isolated in example1. We have to make
	* sure myrsc1 and myrsc2 launch in the same isolated environment.
	*
	* Example: cloned primitives. rsc primitive is container
	* clone myclone iso_parent
	* rsc myrsc1 - last_parent == rsc (this is the iso environment)
	* The individual cloned primitive instances are isolated
	*/
	value = g_hash_table_lookup(last_parent->meta, XML_RSC_ATTR_INCARNATION);
	CRM_ASSERT(value != NULL);

	iso = crm_concat(crm_element_value(last_parent->xml, XML_ATTR_ID), value, '_');
	crm_xml_add(xml, name, iso);
	free(iso);
	} else {
	/*
	* Example: cloned group of containers
	* clone myclone
	* group mygroup
	* rsc myrsc1 - iso_parent (this is the iso environment)
	* rsc myrsc2
	*
	* Example: group of containers
	* group mygroup
	* rsc myrsc1 - iso_parent (this is the iso environment)
	* rsc myrsc2
	*
	* Example: group is container
	* group mygroup - iso_parent ( this is iso environment)
	* rsc myrsc1
	* rsc myrsc2
	*
	* Example: single primitive
	* rsc myrsc1 - iso_parent (this is the iso environment)
	*/
	value = g_hash_table_lookup(iso_parent->meta, XML_RSC_ATTR_INCARNATION);
	if (value) {
	crm_xml_add(xml, name, iso_parent->id);
	iso = crm_concat(crm_element_value(iso_parent->xml, XML_ATTR_ID), value, '_');
	crm_xml_add(xml, name, iso);
	free(iso);
	} else {
	crm_xml_add(xml, name, iso_parent->id);
	}
	}
	free(name);

	break;
	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:42 PM (2 h, 18 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002759
Default Alt Text: (353 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions