No OneTemporary
Actions

Size

181 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crmd/lrm.c b/crmd/lrm.c
	index c010060a54..95ab6fb7c2 100644
	--- a/crmd/lrm.c
	+++ b/crmd/lrm.c
	@@ -1,2251 +1,2269 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/services.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	#define START_DELAY_THRESHOLD 5 * 60 * 1000
	#define MAX_LRM_REG_FAILS 30

	struct delete_event_s {
	int rc;
	const char *rsc;
	lrm_state_t *lrm_state;
	};

	gboolean process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op);
	static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
	static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
	static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
	static int delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
	const char *user_name);

	static lrmd_event_data_t construct_op(lrm_state_t lrm_state, xmlNode * rsc_op,
	const char rsc_id, const char operation);
	static void do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operation,
	xmlNode * msg, xmlNode * request);

	void send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id);

	static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
	int log_level);

	static void
	lrm_connection_destroy(void)
	{
	if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	crm_crit("LRM Connection failed");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	clear_bit(fsa_input_register, R_LRM_CONNECTED);

	} else {
	crm_info("LRM Connection disconnected");
	}

	}

	static char *
	make_stop_id(const char *rsc, int call_id)
	{
	char *op_id = NULL;

	op_id = calloc(1, strlen(rsc) + 34);
	if (op_id != NULL) {
	snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id);
	}
	return op_id;
	}

	static void
	copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") == NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") != NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int target_rc = 0;
	rsc_history_t *entry = NULL;

	if (op->rsc_deleted) {
	crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
	delete_rsc_status(lrm_state, op->rsc_id, cib_quorum_override, NULL);
	return;
	}

	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	return;
	}

	crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);

	entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
	if (entry == NULL && rsc) {
	entry = calloc(1, sizeof(rsc_history_t));
	entry->id = strdup(op->rsc_id);
	g_hash_table_insert(lrm_state->resource_history, entry->id, entry);

	entry->rsc.id = entry->id;
	entry->rsc.type = strdup(rsc->type);
	entry->rsc.class = strdup(rsc->class);
	if (rsc->provider) {
	entry->rsc.provider = strdup(rsc->provider);
	} else {
	entry->rsc.provider = NULL;
	}

	} else if (entry == NULL) {
	crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
	return;
	}

	entry->last_callid = op->call_id;
	target_rc = rsc_op_expected_rc(op);
	if (op->op_status == PCMK_LRM_OP_CANCELLED) {
	if (op->interval > 0) {
	GList gIter, gIterNext;

	crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type,
	op->interval);

	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIterNext) {
	lrmd_event_data_t *existing = gIter->data;

	gIterNext = gIter->next;

	if (crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
	&& safe_str_eq(op->op_type, existing->op_type)
	&& op->interval == existing->interval) {
	lrmd_free_event(existing);
	entry->recurring_op_list = g_list_delete_link(entry->recurring_op_list, gIter);
	}
	}
	return;

	} else {
	crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
	op->rc, op->op_status);
	}

	} else if (did_rsc_op_fail(op, target_rc)) {
	/* We must store failed monitors here
	* - otherwise the block below will cause them to be forgetten them when a stop happens
	*/
	if (entry->failed) {
	lrmd_free_event(entry->failed);
	}
	entry->failed = lrmd_copy_event(op);

	} else if (op->interval == 0) {
	if (entry->last) {
	lrmd_free_event(entry->last);
	}
	entry->last = lrmd_copy_event(op);

	if (op->params &&
	(safe_str_eq(CRMD_ACTION_START, op->op_type) \|\|
	safe_str_eq("reload", op->op_type) \|\|
	safe_str_eq(CRMD_ACTION_STATUS, op->op_type))) {

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}
	entry->stop_params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str,
	g_hash_destroy_str);

	g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
	}
	}

	if (op->interval > 0) {
	GListPtr iter = NULL;

	for(iter = entry->recurring_op_list; iter; iter = iter->next) {
	lrmd_event_data_t *o = iter->data;

	/* op->rsc_id is implied */
	if(op->interval == o->interval && strcmp(op->op_type, o->op_type) == 0) {
	crm_trace("Removing existing recurring op entry: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
	entry->recurring_op_list = g_list_remove(entry->recurring_op_list, o);
	break;
	}
	}

	crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
	entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));

	} else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
	GList *gIter = entry->recurring_op_list;

	crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
	g_list_length(gIter), op->rsc_id, op->op_type, op->interval);
	for (; gIter != NULL; gIter = gIter->next) {
	lrmd_free_event(gIter->data);
	}
	g_list_free(entry->recurring_op_list);
	entry->recurring_op_list = NULL;
	}
	}

	void
	lrm_op_callback(lrmd_event_data_t * op)
	{
	const char *nodename = NULL;
	lrm_state_t *lrm_state = NULL;

	CRM_CHECK(op != NULL, return);

	/* determine the node name for this connection. */
	nodename = op->remote_nodename ? op->remote_nodename : fsa_our_uname;

	if (op->type == lrmd_event_disconnect && (safe_str_eq(nodename, fsa_our_uname))) {
	/* if this is the local lrmd ipc connection, set the right bits in the
	* crmd when the connection goes down */
	lrm_connection_destroy();
	return;
	} else if (op->type != lrmd_event_exec_complete) {
	/* we only need to process execution results */
	return;
	}

	lrm_state = lrm_state_find(nodename);
	CRM_ASSERT(lrm_state != NULL);

	process_lrm_event(lrm_state, op);
	}

	/* A_LRM_CONNECT */
	void
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* This only pertains to local lrmd connections. Remote connections are handled as
	* resources within the pengine. Connecting and disconnecting from remote lrmd instances
	* handled differently than the local. */

	lrm_state_t *lrm_state = NULL;

	if(fsa_our_uname == NULL) {
	return; /* Nothing to do */
	}
	lrm_state = lrm_state_find_or_create(fsa_our_uname);
	if (lrm_state == NULL) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	if (action & A_LRM_DISCONNECT) {
	if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
	if (action == A_LRM_DISCONNECT) {
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	clear_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_info("Disconnecting from the LRM");
	lrm_state_disconnect(lrm_state);
	lrm_state_reset_tables(lrm_state);
	crm_notice("Disconnected from the LRM");
	}

	if (action & A_LRM_CONNECT) {
	int ret = pcmk_ok;

	crm_debug("Connecting to the LRM");
	ret = lrm_state_ipc_connect(lrm_state);

	if (ret != pcmk_ok) {
	if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
	crm_warn("Failed to sign on to the LRM %d"
	" (%d max) times", lrm_state->num_lrm_register_fails, MAX_LRM_REG_FAILS);

	crm_timer_start(wait_timer);
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	if (ret != pcmk_ok) {
	crm_err("Failed to sign on to the LRM %d" " (max) times",
	lrm_state->num_lrm_register_fails);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	set_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_info("LRM connection established");
	}

	if (action & ~(A_LRM_CONNECT \| A_LRM_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
	}
	}

	static gboolean
	lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
	{
	int counter = 0;
	gboolean rc = TRUE;
	const char *when = "lrm disconnect";

	GHashTableIter gIter;
	const char *key = NULL;
	rsc_history_t *entry = NULL;
	struct recurring_op_s *pending = NULL;

	crm_debug("Checking for active resources before exit");

	if (cur_state == S_TERMINATE) {
	log_level = LOG_ERR;
	when = "shutdown";

	} else if (is_set(fsa_input_register, R_SHUTDOWN)) {
	when = "shutdown... waiting";
	}

	if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
	guint removed = g_hash_table_foreach_remove(
	lrm_state->pending_ops, stop_recurring_actions, lrm_state);

	crm_notice("Stopped %u recurring operations at %s (%u ops remaining)",
	removed, when, g_hash_table_size(lrm_state->pending_ops));
	}

	if (lrm_state->pending_ops) {
	g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
	/* Ignore recurring actions in the shutdown calculations */
	if (pending->interval == 0) {
	counter++;
	}
	}
	}

	if (counter > 0) {
	do_crm_log(log_level, "%d pending LRM operations at %s", counter, when);

	if (cur_state == S_TERMINATE \|\| !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
	g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&gIter, (gpointer)&key, (gpointer)&pending)) {
	do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
	}

	} else {
	rc = FALSE;
	}
	return rc;
	}

	if (lrm_state->resource_history == NULL) {
	return rc;
	}

	if (cur_state == S_TERMINATE \|\| is_set(fsa_input_register, R_SHUTDOWN)) {
	/* At this point we're not waiting, we're just shutting down */
	when = "shutdown";
	}

	counter = 0;
	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
	if (is_rsc_active(lrm_state, entry->id) == FALSE) {
	continue;
	}

	counter++;
	crm_trace("Found %s active", entry->id);
	if (lrm_state->pending_ops) {
	GHashTableIter hIter;

	g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&hIter, (gpointer)&key, (gpointer)&pending)) {
	if (crm_str_eq(entry->id, pending->rsc_id, TRUE)) {
	crm_notice("%sction %s (%s) incomplete at %s",
	pending->interval == 0 ? "A" : "Recurring a",
	key, pending->op_key, when);
	}
	}
	}
	}

	if (counter) {
	crm_err("%d resources were active at %s.", counter, when);
	}

	return rc;
	}

	static char *
	get_rsc_metadata(const char type, const char class, const char *provider)
	{
	int rc = 0;
	char *metadata = NULL;

	/* Always use a local connection for this operation */
	lrm_state_t *lrm_state = lrm_state_find(fsa_our_uname);

	CRM_CHECK(type != NULL, return NULL);
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(lrm_state != NULL, return NULL);

	if (provider == NULL) {
	provider = "heartbeat";
	}

	crm_trace("Retreiving metadata for %s::%s:%s", type, class, provider);
	rc = lrm_state_get_metadata(lrm_state, class, provider, type, &metadata, 0);

	if (metadata == NULL) {
	crm_warn("No metadata found for %s::%s:%s: %s (%d)", type, class, provider, pcmk_strerror(rc), rc);
	}

	return metadata;
	}

	typedef struct reload_data_s {
	char *key;
	char *metadata;
	time_t last_query;
	gboolean can_reload;
	GListPtr restart_list;
	} reload_data_t;

	static void
	g_hash_destroy_reload(gpointer data)
	{
	reload_data_t *reload = data;

	free(reload->key);
	free(reload->metadata);
	g_list_free_full(reload->restart_list, free);
	free(reload);
	}

	GHashTable *reload_hash = NULL;
	static GListPtr
	get_rsc_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int len = 0;
	char *key = NULL;
	char *copy = NULL;
	const char *value = NULL;
	const char *provider = NULL;

	xmlNode *param = NULL;
	xmlNode *params = NULL;
	xmlNode *actions = NULL;
	xmlNode *metadata = NULL;

	time_t now = time(NULL);
	reload_data_t *reload = NULL;

	if (reload_hash == NULL) {
	reload_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_reload);
	}

	provider = rsc->provider;
	if (provider == NULL) {
	provider = "heartbeat";
	}

	len = strlen(rsc->type) + strlen(rsc->class) + strlen(provider) + 4;
	key = malloc(len);
	if(key) {
	snprintf(key, len, "%s::%s:%s", rsc->type, rsc->class, provider);
	reload = g_hash_table_lookup(reload_hash, key);
	}

	if (reload && ((now - 9) > reload->last_query)
	&& safe_str_eq(op->op_type, RSC_START)) {
	reload = NULL; /* re-query */
	}

	if (reload == NULL) {
	xmlNode *action = NULL;

	reload = calloc(1, sizeof(reload_data_t));
	g_hash_table_replace(reload_hash, key, reload);

	reload->last_query = now;
	reload->key = key;
	key = NULL;
	reload->metadata = get_rsc_metadata(rsc->type, rsc->class, provider);

	if(reload->metadata == NULL) {
	goto cleanup;
	}

	metadata = string2xml(reload->metadata);
	if (metadata == NULL) {
	crm_err("Metadata for %s::%s:%s is not valid XML",
	rsc->provider, rsc->class, rsc->type);
	goto cleanup;
	}

	actions = find_xml_node(metadata, "actions", TRUE);

	for (action = __xml_first_child(actions); action != NULL; action = __xml_next(action)) {
	if (crm_str_eq((const char *)action->name, "action", TRUE)) {
	value = crm_element_value(action, "name");
	if (safe_str_eq("reload", value)) {
	reload->can_reload = TRUE;
	break;
	}
	}
	}

	if (reload->can_reload == FALSE) {
	goto cleanup;
	}

	params = find_xml_node(metadata, "parameters", TRUE);
	for (param = __xml_first_child(params); param != NULL; param = __xml_next(param)) {
	if (crm_str_eq((const char *)param->name, "parameter", TRUE)) {
	value = crm_element_value(param, "unique");
	if (crm_is_true(value)) {
	value = crm_element_value(param, "name");
	if (value == NULL) {
	crm_err("%s: NULL param", key);
	continue;
	}
	crm_debug("Attr %s is not reloadable", value);
	copy = strdup(value);
	CRM_LOG_ASSERT(copy != NULL);
	if(copy == NULL) { continue; };
	reload->restart_list = g_list_append(reload->restart_list, copy);
	}
	}
	}
	}

	cleanup:
	free(key);
	free_xml(metadata);
	return reload->restart_list;
	}

	static void
	append_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, xmlNode * update,
	const char *version)
	{
	int len = 0;
	char *list = NULL;
	char *digest = NULL;
	const char *value = NULL;
	xmlNode *restart = NULL;
	GListPtr restart_list = NULL;
	GListPtr lpc = NULL;

	if (op->interval > 0) {
	/* monitors are not reloadable */
	return;

	} else if (op->params == NULL) {
	crm_debug("%s has no parameters", ID(update));
	return;

	} else if (rsc == NULL) {
	return;

	} else if (crm_str_eq(CRMD_ACTION_STOP, op->op_type, TRUE)) {
	/* Stopped resources don't need to be reloaded */
	return;

	} else if (compare_version("1.0.8", version) > 0) {
	/* Caller version does not support reloads */
	return;
	}

	restart_list = get_rsc_restart_list(rsc, op);
	if (restart_list == NULL) {
	/* Resource does not support reloads */
	return;
	}

	restart = create_xml_node(NULL, XML_TAG_PARAMS);
	for (lpc = restart_list; lpc != NULL; lpc = lpc->next) {
	const char param = (const char )lpc->data;

	int start = len;

	CRM_LOG_ASSERT(param != NULL);
	if(param == NULL) { continue; };

	value = g_hash_table_lookup(op->params, param);
	if (value != NULL) {
	crm_xml_add(restart, param, value);
	}
	len += strlen(param) + 2;
	list = realloc_safe(list, len + 1);
	sprintf(list + start, " %s ", param);
	}

	digest = calculate_operation_digest(restart, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list);
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	crm_trace("%s: %s, %s", rsc->id, digest, list);
	crm_log_xml_trace(restart, "restart digest source");

	free_xml(restart);
	free(digest);
	free(list);
	}

	static gboolean
	build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op,
	const char *src)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	const char *caller_version = CRM_FEATURE_SET;

	if (op == NULL) {
	return FALSE;

	} else if (AM_I_DC) {

	} else if (fsa_our_dc_version != NULL) {
	caller_version = fsa_our_dc_version;
	} else if (op->params == NULL) {
	caller_version = fsa_our_dc_version;
	} else {
	/* there is a small risk in formerly mixed clusters that
	* it will be sub-optimal.
	* however with our upgrade policy, the update we send
	* should still be completely supported anyway
	*/
	caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
	crm_debug("Falling back to operation originator version: %s", caller_version);
	}

	target_rc = rsc_op_expected_rc(op);
	xml_op = create_operation_update(parent, op, caller_version, target_rc, fsa_our_uname, src, LOG_DEBUG);

	if (xml_op) {
	append_restart_list(rsc, op, xml_op, caller_version);
	}
	return TRUE;
	}

	static gboolean
	is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
	{
	rsc_history_t *entry = NULL;

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	if (entry == NULL \|\| entry->last == NULL) {
	return FALSE;
	}

	crm_trace("Processing %s: %s.%d=%d",
	rsc_id, entry->last->op_type, entry->last->interval, entry->last->rc);
	if (entry->last->rc == PCMK_OCF_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) {
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_OK
	&& safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) {
	/* a stricter check is too complex...
	* leave that to the PE
	*/
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
	return FALSE;

	} else if (entry->last->interval == 0 && entry->last->rc == PCMK_OCF_NOT_CONFIGURED) {
	/* Badly configured resources can't be reliably stopped */
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
	{
	GHashTableIter iter;
	rsc_history_t *entry = NULL;

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {

	GList *gIter = NULL;
	xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);

	crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.class);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);

	if (entry->last && entry->last->params) {
	const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
	if (container) {
	crm_trace("Resource %s is a part of container resource %s", entry->id, container);
	crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
	}
	}
	build_operation_update(xml_rsc, &(entry->rsc), entry->failed, __FUNCTION__);
	build_operation_update(xml_rsc, &(entry->rsc), entry->last, __FUNCTION__);
	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
	build_operation_update(xml_rsc, &(entry->rsc), gIter->data, __FUNCTION__);
	}
	}

	return FALSE;
	}

	xmlNode *
	do_lrm_query_internal(lrm_state_t * lrm_state, gboolean is_replace)
	{
	xmlNode *xml_state = NULL;
	xmlNode *xml_data = NULL;
	xmlNode *rsc_list = NULL;
	const char *uuid = NULL;

	if (safe_str_eq(lrm_state->node_name, fsa_our_uname)) {
	crm_node_t *peer = crm_get_peer(0, lrm_state->node_name);
	xml_state = do_update_node_cib(peer, node_update_cluster\|node_update_peer, NULL, __FUNCTION__);
	/* The next two lines shouldn't be necessary for newer DCs */
	crm_xml_add(xml_state, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
	crm_xml_add(xml_state, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
	uuid = fsa_our_uuid;

	} else {
	xml_state = create_xml_node(NULL, XML_CIB_TAG_STATE);
	crm_xml_add(xml_state, XML_NODE_IS_REMOTE, "true");
	crm_xml_add(xml_state, XML_ATTR_ID, lrm_state->node_name);
	crm_xml_add(xml_state, XML_ATTR_UNAME, lrm_state->node_name);
	uuid = lrm_state->node_name;
	}

	xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
	crm_xml_add(xml_data, XML_ATTR_ID, uuid);
	rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);

	/* Build a list of active (not always running) resources */
	build_active_RAs(lrm_state, rsc_list);

	crm_log_xml_trace(xml_state, "Current state of the LRM");

	return xml_state;
	}

	xmlNode *
	do_lrm_query(gboolean is_replace, const char *node_name)
	{
	lrm_state_t *lrm_state = lrm_state_find(node_name);

	if (!lrm_state) {
	crm_err("Could not query lrm state for lrmd node %s", node_name);
	return NULL;
	}
	return do_lrm_query_internal(lrm_state, is_replace);
	}

	static void
	notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, from_host, rsc_id, rc == pcmk_ok ? "" : " not");

	op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
	CRM_ASSERT(op != NULL);

	if (rc == pcmk_ok) {
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	} else {
	op->op_status = PCMK_LRM_OP_ERROR;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	}

	send_direct_ack(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);

	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	/* this isn't expected - trigger a new transition */
	time_t now = time(NULL);
	char *now_s = crm_itoa(now);

	crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"last-lrm-refresh", now_s, FALSE, NULL, NULL);

	free(now_s);
	}
	}

	static gboolean
	lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	struct delete_event_s *event = user_data;
	struct pending_deletion_op_s *op = value;

	if (crm_str_eq(event->rsc, op->rsc, TRUE)) {
	notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
	return TRUE;
	}
	return FALSE;
	}

	static gboolean
	lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
	{
	const char *rsc = user_data;
	struct recurring_op_s *pending = value;

	if (crm_str_eq(rsc, pending->rsc_id, TRUE)) {
	crm_info("Removing op %s:%d for deleted resource %s",
	pending->op_key, pending->call_id, rsc);
	return TRUE;
	}
	return FALSE;
	}

	/*
	* Remove the rsc from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/
	#define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"

	static int
	delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
	const char *user_name)
	{
	char *rsc_xpath = NULL;
	int max = 0;
	int rc = pcmk_ok;

	CRM_CHECK(rsc_id != NULL, return -ENXIO);

	max = strlen(rsc_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + 1;
	rsc_xpath = calloc(1, max);
	snprintf(rsc_xpath, max, rsc_template, lrm_state->node_name, rsc_id);

	rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
	NULL, NULL, call_options \| cib_xpath, user_name);

	free(rsc_xpath);
	return rc;
	}

	static void
	delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
	GHashTableIter * rsc_gIter, int rc, const char *user_name)
	{
	struct delete_event_s event;

	CRM_CHECK(rsc_id != NULL, return);

	if (rc == pcmk_ok) {
	char *rsc_id_copy = strdup(rsc_id);

	if (rsc_gIter)
	g_hash_table_iter_remove(rsc_gIter);
	else
	g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
	crm_debug("sync: Sending delete op for %s", rsc_id_copy);
	delete_rsc_status(lrm_state, rsc_id_copy, cib_quorum_override, user_name);

	g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
	free(rsc_id_copy);
	}

	if (input) {
	notify_deleted(lrm_state, input, rsc_id, rc);
	}

	event.rc = rc;
	event.rsc = rsc_id;
	event.lrm_state = lrm_state;
	g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
	}

	/*
	* Remove the op from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/

	#define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
	#define op_call_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']"

	static void
	delete_op_entry(lrm_state_t * lrm_state, lrmd_event_data_t * op, const char *rsc_id,
	const char *key, int call_id)
	{
	xmlNode *xml_top = NULL;

	if (op != NULL) {
	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);

	if (op->interval > 0) {
	char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval);

	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, XML_ATTR_ID, op_id);
	free(op_id);
	}

	crm_debug("async: Sending delete op for %s_%s_%d (call=%d)",
	op->rsc_id, op->op_type, op->interval, op->call_id);

	fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, cib_quorum_override);

	} else if (rsc_id != NULL && key != NULL) {
	int max = 0;
	char *op_xpath = NULL;

	if (call_id > 0) {
	max =
	strlen(op_call_template) + strlen(rsc_id) + strlen(lrm_state->node_name) +
	strlen(key) + 10;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_call_template, lrm_state->node_name, rsc_id, key, call_id);

	} else {
	max =
	strlen(op_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + strlen(key) +
	1;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_template, lrm_state->node_name, rsc_id, key);
	}

	crm_debug("sync: Sending delete op for %s (call=%d)", rsc_id, call_id);
	fsa_cib_conn->cmds->delete(fsa_cib_conn, op_xpath, NULL, cib_quorum_override \| cib_xpath);

	free(op_xpath);

	} else {
	crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key);
	return;
	}

	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	void
	lrm_clear_last_failure(const char rsc_id, const char node_name)
	{
	char *attr = NULL;
	GHashTableIter iter;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;
	rsc_history_t *entry = NULL;

	attr = generate_op_key(rsc_id, "last_failure", 0);

	/* This clears last failure for every lrm state that has this rsc.*/
	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (node_name != NULL) {
	if (strcmp(node_name, lrm_state->node_name) != 0) {
	/* filter by node_name if node_name is present */
	continue;
	}
	}

	delete_op_entry(lrm_state, NULL, rsc_id, attr, 0);

	if (!lrm_state->resource_history) {
	continue;
	}

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
	if (crm_str_eq(rsc_id, entry->id, TRUE)) {
	lrmd_free_event(entry->failed);
	entry->failed = NULL;
	}
	}
	}
	free(attr);
	g_list_free(lrm_state_list);
	}

	/* Returns: gboolean - cancellation is in progress */
	static gboolean
	cancel_op(lrm_state_t * lrm_state, const char rsc_id, const char key, int op, gboolean remove)
	{
	int rc = pcmk_ok;
	char *local_key = NULL;
	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != 0, return FALSE);
	CRM_CHECK(rsc_id != NULL, return FALSE);
	if (key == NULL) {
	local_key = make_stop_id(rsc_id, op);
	key = local_key;
	}
	pending = g_hash_table_lookup(lrm_state->pending_ops, key);

	if (pending) {
	if (remove && pending->remove == FALSE) {
	pending->remove = TRUE;
	crm_debug("Scheduling %s for removal", key);
	}

	if (pending->cancelled) {
	crm_debug("Operation %s already cancelled", key);
	free(local_key);
	return FALSE;
	}

	pending->cancelled = TRUE;

	} else {
	crm_info("No pending op found for %s", key);
	free(local_key);
	return FALSE;
	}

	crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
	rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type, pending->interval);
	if (rc == pcmk_ok) {
	crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
	free(local_key);
	return TRUE;
	}

	crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
	/* The caller needs to make sure the entry is
	* removed from the pending_ops list
	*
	* Usually by returning TRUE inside the worker function
	* supplied to g_hash_table_foreach_remove()
	*
	* Not removing the entry from pending_ops will block
	* the node from shutting down
	*/
	free(local_key);
	return FALSE;
	}

	struct cancel_data {
	gboolean done;
	gboolean remove;
	const char *key;
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct cancel_data *data = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (crm_str_eq(op->op_key, data->key, TRUE)) {
	data->done = TRUE;
	remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
	}
	return remove;
	}

	static gboolean
	cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
	{
	guint removed = 0;
	struct cancel_data data;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(key != NULL, return FALSE);

	data.key = key;
	data.rsc = rsc;
	data.done = FALSE;
	data.remove = remove;
	data.lrm_state = lrm_state;

	removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
	crm_trace("Removed %u op cache entries, new size: %u",
	removed, g_hash_table_size(lrm_state->pending_ops));
	return data.done;
	}

	static lrmd_rsc_info_t *
	get_lrm_resource(lrm_state_t * lrm_state, xmlNode * resource, xmlNode * op_msg, gboolean do_create)
	{
	lrmd_rsc_info_t *rsc = NULL;
	const char *id = ID(resource);
	const char *type = crm_element_value(resource, XML_ATTR_TYPE);
	const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER);
	const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG);

	crm_trace("Retrieving %s from the LRM.", id);
	CRM_CHECK(id != NULL, return NULL);

	rsc = lrm_state_get_rsc_info(lrm_state, id, 0);

	if (!rsc && long_id) {
	rsc = lrm_state_get_rsc_info(lrm_state, long_id, 0);
	}

	if (!rsc && do_create) {
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(type != NULL, return NULL);

	crm_trace("Adding rsc %s before operation", id);

	lrm_state_register_rsc(lrm_state, id, class, provider, type, lrmd_opt_drop_recurring);

	rsc = lrm_state_get_rsc_info(lrm_state, id, 0);

	if (!rsc) {
	fsa_data_t *msg_data = NULL;

	crm_err("Could not add resource %s to LRM %s", id, lrm_state->node_name);
	/* only register this as a internal error if this involves the local
	* lrmd. Otherwise we're likely dealing with an unresponsive remote-node
	* which is not a FSA failure. */
	if (lrm_state_is_local(lrm_state) == TRUE) {
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}
	}

	return rsc;
	}

	static void
	delete_resource(lrm_state_t * lrm_state,
	const char *id,
	lrmd_rsc_info_t * rsc,
	GHashTableIter * gIter,
	const char *sys,
	const char *host,
	const char *user,
	ha_msg_input_t * request,
	gboolean unregister)
	{
	int rc = pcmk_ok;

	crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host);

	if (rsc && unregister) {
	rc = lrm_state_unregister_rsc(lrm_state, id, 0);
	}

	if (rc == pcmk_ok) {
	crm_trace("Resource '%s' deleted", id);
	} else if (rc == -EINPROGRESS) {
	crm_info("Deletion of resource '%s' pending", id);
	if (request) {
	struct pending_deletion_op_s *op = NULL;
	char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);

	op = calloc(1, sizeof(struct pending_deletion_op_s));
	op->rsc = strdup(rsc->id);
	op->input = copy_ha_msg_input(request);
	g_hash_table_insert(lrm_state->deletion_ops, ref, op);
	}
	return;
	} else {
	crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d",
	id, sys, user ? user : "internal", host, rc);
	}

	delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
	}

	static int
	get_fake_call_id(lrm_state_t lrm_state, const char rsc_id)
	{
	int call_id = 0;
	rsc_history_t *entry = NULL;

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);

	/* Make sure the call id is greater than the last successful operation,
	* otherwise the failure will not result in a possible recovery of the resource
	* as it could appear the failure occurred before the successful start */
	if (entry) {
	call_id = entry->last_callid + 1;
	}

	if (call_id < 0) {
	call_id = 1;
	}
	return call_id;
	}

	+static void
	+force_reprobe(lrm_state_t lrm_state, const char from_sys, const char from_host, const char user_name, gboolean is_remote_node)
	+{
	+ GHashTableIter gIter;
	+ rsc_history_t *entry = NULL;
	+
	+
	+ crm_info("clearing resource history on node %s", lrm_state->node_name);
	+ g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	+ while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	+ /* only unregister the resource during a reprobe if it is not a remote connection
	+ * resource. otherwise unregistering the connection will terminate remote-node
	+ * membership */
	+ gboolean unregister = TRUE;
	+
	+ if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
	+ lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
	+ if (remote_lrm_state) {
	+ /* when forcing a reprobe, make sure to clear remote node before
	+ * clearing the remote node's connection resource */
	+ force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE);
	+ }
	+ unregister = FALSE;
	+ }
	+
	+ delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, from_host,
	+ user_name, NULL, unregister);
	+ }
	+
	+ /* Now delete the copy in the CIB */
	+ erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);
	+
	+ /* And finally, _delete_ the value in attrd
	+ * Setting it to FALSE results in the PE sending us back here again
	+ */
	+ update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
	+}
	+

	/* A_LRM_INVOKE */
	void
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean create_rsc = TRUE;
	lrm_state_t *lrm_state = NULL;
	const char *crm_op = NULL;
	const char *from_sys = NULL;
	const char *from_host = NULL;
	const char *operation = NULL;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *user_name = NULL;
	const char *target_node = NULL;
	gboolean is_remote_node = FALSE;
	gboolean crm_rsc_delete = FALSE;

	if (input->xml != NULL) {
	/* Remote node operations are routed here to their remote connections */
	target_node = crm_element_value(input->xml, XML_LRM_ATTR_TARGET);
	}
	if (target_node == NULL) {
	target_node = fsa_our_uname;
	} else if (safe_str_neq(target_node, fsa_our_uname)) {
	is_remote_node = TRUE;
	}

	lrm_state = lrm_state_find(target_node);

	if (lrm_state == NULL && is_remote_node) {
	crm_err("no lrmd connection for remote node %s found on cluster node %s. Can not process request.",
	target_node, fsa_our_uname);
	return;
	}

	CRM_ASSERT(lrm_state != NULL);

	#if ENABLE_ACL
	user_name = crm_acl_get_set_user(input->msg, F_CRM_USER, NULL);
	crm_trace("LRM command from user '%s'", user_name);
	#endif

	crm_op = crm_element_value(input->msg, F_CRM_TASK);
	from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
	}

	crm_trace("LRM command from: %s", from_sys);

	if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
	/* remember this delete op came from crm_resource */
	crm_rsc_delete = TRUE;
	operation = CRMD_ACTION_DELETE;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	operation = CRM_OP_LRM_REFRESH;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
	lrmd_event_data_t *op = NULL;
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* The lrmd can not fail a resource, it does not understand the
	* concept of success or failure in relation to a resource, it simply
	* executes operations and reports the results. We determine what a failure is.
	* Becaues of this, if we want to fail a resource we have to fake what we
	* understand a failure to look like.
	*
	* To do this we create a fake lrmd operation event for the resource
	* we want to fail. We then pass that event to the lrmd client callback
	* so it will be processed as if it actually came from the lrmd. */
	op = construct_op(lrm_state, input->xml, ID(xml_rsc), "asyncmon");
	CRM_ASSERT(op != NULL);

	free((char *)op->user_data);
	op->user_data = NULL;
	op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
	op->interval = 0;
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	#if ENABLE_ACL
	if (user_name && is_privileged(user_name) == FALSE) {
	crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;
	}
	#endif

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);
	if (rsc) {
	crm_info("Failing resource %s...", rsc->id);
	process_lrm_event(lrm_state, op);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	lrmd_free_rsc_info(rsc);
	} else {
	crm_info("Cannot find/create resource in order to fail it...");
	crm_log_xml_warn(input->msg, "bad input");
	}

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;

	} else if (input->xml != NULL) {
	operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
	}

	if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	int rc = pcmk_ok;
	xmlNode *fragment = do_lrm_query_internal(lrm_state, TRUE);

	fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
	crm_info("Forced a local LRM refresh: call=%d", rc);

	if(strcmp(CRM_SYSTEM_CRMD, from_sys) != 0) {
	xmlNode *reply = create_request(
	CRM_OP_INVOKE_LRM, fragment,
	from_host, from_sys, CRM_SYSTEM_LRMD, fsa_our_uuid);

	crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}
	free_xml(reply);
	}

	free_xml(fragment);

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
	xmlNode *data = do_lrm_query_internal(lrm_state, FALSE);
	xmlNode *reply = create_reply(input->msg, data);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_err("Unable to route reply");
	crm_log_xml_err(reply, "reply");
	}
	free_xml(reply);
	free_xml(data);

	} else if (safe_str_eq(operation, CRM_OP_PROBED)) {
	update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE, user_name, is_remote_node);

	} else if (safe_str_eq(operation, CRM_OP_REPROBE) \|\| safe_str_eq(crm_op, CRM_OP_REPROBE)) {
	- GHashTableIter gIter;
	- rsc_history_t *entry = NULL;
	-
	crm_notice("Forcing the status of all resources to be redetected");

	- g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	- while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	- /* only unregister the resource during a reprobe if it is not a remote connection
	- * resource. otherwise unregistering the connection will terminate remote-node
	- * membership */
	- gboolean unregister = is_remote_lrmd_ra(NULL, NULL, entry->id) ? FALSE : TRUE;
	-
	- delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, from_host,
	- user_name, NULL, unregister);
	- }
	-
	- /* Now delete the copy in the CIB */
	- erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);
	-
	- /* And finally, _delete_ the value in attrd
	- * Setting it to FALSE results in the PE sending us back here again
	- */
	- update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
	+ force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node);

	if(strcmp(CRM_SYSTEM_TENGINE, from_sys) != 0
	&& strcmp(CRM_SYSTEM_TENGINE, from_sys) != 0) {
	xmlNode *reply = create_request(
	CRM_OP_INVOKE_LRM, NULL,
	from_host, from_sys, CRM_SYSTEM_LRMD, fsa_our_uuid);

	crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}
	free_xml(reply);
	}

	} else if (operation != NULL) {
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *params = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* only the first 16 chars are used by the LRM */
	params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);

	if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	create_rsc = FALSE;
	}

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);

	if (rsc == NULL && create_rsc) {
	lrmd_event_data_t *op = NULL;

	/* if the operation couldn't complete because we can't register
	* the resource, return a generic error */
	op = construct_op(lrm_state, input->xml, ID(xml_rsc), operation);
	CRM_ASSERT(op != NULL);

	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);

	crm_err("Invalid resource definition");
	crm_log_xml_warn(input->msg, "bad input");

	} else if (rsc == NULL) {
	lrmd_event_data_t *op = NULL;

	crm_notice("Not creating resource for a %s event: %s", operation, ID(input->xml));
	delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok, user_name);

	op = construct_op(lrm_state, input->xml, ID(xml_rsc), operation);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	CRM_ASSERT(op != NULL);
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);

	} else if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
	char *op_key = NULL;
	char *meta_key = NULL;
	int call = 0;
	const char *call_id = NULL;
	const char *op_task = NULL;
	const char *op_interval = NULL;
	gboolean in_progress = FALSE;

	CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	op_interval = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
	op_task = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
	call_id = crm_element_value(params, meta_key);
	free(meta_key);

	CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);
	CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	op_key = generate_op_key(rsc->id, op_task, crm_parse_int(op_interval, "0"));

	crm_debug("PE requested op %s (call=%s) be cancelled",
	op_key, call_id ? call_id : "NA");
	call = crm_parse_int(call_id, "0");
	if (call == 0) {
	/* the normal case when the PE cancels a recurring op */
	in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);

	} else {
	/* the normal case when the PE cancels an orphan op */
	in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
	}

	if (in_progress == FALSE) {
	lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc->id, op_task);

	crm_info("Nothing known about operation %d for %s", call, op_key);
	delete_op_entry(lrm_state, NULL, rsc->id, op_key, call);

	CRM_ASSERT(op != NULL);

	op->rc = PCMK_OCF_OK;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(from_host, from_sys, rsc, op, rsc->id);
	lrmd_free_event(op);

	/* needed?? surely not otherwise the cancel_op_(_key) wouldn't
	* have failed in the first place
	*/
	g_hash_table_remove(lrm_state->pending_ops, op_key);
	}

	free(op_key);

	} else if (rsc != NULL && safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	gboolean unregister = TRUE;

	#if ENABLE_ACL
	int cib_rc = delete_rsc_status(lrm_state, rsc->id, cib_dryrun \| cib_sync_call, user_name);
	if (cib_rc != pcmk_ok) {
	lrmd_event_data_t *op = NULL;

	crm_err
	("Attempted deletion of resource status '%s' from CIB for %s (user=%s) on %s failed: (rc=%d) %s",
	rsc->id, from_sys, user_name ? user_name : "unknown", from_host, cib_rc,
	pcmk_strerror(cib_rc));

	op = construct_op(lrm_state, input->xml, rsc->id, operation);
	op->op_status = PCMK_LRM_OP_ERROR;

	if (cib_rc == -EACCES) {
	op->rc = PCMK_OCF_INSUFFICIENT_PRIV;
	} else {
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	}
	send_direct_ack(from_host, from_sys, NULL, op, rsc->id);
	lrmd_free_event(op);
	return;
	}
	#endif
	if (crm_rsc_delete == TRUE && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
	unregister = FALSE;
	}

	delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys, from_host, user_name, input, unregister);

	} else if (rsc != NULL) {
	do_lrm_rsc_op(lrm_state, rsc, operation, input->xml, input->msg);
	}

	lrmd_free_rsc_info(rsc);

	} else {
	crm_err("Operation was neither a lrm_query, nor a rsc op. %s", crm_str(crm_op));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	static lrmd_event_data_t *
	construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, const char rsc_id, const char operation)
	{
	lrmd_event_data_t *op = NULL;
	const char *op_delay = NULL;
	const char *op_timeout = NULL;
	const char *op_interval = NULL;
	GHashTable *params = NULL;

	const char *transition = NULL;

	CRM_ASSERT(rsc_id != NULL);

	op = calloc(1, sizeof(lrmd_event_data_t));
	op->type = lrmd_event_exec_complete;
	op->op_type = strdup(operation);
	op->op_status = PCMK_LRM_OP_PENDING;
	op->rc = -1;
	op->rsc_id = strdup(rsc_id);
	op->interval = 0;
	op->timeout = 0;
	op->start_delay = 0;

	if (rsc_op == NULL) {
	CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
	op->user_data = NULL;
	/* the stop_all_resources() case
	* by definition there is no DC (or they'd be shutting
	* us down).
	* So we should put our version here.
	*/
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));

	crm_trace("Constructed %s op for %s", operation, rsc_id);
	return op;
	}

	params = xml2list(rsc_op);
	g_hash_table_remove(params, CRM_META "_op_target_rc");

	op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
	op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
	op_interval = crm_meta_value(params, XML_LRM_ATTR_INTERVAL);

	op->interval = crm_parse_int(op_interval, "0");
	op->timeout = crm_parse_int(op_timeout, "0");
	op->start_delay = crm_parse_int(op_delay, "0");

	if (safe_str_neq(operation, RSC_STOP)) {
	op->params = params;

	} else {
	rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);

	/* If we do not have stop parameters cached, use
	* whatever we are given */
	if (!entry \|\| !entry->stop_params) {
	op->params = params;
	} else {
	/* Copy the cached parameter list so that we stop the resource
	* with the old attributes, not the new ones */
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_foreach(params, copy_meta_keys, op->params);
	g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
	g_hash_table_destroy(params);
	params = NULL;
	}
	}

	/* sanity */
	if (op->interval < 0) {
	op->interval = 0;
	}
	if (op->timeout <= 0) {
	op->timeout = op->interval;
	}
	if (op->start_delay < 0) {
	op->start_delay = 0;
	}

	transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
	CRM_CHECK(transition != NULL, return op);

	op->user_data = strdup(transition);

	if (op->interval != 0) {
	if (safe_str_eq(operation, CRMD_ACTION_START)
	\|\| safe_str_eq(operation, CRMD_ACTION_STOP)) {
	crm_err("Start and Stop actions cannot have an interval: %d", op->interval);
	op->interval = 0;
	}
	}

	crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval);

	return op;
	}

	void
	send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id)
	{
	xmlNode *reply = NULL;
	xmlNode update, iter;
	crm_node_t *peer = NULL;

	CRM_CHECK(op != NULL, return);
	if (op->rsc_id == NULL) {
	CRM_ASSERT(rsc_id != NULL);
	op->rsc_id = strdup(rsc_id);
	}
	if (to_sys == NULL) {
	to_sys = CRM_SYSTEM_TENGINE;
	}

	peer = crm_get_peer(0, fsa_our_uname);
	update = do_update_node_cib(peer, node_update_none, NULL, __FUNCTION__);

	iter = create_xml_node(update, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);

	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);
	reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);

	crm_log_xml_trace(update, "ACK Update");

	crm_debug("ACK'ing resource op %s_%s_%d from %s: %s",
	op->rsc_id, op->op_type, op->interval, op->user_data,
	crm_element_value(reply, XML_ATTR_REFERENCE));

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}

	free_xml(update);
	free_xml(reply);
	}

	gboolean
	verify_stopped(enum crmd_fsa_state cur_state, int log_level)
	{
	gboolean res = TRUE;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;

	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
	/* keep iterating through all even when false is returned */
	res = FALSE;
	}
	}

	set_bit(fsa_input_register, R_SENT_RSC_STOP);
	g_list_free(lrm_state_list); lrm_state_list = NULL;
	return res;
	}

	struct stop_recurring_action_s {
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct stop_recurring_action_s *event = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0 && crm_str_eq(op->rsc_id, event->rsc->id, TRUE)) {
	crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, key);
	remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static gboolean
	stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	lrm_state_t *lrm_state = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0) {
	crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, key);
	remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static void
	do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char operation, xmlNode msg,
	xmlNode * request)
	{
	int call_id = 0;
	char *op_id = NULL;
	lrmd_event_data_t *op = NULL;
	lrmd_key_value_t *params = NULL;
	fsa_data_t *msg_data = NULL;
	const char *transition = NULL;
	gboolean stop_recurring = FALSE;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(operation != NULL, return);

	if (msg != NULL) {
	transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
	if (transition == NULL) {
	crm_log_xml_err(msg, "Missing transition number");
	}
	}

	op = construct_op(lrm_state, msg, rsc->id, operation);
	CRM_CHECK(op != NULL, return);

	if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
	&& op->interval == 0
	&& strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {

	/* pcmk remote connections are a special use case.
	* We never ever want to stop monitoring a connection resource until
	* the entire migration has completed. If the connection is ever unexpected
	* severed, even during a migration, this is an event we must detect.*/
	stop_recurring = FALSE;

	} else if (op->interval == 0
	&& strcmp(operation, CRMD_ACTION_STATUS) != 0
	&& strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {

	/* stop any previous monitor operations before changing the resource state */
	stop_recurring = TRUE;
	}

	if (stop_recurring == TRUE) {
	guint removed = 0;
	struct stop_recurring_action_s data;

	data.rsc = rsc;
	data.lrm_state = lrm_state;
	removed = g_hash_table_foreach_remove(
	lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);

	crm_debug("Stopped %u recurring operations in preparation for %s_%s_%d",
	removed, rsc->id, operation, op->interval);
	}

	/* now do the op */
	crm_info("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval);

	if (fsa_state != S_NOT_DC && fsa_state != S_POLICY_ENGINE && fsa_state != S_TRANSITION_ENGINE) {
	if (safe_str_neq(operation, "fail")
	&& safe_str_neq(operation, CRMD_ACTION_STOP)) {
	crm_info("Discarding attempt to perform action %s on %s in state %s",
	operation, rsc->id, fsa_state2string(fsa_state));
	op->rc = 99;
	op->op_status = PCMK_LRM_OP_ERROR;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	lrmd_free_event(op);
	free(op_id);
	return;
	}
	}

	op_id = generate_op_key(rsc->id, op->op_type, op->interval);

	if (op->interval > 0) {
	/* cancel it so we can then restart it without conflict */
	cancel_op_key(lrm_state, rsc, op_id, FALSE);
	}

	if (op->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, op->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	params = lrmd_key_value_add(params, key, value);
	}
	}

	call_id = lrm_state_exec(lrm_state,
	rsc->id,
	op->op_type,
	op->user_data, op->interval, op->timeout, op->start_delay, params);

	if (call_id <= 0 && lrm_state_is_local(lrm_state)) {
	crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);

	} else if (call_id <= 0) {

	crm_err("Operation %s on resource %s failed to execute on remote node %s: %d", operation, rsc->id, lrm_state->node_name, call_id);
	op->call_id = get_fake_call_id(lrm_state, rsc->id);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;
	process_lrm_event(lrm_state, op);

	} else {
	/* record all operations so we can wait
	* for them to complete during shutdown
	*/
	char *call_id_s = make_stop_id(rsc->id, call_id);
	struct recurring_op_s *pending = NULL;

	pending = calloc(1, sizeof(struct recurring_op_s));
	crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);

	pending->call_id = call_id;
	pending->interval = op->interval;
	pending->op_type = strdup(operation);
	pending->op_key = strdup(op_id);
	pending->rsc_id = strdup(rsc->id);
	g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);

	if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) {
	char *uuid = NULL;
	int dummy = 0, target_rc = 0;

	crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc);
	free(uuid);

	op->rc = target_rc;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	}
	}

	free(op_id);
	lrmd_free_event(op);
	return;
	}

	int last_resource_update = 0;

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource update %d complete: rc=%d", call_id, rc);
	break;
	default:
	crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
	}

	if (call_id == last_resource_update) {
	last_resource_update = 0;
	trigger_fsa(fsa_source);
	}
	}

	static void
	remote_node_init_status(const char *node_name, int call_opt)
	{
	int call_id = 0;
	xmlNode *update = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	simple_remote_node_status(node_name, update,__FUNCTION__);

	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
	if (call_id != pcmk_ok) {
	crm_debug("Failed to init status section for remote-node %s", node_name);
	}
	free_xml(update);
	}

	static void
	remote_node_clear_status(const char *node_name, int call_opt)
	{
	if (node_name == NULL) {
	return;
	}
	remote_node_init_status(node_name, call_opt);
	erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt);
	erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt);
	}

	static int
	do_update_resource(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	/*
	<status>
	<nodes_status id=uname>
	<lrm>
	<lrm_resources>
	<lrm_resource id=...>
	</...>
	*/
	int rc = pcmk_ok;
	xmlNode update, iter = NULL;
	int call_opt = cib_quorum_override;
	const char *uuid = NULL;

	CRM_CHECK(op != NULL, return 0);

	if (fsa_state == S_ELECTION \|\| fsa_state == S_PENDING) {
	crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state));
	call_opt \|= cib_scope_local;
	}

	iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
	update = iter;
	iter = create_xml_node(iter, XML_CIB_TAG_STATE);

	if (safe_str_eq(lrm_state->node_name, fsa_our_uname)) {
	uuid = fsa_our_uuid;

	} else {
	/* remote nodes uuid and uname are equal */
	uuid = lrm_state->node_name;
	crm_xml_add(iter, XML_NODE_IS_REMOTE, "true");
	}

	CRM_LOG_ASSERT(uuid != NULL);
	if(uuid == NULL) {
	rc = -EINVAL;
	goto done;
	}

	crm_xml_add(iter, XML_ATTR_UUID, uuid);
	crm_xml_add(iter, XML_ATTR_UNAME, lrm_state->node_name);
	crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);

	iter = create_xml_node(iter, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, uuid);

	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);

	if (rsc) {
	const char *container = NULL;

	crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
	crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class);
	crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);

	if (op->params) {
	container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
	}
	if (container) {
	crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
	crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
	}

	CRM_CHECK(rsc->type != NULL, crm_err("Resource %s has no value for type", op->rsc_id));
	CRM_CHECK(rsc->class != NULL, crm_err("Resource %s has no value for class", op->rsc_id));

	/* check to see if we need to initialize remote-node related status sections */
	if (safe_str_eq(op->op_type, "start") && op->rc == 0 && op->op_status == PCMK_LRM_OP_DONE) {
	const char *remote_node = g_hash_table_lookup(op->params, CRM_META"_remote_node");

	if (remote_node) {
	/* A container for a remote-node has started, initalize remote-node's status */
	crm_info("Initalizing lrm status for container remote-node %s. Container successfully started.", remote_node);
	remote_node_clear_status(remote_node, call_opt);
	} else if (container == FALSE && safe_str_eq(rsc->type, "remote") && safe_str_eq(rsc->provider, "pacemaker")) {
	/* baremetal remote node connection resource has started, initalize remote-node's status */
	crm_info("Initializing lrm status for baremetal remote-node %s", rsc->id);
	remote_node_clear_status(rsc->id, call_opt);
	}
	}

	} else {
	crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id);
	send_direct_ack(NULL, NULL, rsc, op, op->rsc_id);
	goto cleanup;
	}

	crm_log_xml_trace(update, __FUNCTION__);

	/* make it an asyncronous call and be done with it
	*
	* Best case:
	* the resource state will be discovered during
	* the next signup or election.
	*
	* Bad case:
	* we are shutting down and there is no DC at the time,
	* but then why were we shutting down then anyway?
	* (probably because of an internal error)
	*
	* Worst case:
	* we get shot for having resources "running" when the really weren't
	*
	* the alternative however means blocking here for too long, which
	* isnt acceptable
	*/
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);

	if (rc > 0) {
	last_resource_update = rc;
	}
	done:
	/* the return code is a call number, not an error code */
	crm_trace("Sent resource state update message: %d for %s=%d on %s", rc,
	op->op_type, op->interval, op->rsc_id);
	fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);

	cleanup:
	free_xml(update);
	return rc;
	}

	void
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
	{
	CRM_CHECK(FALSE, return);
	}

	gboolean
	process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op)
	{
	char *op_id = NULL;
	char *op_key = NULL;

	int update_id = 0;
	gboolean removed = FALSE;
	lrmd_rsc_info_t *rsc = NULL;

	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != NULL, return FALSE);

	CRM_CHECK(op->rsc_id != NULL, return FALSE);
	op_id = make_stop_id(op->rsc_id, op->call_id);
	pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
	op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
	rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);

	if (op->op_status == PCMK_LRM_OP_ERROR) {
	switch(op->rc) {
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_MASTER:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_MASTER:
	/* Leave it up to the TE/PE to decide if this is an error */
	op->op_status = PCMK_LRM_OP_DONE;
	break;
	default:
	/* Nothing to do */
	break;
	}
	}

	if (op->op_status != PCMK_LRM_OP_CANCELLED) {
	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	/* Keep notify ops out of the CIB */
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
	} else {
	update_id = do_update_resource(lrm_state, rsc, op);
	}
	} else if (op->interval == 0) {
	/* This will occur when "crm resource cleanup" is called while actions are in-flight */
	crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else if (pending == NULL) {
	/* We don't need to do anything for cancelled ops
	* that are not in our pending op list. There are no
	* transition actions waiting on these operations. */

	} else if (op->user_data == NULL) {
	/* At this point we have a pending entry, but no transition
	* key present in the user_data field. report this */
	crm_err("Op %s (call=%d): No user data", op_key, op->call_id);

	} else if (pending->remove) {
	/* The tengine canceled this op, we have been waiting for the cancel to finish. */
	delete_op_entry(lrm_state, op, op->rsc_id, op_key, op->call_id);

	} else if (pending && op->rsc_deleted) {
	/* The tengine initiated this op, but it was cancelled outside of the
	* tengine's control during a resource cleanup/re-probe request. The tengine
	* must be alerted that this operation completed, otherwise the tengine
	* will continue waiting for this update to occur until it is timed out.
	* We don't want this update going to the cib though, so use a direct ack. */
	crm_trace("Op %s (call=%d): cancelled due to rsc deletion", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else {
	/* Before a stop is called, no need to direct ack */
	crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id);
	}

	if ((op->interval == 0) && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
	removed = TRUE;
	crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
	op_key, op->call_id, op_id, g_hash_table_size(lrm_state->pending_ops));

	} else if(op->interval != 0 && op->op_status == PCMK_LRM_OP_CANCELLED) {
	removed = TRUE;
	g_hash_table_remove(lrm_state->pending_ops, op_id);
	}

	switch (op->op_status) {
	case PCMK_LRM_OP_CANCELLED:
	crm_info("Operation %s: %s (node=%s, call=%d, confirmed=%s)",
	op_key, services_lrm_status_str(op->op_status), lrm_state->node_name,
	op->call_id, removed ? "true" : "false");
	break;

	case PCMK_LRM_OP_DONE:
	crm_notice("Operation %s: %s (node=%s, call=%d, rc=%d, cib-update=%d, confirmed=%s)",
	op_key, services_ocf_exitcode_str(op->rc), lrm_state->node_name,
	op->call_id, op->rc, update_id, removed ? "true" : "false");
	break;

	case PCMK_LRM_OP_TIMEOUT:
	crm_err("Operation %s: %s (node=%s, call=%d, timeout=%dms)",
	op_key, services_lrm_status_str(op->op_status), lrm_state->node_name, op->call_id, op->timeout);
	break;

	default:
	crm_err("Operation %s (node=%s, call=%d, status=%d, cib-update=%d, confirmed=%s) %s",
	op_key, lrm_state->node_name, op->call_id, op->op_status, update_id, removed ? "true" : "false",
	services_lrm_status_str(op->op_status));
	}

	if (op->output) {
	char *prefix =
	crm_strdup_printf("%s-%s_%s_%d:%d", lrm_state->node_name, op->rsc_id, op->op_type, op->interval, op->call_id);

	if (op->rc) {
	crm_log_output(LOG_NOTICE, prefix, op->output);
	} else {
	crm_log_output(LOG_DEBUG, prefix, op->output);
	}
	free(prefix);
	}

	if (op->rsc_deleted) {
	crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
	delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
	}

	/* If a shutdown was escalated while operations were pending,
	* then the FSA will be stalled right now... allow it to continue
	*/
	mainloop_set_trigger(fsa_source);
	update_history_cache(lrm_state, rsc, op);

	lrmd_free_rsc_info(rsc);
	free(op_key);
	free(op_id);

	return TRUE;
	}
	diff --git a/tools/crm_resource.c b/tools/crm_resource.c
	index 8f130d63b1..71bb4fff4b 100644
	--- a/tools/crm_resource.c
	+++ b/tools/crm_resource.c
	@@ -1,2985 +1,3010 @@

	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <time.h>

	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>

	#include <crm/cib.h>
	#include <crm/attrd.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>

	#include "fake_transition.h"
	extern xmlNode do_calculations(pe_working_set_t data_set, xmlNode * xml_input, crm_time_t * now);

	bool scope_master = FALSE;
	gboolean do_force = FALSE;
	gboolean BE_QUIET = FALSE;
	const char *attr_set_type = XML_TAG_ATTR_SETS;
	char *host_id = NULL;
	const char *rsc_id = NULL;
	const char *host_uname = NULL;
	const char *prop_name = NULL;
	const char *prop_value = NULL;
	const char *rsc_type = NULL;
	const char *prop_id = NULL;
	const char *prop_set = NULL;
	char *move_lifetime = NULL;
	char rsc_cmd = 'L';
	const char *rsc_long_cmd = NULL;
	char *our_pid = NULL;
	crm_ipc_t *crmd_channel = NULL;
	char *xml_file = NULL;
	int cib_options = cib_sync_call;
	int crmd_replies_needed = 1; /* The welcome message */
	GMainLoop *mainloop = NULL;
	gboolean print_pending = FALSE;

	extern void cleanup_alloc_calculations(pe_working_set_t * data_set);

	#define CMD_ERR(fmt, args...) do { \
	crm_warn(fmt, ##args); \
	fprintf(stderr, fmt, ##args); \
	} while(0)

	#define message_timeout_ms 60*1000

	static gboolean
	resource_ipc_timeout(gpointer data)
	{
	fprintf(stderr, "No messages received in %d seconds.. aborting\n",
	(int)message_timeout_ms / 1000);
	crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
	return crm_exit(-1);
	}

	static void
	resource_ipc_connection_destroy(gpointer user_data)
	{
	crm_info("Connection to CRMd was terminated");
	crm_exit(1);
	}

	static void
	start_mainloop(void)
	{
	if (crmd_replies_needed == 0) {
	return;
	}

	mainloop = g_main_new(FALSE);
	fprintf(stderr, "Waiting for %d replies from the CRMd", crmd_replies_needed);
	crm_debug("Waiting for %d replies from the CRMd", crmd_replies_needed);

	g_timeout_add(message_timeout_ms, resource_ipc_timeout, NULL);
	g_main_run(mainloop);
	}

	static int
	resource_ipc_callback(const char *buffer, ssize_t length, gpointer userdata)
	{
	xmlNode *msg = string2xml(buffer);

	fprintf(stderr, ".");
	crm_log_xml_trace(msg, "[inbound]");

	crmd_replies_needed--;
	if (crmd_replies_needed == 0) {
	fprintf(stderr, " OK\n");
	crm_debug("Got all the replies we expected");
	return crm_exit(pcmk_ok);
	}

	free_xml(msg);
	return 0;
	}

	struct ipc_client_callbacks crm_callbacks = {
	.dispatch = resource_ipc_callback,
	.destroy = resource_ipc_connection_destroy,
	};

	static int
	do_find_resource(const char rsc, resource_t the_rsc, pe_working_set_t * data_set)
	{
	int found = 0;
	GListPtr lpc = NULL;

	for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
	node_t node = (node_t ) lpc->data;

	crm_trace("resource %s is running on: %s", rsc, node->details->uname);
	if (BE_QUIET) {
	fprintf(stdout, "%s\n", node->details->uname);
	} else {
	const char *state = "";

	if (the_rsc->variant < pe_clone && the_rsc->fns->state(the_rsc, TRUE) == RSC_ROLE_MASTER) {
	state = "Master";
	}
	fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
	}

	found++;
	}

	if (BE_QUIET == FALSE && found == 0) {
	fprintf(stderr, "resource %s is NOT running\n", rsc);
	}

	return found;
	}

	static int
	search_resource(const char rsc, pe_working_set_t data_set)
	{
	int found = 0;
	resource_t *the_rsc = NULL;
	resource_t *parent = NULL;

	if (the_rsc == NULL) {
	the_rsc = pe_find_resource(data_set->resources, rsc);
	}

	if (the_rsc == NULL) {
	return -ENXIO;
	}

	if (the_rsc->variant >= pe_clone) {
	GListPtr gIter = the_rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	found += do_find_resource(rsc, gIter->data, data_set);
	}

	/* The anonymous clone children's common ID is supplied */
	} else if ((parent = uber_parent(the_rsc)) != NULL
	&& parent->variant >= pe_clone
	&& is_not_set(the_rsc->flags, pe_rsc_unique)
	&& the_rsc->clone_name
	&& safe_str_eq(rsc, the_rsc->clone_name)
	&& safe_str_neq(rsc, the_rsc->id)) {
	GListPtr gIter = parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	found += do_find_resource(rsc, gIter->data, data_set);
	}

	} else {
	found += do_find_resource(rsc, the_rsc, data_set);
	}

	return found;
	}

	#define cons_string(x) x?x:"NA"
	static void
	print_cts_constraints(pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	xmlNode *lifetime = NULL;
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	for (xml_obj = __xml_first_child(cib_constraints); xml_obj != NULL;
	xml_obj = __xml_next(xml_obj)) {
	const char *id = crm_element_value(xml_obj, XML_ATTR_ID);

	if (id == NULL) {
	continue;
	}

	lifetime = first_named_child(xml_obj, "lifetime");

	if (test_ruleset(lifetime, NULL, data_set->now) == FALSE) {
	continue;
	}

	if (safe_str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj))) {
	printf("Constraint %s %s %s %s %s %s %s\n",
	crm_element_name(xml_obj),
	cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
	cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));

	} else if (safe_str_eq(XML_CONS_TAG_RSC_LOCATION, crm_element_name(xml_obj))) {
	/* unpack_location(xml_obj, data_set); */
	}
	}
	}

	static void
	print_cts_rsc(resource_t * rsc)
	{
	GListPtr lpc = NULL;
	const char *host = NULL;
	gboolean needs_quorum = TRUE;
	const char *rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	if (safe_str_eq(rclass, "stonith")) {
	xmlNode *op = NULL;

	needs_quorum = FALSE;

	for (op = __xml_first_child(rsc->ops_xml); op != NULL; op = __xml_next(op)) {
	if (crm_str_eq((const char *)op->name, "op", TRUE)) {
	const char *name = crm_element_value(op, "name");

	if (safe_str_neq(name, CRMD_ACTION_START)) {
	const char *value = crm_element_value(op, "requires");

	if (safe_str_eq(value, "nothing")) {
	needs_quorum = FALSE;
	}
	break;
	}
	}
	}
	}

	if (rsc->running_on != NULL && g_list_length(rsc->running_on) == 1) {
	node_t *tmp = rsc->running_on->data;

	host = tmp->details->uname;
	}

	printf("Resource: %s %s %s %s %s %s %s %s %d %lld 0x%.16llx\n",
	crm_element_name(rsc->xml), rsc->id,
	rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
	rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
	rsc->flags);

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	resource_t child = (resource_t ) lpc->data;

	print_cts_rsc(child);
	}
	}

	static void
	print_raw_rsc(resource_t * rsc)
	{
	GListPtr lpc = NULL;
	GListPtr children = rsc->children;

	if (children == NULL) {
	printf("%s\n", rsc->id);
	}

	for (lpc = children; lpc != NULL; lpc = lpc->next) {
	resource_t child = (resource_t ) lpc->data;

	print_raw_rsc(child);
	}
	}

	static int
	do_find_resource_list(pe_working_set_t * data_set, gboolean raw)
	{
	int found = 0;

	GListPtr lpc = NULL;
	int opts = pe_print_printf \| pe_print_rsconly;

	if (print_pending) {
	opts \|= pe_print_pending;
	}

	for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;

	if (is_set(rsc->flags, pe_rsc_orphan)
	&& rsc->fns->active(rsc, TRUE) == FALSE) {
	continue;
	}
	rsc->fns->print(rsc, NULL, opts, stdout);
	found++;
	}

	if (found == 0) {
	printf("NO resources configured\n");
	return -ENXIO;
	}

	return 0;
	}

	static resource_t *
	find_rsc_or_clone(const char rsc, pe_working_set_t data_set)
	{
	resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);

	if (the_rsc == NULL) {
	char *as_clone = crm_concat(rsc, "0", ':');

	the_rsc = pe_find_resource(data_set->resources, as_clone);
	free(as_clone);
	}
	return the_rsc;
	}

	static int
	dump_resource(const char rsc, pe_working_set_t data_set, gboolean expanded)
	{
	char *rsc_xml = NULL;
	resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);
	int opts = pe_print_printf;

	if (the_rsc == NULL) {
	return -ENXIO;
	}

	if (print_pending) {
	opts \|= pe_print_pending;
	}
	the_rsc->fns->print(the_rsc, NULL, opts, stdout);

	if (expanded) {
	rsc_xml = dump_xml_formatted(the_rsc->xml);
	} else {
	if (the_rsc->orig_xml) {
	rsc_xml = dump_xml_formatted(the_rsc->orig_xml);
	} else {
	rsc_xml = dump_xml_formatted(the_rsc->xml);
	}
	}

	fprintf(stdout, "%sxml:\n%s\n", expanded ? "" : "raw ", rsc_xml);

	free(rsc_xml);

	return 0;
	}

	static int
	dump_resource_attr(const char rsc, const char attr, pe_working_set_t * data_set)
	{
	int rc = -ENXIO;
	node_t *current = NULL;
	GHashTable *params = NULL;
	resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);
	const char *value = NULL;

	if (the_rsc == NULL) {
	return -ENXIO;
	}

	if (g_list_length(the_rsc->running_on) == 1) {
	current = the_rsc->running_on->data;

	} else if (g_list_length(the_rsc->running_on) > 1) {
	CMD_ERR("%s is active on more than one node,"
	" returning the default value for %s\n", the_rsc->id, crm_str(value));
	}

	params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
	get_rsc_attributes(params, the_rsc, current, data_set);
	} else if (safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
	get_meta_attributes(params, the_rsc, current, data_set);
	} else {
	unpack_instance_attributes(data_set->input, the_rsc->xml, XML_TAG_UTILIZATION, NULL,
	params, NULL, FALSE, data_set->now);
	}

	crm_debug("Looking up %s in %s", attr, the_rsc->id);
	value = g_hash_table_lookup(params, attr);
	if (value != NULL) {
	fprintf(stdout, "%s\n", value);
	rc = 0;

	} else {
	CMD_ERR("Attribute '%s' not found for '%s'\n", attr, the_rsc->id);
	}

	g_hash_table_destroy(params);
	return rc;
	}

	static int
	find_resource_attr(cib_t * the_cib, const char attr, const char rsc, const char *set_type,
	const char set_name, const char attr_id, const char attr_name, char *value)
	{
	int offset = 0;
	static int xpath_max = 1024;
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;
	char *xpath_string = NULL;

	CRM_ASSERT(value != NULL);
	*value = NULL;

	if(the_cib == NULL) {
	return -ENOTCONN;
	}

	xpath_string = calloc(1, xpath_max);
	offset +=
	snprintf(xpath_string + offset, xpath_max - offset, "%s", get_object_path("resources"));

	offset += snprintf(xpath_string + offset, xpath_max - offset, "//*[@id=\"%s\"]", rsc);

	if (set_type) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s", set_type);
	if (set_name) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "[@id=\"%s\"]", set_name);
	}
	}

	offset += snprintf(xpath_string + offset, xpath_max - offset, "//nvpair[");
	if (attr_id) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "@id=\"%s\"", attr_id);
	}

	if (attr_name) {
	if (attr_id) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, " and ");
	}
	offset += snprintf(xpath_string + offset, xpath_max - offset, "@name=\"%s\"", attr_name);
	}
	offset += snprintf(xpath_string + offset, xpath_max - offset, "]");
	CRM_LOG_ASSERT(offset > 0);

	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);

	if (rc != pcmk_ok) {
	goto bail;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	xmlNode *child = NULL;

	rc = -EINVAL;
	printf("Multiple attributes match name=%s\n", attr_name);

	for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
	printf(" Value: %s \t(id=%s)\n",
	crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
	}

	} else {
	const char *tmp = crm_element_value(xml_search, attr);

	if (tmp) {
	*value = strdup(tmp);
	}
	}

	bail:
	free(xpath_string);
	free_xml(xml_search);
	return rc;
	}

	#include "../pengine/pengine.h"


	static int
	set_resource_attr(const char rsc_id, const char attr_set, const char *attr_id,
	const char attr_name, const char attr_value, bool recursive,
	cib_t * cib, pe_working_set_t * data_set)
	{
	int rc = pcmk_ok;
	static bool need_init = TRUE;

	char *local_attr_id = NULL;
	char *local_attr_set = NULL;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;

	gboolean use_attributes_tag = FALSE;
	resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);

	if (rsc == NULL) {
	return -ENXIO;
	}

	if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
	rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &local_attr_id);
	if (rc == pcmk_ok) {
	printf("WARNING: There is already a meta attribute called %s (id=%s)\n", attr_name,
	local_attr_id);
	}
	}
	rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if (rc == pcmk_ok) {
	crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
	attr_id = local_attr_id;

	} else if (rc != -ENXIO) {
	free(local_attr_id);
	return rc;

	} else {
	const char *value = NULL;
	xmlNode *cib_top = NULL;
	const char *tag = crm_element_name(rsc->xml);

	cib->cmds->query(cib, "/cib", &cib_top,
	cib_sync_call \| cib_scope_local \| cib_xpath \| cib_no_children);
	value = crm_element_value(cib_top, "ignore_dtd");
	if (value != NULL) {
	use_attributes_tag = TRUE;

	} else {
	value = crm_element_value(cib_top, XML_ATTR_VALIDATION);
	if (value && strstr(value, "-0.6")) {
	use_attributes_tag = TRUE;
	}
	}
	free_xml(cib_top);

	if (attr_set == NULL) {
	local_attr_set = crm_concat(rsc_id, attr_set_type, '-');
	attr_set = local_attr_set;
	}
	if (attr_id == NULL) {
	local_attr_id = crm_concat(attr_set, attr_name, '-');
	attr_id = local_attr_id;
	}

	if (use_attributes_tag && safe_str_eq(tag, XML_CIB_TAG_MASTER)) {
	tag = "master_slave"; /* use the old name */
	}

	xml_top = create_xml_node(NULL, tag);
	crm_xml_add(xml_top, XML_ATTR_ID, rsc_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);

	if (use_attributes_tag) {
	xml_obj = create_xml_node(xml_obj, XML_TAG_ATTRS);
	}
	}

	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
	free_xml(xml_top);
	free(local_attr_id);
	free(local_attr_set);

	if(recursive && safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
	GListPtr lpc = NULL;

	if(need_init) {
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	need_init = FALSE;
	unpack_constraints(cib_constraints, data_set);

	for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}
	}

	crm_debug("Looking for dependancies %p", rsc->rsc_cons_lhs);
	set_bit(rsc->flags, pe_rsc_allocating);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) lpc->data;
	resource_t *peer = cons->rsc_lh;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if (cons->score > 0 && is_not_set(peer->flags, pe_rsc_allocating)) {
	/* Don't get into colocation loops */
	crm_debug("Setting %s=%s for dependant resource %s", attr_name, attr_value, peer->id);
	set_resource_attr(peer->id, NULL, NULL, attr_name, attr_value, recursive, cib, data_set);
	}
	}
	}

	return rc;
	}

	static int
	delete_resource_attr(const char rsc_id, const char attr_set, const char *attr_id,
	const char attr_name, cib_t cib, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	int rc = pcmk_ok;
	char *local_attr_id = NULL;
	resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);

	if (rsc == NULL) {
	return -ENXIO;
	}

	rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if (rc == -ENXIO) {
	return pcmk_ok;

	} else if (rc != pcmk_ok) {
	return rc;
	}

	if (attr_id == NULL) {
	attr_id = local_attr_id;
	}

	xml_obj = create_xml_node(NULL, XML_CIB_TAG_NVPAIR);
	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);

	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->delete(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);

	if (rc == pcmk_ok) {
	printf("Deleted %s option: id=%s%s%s%s%s\n", rsc_id, local_attr_id,
	attr_set ? " set=" : "", attr_set ? attr_set : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free_xml(xml_obj);
	free(local_attr_id);
	return rc;
	}

	static int
	dump_resource_prop(const char rsc, const char attr, pe_working_set_t * data_set)
	{
	const char *value = NULL;
	resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);

	if (the_rsc == NULL) {
	return -ENXIO;
	}

	value = crm_element_value(the_rsc->xml, attr);

	if (value != NULL) {
	fprintf(stdout, "%s\n", value);
	return 0;
	}
	return -ENXIO;
	}

	static int
	send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op,
	const char host_uname, const char rsc_id,
	gboolean only_failed, pe_working_set_t * data_set)
	{
	char *key = NULL;
	int rc = -ECOMM;
	xmlNode *cmd = NULL;
	xmlNode *xml_rsc = NULL;
	const char *value = NULL;
	const char *router_node = host_uname;
	xmlNode *params = NULL;
	xmlNode *msg_data = NULL;
	resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);

	if (rsc == NULL) {
	CMD_ERR("Resource %s not found\n", rsc_id);
	return -ENXIO;

	} else if (rsc->variant != pe_native) {
	CMD_ERR("We can only process primitive resources, not %s\n", rsc_id);
	return -EINVAL;

	} else if (host_uname == NULL) {
	CMD_ERR("Please supply a hostname with -H\n");
	return -EINVAL;
	} else {
	node_t *node = pe_find_node(data_set->nodes, host_uname);

	if (node && is_remote_node(node)) {
	if (node->details->remote_rsc == NULL \|\| node->details->remote_rsc->running_on == NULL) {
	CMD_ERR("No lrmd connection detected to remote node %s", host_uname);
	return -ENXIO;
	}
	node = node->details->remote_rsc->running_on->data;
	router_node = node->details->uname;
	}
	}

	key = generate_transition_key(0, getpid(), 0, "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");

	msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
	crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
	free(key);

	crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
	if (safe_str_neq(router_node, host_uname)) {
	crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	}

	xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
	if (rsc->clone_name) {
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
	crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);

	} else {
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
	}

	value = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, value);
	if (value == NULL) {
	CMD_ERR("%s has no type! Aborting...\n", rsc_id);
	return -ENXIO;
	}

	value = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, value);
	if (value == NULL) {
	CMD_ERR("%s has no class! Aborting...\n", rsc_id);
	return -ENXIO;
	}

	value = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, value);

	params = create_xml_node(msg_data, XML_TAG_ATTRS);
	crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	crm_xml_add(params, key, "60000"); /* 1 minute */
	free(key);

	cmd = create_request(op, msg_data, router_node, CRM_SYSTEM_CRMD, crm_system_name, our_pid);

	/* crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */
	free_xml(msg_data);

	if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
	rc = 0;

	} else {
	CMD_ERR("Could not send %s op to the crmd", op);
	rc = -ENOTCONN;
	}

	free_xml(cmd);
	return rc;
	}

	static int
	delete_lrm_rsc(cib_t cib_conn, crm_ipc_t crmd_channel, const char *host_uname,
	resource_t * rsc, pe_working_set_t * data_set)
	{
	int rc = pcmk_ok;
	node_t *node = NULL;

	if (rsc == NULL) {
	return -ENXIO;

	} else if (rsc->children) {
	GListPtr lpc = NULL;

	for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	resource_t child = (resource_t ) lpc->data;

	delete_lrm_rsc(cib_conn, crmd_channel, host_uname, child, data_set);
	}
	return pcmk_ok;

	} else if (host_uname == NULL) {
	GListPtr lpc = NULL;

	for (lpc = data_set->nodes; lpc != NULL; lpc = lpc->next) {
	node = (node_t *) lpc->data;

	if (node->details->online) {
	delete_lrm_rsc(cib_conn, crmd_channel, node->details->uname, rsc, data_set);
	}
	}

	return pcmk_ok;
	}

	node = pe_find_node(data_set->nodes, host_uname);

	if (node && node->details->rsc_discovery_enabled) {
	printf("Cleaning up %s on %s\n", rsc->id, host_uname);
	rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
	} else {
	printf("Resource discovery disabled on %s. Unable to delete lrm state.\n", host_uname);
	}

	if (rc == pcmk_ok) {
	char *attr_name = NULL;
	const char *id = rsc->id;

	if(node && node->details->remote_rsc == NULL && node->details->rsc_discovery_enabled) {
	crmd_replies_needed++;
	}
	if (rsc->clone_name) {
	id = rsc->clone_name;
	}

	attr_name = crm_concat("fail-count", id, '-');
	rc = attrd_update_delegate(NULL, 'D', host_uname, attr_name, NULL, XML_CIB_TAG_STATUS, NULL,
	NULL, NULL, node ? is_remote_node(node) : FALSE);
	free(attr_name);
	}
	return rc;
	}

	static int
	fail_lrm_rsc(crm_ipc_t * crmd_channel, const char *host_uname,
	const char rsc_id, pe_working_set_t data_set)
	{
	crm_warn("Failing: %s", rsc_id);
	return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set);
	}

	static char *
	parse_cli_lifetime(const char *input)
	{
	char *later_s = NULL;
	crm_time_t *now = NULL;
	crm_time_t *later = NULL;
	crm_time_t *duration = NULL;

	if (input == NULL) {
	return NULL;
	}

	duration = crm_time_parse_duration(move_lifetime);
	if (duration == NULL) {
	CMD_ERR("Invalid duration specified: %s\n", move_lifetime);
	CMD_ERR("Please refer to"
	" http://en.wikipedia.org/wiki/ISO_8601#Durations"
	" for examples of valid durations\n");
	return NULL;
	}

	now = crm_time_new(NULL);
	later = crm_time_add(now, duration);
	crm_time_log(LOG_INFO, "now ", now,
	crm_time_log_date \| crm_time_log_timeofday \| crm_time_log_with_timezone);
	crm_time_log(LOG_INFO, "later ", later,
	crm_time_log_date \| crm_time_log_timeofday \| crm_time_log_with_timezone);
	crm_time_log(LOG_INFO, "duration", duration, crm_time_log_date \| crm_time_log_timeofday);
	later_s = crm_time_as_string(later, crm_time_log_date \| crm_time_log_timeofday);
	printf("Migration will take effect until: %s\n", later_s);

	crm_time_free(duration);
	crm_time_free(later);
	crm_time_free(now);
	return later_s;
	}

	static int
	ban_resource(const char rsc_id, const char host, GListPtr allnodes, cib_t * cib_conn)
	{
	char *later_s = NULL;
	int rc = pcmk_ok;
	char *id = NULL;
	xmlNode *fragment = NULL;
	xmlNode *location = NULL;

	if(host == NULL) {
	GListPtr n = allnodes;
	for(; n && rc == pcmk_ok; n = n->next) {
	node_t *target = n->data;

	rc = ban_resource(rsc_id, target->details->uname, NULL, cib_conn);
	}
	return rc;
	}

	later_s = parse_cli_lifetime(move_lifetime);
	if(move_lifetime && later_s == NULL) {
	return -EINVAL;
	}

	fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);

	id = crm_strdup_printf("cli-ban-%s-on-%s", rsc_id, host);
	location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
	crm_xml_add(location, XML_ATTR_ID, id);
	free(id);

	if (BE_QUIET == FALSE) {
	CMD_ERR("WARNING: Creating rsc_location constraint '%s'"
	" with a score of -INFINITY for resource %s"
	" on %s.\n", ID(location), rsc_id, host);
	CMD_ERR("\tThis will prevent %s from %s"
	" on %s until the constraint is removed using"
	" the 'crm_resource --clear' command or manually"
	" with cibadmin\n", rsc_id, scope_master?"being promoted":"running", host);
	CMD_ERR("\tThis will be the case even if %s is"
	" the last node in the cluster\n", host);
	CMD_ERR("\tThis message can be disabled with --quiet\n");
	}

	crm_xml_add(location, XML_COLOC_ATTR_SOURCE, rsc_id);
	if(scope_master) {
	crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_MASTER_S);
	} else {
	crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_STARTED_S);
	}

	if (later_s == NULL) {
	/* Short form */
	crm_xml_add(location, XML_CIB_TAG_NODE, host);
	crm_xml_add(location, XML_RULE_ATTR_SCORE, MINUS_INFINITY_S);

	} else {
	xmlNode *rule = create_xml_node(location, XML_TAG_RULE);
	xmlNode *expr = create_xml_node(rule, XML_TAG_EXPRESSION);

	id = crm_strdup_printf("cli-ban-%s-on-%s-rule", rsc_id, host);
	crm_xml_add(rule, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(rule, XML_RULE_ATTR_SCORE, MINUS_INFINITY_S);
	crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");

	id = crm_strdup_printf("cli-ban-%s-on-%s-expr", rsc_id, host);
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
	crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
	crm_xml_add(expr, XML_EXPR_ATTR_VALUE, host);
	crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");

	expr = create_xml_node(rule, "date_expression");
	id = crm_strdup_printf("cli-ban-%s-on-%s-lifetime", rsc_id, host);
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, "operation", "lt");
	crm_xml_add(expr, "end", later_s);
	}

	crm_log_xml_notice(fragment, "Modify");
	rc = cib_conn->cmds->update(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);

	free_xml(fragment);
	free(later_s);
	return rc;
	}

	static int
	prefer_resource(const char rsc_id, const char host, cib_t * cib_conn)
	{
	char *later_s = parse_cli_lifetime(move_lifetime);
	int rc = pcmk_ok;
	char *id = NULL;
	xmlNode *location = NULL;
	xmlNode *fragment = NULL;

	if(move_lifetime && later_s == NULL) {
	return -EINVAL;
	}

	if(cib_conn == NULL) {
	free(later_s);
	return -ENOTCONN;
	}

	fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);

	id = crm_strdup_printf("cli-prefer-%s", rsc_id);
	location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
	crm_xml_add(location, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(location, XML_COLOC_ATTR_SOURCE, rsc_id);
	if(scope_master) {
	crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_MASTER_S);
	} else {
	crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_STARTED_S);
	}

	if (later_s == NULL) {
	/* Short form */
	crm_xml_add(location, XML_CIB_TAG_NODE, host);
	crm_xml_add(location, XML_RULE_ATTR_SCORE, INFINITY_S);

	} else {
	xmlNode *rule = create_xml_node(location, XML_TAG_RULE);
	xmlNode *expr = create_xml_node(rule, XML_TAG_EXPRESSION);

	id = crm_concat("cli-prefer-rule", rsc_id, '-');
	crm_xml_add(rule, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(rule, XML_RULE_ATTR_SCORE, INFINITY_S);
	crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");

	id = crm_concat("cli-prefer-expr", rsc_id, '-');
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
	crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
	crm_xml_add(expr, XML_EXPR_ATTR_VALUE, host);
	crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");

	expr = create_xml_node(rule, "date_expression");
	id = crm_concat("cli-prefer-lifetime-end", rsc_id, '-');
	crm_xml_add(expr, XML_ATTR_ID, id);
	free(id);

	crm_xml_add(expr, "operation", "lt");
	crm_xml_add(expr, "end", later_s);
	}

	crm_log_xml_info(fragment, "Modify");
	rc = cib_conn->cmds->update(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);

	free_xml(fragment);
	free(later_s);
	return rc;
	}

	static int
	clear_resource(const char rsc_id, const char host, GListPtr allnodes, cib_t * cib_conn)
	{
	char *id = NULL;
	int rc = pcmk_ok;
	xmlNode *fragment = NULL;
	xmlNode *location = NULL;

	if(cib_conn == NULL) {
	return -ENOTCONN;
	}

	fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);

	if(host) {
	id = crm_strdup_printf("cli-ban-%s-on-%s", rsc_id, host);
	location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
	crm_xml_add(location, XML_ATTR_ID, id);
	free(id);

	} else {
	GListPtr n = allnodes;
	for(; n; n = n->next) {
	node_t *target = n->data;

	id = crm_strdup_printf("cli-ban-%s-on-%s", rsc_id, target->details->uname);
	location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
	crm_xml_add(location, XML_ATTR_ID, id);
	free(id);
	}
	}

	id = crm_strdup_printf("cli-prefer-%s", rsc_id);
	location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
	crm_xml_add(location, XML_ATTR_ID, id);
	if(host && do_force == FALSE) {
	crm_xml_add(location, XML_CIB_TAG_NODE, host);
	}
	free(id);

	crm_log_xml_info(fragment, "Delete");
	rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);
	if (rc == -ENXIO) {
	rc = pcmk_ok;

	} else if (rc != pcmk_ok) {
	goto bail;
	}

	bail:
	free_xml(fragment);
	return rc;
	}

	static int
	list_resource_operations(const char rsc_id, const char host_uname, gboolean active,
	pe_working_set_t * data_set)
	{
	resource_t *rsc = NULL;
	int opts = pe_print_printf \| pe_print_rsconly \| pe_print_suppres_nl;
	GListPtr ops = find_operations(rsc_id, host_uname, active, data_set);
	GListPtr lpc = NULL;

	if (print_pending) {
	opts \|= pe_print_pending;
	}

	for (lpc = ops; lpc != NULL; lpc = lpc->next) {
	xmlNode xml_op = (xmlNode ) lpc->data;

	const char *op_rsc = crm_element_value(xml_op, "resource");
	const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
	const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
	const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	int status = crm_parse_int(status_s, "0");

	rsc = pe_find_resource(data_set->resources, op_rsc);
	if(rsc) {
	rsc->fns->print(rsc, "", opts, stdout);
	} else {
	fprintf(stdout, "Unknown resource %s", op_rsc);
	}

	fprintf(stdout, ": %s (node=%s, call=%s, rc=%s",
	op_key ? op_key : ID(xml_op),
	crm_element_value(xml_op, XML_ATTR_UNAME),
	crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	crm_element_value(xml_op, XML_LRM_ATTR_RC));
	if (last) {
	time_t run_at = crm_parse_int(last, "0");

	fprintf(stdout, ", last-rc-change=%s, exec=%sms",
	crm_strip_trailing_newline(ctime(&run_at)), crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
	}
	fprintf(stdout, "): %s\n", services_lrm_status_str(status));
	}
	return pcmk_ok;
	}

	static void
	show_location(resource_t * rsc, const char *prefix)
	{
	GListPtr lpc = NULL;
	GListPtr list = rsc->rsc_location;
	int offset = 0;

	if (prefix) {
	offset = strlen(prefix) - 2;
	}

	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	rsc_to_node_t cons = (rsc_to_node_t ) lpc->data;

	GListPtr lpc2 = NULL;

	for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
	node_t node = (node_t ) lpc2->data;
	char *score = score2char(node->weight);

	fprintf(stdout, "%s: Node %-*s (score=%s, id=%s)\n",
	prefix ? prefix : " ", 71 - offset, node->details->uname, score, cons->id);
	free(score);
	}
	}
	}

	static void
	show_colocation(resource_t * rsc, gboolean dependants, gboolean recursive, int offset)
	{
	char *prefix = NULL;
	GListPtr lpc = NULL;
	GListPtr list = rsc->rsc_cons;

	prefix = calloc(1, (offset * 4) + 1);
	memset(prefix, ' ', offset * 4);

	if (dependants) {
	list = rsc->rsc_cons_lhs;
	}

	if (is_set(rsc->flags, pe_rsc_allocating)) {
	/* Break colocation loops */
	printf("loop %s\n", rsc->id);
	free(prefix);
	return;
	}

	set_bit(rsc->flags, pe_rsc_allocating);
	for (lpc = list; lpc != NULL; lpc = lpc->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) lpc->data;

	char *score = NULL;
	resource_t *peer = cons->rsc_rh;

	if (dependants) {
	peer = cons->rsc_lh;
	}

	if (is_set(peer->flags, pe_rsc_allocating)) {
	if (dependants == FALSE) {
	fprintf(stdout, "%s%-s (id=%s - loop)\n", prefix, 80 - (4 offset), peer->id,
	cons->id);
	}
	continue;
	}

	if (dependants && recursive) {
	show_colocation(peer, dependants, recursive, offset + 1);
	}

	score = score2char(cons->score);
	if (cons->role_rh > RSC_ROLE_STARTED) {
	fprintf(stdout, "%s%-s (score=%s, %s role=%s, id=%s)\n", prefix, 80 - (4 offset),
	peer->id, score, dependants ? "needs" : "with", role2text(cons->role_rh),
	cons->id);
	} else {
	fprintf(stdout, "%s%-s (score=%s, id=%s)\n", prefix, 80 - (4 offset),
	peer->id, score, cons->id);
	}
	show_location(peer, prefix);
	free(score);

	if (!dependants && recursive) {
	show_colocation(peer, dependants, recursive, offset + 1);
	}
	}
	free(prefix);
	}

	static GHashTable *
	generate_resource_params(resource_t * rsc, pe_working_set_t * data_set)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;

	if (!rsc) {
	crm_err("Resource does not exist in config");
	return NULL;
	}

	params =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	meta = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	combined =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
	get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);

	if (params) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	g_hash_table_destroy(params);
	}

	if (meta) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	static bool resource_is_running_on(resource_t rsc, const char host)
	{
	bool found = TRUE;
	GListPtr hIter = NULL;
	GListPtr hosts = NULL;

	if(rsc == NULL) {
	return FALSE;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pe_node_t node = (pe_node_t ) hIter->data;

	if(strcmp(host, node->details->uname) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	} else if(strcmp(host, node->details->id) == 0) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if(host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = FALSE;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = FALSE;
	}

	done:

	g_list_free(hosts);
	return found;
	}

	static GList get_active_resources(const char host, pe_working_set_t *data_set)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = data_set->resources; rIter != NULL; rIter = rIter->next) {
	resource_t rsc = (resource_t ) rIter->data;

	if(resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}

	return active;
	}

	static GList subtract_lists(GList from, GList *items)
	{
	GList *item = NULL;
	GList *result = g_list_copy(from);

	for (item = items; item != NULL; item = item->next) {
	GList *candidate = NULL;
	for (candidate = from; candidate != NULL; candidate = candidate->next) {
	crm_info("Comparing %s with %s", candidate->data, item->data);
	if(strcmp(candidate->data, item->data) == 0) {
	result = g_list_remove(result, candidate->data);
	break;
	}
	}
	}

	return result;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, item->data);
	lpc++;
	}
	}

	static void display_list(GList items, const char tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	fprintf(stdout, "%s%s\n", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as working set input
	*
	* This also updates the working set timestamp to the current time.
	*
	* \param[in] data_set Working set instance to update
	* \param[in] xml XML to use as input
	*
	* \return pcmk_ok on success, -ENOKEY if unable to upgrade XML
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns -ENOKEY ("Required key not available") if that fails,
	* but perhaps -pcmk_err_schema_validation would be better in that case.
	*/
	static int
	update_working_set_xml(pe_working_set_t data_set, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return -ENOKEY;
	}
	data_set->input = *xml;
	data_set->now = crm_time_new(NULL);
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Update a working set's XML input based on a CIB query
	*
	* \param[in] data_set Data set instance to initialize
	* \param[in] cib Connection to the CIB
	*
	* \return pcmk_ok on success, -errno on failure
	* \note On success, caller is responsible for freeing memory allocated for
	* data_set->input and data_set->now.
	*/
	static int
	update_working_set_from_cib(pe_working_set_t * data_set, cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	if (rc != pcmk_ok) {
	fprintf(stderr, "Could not obtain the current CIB: %s (%d)\n", pcmk_strerror(rc), rc);
	return rc;
	}
	rc = update_working_set_xml(data_set, &cib_xml_copy);
	if (rc != pcmk_ok) {
	fprintf(stderr, "Could not upgrade the current CIB XML\n");
	free_xml(cib_xml_copy);
	return rc;
	}
	return pcmk_ok;
	}

	static int
	update_dataset(cib_t cib, pe_working_set_t data_set, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc;

	cleanup_alloc_calculations(data_set);
	rc = update_working_set_from_cib(data_set, cib);
	if (rc != pcmk_ok) {
	return rc;
	}

	if(simulate) {
	pid = crm_itoa(getpid());
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	fprintf(stderr, "Could not create shadow cib: '%s'\n", pid);
	rc = -ENXIO;
	goto cleanup;
	}

	rc = write_xml_file(data_set->input, shadow_file, FALSE);

	if (rc < 0) {
	fprintf(stderr, "Could not populate shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
	goto cleanup;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not connect to shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
	goto cleanup;
	}

	do_calculations(data_set, data_set->input, NULL);
	run_simulation(data_set, shadow_cib, NULL, TRUE);
	rc = update_dataset(shadow_cib, data_set, FALSE);

	} else {
	cluster_status(data_set);
	}

	cleanup:
	/* Do not free data_set->input here, we need rsc->xml to be valid later on */
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	static int
	max_delay_for_resource(pe_working_set_t * data_set, resource_t *rsc)
	{
	int delay = 0;
	int max_delay = 0;

	if(rsc && rsc->children) {
	GList *iter = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	resource_t child = (resource_t )iter->data;

	delay = max_delay_for_resource(data_set, child);
	if(delay > max_delay) {
	double seconds = delay / 1000;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
	max_delay = delay;
	}
	}

	} else if(rsc) {
	char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
	action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
	const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);

	max_delay = crm_int_helper(value, NULL);
	pe_free_action(stop);
	}


	return max_delay;
	}

	static int
	max_delay_in(pe_working_set_t * data_set, GList *resources)
	{
	int max_delay = 0;
	GList *item = NULL;

	for (item = resources; item != NULL; item = item->next) {
	int delay = 0;
	resource_t rsc = pe_find_resource(data_set->resources, (const char )item->data);

	delay = max_delay_for_resource(data_set, rsc);

	if(delay > max_delay) {
	double seconds = delay / 1000;
	crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
	max_delay = delay;
	}
	}

	return 5 + (max_delay / 1000);
	}

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in] rsc The resource to restart
	* \param[in] host The host to restart the resource on (or NULL for all)
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but a two-second
	* granularity is actually used; if 0, a timeout will be
	* calculated based on the resource timeout)
	* \param[in] cib Connection to the CIB for modifying/checking resource
	*
	* \return pcmk_ok on success, -errno on failure (exits on certain failures)
	*/
	static int
	resource_restart(resource_t * rsc, const char host, int timeout_ms, cib_t cib)
	{
	int rc = 0;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	bool is_clone = FALSE;
	char *rsc_id = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pe_working_set_t data_set;

	if(resource_is_running_on(rsc, host) == FALSE) {
	return -ENXIO;
	}

	attr_set_type = XML_TAG_META_SETS;
	rsc_id = strdup(rsc->id);
	if(rsc->variant > pe_group) {
	is_clone = TRUE;
	}

	/*
	grab full cib
	determine resource state of list
	disable or ban
	poll and and watch for resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll and and watch for resources to get stopped
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/


	set_working_set_defaults(&data_set);
	rc = update_dataset(cib, &data_set, FALSE);
	if(rc != pcmk_ok) {
	fprintf(stdout, "Could not get new resource list: %s (%d)\n", pcmk_strerror(rc), rc);
	free(rsc_id);
	return rc;
	}

	restart_target_active = get_active_resources(host, &data_set);
	current_active = get_active_resources(host, &data_set);

	dump_list(current_active, "Origin");

	if(is_clone && host) {
	BE_QUIET = TRUE;
	rc = ban_resource(rsc_id, host, NULL, cib);

	} else {
	rc = set_resource_attr(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, RSC_STOPPED, FALSE, cib, &data_set);
	}
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not set target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
	free(rsc_id);
	return crm_exit(rc);
	}

	rc = update_dataset(cib, &data_set, TRUE);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not determine which resources would be stopped\n");
	goto failure;
	}

	target_active = get_active_resources(host, &data_set);
	dump_list(target_active, "Target");

	list_delta = subtract_lists(current_active, target_active);
	fprintf(stdout, "Waiting for %d resources to stop:\n", g_list_length(list_delta));
	display_list(list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while(g_list_length(list_delta) > 0) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; lpc < step_timeout_s && g_list_length(list_delta) > 0; lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(cib, &data_set, FALSE);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not determine which resources were stopped\n");
	goto failure;
	}

	current_active = get_active_resources(host, &data_set);
	list_delta = subtract_lists(current_active, target_active);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", before, g_list_length(list_delta));
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	fprintf(stderr, "Could not complete shutdown of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
	display_list(list_delta, " * ");
	rc = -ETIME;
	goto failure;
	}

	}

	if(is_clone && host) {
	rc = clear_resource(rsc_id, host, NULL, cib);

	} else {
	rc = delete_resource_attr(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, cib, &data_set);
	}

	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not unset target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
	free(rsc_id);
	return crm_exit(rc);
	}

	target_active = restart_target_active;
	list_delta = subtract_lists(target_active, current_active);
	fprintf(stdout, "Waiting for %d resources to start again:\n", g_list_length(list_delta));
	display_list(list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while(g_list_length(list_delta) > 0) {
	if(timeout_ms == 0) {
	step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; lpc < step_timeout_s && g_list_length(list_delta) > 0; lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(cib, &data_set, FALSE);
	if(rc != pcmk_ok) {
	fprintf(stderr, "Could not determine which resources were started\n");
	goto failure;
	}

	current_active = get_active_resources(host, &data_set);
	list_delta = subtract_lists(target_active, current_active);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	fprintf(stdout, "Could not complete restart of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
	display_list(list_delta, " * ");
	rc = -ETIME;
	goto failure;
	}

	} while(g_list_length(list_delta) > 0);

	free(rsc_id);
	return pcmk_ok;

	failure:
	if(is_clone && host) {
	clear_resource(rsc_id, host, NULL, cib);

	} else {
	delete_resource_attr(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, cib, &data_set);
	}
	free(rsc_id);
	return rc;
	}

	#define action_is_pending(action) \
	((is_set((action)->flags, pe_action_optional) == FALSE) \
	&& (is_set((action)->flags, pe_action_runnable) == TRUE) \
	&& (is_set((action)->flags, pe_action_pseudo) == FALSE))

	/*!
	* \internal
	* \brief Return TRUE if any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return TRUE if any actions in the list are pending, FALSE otherwise
	*/
	static gboolean
	actions_are_pending(GListPtr actions)
	{
	GListPtr action;

	for (action = actions; action != NULL; action = action->next) {
	if (action_is_pending((action_t *) action->data)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Print pending actions to stderr
	*
	* \param[in] actions List of actions to check
	*
	* \return void
	*/
	static void
	print_pending_actions(GListPtr actions)
	{
	GListPtr action;

	fprintf(stderr, "Pending actions:\n");
	for (action = actions; action != NULL; action = action->next) {
	action_t a = (action_t ) action->data;

	if (action_is_pending(a)) {
	fprintf(stderr, "\tAction %d: %s", a->id, a->uuid);
	if (a->node) {
	fprintf(stderr, "\ton %s", a->node->details->uname);
	}
	fprintf(stderr, "\n");
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in] timeout_ms Consider failed if actions do not complete in this time
	* (specified in milliseconds, but one-second granularity
	* is actually used; if 0, a default will be used)
	* \param[in] cib Connection to the CIB
	*
	* \return pcmk_ok on success, -errno on failure
	*/
	static int
	wait_till_stable(int timeout_ms, cib_t * cib)
	{
	pe_working_set_t data_set;
	int rc = -1;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;

	set_working_set_defaults(&data_set);
	do {

	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %d seconds for cluster actions to complete", time_diff);
	} else {
	print_pending_actions(data_set.actions);
	cleanup_alloc_calculations(&data_set);
	return -ETIME;
	}
	if (rc == pcmk_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	cleanup_alloc_calculations(&data_set);
	rc = update_working_set_from_cib(&data_set, cib);
	if (rc != pcmk_ok) {
	cleanup_alloc_calculations(&data_set);
	return rc;
	}
	do_calculations(&data_set, data_set.input, NULL);

	} while (actions_are_pending(data_set.actions));

	return pcmk_ok;
	}

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\t\tThis text"},
	{"version", 0, 0, '$', "\t\tVersion information" },
	{"verbose", 0, 0, 'V', "\t\tIncrease debug output"},
	{"quiet", 0, 0, 'Q', "\t\tPrint only the value on stdout\n"},

	{"resource", 1, 0, 'r', "\tResource ID" },

	{"-spacer-",1, 0, '-', "\nQueries:"},
	{"list", 0, 0, 'L', "\t\tList all cluster resources"},
	{"list-raw", 0, 0, 'l', "\tList the IDs of all instantiated resources (no groups/clones/...)"},
	{"list-cts", 0, 0, 'c', NULL, pcmk_option_hidden},
	{"list-operations", 0, 0, 'O', "\tList active resource operations. Optionally filtered by resource (-r) and/or node (-N)"},
	{"list-all-operations", 0, 0, 'o', "List all resource operations. Optionally filtered by resource (-r) and/or node (-N)"},
	{"pending", 0, 0, 'j', "\t\tDisplay pending state if 'record-pending' is enabled\n"},

	{"list-standards", 0, 0, 0, "\tList supported standards"},
	{"list-ocf-providers", 0, 0, 0, "List all available OCF providers"},
	{"list-agents", 1, 0, 0, "List all agents available for the named standard and/or provider."},
	{"list-ocf-alternatives", 1, 0, 0, "List all available providers for the named OCF agent\n"},
	{"show-metadata", 1, 0, 0, "Show the metadata for the named class:provider:agent"},

	{"query-xml", 0, 0, 'q', "\tQuery the definition of a resource (template expanded)"},
	{"query-xml-raw", 0, 0, 'w', "\tQuery the definition of a resource (raw xml)"},
	{"locate", 0, 0, 'W', "\t\tDisplay the current location(s) of a resource"},
	{"stack", 0, 0, 'A', "\t\tDisplay the prerequisites and dependents of a resource"},
	{"constraints",0, 0, 'a', "\tDisplay the (co)location constraints that apply to a resource"},

	{"-spacer-", 1, 0, '-', "\nCommands:"},
	{"cleanup", 0, 0, 'C', "\t\tDelete the resource history and re-check the current state. Optional: --resource"},
	{"set-parameter", 1, 0, 'p', "Set the named parameter for a resource. See also -m, --meta"},
	{"get-parameter", 1, 0, 'g', "Display the named parameter for a resource. See also -m, --meta"},
	{"delete-parameter",1, 0, 'd', "Delete the named parameter for a resource. See also -m, --meta"},
	{"get-property", 1, 0, 'G', "Display the 'class', 'type' or 'provider' of a resource", pcmk_option_hidden},
	{"set-property", 1, 0, 'S', "(Advanced) Set the class, type or provider of a resource", pcmk_option_hidden},

	{"-spacer-", 1, 0, '-', "\nResource location:"},
	{
	"move", 0, 0, 'M',
	"\t\tMove a resource from its current location to the named destination.\n "
	"\t\t\t\tRequires: --host. Optional: --lifetime, --master\n\n"
	"\t\t\t\tNOTE: This may prevent the resource from running on the previous location node until the implicit constraints expire or are removed with --unban\n"
	},
	{
	"ban", 0, 0, 'B',
	"\t\tPrevent the named resource from running on the named --host. \n"
	"\t\t\t\tRequires: --resource. Optional: --host, --lifetime, --master\n\n"
	"\t\t\t\tIf --host is not specified, it defaults to:\n"
	"\t\t\t\t * the current location for primitives and groups, or\n\n"
	"\t\t\t\t * the current location of the master for m/s resources with master-max=1\n\n"
	"\t\t\t\tAll other situations result in an error as there is no sane default.\n\n"
	"\t\t\t\tNOTE: This will prevent the resource from running on this node until the constraint expires or is removed with --clear\n"
	},
	{
	"clear", 0, 0, 'U', "\t\tRemove all constraints created by the --ban and/or --move commands. \n"
	"\t\t\t\tRequires: --resource. Optional: --host, --master\n\n"
	"\t\t\t\tIf --host is not specified, all constraints created by --ban and --move will be removed for the named resource.\n"
	},
	{"lifetime", 1, 0, 'u', "\tLifespan of constraints created by the --ban and --move commands"},
	{
	"master", 0, 0, 0,
	"\t\tLimit the scope of the --ban, --move and --clear commands to the Master role.\n"
	"\t\t\t\tFor --ban and --move, the previous master can still remain active in the Slave role."
	},

	{"-spacer-", 1, 0, '-', "\nAdvanced Commands:"},
	{"delete", 0, 0, 'D', "\t\t(Advanced) Delete a resource from the CIB"},
	{"fail", 0, 0, 'F', "\t\t(Advanced) Tell the cluster this resource has failed"},
	{"restart", 0, 0, 0, "\t\t(Advanced) Tell the cluster to restart this resource and anything that depends on it"},
	{"wait", 0, 0, 0, "\t\t(Advanced) Wait until the cluster settles into a stable state"},
	{"force-stop", 0, 0, 0, "\t(Advanced) Bypass the cluster and stop a resource on the local node. Additional detail with -V"},
	{"force-start",0, 0, 0, "\t(Advanced) Bypass the cluster and start a resource on the local node. Additional detail with -V"},
	{"force-check",0, 0, 0, "\t(Advanced) Bypass the cluster and check the state of a resource on the local node. Additional detail with -V\n"},

	{"-spacer-", 1, 0, '-', "\nAdditional Options:"},
	{"node", 1, 0, 'N', "\tHost uname"},
	{"recursive", 0, 0, 0, "\tFollow colocation chains when using --set-parameter"},
	{"resource-type", 1, 0, 't', "Resource type (primitive, clone, group, ...)"},
	{"parameter-value", 1, 0, 'v', "Value to use with -p or -S"},
	{"meta", 0, 0, 'm', "\t\tModify a resource's configuration option rather than one which is passed to the resource agent script. For use with -p, -g, -d"},
	{"utilization", 0, 0, 'z', "\tModify a resource's utilization attribute. For use with -p, -g, -d"},
	{"set-name", 1, 0, 's', "\t(Advanced) ID of the instance_attributes object to change"},
	{"nvpair", 1, 0, 'i', "\t(Advanced) ID of the nvpair object to change/delete"},
	{"timeout", 1, 0, 'T', "\t(Advanced) Abort if command does not finish in this time (with --restart or --wait)", pcmk_option_hidden},
	{"force", 0, 0, 'f', "\n" /* Is this actually true anymore?
	"\t\tForce the resource to move by creating a rule for the current location and a score of -INFINITY"
	"\n\t\tThis should be used if the resource's stickiness and constraint scores total more than INFINITY (Currently 100,000)"
	"\n\t\tNOTE: This will prevent the resource from running on this node until the constraint is removed with -U or the --lifetime duration expires\n"*/ },

	{"xml-file", 1, 0, 'x', NULL, pcmk_option_hidden},\

	/* legacy options */
	{"host-uname", 1, 0, 'H', NULL, pcmk_option_hidden},
	{"migrate", 0, 0, 'M', NULL, pcmk_option_hidden},
	{"un-migrate", 0, 0, 'U', NULL, pcmk_option_hidden},
	{"un-move", 0, 0, 'U', NULL, pcmk_option_hidden},

	{"refresh", 0, 0, 'R', NULL, pcmk_option_hidden},
	{"reprobe", 0, 0, 'P', NULL, pcmk_option_hidden},

	{"-spacer-", 1, 0, '-', "\nExamples:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "List the configured resources:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --list", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "List the available OCF agents:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --list-agents ocf", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "List the available OCF agents from the linux-ha project:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --list-agents ocf:heartbeat", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Display the current location of 'myResource':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --locate", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Move 'myResource' to another machine:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --move", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Move 'myResource' to a specific machine:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --move --node altNode", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Allow (but not force) 'myResource' to move back to its original location:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --un-move", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Tell the cluster that 'myResource' failed:", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --fail", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Stop 'myResource' (and anything that depends on it):", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --set-parameter target-role --meta --parameter-value Stopped", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Tell the cluster not to manage 'myResource':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "The cluster will not attempt to start or stop the resource under any circumstances."},
	{"-spacer-", 1, 0, '-', "Useful when performing maintenance tasks on a resource.", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --set-parameter is-managed --meta --parameter-value false", pcmk_option_example},
	{"-spacer-", 1, 0, '-', "Erase the operation history of 'myResource' on 'aNode':", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', "The cluster will 'forget' the existing resource state (including any errors) and attempt to recover the resource."},
	{"-spacer-", 1, 0, '-', "Useful when a resource had failed permanently and has been repaired by an administrator.", pcmk_option_paragraph},
	{"-spacer-", 1, 0, '-', " crm_resource --resource myResource --cleanup --node aNode", pcmk_option_example},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int
	main(int argc, char **argv)
	{
	const char *longname = NULL;
	pe_working_set_t data_set;
	cib_t *cib_conn = NULL;
	bool do_trace = FALSE;
	bool recursive = FALSE;

	/* Not all commands set these appropriately, but the defaults here are
	* sufficient to get the logic right. */
	bool require_resource = TRUE; /* whether command requires that resource be specified */
	bool require_dataset = TRUE; /* whether command requires populated dataset instance */
	bool require_crmd = FALSE; /* whether command requires connection to CRMd */

	int rc = pcmk_ok;
	int option_index = 0;
	int timeout_ms = 0;
	int argerr = 0;
	int flag;

	crm_log_cli_init("crm_resource");
	crm_set_options(NULL, "(query\|command) [options]", long_options,
	"Perform tasks related to cluster resources.\nAllows resources to be queried (definition and location), modified, and moved around the cluster.\n");

	if (argc < 2) {
	crm_help('?', EX_USAGE);
	}

	while (1) {
	flag = crm_get_option_long(argc, argv, &option_index, &longname);
	if (flag == -1)
	break;

	switch (flag) {
	case 0: /* long options with no short equivalent */
	if (safe_str_eq("master", longname)) {
	scope_master = TRUE;

	} else if(safe_str_eq(longname, "recursive")) {
	recursive = TRUE;

	} else if (safe_str_eq("wait", longname)) {
	rsc_cmd = flag;
	rsc_long_cmd = longname;
	require_resource = FALSE;
	require_dataset = FALSE;

	} else if (safe_str_eq("force-stop", longname)
	\|\| safe_str_eq("restart", longname)
	\|\| safe_str_eq("force-start", longname)
	\|\| safe_str_eq("force-check", longname)) {
	rsc_cmd = flag;
	rsc_long_cmd = longname;

	} else if (safe_str_eq("list-ocf-providers", longname)
	\|\| safe_str_eq("list-ocf-alternatives", longname)
	\|\| safe_str_eq("list-standards", longname)) {
	const char *text = NULL;
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;
	lrmd_t *lrmd_conn = lrmd_api_new();

	if (safe_str_eq("list-ocf-providers", longname)
	\|\| safe_str_eq("list-ocf-alternatives", longname)) {
	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, optarg, &list);
	text = "OCF providers";

	} else if (safe_str_eq("list-standards", longname)) {
	rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
	text = "standards";
	}

	if (rc > 0) {
	rc = 0;
	for (iter = list; iter != NULL; iter = iter->next) {
	rc++;
	printf("%s\n", iter->val);
	}
	lrmd_list_freeall(list);

	} else if (optarg) {
	fprintf(stderr, "No %s found for %s\n", text, optarg);
	} else {
	fprintf(stderr, "No %s found\n", text);
	}

	lrmd_api_delete(lrmd_conn);
	return crm_exit(rc);

	} else if (safe_str_eq("show-metadata", longname)) {
	char standard[512];
	char provider[512];
	char type[512];
	char *metadata = NULL;
	lrmd_t *lrmd_conn = lrmd_api_new();

	rc = sscanf(optarg, "%[^:]:%[^:]:%s", standard, provider, type);
	if (rc == 3) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, provider, type,
	&metadata, 0);

	} else if (rc == 2) {
	rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, NULL, provider,
	&metadata, 0);

	} else if (rc < 2) {
	fprintf(stderr,
	"Please specify standard:type or standard:provider:type, not %s\n",
	optarg);
	rc = -EINVAL;
	}

	if (metadata) {
	printf("%s\n", metadata);
	} else {
	fprintf(stderr, "Metadata query for %s failed: %d\n", optarg, rc);
	}
	lrmd_api_delete(lrmd_conn);
	return crm_exit(rc);

	} else if (safe_str_eq("list-agents", longname)) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;
	char standard[512];
	char provider[512];
	lrmd_t *lrmd_conn = lrmd_api_new();

	rc = sscanf(optarg, "%[^:]:%s", standard, provider);
	if (rc == 1) {
	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, optarg, NULL);
	provider[0] = '*';
	provider[1] = 0;

	} else if (rc == 2) {
	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, standard, provider);
	}

	if (rc > 0) {
	rc = 0;
	for (iter = list; iter != NULL; iter = iter->next) {
	printf("%s\n", iter->val);
	rc++;
	}
	lrmd_list_freeall(list);
	rc = 0;
	} else {
	fprintf(stderr, "No agents found for standard=%s, provider=%s\n", standard,
	provider);
	rc = -1;
	}
	lrmd_api_delete(lrmd_conn);
	return crm_exit(rc);

	} else {
	crm_err("Unhandled long option: %s", longname);
	}
	break;
	case 'V':
	do_trace = TRUE;
	crm_bump_log_level(argc, argv);
	break;
	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'x':
	xml_file = strdup(optarg);
	break;
	case 'Q':
	BE_QUIET = TRUE;
	break;
	case 'm':
	attr_set_type = XML_TAG_META_SETS;
	break;
	case 'z':
	attr_set_type = XML_TAG_UTILIZATION;
	break;
	case 'u':
	move_lifetime = strdup(optarg);
	break;
	case 'f':
	do_force = TRUE;
	break;
	case 'i':
	prop_id = optarg;
	break;
	case 's':
	prop_set = optarg;
	break;
	case 'r':
	rsc_id = optarg;
	break;
	case 'v':
	prop_value = optarg;
	break;
	case 't':
	rsc_type = optarg;
	break;
	case 'T':
	timeout_ms = crm_get_msec(optarg);
	break;
	case 'C':
	case 'R':
	case 'P':
	rsc_cmd = 'C';
	require_resource = FALSE;
	require_crmd = TRUE;
	break;
	case 'F':
	rsc_cmd = flag;
	require_crmd = TRUE;
	break;
	case 'L':
	case 'c':
	case 'l':
	case 'q':
	case 'w':
	case 'D':
	case 'W':
	case 'M':
	case 'U':
	case 'B':
	case 'O':
	case 'o':
	case 'A':
	case 'a':
	rsc_cmd = flag;
	break;
	case 'j':
	print_pending = TRUE;
	break;
	case 'p':
	case 'g':
	case 'd':
	case 'S':
	case 'G':
	prop_name = optarg;
	rsc_cmd = flag;
	break;
	case 'h':
	case 'H':
	case 'N':
	crm_trace("Option %c => %s", flag, optarg);
	host_uname = optarg;
	break;

	default:
	CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc && argv[optind] != NULL) {
	CMD_ERR("non-option ARGV-elements: ");
	while (optind < argc && argv[optind] != NULL) {
	CMD_ERR("%s ", argv[optind++]);
	++argerr;
	}
	CMD_ERR("\n");
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	our_pid = calloc(1, 11);
	if (our_pid != NULL) {
	snprintf(our_pid, 10, "%d", getpid());
	our_pid[10] = '\0';
	}

	if (do_force) {
	crm_debug("Forcing...");
	cib_options \|= cib_quorum_override;
	}

	data_set.input = NULL; /* make clean-up easier */

	/* If user specified resource, look for it, even if it's optional for command */
	if (rsc_id) {
	require_resource = TRUE;
	}

	/* We need a dataset to find a resource, even if command doesn't need it */
	if (require_resource) {
	require_dataset = TRUE;
	}

	/* Establish a connection to the CIB */
	cib_conn = cib_new();
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	if (rc != pcmk_ok) {
	CMD_ERR("Error signing on to the CIB service: %s\n", pcmk_strerror(rc));
	return crm_exit(rc);
	}

	/* Populate working set from XML file if specified or CIB query otherwise */
	if (require_dataset) {
	xmlNode *cib_xml_copy = NULL;

	if (xml_file != NULL) {
	cib_xml_copy = filename2xml(xml_file);

	} else {
	rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	}

	if(rc != pcmk_ok) {
	goto bail;
	}

	/* Populate the working set instance */
	set_working_set_defaults(&data_set);
	rc = update_working_set_xml(&data_set, &cib_xml_copy);
	if (rc != pcmk_ok) {
	goto bail;
	}
	cluster_status(&data_set);

	/* Set rc to -ENXIO if no resource matching rsc_id is found.
	* This does not bail, but is handled later for certain commands.
	* That handling could be done here instead if all flags above set
	* require_resource appropriately. */
	if (require_resource && rsc_id && (find_rsc_or_clone(rsc_id, &data_set) == NULL)) {
	rc = -ENXIO;
	}
	}

	/* Establish a connection to the CRMd if needed */
	if (require_crmd) {
	xmlNode *xml = NULL;
	mainloop_io_t *source =
	mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
	crmd_channel = mainloop_get_ipc_client(source);

	if (crmd_channel == NULL) {
	CMD_ERR("Error signing on to the CRMd service\n");
	rc = -ENOTCONN;
	goto bail;
	}

	xml = create_hello_message(our_pid, crm_system_name, "0", "1");
	crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
	free_xml(xml);
	}

	/* Handle rsc_cmd appropriately */
	if (rsc_cmd == 'L') {
	rc = pcmk_ok;
	do_find_resource_list(&data_set, FALSE);

	} else if (rsc_cmd == 'l') {
	int found = 0;
	GListPtr lpc = NULL;

	rc = pcmk_ok;
	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;

	found++;
	print_raw_rsc(rsc);
	}

	if (found == 0) {
	printf("NO resources configured\n");
	rc = -ENXIO;
	goto bail;
	}

	} else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "restart")) {
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);

	rc = resource_restart(rsc, host_uname, timeout_ms, cib_conn);

	} else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "wait")) {
	rc = wait_till_stable(timeout_ms, cib_conn);

	} else if (rsc_cmd == 0 && rsc_long_cmd) { /* force-(stop\|start\|check) */
	svc_action_t *op = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	const char *action = NULL;
	GHashTable *params = NULL;
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);

	if (rsc == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}

	if (safe_str_eq(rsc_long_cmd, "force-stop")) {
	action = "stop";
	} else if (safe_str_eq(rsc_long_cmd, "force-start")) {
	action = "start";
	if(rsc->variant >= pe_clone) {
	rc = search_resource(rsc_id, &data_set);
	if(rc > 0 && do_force == FALSE) {
	CMD_ERR("It is not safe to start %s here: the cluster claims it is already active\n", rsc_id);
	CMD_ERR("Try setting target-role=stopped first or specifying --force\n");
	crm_exit(EPERM);
	}
	}

	} else if (safe_str_eq(rsc_long_cmd, "force-check")) {
	action = "monitor";
	}

	if(rsc->variant == pe_clone \|\| rsc->variant == pe_master) {
	/* Grab the first child resource in the hope its not a group */
	rsc = rsc->children->data;
	}

	if(rsc->variant == pe_group) {
	CMD_ERR("Sorry, --%s doesn't support group resources\n", rsc_long_cmd);
	crm_exit(EOPNOTSUPP);
	}

	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	if(safe_str_eq(rclass, "stonith")){
	CMD_ERR("Sorry, --%s doesn't support %s resources yet\n", rsc_long_cmd, rclass);
	crm_exit(EOPNOTSUPP);
	}

	params = generate_resource_params(rsc, &data_set);
	op = resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params);

	if(do_trace) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	if(op == NULL) {
	/* Re-run but with stderr enabled so we can display a sane error message */
	crm_enable_stderr(TRUE);
	resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params);
	return crm_exit(EINVAL);

	} else if (services_action_sync(op)) {
	int more, lpc, last;
	char *local_copy = NULL;

	if (op->status == PCMK_LRM_OP_DONE) {
	printf("Operation %s for %s (%s:%s:%s) returned %d\n",
	action, rsc->id, rclass, rprov ? rprov : "", rtype, op->rc);
	} else {
	printf("Operation %s for %s (%s:%s:%s) failed: %d\n",
	action, rsc->id, rclass, rprov ? rprov : "", rtype, op->status);
	}

	if (op->stdout_data) {
	local_copy = strdup(op->stdout_data);
	more = strlen(local_copy);
	last = 0;

	for (lpc = 0; lpc < more; lpc++) {
	if (local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	printf(" > stdout: %s\n", local_copy + last);
	last = lpc + 1;
	}
	}
	free(local_copy);
	}
	if (op->stderr_data) {
	local_copy = strdup(op->stderr_data);
	more = strlen(local_copy);
	last = 0;

	for (lpc = 0; lpc < more; lpc++) {
	if (local_copy[lpc] == '\n' \|\| local_copy[lpc] == 0) {
	local_copy[lpc] = 0;
	printf(" > stderr: %s\n", local_copy + last);
	last = lpc + 1;
	}
	}
	free(local_copy);
	}
	}
	rc = op->rc;
	services_action_free(op);
	return crm_exit(rc);

	} else if (rsc_cmd == 'A' \|\| rsc_cmd == 'a') {
	GListPtr lpc = NULL;
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);

	if (rsc == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}

	unpack_constraints(cib_constraints, &data_set);

	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}

	show_colocation(rsc, TRUE, rsc_cmd == 'A', 1);

	fprintf(stdout, "* %s\n", rsc->id);
	show_location(rsc, NULL);

	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t r = (resource_t ) lpc->data;

	clear_bit(r->flags, pe_rsc_allocating);
	}

	show_colocation(rsc, FALSE, rsc_cmd == 'A', 1);

	} else if (rsc_cmd == 'c') {
	int found = 0;
	GListPtr lpc = NULL;

	rc = pcmk_ok;
	for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
	resource_t rsc = (resource_t ) lpc->data;

	print_cts_rsc(rsc);
	found++;
	}
	print_cts_constraints(&data_set);

	} else if (rsc_cmd == 'F') {
	rc = fail_lrm_rsc(crmd_channel, host_uname, rsc_id, &data_set);
	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'O') {
	rc = list_resource_operations(rsc_id, host_uname, TRUE, &data_set);

	} else if (rsc_cmd == 'o') {
	rc = list_resource_operations(rsc_id, host_uname, FALSE, &data_set);

	/* All remaining commands require that resource exist */
	} else if (rc == -ENXIO) {
	CMD_ERR("Resource '%s' not found: %s\n", crm_str(rsc_id), pcmk_strerror(rc));

	} else if (rsc_cmd == 'W') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = search_resource(rsc_id, &data_set);
	if (rc >= 0) {
	rc = pcmk_ok;
	}

	} else if (rsc_cmd == 'q') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource(rsc_id, &data_set, TRUE);

	} else if (rsc_cmd == 'w') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource(rsc_id, &data_set, FALSE);

	} else if (rsc_cmd == 'U') {
	node_t *dest = NULL;

	if (rsc_id == NULL) {
	CMD_ERR("No value specified for --resource\n");
	rc = -ENXIO;
	goto bail;
	}

	if (host_uname) {
	dest = pe_find_node(data_set.nodes, host_uname);
	if (dest == NULL) {
	CMD_ERR("Unknown node: %s\n", host_uname);
	rc = -ENXIO;
	goto bail;
	}
	rc = clear_resource(rsc_id, dest->details->uname, NULL, cib_conn);

	} else {
	rc = clear_resource(rsc_id, NULL, data_set.nodes, cib_conn);
	}

	} else if (rsc_cmd == 'M' && host_uname) {

	int count = 0;
	node_t *current = NULL;
	node_t *dest = pe_find_node(data_set.nodes, host_uname);
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
	gboolean cur_is_dest = FALSE;

	rc = -EINVAL;

	if (rsc == NULL) {
	CMD_ERR("Resource '%s' not moved: not found\n", rsc_id);
	rc = -ENXIO;
	goto bail;

	} else if (scope_master && rsc->variant < pe_master) {
	resource_t *p = uber_parent(rsc);
	if(p->variant == pe_master) {
	CMD_ERR("Using parent '%s' for --move command instead of '%s'.\n", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	CMD_ERR("Ignoring '--master' option: not valid for %s resources.\n",
	get_resource_typename(rsc->variant));
	scope_master = FALSE;
	}
	}

	if(rsc->variant == pe_master) {
	GListPtr iter = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	resource_t child = (resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if(child_role == RSC_ROLE_MASTER) {
	rsc = child;
	count++;
	}
	}

	if(scope_master == FALSE && count == 0) {
	count = g_list_length(rsc->running_on);
	}

	} else if (rsc->variant > pe_group) {
	count = g_list_length(rsc->running_on);

	} else if (g_list_length(rsc->running_on) > 1) {
	CMD_ERR("Resource '%s' not moved: active on multiple nodes\n", rsc_id);
	goto bail;
	}

	if(dest == NULL) {
	CMD_ERR("Error performing operation: node '%s' is unknown\n", host_uname);
	rc = -ENXIO;
	goto bail;
	}

	if(g_list_length(rsc->running_on) == 1) {
	current = rsc->running_on->data;
	}

	if(current == NULL) {
	/* Nothing to check */

	} else if(scope_master && rsc->fns->state(rsc, TRUE) != RSC_ROLE_MASTER) {
	crm_trace("%s is already active on %s but not in correct state", rsc_id, dest->details->uname);
	} else if (safe_str_eq(current->details->uname, dest->details->uname)) {
	cur_is_dest = TRUE;
	if (do_force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.\n",
	rsc_id, scope_master?"promoted":"active", dest->details->uname);
	} else {
	CMD_ERR("Error performing operation: %s is already %s on %s\n",
	rsc_id, scope_master?"promoted":"active", dest->details->uname);
	goto bail;
	}
	}

	/* Clear any previous constraints for 'dest' */
	clear_resource(rsc_id, dest->details->uname, data_set.nodes, cib_conn);

	/* Record an explicit preference for 'dest' */
	rc = prefer_resource(rsc_id, dest->details->uname, cib_conn);

	crm_trace("%s%s now prefers node %s%s",
	rsc->id, scope_master?" (master)":"", dest->details->uname, do_force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if(do_force && (cur_is_dest == FALSE)) {
	/* Ban the original location if possible */
	if(current) {
	ban_resource(rsc_id, current->details->uname, NULL, cib_conn);

	} else if(count > 1) {
	CMD_ERR("Resource '%s' is currently %s in %d locations. One may now move one to %s\n",
	rsc_id, scope_master?"promoted":"active", count, dest->details->uname);
	CMD_ERR("You can prevent '%s' from being %s at a specific location with:"
	" --ban %s--host <name>\n", rsc_id, scope_master?"promoted":"active", scope_master?"--master ":"");

	} else {
	crm_trace("Not banning %s from it's current location: not active", rsc_id);
	}
	}

	} else if (rsc_cmd == 'B' && host_uname) {
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
	node_t *dest = pe_find_node(data_set.nodes, host_uname);

	rc = -ENXIO;
	if (rsc_id == NULL) {
	CMD_ERR("No value specified for --resource\n");
	goto bail;
	} else if(rsc == NULL) {
	CMD_ERR("Resource '%s' not moved: unknown\n", rsc_id);

	} else if (dest == NULL) {
	CMD_ERR("Error performing operation: node '%s' is unknown\n", host_uname);
	goto bail;
	}
	rc = ban_resource(rsc_id, dest->details->uname, NULL, cib_conn);

	} else if (rsc_cmd == 'B' \|\| rsc_cmd == 'M') {
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);

	rc = -ENXIO;
	if (rsc_id == NULL) {
	CMD_ERR("No value specified for --resource\n");
	goto bail;
	}

	rc = -EINVAL;
	if(rsc == NULL) {
	CMD_ERR("Resource '%s' not moved: unknown\n", rsc_id);

	} else if(g_list_length(rsc->running_on) == 1) {
	node_t *current = rsc->running_on->data;
	rc = ban_resource(rsc_id, current->details->uname, NULL, cib_conn);

	} else if(rsc->variant == pe_master) {
	int count = 0;
	GListPtr iter = NULL;
	node_t *current = NULL;

	for(iter = rsc->children; iter; iter = iter->next) {
	resource_t child = (resource_t )iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if(child_role == RSC_ROLE_MASTER) {
	count++;
	current = child->running_on->data;
	}
	}

	if(count == 1 && current) {
	rc = ban_resource(rsc_id, current->details->uname, NULL, cib_conn);

	} else {
	CMD_ERR("Resource '%s' not moved: active in %d locations (promoted in %d).\n", rsc_id, g_list_length(rsc->running_on), count);
	CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --host <name>\n", rsc_id);
	CMD_ERR("You can prevent '%s' from being promoted at a specific location with:"
	" --ban --master --host <name>\n", rsc_id);
	}

	} else {
	CMD_ERR("Resource '%s' not moved: active in %d locations.\n", rsc_id, g_list_length(rsc->running_on));
	CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --host <name>\n", rsc_id);
	}

	} else if (rsc_cmd == 'G') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource_prop(rsc_id, prop_name, &data_set);

	} else if (rsc_cmd == 'S') {
	xmlNode *msg_data = NULL;

	if (prop_value == NULL \|\| strlen(prop_value) == 0) {
	CMD_ERR("You need to supply a value with the -v option\n");
	rc = -EINVAL;
	goto bail;

	} else if (cib_conn == NULL) {
	rc = -ENOTCONN;
	goto bail;
	}

	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	CRM_LOG_ASSERT(rsc_type != NULL);
	CRM_LOG_ASSERT(prop_name != NULL);
	CRM_LOG_ASSERT(prop_value != NULL);

	msg_data = create_xml_node(NULL, rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
	crm_xml_add(msg_data, prop_name, prop_value);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
	free_xml(msg_data);

	} else if (rsc_cmd == 'g') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	rc = dump_resource_attr(rsc_id, prop_name, &data_set);

	} else if (rsc_cmd == 'p') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	if (prop_value == NULL \|\| strlen(prop_value) == 0) {
	CMD_ERR("You need to supply a value with the -v option\n");
	rc = -EINVAL;
	goto bail;
	}

	/* coverity[var_deref_model] False positive */
	rc = set_resource_attr(rsc_id, prop_set, prop_id, prop_name,
	prop_value, recursive, cib_conn, &data_set);

	} else if (rsc_cmd == 'd') {
	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;
	}
	/* coverity[var_deref_model] False positive */
	rc = delete_resource_attr(rsc_id, prop_set, prop_id, prop_name, cib_conn, &data_set);

	} else if (rsc_cmd == 'C' && rsc_id) {
	resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);

	crm_debug("Re-checking the state of %s on %s", rsc_id, host_uname);
	if(rsc) {
	crmd_replies_needed = 0;
	rc = delete_lrm_rsc(cib_conn, crmd_channel, host_uname, rsc, &data_set);
	} else {
	rc = -ENODEV;
	}

	if (rc == pcmk_ok) {
	start_mainloop();
	}

	} else if (rsc_cmd == 'C') {
	#if HAVE_ATOMIC_ATTRD
	- xmlNode *cmd = create_request(CRM_OP_REPROBE, NULL, host_uname,
	- CRM_SYSTEM_CRMD, crm_system_name, our_pid);
	+ const char *router_node = host_uname;
	+ xmlNode *msg_data = NULL;
	+ xmlNode *cmd = NULL;
	+
	+ if (host_uname) {
	+ node_t *node = pe_find_node(data_set.nodes, host_uname);
	+
	+ if (node && is_remote_node(node)) {
	+ if (node->details->remote_rsc == NULL \|\| node->details->remote_rsc->running_on == NULL) {
	+ CMD_ERR("No lrmd connection detected to remote node %s", host_uname);
	+ return -ENXIO;
	+ }
	+ node = node->details->remote_rsc->running_on->data;
	+ router_node = node->details->uname;
	+
	+ }
	+ }
	+
	+ msg_data = create_xml_node(NULL, "crm-resource-reprobe-op");
	+ crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
	+ if (safe_str_neq(router_node, host_uname)) {
	+ crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	+ }
	+
	+ cmd = create_request(CRM_OP_REPROBE, msg_data, router_node,
	+ CRM_SYSTEM_CRMD, crm_system_name, our_pid);
	+ free_xml(msg_data);

	crm_debug("Re-checking the state of all resources on %s", host_uname?host_uname:"all nodes");

	rc = attrd_update_delegate(
	NULL, 'u', host_uname, "fail-count-*", NULL, XML_CIB_TAG_STATUS, NULL, NULL, NULL, FALSE);

	if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
	start_mainloop();
	}

	free_xml(cmd);
	#else
	GListPtr rIter = NULL;

	crmd_replies_needed = 0;
	for (rIter = data_set.resources; rIter; rIter = rIter->next) {
	resource_t *rsc = rIter->data;
	delete_lrm_rsc(cib_conn, crmd_channel, host_uname, rsc, &data_set);
	}

	start_mainloop();
	#endif

	} else if (rsc_cmd == 'D') {
	xmlNode *msg_data = NULL;

	if (rsc_id == NULL) {
	CMD_ERR("Must supply a resource id with -r\n");
	rc = -ENXIO;
	goto bail;

	}
	if (rsc_type == NULL) {
	CMD_ERR("You need to specify a resource type with -t");
	rc = -ENXIO;
	goto bail;

	} else if (cib_conn == NULL) {
	rc = -ENOTCONN;
	goto bail;
	}

	msg_data = create_xml_node(NULL, rsc_type);
	crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);

	rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
	free_xml(msg_data);

	} else {
	CMD_ERR("Unknown command: %c\n", rsc_cmd);
	}

	bail:

	if (data_set.input != NULL) {
	cleanup_alloc_calculations(&data_set);
	}
	if (cib_conn != NULL) {
	cib_conn->cmds->signoff(cib_conn);
	cib_delete(cib_conn);
	}

	if (rc == -pcmk_err_no_quorum) {
	CMD_ERR("Error performing operation: %s\n", pcmk_strerror(rc));
	CMD_ERR("Try using -f\n");

	} else if (rc != pcmk_ok) {
	CMD_ERR("Error performing operation: %s\n", pcmk_strerror(rc));
	}

	return crm_exit(rc);
	}

File Metadata

Mime Type: text/x-diff
Expires: Wed, Jun 25, 2:41 AM (14 h, 57 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1951902
Default Alt Text: (181 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions