No OneTemporary
Actions

Size

212 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crmd/lrm.c b/crmd/lrm.c
	index ead2e05112..3bc48972a6 100644
	--- a/crmd/lrm.c
	+++ b/crmd/lrm.c
	@@ -1,2251 +1,2250 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/services.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	#define START_DELAY_THRESHOLD 5 * 60 * 1000
	#define MAX_LRM_REG_FAILS 30

	struct delete_event_s {
	int rc;
	const char *rsc;
	lrm_state_t *lrm_state;
	};

	gboolean process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op);
	static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
	static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
	static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
	static int delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
	const char *user_name);

	static lrmd_event_data_t construct_op(lrm_state_t lrm_state, xmlNode * rsc_op,
	const char rsc_id, const char operation);
	static void do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operation,
	xmlNode * msg, xmlNode * request);

	void send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id);

	static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
	int log_level);

	static void
	lrm_connection_destroy(void)
	{
	if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	crm_crit("LRM Connection failed");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	clear_bit(fsa_input_register, R_LRM_CONNECTED);

	} else {
	crm_info("LRM Connection disconnected");
	}

	}

	static char *
	make_stop_id(const char *rsc, int call_id)
	{
	char *op_id = NULL;

	op_id = calloc(1, strlen(rsc) + 34);
	if (op_id != NULL) {
	snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id);
	}
	return op_id;
	}

	static void
	copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") == NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") != NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int target_rc = 0;
	rsc_history_t *entry = NULL;

	if (op->rsc_deleted) {
	crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
	delete_rsc_status(lrm_state, op->rsc_id, cib_quorum_override, NULL);
	return;
	}

	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	return;
	}

	crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);

	entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
	if (entry == NULL && rsc) {
	entry = calloc(1, sizeof(rsc_history_t));
	entry->id = strdup(op->rsc_id);
	g_hash_table_insert(lrm_state->resource_history, entry->id, entry);

	entry->rsc.id = entry->id;
	entry->rsc.type = strdup(rsc->type);
	entry->rsc.class = strdup(rsc->class);
	if (rsc->provider) {
	entry->rsc.provider = strdup(rsc->provider);
	} else {
	entry->rsc.provider = NULL;
	}

	} else if (entry == NULL) {
	crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
	return;
	}

	entry->last_callid = op->call_id;
	target_rc = rsc_op_expected_rc(op);
	if (op->op_status == PCMK_LRM_OP_CANCELLED) {
	if (op->interval > 0) {
	GList gIter, gIterNext;

	crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type,
	op->interval);

	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIterNext) {
	lrmd_event_data_t *existing = gIter->data;

	gIterNext = gIter->next;

	if (crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
	&& safe_str_eq(op->op_type, existing->op_type)
	&& op->interval == existing->interval) {
	lrmd_free_event(existing);
	entry->recurring_op_list = g_list_delete_link(entry->recurring_op_list, gIter);
	}
	}
	return;

	} else {
	crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
	op->rc, op->op_status);
	}

	} else if (did_rsc_op_fail(op, target_rc)) {
	/* We must store failed monitors here
	* - otherwise the block below will cause them to be forgetten them when a stop happens
	*/
	if (entry->failed) {
	lrmd_free_event(entry->failed);
	}
	entry->failed = lrmd_copy_event(op);

	} else if (op->interval == 0) {
	if (entry->last) {
	lrmd_free_event(entry->last);
	}
	entry->last = lrmd_copy_event(op);

	if (op->params &&
	(safe_str_eq(CRMD_ACTION_START, op->op_type) \|\|
	safe_str_eq(CRMD_ACTION_STATUS, op->op_type))) {

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}
	entry->stop_params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str,
	g_hash_destroy_str);

	g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
	}
	}

	if (op->interval > 0) {
	GListPtr iter = NULL;

	for(iter = entry->recurring_op_list; iter; iter = iter->next) {
	lrmd_event_data_t *o = iter->data;

	/* op->rsc_id is implied */
	if(op->interval == o->interval && strcmp(op->op_type, o->op_type) == 0) {
	crm_trace("Removing existing recurring op entry: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
	entry->recurring_op_list = g_list_remove(entry->recurring_op_list, o);
	break;
	}
	}

	crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
	entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));

	} else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
	GList *gIter = entry->recurring_op_list;

	crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
	g_list_length(gIter), op->rsc_id, op->op_type, op->interval);
	for (; gIter != NULL; gIter = gIter->next) {
	lrmd_free_event(gIter->data);
	}
	g_list_free(entry->recurring_op_list);
	entry->recurring_op_list = NULL;
	}
	}

	void
	lrm_op_callback(lrmd_event_data_t * op)
	{
	const char *nodename = NULL;
	lrm_state_t *lrm_state = NULL;

	CRM_CHECK(op != NULL, return);

	/* determine the node name for this connection. */
	nodename = op->remote_nodename ? op->remote_nodename : fsa_our_uname;

	if (op->type == lrmd_event_disconnect && (safe_str_eq(nodename, fsa_our_uname))) {
	/* if this is the local lrmd ipc connection, set the right bits in the
	* crmd when the connection goes down */
	lrm_connection_destroy();
	return;
	} else if (op->type != lrmd_event_exec_complete) {
	/* we only need to process execution results */
	return;
	}

	lrm_state = lrm_state_find(nodename);
	CRM_ASSERT(lrm_state != NULL);

	process_lrm_event(lrm_state, op);
	}

	/* A_LRM_CONNECT */
	void
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* This only pertains to local lrmd connections. Remote connections are handled as
	* resources within the pengine. Connecting and disconnecting from remote lrmd instances
	* handled differently than the local. */

	lrm_state_t *lrm_state = NULL;

	if(fsa_our_uname == NULL) {
	return; /* Nothing to do */
	}
	lrm_state = lrm_state_find_or_create(fsa_our_uname);
	if (lrm_state == NULL) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	if (action & A_LRM_DISCONNECT) {
	if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
	if (action == A_LRM_DISCONNECT) {
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	clear_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_info("Disconnecting from the LRM");
	lrm_state_disconnect(lrm_state);
	lrm_state_reset_tables(lrm_state);
	crm_notice("Disconnected from the LRM");
	}

	if (action & A_LRM_CONNECT) {
	int ret = pcmk_ok;

	crm_debug("Connecting to the LRM");
	ret = lrm_state_ipc_connect(lrm_state);

	if (ret != pcmk_ok) {
	if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
	crm_warn("Failed to sign on to the LRM %d"
	" (%d max) times", lrm_state->num_lrm_register_fails, MAX_LRM_REG_FAILS);

	crm_timer_start(wait_timer);
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	if (ret != pcmk_ok) {
	crm_err("Failed to sign on to the LRM %d" " (max) times",
	lrm_state->num_lrm_register_fails);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	set_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_info("LRM connection established");
	}

	if (action & ~(A_LRM_CONNECT \| A_LRM_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
	}
	}

	static gboolean
	lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
	{
	int counter = 0;
	gboolean rc = TRUE;
	const char *when = "lrm disconnect";

	GHashTableIter gIter;
	const char *key = NULL;
	rsc_history_t *entry = NULL;
	struct recurring_op_s *pending = NULL;

	crm_debug("Checking for active resources before exit");

	if (cur_state == S_TERMINATE) {
	log_level = LOG_ERR;
	when = "shutdown";

	} else if (is_set(fsa_input_register, R_SHUTDOWN)) {
	when = "shutdown... waiting";
	}

	if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
	guint removed = g_hash_table_foreach_remove(
	lrm_state->pending_ops, stop_recurring_actions, lrm_state);

	crm_notice("Stopped %u recurring operations at %s (%u ops remaining)",
	removed, when, g_hash_table_size(lrm_state->pending_ops));
	}

	if (lrm_state->pending_ops) {
	g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
	/* Ignore recurring actions in the shutdown calculations */
	if (pending->interval == 0) {
	counter++;
	}
	}
	}

	if (counter > 0) {
	do_crm_log(log_level, "%d pending LRM operations at %s", counter, when);

	if (cur_state == S_TERMINATE \|\| !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
	g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&gIter, (gpointer)&key, (gpointer)&pending)) {
	do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
	}

	} else {
	rc = FALSE;
	}
	return rc;
	}

	if (lrm_state->resource_history == NULL) {
	return rc;
	}

	if (cur_state == S_TERMINATE \|\| is_set(fsa_input_register, R_SHUTDOWN)) {
	/* At this point we're not waiting, we're just shutting down */
	when = "shutdown";
	}

	counter = 0;
	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
	if (is_rsc_active(lrm_state, entry->id) == FALSE) {
	continue;
	}

	counter++;
	crm_trace("Found %s active", entry->id);
	if (lrm_state->pending_ops) {
	GHashTableIter hIter;

	g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
	while (g_hash_table_iter_next(&hIter, (gpointer)&key, (gpointer)&pending)) {
	if (crm_str_eq(entry->id, pending->rsc_id, TRUE)) {
	crm_notice("%sction %s (%s) incomplete at %s",
	pending->interval == 0 ? "A" : "Recurring a",
	key, pending->op_key, when);
	}
	}
	}
	}

	if (counter) {
	crm_err("%d resources were active at %s.", counter, when);
	}

	return rc;
	}

	static char *
	get_rsc_metadata(const char type, const char class, const char *provider)
	{
	int rc = 0;
	char *metadata = NULL;

	/* Always use a local connection for this operation */
	lrm_state_t *lrm_state = lrm_state_find(fsa_our_uname);

	CRM_CHECK(type != NULL, return NULL);
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(lrm_state != NULL, return NULL);

	if (provider == NULL) {
	provider = "heartbeat";
	}

	crm_trace("Retreiving metadata for %s::%s:%s", type, class, provider);
	rc = lrm_state_get_metadata(lrm_state, class, provider, type, &metadata, 0);

	if (metadata) {
	/* copy the metadata because the LRM likes using
	* g_alloc instead of cl_malloc
	*/
	char *m_copy = strdup(metadata);

	g_free(metadata);
	metadata = m_copy;

	} else {
	crm_warn("No metadata found for %s::%s:%s: %s (%d)", type, class, provider, pcmk_strerror(rc), rc);
	}

	return metadata;
	}

	typedef struct reload_data_s {
	char *key;
	char *metadata;
	time_t last_query;
	gboolean can_reload;
	GListPtr restart_list;
	} reload_data_t;

	static void
	g_hash_destroy_reload(gpointer data)
	{
	reload_data_t *reload = data;

	free(reload->key);
	free(reload->metadata);
	g_list_free_full(reload->restart_list, free);
	free(reload);
	}

	GHashTable *reload_hash = NULL;
	static GListPtr
	get_rsc_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int len = 0;
	char *key = NULL;
	char *copy = NULL;
	const char *value = NULL;
	const char *provider = NULL;

	xmlNode *param = NULL;
	xmlNode *params = NULL;
	xmlNode *actions = NULL;
	xmlNode *metadata = NULL;

	time_t now = time(NULL);
	reload_data_t *reload = NULL;

	if (reload_hash == NULL) {
	reload_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_reload);
	}

	provider = rsc->provider;
	if (provider == NULL) {
	provider = "heartbeat";
	}

	len = strlen(rsc->type) + strlen(rsc->class) + strlen(provider) + 4;
	key = malloc(len);
	if(key) {
	snprintf(key, len, "%s::%s:%s", rsc->type, rsc->class, provider);
	reload = g_hash_table_lookup(reload_hash, key);
	}

	if (reload && ((now - 9) > reload->last_query)
	&& safe_str_eq(op->op_type, RSC_START)) {
	reload = NULL; /* re-query */
	}

	if (reload == NULL) {
	xmlNode *action = NULL;

	reload = calloc(1, sizeof(reload_data_t));
	g_hash_table_replace(reload_hash, key, reload);

	reload->last_query = now;
	reload->key = key;
	key = NULL;
	reload->metadata = get_rsc_metadata(rsc->type, rsc->class, provider);

	if(reload->metadata == NULL) {
	goto cleanup;
	}

	metadata = string2xml(reload->metadata);
	if (metadata == NULL) {
	crm_err("Metadata for %s::%s:%s is not valid XML",
	rsc->provider, rsc->class, rsc->type);
	goto cleanup;
	}

	actions = find_xml_node(metadata, "actions", TRUE);

	for (action = __xml_first_child(actions); action != NULL; action = __xml_next(action)) {
	if (crm_str_eq((const char *)action->name, "action", TRUE)) {
	value = crm_element_value(action, "name");
	if (safe_str_eq("reload", value)) {
	reload->can_reload = TRUE;
	break;
	}
	}
	}

	if (reload->can_reload == FALSE) {
	goto cleanup;
	}

	params = find_xml_node(metadata, "parameters", TRUE);
	for (param = __xml_first_child(params); param != NULL; param = __xml_next(param)) {
	if (crm_str_eq((const char *)param->name, "parameter", TRUE)) {
	value = crm_element_value(param, "unique");
	if (crm_is_true(value)) {
	value = crm_element_value(param, "name");
	if (value == NULL) {
	crm_err("%s: NULL param", key);
	continue;
	}
	crm_debug("Attr %s is not reloadable", value);
	copy = strdup(value);
	CRM_LOG_ASSERT(copy != NULL);
	if(copy == NULL) { continue; };
	reload->restart_list = g_list_append(reload->restart_list, copy);
	}
	}
	}
	}

	cleanup:
	free(key);
	free_xml(metadata);
	return reload->restart_list;
	}

	static void
	append_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, xmlNode * update,
	const char *version)
	{
	int len = 0;
	char *list = NULL;
	char *digest = NULL;
	const char *value = NULL;
	xmlNode *restart = NULL;
	GListPtr restart_list = NULL;
	GListPtr lpc = NULL;

	if (op->interval > 0) {
	/* monitors are not reloadable */
	return;

	} else if (op->params == NULL) {
	crm_debug("%s has no parameters", ID(update));
	return;

	} else if (rsc == NULL) {
	return;

	} else if (crm_str_eq(CRMD_ACTION_STOP, op->op_type, TRUE)) {
	/* Stopped resources don't need to be reloaded */
	return;

	} else if (compare_version("1.0.8", version) > 0) {
	/* Caller version does not support reloads */
	return;
	}

	restart_list = get_rsc_restart_list(rsc, op);
	if (restart_list == NULL) {
	/* Resource does not support reloads */
	return;
	}

	restart = create_xml_node(NULL, XML_TAG_PARAMS);
	for (lpc = restart_list; lpc != NULL; lpc = lpc->next) {
	const char param = (const char )lpc->data;

	int start = len;

	CRM_LOG_ASSERT(param != NULL);
	if(param == NULL) { continue; };

	value = g_hash_table_lookup(op->params, param);
	if (value != NULL) {
	crm_xml_add(restart, param, value);
	}
	len += strlen(param) + 2;
	list = realloc_safe(list, len + 1);
	sprintf(list + start, " %s ", param);
	}

	digest = calculate_operation_digest(restart, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list);
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	crm_trace("%s: %s, %s", rsc->id, digest, list);
	crm_log_xml_trace(restart, "restart digest source");

	free_xml(restart);
	free(digest);
	free(list);
	}

	static gboolean
	build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op,
	const char *src)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	const char *caller_version = CRM_FEATURE_SET;

	if (op == NULL) {
	return FALSE;

	} else if (AM_I_DC) {

	} else if (fsa_our_dc_version != NULL) {
	caller_version = fsa_our_dc_version;
	} else if (op->params == NULL) {
	caller_version = fsa_our_dc_version;
	} else {
	/* there is a small risk in formerly mixed clusters that
	* it will be sub-optimal.
	* however with our upgrade policy, the update we send
	* should still be completely supported anyway
	*/
	caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
	crm_debug("Falling back to operation originator version: %s", caller_version);
	}

	target_rc = rsc_op_expected_rc(op);
	- xml_op = create_operation_update(parent, op, caller_version, target_rc, src, LOG_DEBUG);
	- crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, fsa_our_uname); /* For context during triage */
	+ xml_op = create_operation_update(parent, op, caller_version, target_rc, fsa_our_uname, src, LOG_DEBUG);

	if (xml_op) {
	append_restart_list(rsc, op, xml_op, caller_version);
	}
	return TRUE;
	}

	static gboolean
	is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
	{
	rsc_history_t *entry = NULL;

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	if (entry == NULL \|\| entry->last == NULL) {
	return FALSE;
	}

	crm_trace("Processing %s: %s.%d=%d",
	rsc_id, entry->last->op_type, entry->last->interval, entry->last->rc);
	if (entry->last->rc == PCMK_OCF_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) {
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_OK
	&& safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) {
	/* a stricter check is too complex...
	* leave that to the PE
	*/
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
	return FALSE;

	} else if (entry->last->interval == 0 && entry->last->rc == PCMK_OCF_NOT_CONFIGURED) {
	/* Badly configured resources can't be reliably stopped */
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
	{
	GHashTableIter iter;
	rsc_history_t *entry = NULL;

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {

	GList *gIter = NULL;
	xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);

	crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.class);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);

	if (entry->last && entry->last->params) {
	const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
	if (container) {
	crm_trace("Resource %s is a part of container resource %s", entry->id, container);
	crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
	}
	}
	build_operation_update(xml_rsc, &(entry->rsc), entry->failed, __FUNCTION__);
	build_operation_update(xml_rsc, &(entry->rsc), entry->last, __FUNCTION__);
	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
	build_operation_update(xml_rsc, &(entry->rsc), gIter->data, __FUNCTION__);
	}
	}

	return FALSE;
	}

	xmlNode *
	do_lrm_query_internal(lrm_state_t * lrm_state, gboolean is_replace)
	{
	xmlNode *xml_state = NULL;
	xmlNode *xml_data = NULL;
	xmlNode *rsc_list = NULL;
	const char *uuid = NULL;

	if (safe_str_eq(lrm_state->node_name, fsa_our_uname)) {
	crm_node_t *peer = crm_get_peer(0, lrm_state->node_name);
	xml_state = do_update_node_cib(peer, node_update_cluster\|node_update_peer, NULL, __FUNCTION__);
	/* The next two lines shouldn't be necessary for newer DCs */
	crm_xml_add(xml_state, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
	crm_xml_add(xml_state, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
	uuid = fsa_our_uuid;

	} else {
	xml_state = create_xml_node(NULL, XML_CIB_TAG_STATE);
	crm_xml_add(xml_state, XML_NODE_IS_REMOTE, "true");
	crm_xml_add(xml_state, XML_ATTR_ID, lrm_state->node_name);
	crm_xml_add(xml_state, XML_ATTR_UNAME, lrm_state->node_name);
	uuid = lrm_state->node_name;
	}

	xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
	crm_xml_add(xml_data, XML_ATTR_ID, uuid);
	rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);

	/* Build a list of active (not always running) resources */
	build_active_RAs(lrm_state, rsc_list);

	crm_log_xml_trace(xml_state, "Current state of the LRM");

	return xml_state;
	}

	xmlNode *
	do_lrm_query(gboolean is_replace, const char *node_name)
	{
	lrm_state_t *lrm_state = lrm_state_find(node_name);

	if (!lrm_state) {
	crm_err("Could not query lrm state for lrmd node %s", node_name);
	return NULL;
	}
	return do_lrm_query_internal(lrm_state, is_replace);
	}

	static void
	notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, from_host, rsc_id, rc == pcmk_ok ? "" : " not");

	op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
	CRM_ASSERT(op != NULL);

	if (rc == pcmk_ok) {
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	} else {
	op->op_status = PCMK_LRM_OP_ERROR;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	}

	send_direct_ack(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);

	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	/* this isn't expected - trigger a new transition */
	time_t now = time(NULL);
	char *now_s = crm_itoa(now);

	crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"last-lrm-refresh", now_s, FALSE, NULL, NULL);

	free(now_s);
	}
	}

	static gboolean
	lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	struct delete_event_s *event = user_data;
	struct pending_deletion_op_s *op = value;

	if (crm_str_eq(event->rsc, op->rsc, TRUE)) {
	notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
	return TRUE;
	}
	return FALSE;
	}

	static gboolean
	lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
	{
	const char *rsc = user_data;
	struct recurring_op_s *pending = value;

	if (crm_str_eq(rsc, pending->rsc_id, TRUE)) {
	crm_info("Removing op %s:%d for deleted resource %s",
	pending->op_key, pending->call_id, rsc);
	return TRUE;
	}
	return FALSE;
	}

	/*
	* Remove the rsc from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/
	#define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"

	static int
	delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
	const char *user_name)
	{
	char *rsc_xpath = NULL;
	int max = 0;
	int rc = pcmk_ok;

	CRM_CHECK(rsc_id != NULL, return -ENXIO);

	max = strlen(rsc_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + 1;
	rsc_xpath = calloc(1, max);
	snprintf(rsc_xpath, max, rsc_template, lrm_state->node_name, rsc_id);

	rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
	NULL, NULL, call_options \| cib_xpath, user_name);

	free(rsc_xpath);
	return rc;
	}

	static void
	delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
	GHashTableIter * rsc_gIter, int rc, const char *user_name)
	{
	struct delete_event_s event;

	CRM_CHECK(rsc_id != NULL, return);

	if (rc == pcmk_ok) {
	char *rsc_id_copy = strdup(rsc_id);

	if (rsc_gIter)
	g_hash_table_iter_remove(rsc_gIter);
	else
	g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
	crm_debug("sync: Sending delete op for %s", rsc_id_copy);
	delete_rsc_status(lrm_state, rsc_id_copy, cib_quorum_override, user_name);

	g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
	free(rsc_id_copy);
	}

	if (input) {
	notify_deleted(lrm_state, input, rsc_id, rc);
	}

	event.rc = rc;
	event.rsc = rsc_id;
	event.lrm_state = lrm_state;
	g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
	}

	/*
	* Remove the op from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/

	#define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
	#define op_call_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']"

	static void
	delete_op_entry(lrm_state_t * lrm_state, lrmd_event_data_t * op, const char *rsc_id,
	const char *key, int call_id)
	{
	xmlNode *xml_top = NULL;

	if (op != NULL) {
	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);

	if (op->interval > 0) {
	char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval);

	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, XML_ATTR_ID, op_id);
	free(op_id);
	}

	crm_debug("async: Sending delete op for %s_%s_%d (call=%d)",
	op->rsc_id, op->op_type, op->interval, op->call_id);

	fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, cib_quorum_override);

	} else if (rsc_id != NULL && key != NULL) {
	int max = 0;
	char *op_xpath = NULL;

	if (call_id > 0) {
	max =
	strlen(op_call_template) + strlen(rsc_id) + strlen(lrm_state->node_name) +
	strlen(key) + 10;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_call_template, lrm_state->node_name, rsc_id, key, call_id);

	} else {
	max =
	strlen(op_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + strlen(key) +
	1;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_template, lrm_state->node_name, rsc_id, key);
	}

	crm_debug("sync: Sending delete op for %s (call=%d)", rsc_id, call_id);
	fsa_cib_conn->cmds->delete(fsa_cib_conn, op_xpath, NULL, cib_quorum_override \| cib_xpath);

	free(op_xpath);

	} else {
	crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key);
	return;
	}

	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	void
	lrm_clear_last_failure(const char rsc_id, const char node_name)
	{
	char *attr = NULL;
	GHashTableIter iter;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;
	rsc_history_t *entry = NULL;

	attr = generate_op_key(rsc_id, "last_failure", 0);

	/* This clears last failure for every lrm state that has this rsc.*/
	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (node_name != NULL) {
	if (strcmp(node_name, lrm_state->node_name) != 0) {
	/* filter by node_name if node_name is present */
	continue;
	}
	}

	delete_op_entry(lrm_state, NULL, rsc_id, attr, 0);

	if (!lrm_state->resource_history) {
	continue;
	}

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
	if (crm_str_eq(rsc_id, entry->id, TRUE)) {
	lrmd_free_event(entry->failed);
	entry->failed = NULL;
	}
	}
	}
	free(attr);
	g_list_free(lrm_state_list);
	}

	/* Returns: gboolean - cancellation is in progress */
	static gboolean
	cancel_op(lrm_state_t * lrm_state, const char rsc_id, const char key, int op, gboolean remove)
	{
	int rc = pcmk_ok;
	char *local_key = NULL;
	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != 0, return FALSE);
	CRM_CHECK(rsc_id != NULL, return FALSE);
	if (key == NULL) {
	local_key = make_stop_id(rsc_id, op);
	key = local_key;
	}
	pending = g_hash_table_lookup(lrm_state->pending_ops, key);

	if (pending) {
	if (remove && pending->remove == FALSE) {
	pending->remove = TRUE;
	crm_debug("Scheduling %s for removal", key);
	}

	if (pending->cancelled) {
	crm_debug("Operation %s already cancelled", key);
	free(local_key);
	return FALSE;
	}

	pending->cancelled = TRUE;

	} else {
	crm_info("No pending op found for %s", key);
	free(local_key);
	return FALSE;
	}

	crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
	rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type, pending->interval);
	if (rc == pcmk_ok) {
	crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
	free(local_key);
	return TRUE;
	}

	crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
	/* The caller needs to make sure the entry is
	* removed from the pending_ops list
	*
	* Usually by returning TRUE inside the worker function
	* supplied to g_hash_table_foreach_remove()
	*
	* Not removing the entry from pending_ops will block
	* the node from shutting down
	*/
	free(local_key);
	return FALSE;
	}

	struct cancel_data {
	gboolean done;
	gboolean remove;
	const char *key;
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct cancel_data *data = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (crm_str_eq(op->op_key, data->key, TRUE)) {
	data->done = TRUE;
	remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
	}
	return remove;
	}

	static gboolean
	cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
	{
	guint removed = 0;
	struct cancel_data data;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(key != NULL, return FALSE);

	data.key = key;
	data.rsc = rsc;
	data.done = FALSE;
	data.remove = remove;
	data.lrm_state = lrm_state;

	removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
	crm_trace("Removed %u op cache entries, new size: %u",
	removed, g_hash_table_size(lrm_state->pending_ops));
	return data.done;
	}

	static lrmd_rsc_info_t *
	get_lrm_resource(lrm_state_t * lrm_state, xmlNode * resource, xmlNode * op_msg, gboolean do_create)
	{
	lrmd_rsc_info_t *rsc = NULL;
	const char *id = ID(resource);
	const char *type = crm_element_value(resource, XML_ATTR_TYPE);
	const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER);
	const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG);

	crm_trace("Retrieving %s from the LRM.", id);
	CRM_CHECK(id != NULL, return NULL);

	rsc = lrm_state_get_rsc_info(lrm_state, id, 0);

	if (!rsc && long_id) {
	rsc = lrm_state_get_rsc_info(lrm_state, long_id, 0);
	}

	if (!rsc && do_create) {
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(type != NULL, return NULL);

	crm_trace("Adding rsc %s before operation", id);

	lrm_state_register_rsc(lrm_state, id, class, provider, type, lrmd_opt_drop_recurring);

	rsc = lrm_state_get_rsc_info(lrm_state, id, 0);

	if (!rsc) {
	fsa_data_t *msg_data = NULL;

	crm_err("Could not add resource %s to LRM %s", id, lrm_state->node_name);
	/* only register this as a internal error if this involves the local
	* lrmd. Otherwise we're likely dealing with an unresponsive remote-node
	* which is not a FSA failure. */
	if (lrm_state_is_local(lrm_state) == TRUE) {
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}
	}

	return rsc;
	}

	static void
	delete_resource(lrm_state_t * lrm_state,
	const char *id,
	lrmd_rsc_info_t * rsc,
	GHashTableIter * gIter,
	const char *sys,
	const char *host,
	const char *user,
	ha_msg_input_t * request,
	gboolean unregister)
	{
	int rc = pcmk_ok;

	crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host);

	if (rsc && unregister) {
	rc = lrm_state_unregister_rsc(lrm_state, id, 0);
	}

	if (rc == pcmk_ok) {
	crm_trace("Resource '%s' deleted", id);
	} else if (rc == -EINPROGRESS) {
	crm_info("Deletion of resource '%s' pending", id);
	if (request) {
	struct pending_deletion_op_s *op = NULL;
	char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);

	op = calloc(1, sizeof(struct pending_deletion_op_s));
	op->rsc = strdup(rsc->id);
	op->input = copy_ha_msg_input(request);
	g_hash_table_insert(lrm_state->deletion_ops, ref, op);
	}
	return;
	} else {
	crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d",
	id, sys, user ? user : "internal", host, rc);
	}

	delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
	}

	static int
	get_fake_call_id(lrm_state_t lrm_state, const char rsc_id)
	{
	int call_id = 0;
	rsc_history_t *entry = NULL;

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);

	/* Make sure the call id is greater than the last successful operation,
	* otherwise the failure will not result in a possible recovery of the resource
	* as it could appear the failure occurred before the successful start */
	if (entry) {
	call_id = entry->last_callid + 1;
	}

	if (call_id < 0) {
	call_id = 1;
	}
	return call_id;
	}


	/* A_LRM_INVOKE */
	void
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean create_rsc = TRUE;
	lrm_state_t *lrm_state = NULL;
	const char *crm_op = NULL;
	const char *from_sys = NULL;
	const char *from_host = NULL;
	const char *operation = NULL;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *user_name = NULL;
	const char *target_node = NULL;
	gboolean is_remote_node = FALSE;
	gboolean crm_rsc_delete = FALSE;

	if (input->xml != NULL) {
	/* Remote node operations are routed here to their remote connections */
	target_node = crm_element_value(input->xml, XML_LRM_ATTR_TARGET);
	}
	if (target_node == NULL) {
	target_node = fsa_our_uname;
	} else if (safe_str_neq(target_node, fsa_our_uname)) {
	is_remote_node = TRUE;
	}

	lrm_state = lrm_state_find(target_node);

	if (lrm_state == NULL && is_remote_node) {
	crm_err("no lrmd connection for remote node %s found on cluster node %s. Can not process request.",
	target_node, fsa_our_uname);
	return;
	}

	CRM_ASSERT(lrm_state != NULL);

	#if ENABLE_ACL
	user_name = crm_acl_get_set_user(input->msg, F_CRM_USER, NULL);
	crm_trace("LRM command from user '%s'", user_name);
	#endif

	crm_op = crm_element_value(input->msg, F_CRM_TASK);
	from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
	}

	crm_trace("LRM command from: %s", from_sys);

	if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
	/* remember this delete op came from crm_resource */
	crm_rsc_delete = TRUE;
	operation = CRMD_ACTION_DELETE;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	operation = CRM_OP_LRM_REFRESH;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
	lrmd_event_data_t *op = NULL;
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* The lrmd can not fail a resource, it does not understand the
	* concept of success or failure in relation to a resource, it simply
	* executes operations and reports the results. We determine what a failure is.
	* Becaues of this, if we want to fail a resource we have to fake what we
	* understand a failure to look like.
	*
	* To do this we create a fake lrmd operation event for the resource
	* we want to fail. We then pass that event to the lrmd client callback
	* so it will be processed as if it actually came from the lrmd. */
	op = construct_op(lrm_state, input->xml, ID(xml_rsc), "asyncmon");
	CRM_ASSERT(op != NULL);

	free((char *)op->user_data);
	op->user_data = NULL;
	op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
	op->interval = 0;
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	#if ENABLE_ACL
	if (user_name && is_privileged(user_name) == FALSE) {
	crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;
	}
	#endif

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);
	if (rsc) {
	crm_info("Failing resource %s...", rsc->id);
	process_lrm_event(lrm_state, op);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	lrmd_free_rsc_info(rsc);
	} else {
	crm_info("Cannot find/create resource in order to fail it...");
	crm_log_xml_warn(input->msg, "bad input");
	}

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;

	} else if (input->xml != NULL) {
	operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
	}

	if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	int rc = pcmk_ok;
	xmlNode *fragment = do_lrm_query_internal(lrm_state, TRUE);

	fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
	crm_info("Forced a local LRM refresh: call=%d", rc);

	if(strcmp(CRM_SYSTEM_CRMD, from_sys) != 0) {
	xmlNode *reply = create_request(
	CRM_OP_INVOKE_LRM, fragment,
	from_host, from_sys, CRM_SYSTEM_LRMD, fsa_our_uuid);

	crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}
	free_xml(reply);
	}

	free_xml(fragment);

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
	xmlNode *data = do_lrm_query_internal(lrm_state, FALSE);
	xmlNode *reply = create_reply(input->msg, data);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_err("Unable to route reply");
	crm_log_xml_err(reply, "reply");
	}
	free_xml(reply);
	free_xml(data);

	} else if (safe_str_eq(operation, CRM_OP_PROBED)) {
	update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE, user_name, is_remote_node);

	} else if (safe_str_eq(operation, CRM_OP_REPROBE) \|\| safe_str_eq(crm_op, CRM_OP_REPROBE)) {
	GHashTableIter gIter;
	rsc_history_t *entry = NULL;

	crm_notice("Forcing the status of all resources to be redetected");

	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	/* only unregister the resource during a reprobe if it is not a remote connection
	* resource. otherwise unregistering the connection will terminate remote-node
	* membership */
	gboolean unregister = is_remote_lrmd_ra(NULL, NULL, entry->id) ? FALSE : TRUE;

	delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, from_host,
	user_name, NULL, unregister);
	}

	/* Now delete the copy in the CIB */
	erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);

	/* And finally, _delete_ the value in attrd
	* Setting it to FALSE results in the PE sending us back here again
	*/
	update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);

	if(strcmp(CRM_SYSTEM_TENGINE, from_sys) != 0
	&& strcmp(CRM_SYSTEM_TENGINE, from_sys) != 0) {
	xmlNode *reply = create_request(
	CRM_OP_INVOKE_LRM, NULL,
	from_host, from_sys, CRM_SYSTEM_LRMD, fsa_our_uuid);

	crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}
	free_xml(reply);
	}

	} else if (operation != NULL) {
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *params = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* only the first 16 chars are used by the LRM */
	params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);

	if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	create_rsc = FALSE;
	}

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);

	if (rsc == NULL && create_rsc) {
	lrmd_event_data_t *op = NULL;

	/* if the operation couldn't complete because we can't register
	* the resource, return a generic error */
	op = construct_op(lrm_state, input->xml, ID(xml_rsc), operation);
	CRM_ASSERT(op != NULL);

	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);

	crm_err("Invalid resource definition");
	crm_log_xml_warn(input->msg, "bad input");

	} else if (rsc == NULL) {
	lrmd_event_data_t *op = NULL;

	crm_notice("Not creating resource for a %s event: %s", operation, ID(input->xml));
	delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok, user_name);

	op = construct_op(lrm_state, input->xml, ID(xml_rsc), operation);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_OK;
	CRM_ASSERT(op != NULL);
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);

	} else if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
	char *op_key = NULL;
	char *meta_key = NULL;
	int call = 0;
	const char *call_id = NULL;
	const char *op_task = NULL;
	const char *op_interval = NULL;
	gboolean in_progress = FALSE;

	CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	op_interval = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
	op_task = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
	call_id = crm_element_value(params, meta_key);
	free(meta_key);

	CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);
	CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	op_key = generate_op_key(rsc->id, op_task, crm_parse_int(op_interval, "0"));

	crm_debug("PE requested op %s (call=%s) be cancelled",
	op_key, call_id ? call_id : "NA");
	call = crm_parse_int(call_id, "0");
	if (call == 0) {
	/* the normal case when the PE cancels a recurring op */
	in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);

	} else {
	/* the normal case when the PE cancels an orphan op */
	in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
	}

	if (in_progress == FALSE) {
	lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc->id, op_task);

	crm_info("Nothing known about operation %d for %s", call, op_key);
	delete_op_entry(lrm_state, NULL, rsc->id, op_key, call);

	CRM_ASSERT(op != NULL);

	op->rc = PCMK_OCF_OK;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(from_host, from_sys, rsc, op, rsc->id);
	lrmd_free_event(op);

	/* needed?? surely not otherwise the cancel_op_(_key) wouldn't
	* have failed in the first place
	*/
	g_hash_table_remove(lrm_state->pending_ops, op_key);
	}

	free(op_key);

	} else if (rsc != NULL && safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	gboolean unregister = TRUE;

	#if ENABLE_ACL
	int cib_rc = delete_rsc_status(lrm_state, rsc->id, cib_dryrun \| cib_sync_call, user_name);
	if (cib_rc != pcmk_ok) {
	lrmd_event_data_t *op = NULL;

	crm_err
	("Attempted deletion of resource status '%s' from CIB for %s (user=%s) on %s failed: (rc=%d) %s",
	rsc->id, from_sys, user_name ? user_name : "unknown", from_host, cib_rc,
	pcmk_strerror(cib_rc));

	op = construct_op(lrm_state, input->xml, rsc->id, operation);
	op->op_status = PCMK_LRM_OP_ERROR;

	if (cib_rc == -EACCES) {
	op->rc = PCMK_OCF_INSUFFICIENT_PRIV;
	} else {
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	}
	send_direct_ack(from_host, from_sys, NULL, op, rsc->id);
	lrmd_free_event(op);
	return;
	}
	#endif
	if (crm_rsc_delete == TRUE && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
	unregister = FALSE;
	}

	delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys, from_host, user_name, input, unregister);

	} else if (rsc != NULL) {
	do_lrm_rsc_op(lrm_state, rsc, operation, input->xml, input->msg);
	}

	lrmd_free_rsc_info(rsc);

	} else {
	crm_err("Operation was neither a lrm_query, nor a rsc op. %s", crm_str(crm_op));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	static lrmd_event_data_t *
	construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, const char rsc_id, const char operation)
	{
	lrmd_event_data_t *op = NULL;
	const char *op_delay = NULL;
	const char *op_timeout = NULL;
	const char *op_interval = NULL;
	GHashTable *params = NULL;

	const char *transition = NULL;

	CRM_ASSERT(rsc_id != NULL);

	op = calloc(1, sizeof(lrmd_event_data_t));
	op->type = lrmd_event_exec_complete;
	op->op_type = strdup(operation);
	op->op_status = PCMK_LRM_OP_PENDING;
	op->rc = -1;
	op->rsc_id = strdup(rsc_id);
	op->interval = 0;
	op->timeout = 0;
	op->start_delay = 0;

	if (rsc_op == NULL) {
	CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
	op->user_data = NULL;
	/* the stop_all_resources() case
	* by definition there is no DC (or they'd be shutting
	* us down).
	* So we should put our version here.
	*/
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));

	crm_trace("Constructed %s op for %s", operation, rsc_id);
	return op;
	}

	params = xml2list(rsc_op);
	g_hash_table_remove(params, CRM_META "_op_target_rc");

	op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
	op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
	op_interval = crm_meta_value(params, XML_LRM_ATTR_INTERVAL);

	op->interval = crm_parse_int(op_interval, "0");
	op->timeout = crm_parse_int(op_timeout, "0");
	op->start_delay = crm_parse_int(op_delay, "0");

	if (safe_str_neq(operation, RSC_STOP)) {
	op->params = params;

	} else {
	rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);

	/* If we do not have stop parameters cached, use
	* whatever we are given */
	if (!entry \|\| !entry->stop_params) {
	op->params = params;
	} else {
	/* Copy the cached parameter list so that we stop the resource
	* with the old attributes, not the new ones */
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_foreach(params, copy_meta_keys, op->params);
	g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
	g_hash_table_destroy(params);
	params = NULL;
	}
	}

	/* sanity */
	if (op->interval < 0) {
	op->interval = 0;
	}
	if (op->timeout <= 0) {
	op->timeout = op->interval;
	}
	if (op->start_delay < 0) {
	op->start_delay = 0;
	}

	transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
	CRM_CHECK(transition != NULL, return op);

	op->user_data = strdup(transition);

	if (op->interval != 0) {
	if (safe_str_eq(operation, CRMD_ACTION_START)
	\|\| safe_str_eq(operation, CRMD_ACTION_STOP)) {
	crm_err("Start and Stop actions cannot have an interval: %d", op->interval);
	op->interval = 0;
	}
	}

	crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval);

	return op;
	}

	void
	send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id)
	{
	xmlNode *reply = NULL;
	xmlNode update, iter;
	crm_node_t *peer = NULL;

	CRM_CHECK(op != NULL, return);
	if (op->rsc_id == NULL) {
	CRM_ASSERT(rsc_id != NULL);
	op->rsc_id = strdup(rsc_id);
	}
	if (to_sys == NULL) {
	to_sys = CRM_SYSTEM_TENGINE;
	}

	peer = crm_get_peer(0, fsa_our_uname);
	update = do_update_node_cib(peer, node_update_none, NULL, __FUNCTION__);

	iter = create_xml_node(update, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);

	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);
	reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);

	crm_log_xml_trace(update, "ACK Update");

	crm_debug("ACK'ing resource op %s_%s_%d from %s: %s",
	op->rsc_id, op->op_type, op->interval, op->user_data,
	crm_element_value(reply, XML_ATTR_REFERENCE));

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}

	free_xml(update);
	free_xml(reply);
	}

	gboolean
	verify_stopped(enum crmd_fsa_state cur_state, int log_level)
	{
	gboolean res = TRUE;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;

	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
	/* keep iterating through all even when false is returned */
	res = FALSE;
	}
	}

	set_bit(fsa_input_register, R_SENT_RSC_STOP);
	g_list_free(lrm_state_list); lrm_state_list = NULL;
	return res;
	}

	struct stop_recurring_action_s {
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct stop_recurring_action_s *event = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0 && crm_str_eq(op->rsc_id, event->rsc->id, TRUE)) {
	crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, key);
	remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static gboolean
	stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	lrm_state_t *lrm_state = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0) {
	crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, key);
	remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static void
	do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char operation, xmlNode msg,
	xmlNode * request)
	{
	int call_id = 0;
	char *op_id = NULL;
	lrmd_event_data_t *op = NULL;
	lrmd_key_value_t *params = NULL;
	fsa_data_t *msg_data = NULL;
	const char *transition = NULL;
	gboolean stop_recurring = FALSE;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(operation != NULL, return);

	if (msg != NULL) {
	transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
	if (transition == NULL) {
	crm_log_xml_err(msg, "Missing transition number");
	}
	}

	op = construct_op(lrm_state, msg, rsc->id, operation);
	CRM_CHECK(op != NULL, return);

	if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
	&& op->interval == 0
	&& strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {

	/* pcmk remote connections are a special use case.
	* We never ever want to stop monitoring a connection resource until
	* the entire migration has completed. If the connection is ever unexpected
	* severed, even during a migration, this is an event we must detect.*/
	stop_recurring = FALSE;

	} else if (op->interval == 0
	&& strcmp(operation, CRMD_ACTION_STATUS) != 0
	&& strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {

	/* stop any previous monitor operations before changing the resource state */
	stop_recurring = TRUE;
	}

	if (stop_recurring == TRUE) {
	guint removed = 0;
	struct stop_recurring_action_s data;

	data.rsc = rsc;
	data.lrm_state = lrm_state;
	removed = g_hash_table_foreach_remove(
	lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);

	crm_debug("Stopped %u recurring operations in preparation for %s_%s_%d",
	removed, rsc->id, operation, op->interval);
	}

	/* now do the op */
	crm_info("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval);

	if (fsa_state != S_NOT_DC && fsa_state != S_POLICY_ENGINE && fsa_state != S_TRANSITION_ENGINE) {
	if (safe_str_neq(operation, "fail")
	&& safe_str_neq(operation, CRMD_ACTION_STOP)) {
	crm_info("Discarding attempt to perform action %s on %s in state %s",
	operation, rsc->id, fsa_state2string(fsa_state));
	op->rc = 99;
	op->op_status = PCMK_LRM_OP_ERROR;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	lrmd_free_event(op);
	free(op_id);
	return;
	}
	}

	op_id = generate_op_key(rsc->id, op->op_type, op->interval);

	if (op->interval > 0) {
	/* cancel it so we can then restart it without conflict */
	cancel_op_key(lrm_state, rsc, op_id, FALSE);
	}

	if (op->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, op->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	params = lrmd_key_value_add(params, key, value);
	}
	}

	call_id = lrm_state_exec(lrm_state,
	rsc->id,
	op->op_type,
	op->user_data, op->interval, op->timeout, op->start_delay, params);

	if (call_id <= 0 && lrm_state_is_local(lrm_state)) {
	crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);

	} else if (call_id <= 0) {

	crm_err("Operation %s on resource %s failed to execute on remote node %s: %d", operation, rsc->id, lrm_state->node_name, call_id);
	op->call_id = get_fake_call_id(lrm_state, rsc->id);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_OCF_UNKNOWN_ERROR;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;
	process_lrm_event(lrm_state, op);

	} else {
	/* record all operations so we can wait
	* for them to complete during shutdown
	*/
	char *call_id_s = make_stop_id(rsc->id, call_id);
	struct recurring_op_s *pending = NULL;

	pending = calloc(1, sizeof(struct recurring_op_s));
	crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);

	pending->call_id = call_id;
	pending->interval = op->interval;
	pending->op_type = strdup(operation);
	pending->op_key = strdup(op_id);
	pending->rsc_id = strdup(rsc->id);
	g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);

	if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) {
	char *uuid = NULL;
	int dummy = 0, target_rc = 0;

	crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc);
	free(uuid);

	op->rc = target_rc;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	}
	}

	free(op_id);
	lrmd_free_event(op);
	return;
	}

	int last_resource_update = 0;

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource update %d complete: rc=%d", call_id, rc);
	break;
	default:
	crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
	}

	if (call_id == last_resource_update) {
	last_resource_update = 0;
	trigger_fsa(fsa_source);
	}
	}

	static void
	remote_node_init_status(const char *node_name, int call_opt)
	{
	int call_id = 0;
	xmlNode *update = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	simple_remote_node_status(node_name, update,__FUNCTION__);

	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
	if (call_id != pcmk_ok) {
	crm_debug("Failed to init status section for remote-node %s", node_name);
	}
	free_xml(update);
	}

	static void
	remote_node_clear_status(const char *node_name, int call_opt)
	{
	if (node_name == NULL) {
	return;
	}
	remote_node_init_status(node_name, call_opt);
	erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt);
	erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt);
	}

	static int
	do_update_resource(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	/*
	<status>
	<nodes_status id=uname>
	<lrm>
	<lrm_resources>
	<lrm_resource id=...>
	</...>
	*/
	int rc = pcmk_ok;
	xmlNode update, iter = NULL;
	int call_opt = cib_quorum_override;
	const char *uuid = NULL;

	CRM_CHECK(op != NULL, return 0);

	if (fsa_state == S_ELECTION \|\| fsa_state == S_PENDING) {
	crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state));
	call_opt \|= cib_scope_local;
	}

	iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
	update = iter;
	iter = create_xml_node(iter, XML_CIB_TAG_STATE);

	if (safe_str_eq(lrm_state->node_name, fsa_our_uname)) {
	uuid = fsa_our_uuid;

	} else {
	/* remote nodes uuid and uname are equal */
	uuid = lrm_state->node_name;
	crm_xml_add(iter, XML_NODE_IS_REMOTE, "true");
	}

	CRM_LOG_ASSERT(uuid != NULL);
	if(uuid == NULL) {
	rc = -EINVAL;
	goto done;
	}

	crm_xml_add(iter, XML_ATTR_UUID, uuid);
	crm_xml_add(iter, XML_ATTR_UNAME, lrm_state->node_name);
	crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);

	iter = create_xml_node(iter, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, uuid);

	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);

	if (rsc) {
	const char *container = NULL;

	crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
	crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class);
	crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);

	if (op->params) {
	container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
	}
	if (container) {
	crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
	crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
	}

	CRM_CHECK(rsc->type != NULL, crm_err("Resource %s has no value for type", op->rsc_id));
	CRM_CHECK(rsc->class != NULL, crm_err("Resource %s has no value for class", op->rsc_id));

	/* check to see if we need to initialize remote-node related status sections */
	if (safe_str_eq(op->op_type, "start") && op->rc == 0 && op->op_status == PCMK_LRM_OP_DONE) {
	const char *remote_node = g_hash_table_lookup(op->params, CRM_META"_remote_node");

	if (remote_node) {
	/* A container for a remote-node has started, initalize remote-node's status */
	crm_info("Initalizing lrm status for container remote-node %s. Container successfully started.", remote_node);
	remote_node_clear_status(remote_node, call_opt);
	} else if (container == FALSE && safe_str_eq(rsc->type, "remote") && safe_str_eq(rsc->provider, "pacemaker")) {
	/* baremetal remote node connection resource has started, initalize remote-node's status */
	crm_info("Initializing lrm status for baremetal remote-node %s", rsc->id);
	remote_node_clear_status(rsc->id, call_opt);
	}
	}

	} else {
	crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id);
	send_direct_ack(NULL, NULL, rsc, op, op->rsc_id);
	goto cleanup;
	}

	crm_log_xml_trace(update, __FUNCTION__);

	/* make it an asyncronous call and be done with it
	*
	* Best case:
	* the resource state will be discovered during
	* the next signup or election.
	*
	* Bad case:
	* we are shutting down and there is no DC at the time,
	* but then why were we shutting down then anyway?
	* (probably because of an internal error)
	*
	* Worst case:
	* we get shot for having resources "running" when the really weren't
	*
	* the alternative however means blocking here for too long, which
	* isnt acceptable
	*/
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);

	if (rc > 0) {
	last_resource_update = rc;
	}
	done:
	/* the return code is a call number, not an error code */
	crm_trace("Sent resource state update message: %d for %s=%d on %s", rc,
	op->op_type, op->interval, op->rsc_id);
	fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);

	cleanup:
	free_xml(update);
	return rc;
	}

	void
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
	{
	CRM_CHECK(FALSE, return);
	}

	gboolean
	process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op)
	{
	char *op_id = NULL;
	char *op_key = NULL;

	int update_id = 0;
	gboolean removed = FALSE;
	lrmd_rsc_info_t *rsc = NULL;

	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != NULL, return FALSE);

	CRM_CHECK(op->rsc_id != NULL, return FALSE);
	op_id = make_stop_id(op->rsc_id, op->call_id);
	pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
	op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
	rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);

	if (op->op_status == PCMK_LRM_OP_ERROR
	&& (op->rc == PCMK_OCF_RUNNING_MASTER \|\| op->rc == PCMK_OCF_NOT_RUNNING)) {
	/* Leave it up to the TE/PE to decide if this is an error */
	op->op_status = PCMK_LRM_OP_DONE;
	}

	if (op->op_status != PCMK_LRM_OP_CANCELLED) {
	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	/* Keep notify ops out of the CIB */
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
	} else {
	update_id = do_update_resource(lrm_state, rsc, op);
	}
	} else if (op->interval == 0) {
	/* This will occur when "crm resource cleanup" is called while actions are in-flight */
	crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else if (pending == NULL) {
	/* We don't need to do anything for cancelled ops
	* that are not in our pending op list. There are no
	* transition actions waiting on these operations. */

	} else if (op->user_data == NULL) {
	/* At this point we have a pending entry, but no transition
	* key present in the user_data field. report this */
	crm_err("Op %s (call=%d): No user data", op_key, op->call_id);

	} else if (pending->remove) {
	/* The tengine canceled this op, we have been waiting for the cancel to finish. */
	delete_op_entry(lrm_state, op, op->rsc_id, op_key, op->call_id);

	} else if (pending && op->rsc_deleted) {
	/* The tengine initiated this op, but it was cancelled outside of the
	* tengine's control during a resource cleanup/re-probe request. The tengine
	* must be alerted that this operation completed, otherwise the tengine
	* will continue waiting for this update to occur until it is timed out.
	* We don't want this update going to the cib though, so use a direct ack. */
	crm_trace("Op %s (call=%d): cancelled due to rsc deletion", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else {
	/* Before a stop is called, no need to direct ack */
	crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id);
	}

	if ((op->interval == 0) && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
	removed = TRUE;
	crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
	op_key, op->call_id, op_id, g_hash_table_size(lrm_state->pending_ops));

	} else if(op->interval != 0 && op->op_status == PCMK_LRM_OP_CANCELLED) {
	removed = TRUE;
	g_hash_table_remove(lrm_state->pending_ops, op_id);
	}

	switch (op->op_status) {
	case PCMK_LRM_OP_CANCELLED:
	crm_info("Operation %s: %s (node=%s, call=%d, confirmed=%s)",
	op_key, services_lrm_status_str(op->op_status), lrm_state->node_name,
	op->call_id, removed ? "true" : "false");
	break;

	case PCMK_LRM_OP_DONE:
	crm_notice("Operation %s: %s (node=%s, call=%d, rc=%d, cib-update=%d, confirmed=%s)",
	op_key, services_ocf_exitcode_str(op->rc), lrm_state->node_name,
	op->call_id, op->rc, update_id, removed ? "true" : "false");
	break;

	case PCMK_LRM_OP_TIMEOUT:
	crm_err("Operation %s: %s (node=%s, call=%d, timeout=%dms)",
	op_key, services_lrm_status_str(op->op_status), lrm_state->node_name, op->call_id, op->timeout);
	break;

	default:
	crm_err("Operation %s (node=%s, call=%d, status=%d, cib-update=%d, confirmed=%s) %s",
	op_key, lrm_state->node_name, op->call_id, op->op_status, update_id, removed ? "true" : "false",
	services_lrm_status_str(op->op_status));
	}

	if (op->output) {
	char *prefix =
	g_strdup_printf("%s-%s_%s_%d:%d", lrm_state->node_name, op->rsc_id, op->op_type, op->interval, op->call_id);

	if (op->rc) {
	crm_log_output(LOG_NOTICE, prefix, op->output);
	} else {
	crm_log_output(LOG_DEBUG, prefix, op->output);
	}
	g_free(prefix);
	}

	if (op->rsc_deleted) {
	crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
	delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
	}

	/* If a shutdown was escalated while operations were pending,
	* then the FSA will be stalled right now... allow it to continue
	*/
	mainloop_set_trigger(fsa_source);
	update_history_cache(lrm_state, rsc, op);

	lrmd_free_rsc_info(rsc);
	free(op_key);
	free(op_id);

	return TRUE;
	}
	diff --git a/crmd/te_actions.c b/crmd/te_actions.c
	index a3aa78b6ea..c001b60886 100644
	--- a/crmd/te_actions.c
	+++ b/crmd/te_actions.c
	@@ -1,734 +1,733 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>

	#include <crm/common/xml.h>
	#include <tengine.h>

	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crm/cluster.h>
	#include <throttle.h>

	char *te_uuid = NULL;
	GHashTable *te_targets = NULL;
	void send_rsc_command(crm_action_t * action);
	static void te_update_job_count(crm_action_t * action, int offset);

	static void
	te_start_action_timer(crm_graph_t * graph, crm_action_t * action)
	{
	action->timer = calloc(1, sizeof(crm_action_timer_t));
	action->timer->timeout = action->timeout;
	action->timer->reason = timeout_action;
	action->timer->action = action;
	action->timer->source_id = g_timeout_add(action->timer->timeout + graph->network_delay,
	action_timer_callback, (void *)action->timer);

	CRM_ASSERT(action->timer->source_id != 0);
	}

	static gboolean
	te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
	{
	crm_debug("Pseudo action %d fired and confirmed", pseudo->id);
	te_action_confirmed(pseudo);
	update_graph(graph, pseudo);
	trigger_graph();
	return TRUE;
	}

	void
	send_stonith_update(crm_action_t * action, const char target, const char uuid)
	{
	int rc = pcmk_ok;
	crm_node_t *peer = NULL;

	/* zero out the node-status & remove all LRM status info */
	xmlNode *node_state = NULL;

	CRM_CHECK(target != NULL, return);
	CRM_CHECK(uuid != NULL, return);

	/* Make sure the membership and join caches are accurate */
	peer = crm_get_peer_full(0, target, CRM_GET_PEER_CLUSTER \| CRM_GET_PEER_REMOTE);

	CRM_CHECK(peer != NULL, return);

	if (peer->uuid == NULL) {
	crm_info("Recording uuid '%s' for node '%s'", uuid, target);
	peer->uuid = strdup(uuid);
	}

	crmd_peer_down(peer, TRUE);
	node_state =
	do_update_node_cib(peer,
	node_update_cluster \| node_update_peer \| node_update_join \|
	node_update_expected, NULL, __FUNCTION__);

	/* Force our known ID */
	crm_xml_add(node_state, XML_ATTR_UUID, uuid);

	rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
	cib_quorum_override \| cib_scope_local \| cib_can_create);

	/* Delay processing the trigger until the update completes */
	crm_debug("Sending fencing update %d for %s", rc, target);
	fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated);

	/* Make sure it sticks */
	/* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override\|cib_scope_local); */

	erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local);
	erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);

	free_xml(node_state);
	return;
	}

	static gboolean
	te_fence_node(crm_graph_t * graph, crm_action_t * action)
	{
	int rc = 0;
	const char *id = NULL;
	const char *uuid = NULL;
	const char *target = NULL;
	const char *type = NULL;
	gboolean invalid_action = FALSE;
	enum stonith_call_options options = st_opt_none;

	id = ID(action->xml);
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
	type = crm_meta_value(action->params, "stonith_action");

	CRM_CHECK(id != NULL, invalid_action = TRUE);
	CRM_CHECK(uuid != NULL, invalid_action = TRUE);
	CRM_CHECK(type != NULL, invalid_action = TRUE);
	CRM_CHECK(target != NULL, invalid_action = TRUE);

	if (invalid_action) {
	crm_log_xml_warn(action->xml, "BadAction");
	return FALSE;
	}

	crm_notice("Executing %s fencing operation (%s) on %s (timeout=%d)",
	type, id, target, transition_graph->stonith_timeout);

	/* Passing NULL means block until we can connect... */
	te_connect_stonith(NULL);

	if (crmd_join_phase_count(crm_join_confirmed) == 1) {
	options \|= st_opt_allow_suicide;
	}

	rc = stonith_api->cmds->fence(stonith_api, options, target, type,
	transition_graph->stonith_timeout / 1000, 0);

	stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000,
	st_opt_timeout_updates,
	generate_transition_key(transition_graph->id, action->id,
	0, te_uuid),
	"tengine_stonith_callback", tengine_stonith_callback);

	return TRUE;
	}

	static int
	get_target_rc(crm_action_t * action)
	{
	const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);

	if (target_rc_s != NULL) {
	return crm_parse_int(target_rc_s, "0");
	}
	return 0;
	}

	static gboolean
	te_crm_command(crm_graph_t * graph, crm_action_t * action)
	{
	char *counter = NULL;
	xmlNode *cmd = NULL;
	gboolean is_local = FALSE;

	const char *id = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;

	id = ID(action->xml);
	task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	}

	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
	crm_err("Corrupted command (id=%s) %s: no node", crm_str(id), crm_str(task));
	return FALSE);

	crm_info("Executing crm-event (%s): %s on %s%s%s",
	crm_str(id), crm_str(task), on_node,
	is_local ? " (local)" : "", no_wait ? " - no waiting" : "");

	if (safe_str_eq(router_node, fsa_our_uname)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	if (is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
	/* defer until everything else completes */
	crm_info("crm-event (%s) is a local shutdown", crm_str(id));
	graph->completion_action = tg_shutdown;
	graph->abort_reason = "local shutdown";
	te_action_confirmed(action);
	update_graph(graph, action);
	trigger_graph();
	return TRUE;

	} else if (safe_str_eq(task, CRM_OP_SHUTDOWN)) {
	crm_node_t *peer = crm_get_peer(0, router_node);
	crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
	}

	cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);

	counter =
	generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
	crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);

	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
	free(counter);
	free_xml(cmd);

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return FALSE;

	} else if (no_wait) {
	te_action_confirmed(action);
	update_graph(graph, action);
	trigger_graph();

	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %dms instead",
	action->id, task, on_node, action->timeout, graph->network_delay);
	action->timeout = graph->network_delay;
	}
	te_start_action_timer(graph, action);
	}

	return TRUE;
	}

	gboolean
	cib_action_update(crm_action_t * action, int status, int op_rc)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *state = NULL;
	xmlNode *rsc = NULL;
	xmlNode *xml_op = NULL;
	xmlNode *action_rsc = NULL;

	int rc = pcmk_ok;

	const char *name = NULL;
	const char *value = NULL;
	const char *rsc_id = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);

	int call_options = cib_quorum_override \| cib_scope_local;
	int target_rc = get_target_rc(action);

	if (status == PCMK_LRM_OP_PENDING) {
	crm_debug("%s %d: Recording pending operation %s on %s",
	crm_element_name(action->xml), action->id, task_uuid, target);
	} else {
	crm_warn("%s %d: %s on %s timed out",
	crm_element_name(action->xml), action->id, task_uuid, target);
	}

	action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
	if (action_rsc == NULL) {
	return FALSE;
	}

	rsc_id = ID(action_rsc);
	CRM_CHECK(rsc_id != NULL, crm_log_xml_err(action->xml, "Bad:action");
	return FALSE);

	/*
	update the CIB

	<node_state id="hadev">
	<lrm>
	<lrm_resources>
	<lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
	*/

	state = create_xml_node(NULL, XML_CIB_TAG_STATE);

	crm_xml_add(state, XML_ATTR_UUID, target_uuid);
	crm_xml_add(state, XML_ATTR_UNAME, target);

	rsc = create_xml_node(state, XML_CIB_TAG_LRM);
	crm_xml_add(rsc, XML_ATTR_ID, target_uuid);

	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
	crm_xml_add(rsc, XML_ATTR_ID, rsc_id);

	name = XML_ATTR_TYPE;
	value = crm_element_value(action_rsc, name);
	crm_xml_add(rsc, name, value);
	name = XML_AGENT_ATTR_CLASS;
	value = crm_element_value(action_rsc, name);
	crm_xml_add(rsc, name, value);
	name = XML_AGENT_ATTR_PROVIDER;
	value = crm_element_value(action_rsc, name);
	crm_xml_add(rsc, name, value);

	op = convert_graph_action(NULL, action, status, op_rc);
	op->call_id = -1;
	op->user_data = generate_transition_key(transition_graph->id, action->id, target_rc, te_uuid);

	- xml_op = create_operation_update(rsc, op, CRM_FEATURE_SET, target_rc, __FUNCTION__, LOG_INFO);
	- crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, target); /* For context during triage */
	+ xml_op = create_operation_update(rsc, op, CRM_FEATURE_SET, target_rc, target, __FUNCTION__, LOG_INFO);
	lrmd_free_event(op);

	crm_trace("Updating CIB with \"%s\" (%s): %s %s on %s",
	status < 0 ? "new action" : XML_ATTR_TIMEOUT,
	crm_element_name(action->xml), crm_str(task), rsc_id, target);
	crm_log_xml_trace(xml_op, "Op");

	rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);

	crm_trace("Updating CIB with %s action %d: %s on %s (call_id=%d)",
	services_lrm_status_str(status), action->id, task_uuid, target, rc);

	fsa_register_cib_callback(rc, FALSE, NULL, cib_action_updated);
	free_xml(state);

	action->sent_update = TRUE;

	if (rc < pcmk_ok) {
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	te_rsc_command(crm_graph_t * graph, crm_action_t * action)
	{
	/* never overwrite stop actions in the CIB with
	* anything other than completed results
	*
	* Writing pending stops makes it look like the
	* resource is running again
	*/
	xmlNode *cmd = NULL;
	xmlNode *rsc_op = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;
	gboolean is_local = FALSE;

	char *counter = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;
	const char *task_uuid = NULL;

	CRM_ASSERT(action != NULL);
	CRM_ASSERT(action->xml != NULL);

	action->executed = FALSE;
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
	crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), crm_str(task));
	return FALSE);

	rsc_op = action->xml;
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	}

	counter =
	generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
	crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);

	if (safe_str_eq(router_node, fsa_our_uname)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_notice("Initiating action %d: %s %s on %s%s%s",
	action->id, task, task_uuid, on_node,
	is_local ? " (local)" : "", no_wait ? " - no waiting" : "");

	cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
	CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);

	if (is_local) {
	/* shortcut local resource commands */
	ha_msg_input_t data = {
	.msg = cmd,
	.xml = rsc_op,
	};

	fsa_data_t msg = {
	.id = 0,
	.data = &data,
	.data_type = fsa_dt_ha_msg,
	.fsa_input = I_NULL,
	.fsa_cause = C_FSA_INTERNAL,
	.actions = A_LRM_INVOKE,
	.origin = __FUNCTION__,
	};

	do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);

	} else {
	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
	}

	free(counter);
	free_xml(cmd);

	action->executed = TRUE;

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return FALSE;

	} else if (no_wait) {
	crm_info("Action %d confirmed - no wait", action->id);
	action->confirmed = TRUE; /* Just mark confirmed.
	* Don't bump the job count only to immediately decrement it
	*/
	update_graph(transition_graph, action);
	trigger_graph();

	} else if (action->confirmed == TRUE) {
	crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
	action->id, task, task_uuid, on_node, action->timeout);
	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %dms instead",
	action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
	action->timeout = graph->network_delay;
	}
	te_update_job_count(action, 1);
	te_start_action_timer(graph, action);
	}

	value = crm_meta_value(action->params, XML_OP_ATTR_PENDING);
	if (crm_is_true(value)
	&& safe_str_neq(task, CRMD_ACTION_CANCEL)
	&& safe_str_neq(task, CRMD_ACTION_DELETE)) {
	/* write a "pending" entry to the CIB, inhibit notification */
	crm_debug("Recording pending op %s in the CIB", task_uuid);
	cib_action_update(action, PCMK_LRM_OP_PENDING, PCMK_OCF_UNKNOWN);
	}

	return TRUE;
	}

	struct te_peer_s
	{
	char *name;
	int jobs;
	int migrate_jobs;
	};

	static void te_peer_free(gpointer p)
	{
	struct te_peer_s *peer = p;

	free(peer->name);
	free(peer);
	}

	void te_reset_job_counts(void)
	{
	GHashTableIter iter;
	struct te_peer_s *peer = NULL;

	if(te_targets == NULL) {
	te_targets = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, te_peer_free);
	}

	g_hash_table_iter_init(&iter, te_targets);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
	peer->jobs = 0;
	peer->migrate_jobs = 0;
	}
	}

	static void
	te_update_job_count_on(const char *target, int offset, bool migrate)
	{
	struct te_peer_s *r = NULL;

	if(target == NULL \|\| te_targets == NULL) {
	return;
	}

	r = g_hash_table_lookup(te_targets, target);
	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	r->jobs += offset;
	if(migrate) {
	r->migrate_jobs += offset;
	}
	crm_trace("jobs[%s] = %d", target, r->jobs);
	}

	static void
	te_update_job_count(crm_action_t * action, int offset)
	{
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	if (action->type != action_type_rsc \|\| target == NULL) {
	/* No limit on these */
	return;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all action occuring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL) &&
	(safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED))) {

	const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	te_update_job_count_on(t1, offset, TRUE);
	te_update_job_count_on(t2, offset, TRUE);
	return;
	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	te_update_job_count_on(target, offset, FALSE);
	}

	static gboolean
	te_should_perform_action_on(crm_graph_t * graph, crm_action_t * action, const char *target)
	{
	int limit = 0;
	struct te_peer_s *r = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	if(target == NULL) {
	/* No limit on these */
	return TRUE;

	} else if(te_targets == NULL) {
	return FALSE;
	}

	r = g_hash_table_lookup(te_targets, target);
	limit = throttle_get_job_limit(target);

	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	if(limit <= r->jobs) {
	crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
	target, limit, r->jobs, id);
	return FALSE;

	} else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
	if (safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
	target, graph->migration_limit, r->migrate_jobs, id);
	return FALSE;
	}
	}

	crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);

	return TRUE;
	}

	static gboolean
	te_should_perform_action(crm_graph_t * graph, crm_action_t * action)
	{
	const char *target = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);

	if (action->type != action_type_rsc) {
	/* No limit on these */
	return TRUE;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all action occuring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL) &&
	(safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED))) {

	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	if(te_should_perform_action_on(graph, action, target) == FALSE) {
	return FALSE;
	}

	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	return te_should_perform_action_on(graph, action, target);
	}

	void
	te_action_confirmed(crm_action_t * action)
	{
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	if (action->confirmed == FALSE && action->type == action_type_rsc && target != NULL) {
	te_update_job_count(action, -1);
	}
	action->confirmed = TRUE;
	}


	crm_graph_functions_t te_graph_fns = {
	te_pseudo_action,
	te_rsc_command,
	te_crm_command,
	te_fence_node,
	te_should_perform_action,
	};

	void
	notify_crmd(crm_graph_t * graph)
	{
	const char *type = "unknown";
	enum crmd_fsa_input event = I_NULL;

	crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));

	CRM_CHECK(graph->complete, graph->complete = TRUE);

	switch (graph->completion_action) {
	case tg_stop:
	type = "stop";
	if (fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;
	case tg_done:
	type = "done";
	if (fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;

	case tg_restart:
	type = "restart";
	if (fsa_state == S_TRANSITION_ENGINE) {
	if (too_many_st_failures() == FALSE) {
	if (transition_timer->period_ms > 0) {
	crm_timer_stop(transition_timer);
	crm_timer_start(transition_timer);
	} else {
	event = I_PE_CALC;
	}
	} else {
	event = I_TE_SUCCESS;
	}

	} else if (fsa_state == S_POLICY_ENGINE) {
	register_fsa_action(A_PE_INVOKE);
	}
	break;

	case tg_shutdown:
	type = "shutdown";
	if (is_set(fsa_input_register, R_SHUTDOWN)) {
	event = I_STOP;

	} else {
	crm_err("We didn't ask to be shut down, yet our" " PE is telling us too.");
	event = I_TERMINATE;
	}
	}

	crm_debug("Transition %d status: %s - %s", graph->id, type, crm_str(graph->abort_reason));

	graph->abort_reason = NULL;
	graph->completion_action = tg_done;
	clear_bit(fsa_input_register, R_IN_TRANSITION);

	if (event != I_NULL) {
	register_fsa_input(C_FSA_INTERNAL, event, NULL);

	} else if (fsa_source) {
	mainloop_set_trigger(fsa_source);
	}
	}
	diff --git a/include/crm_internal.h b/include/crm_internal.h
	index 169d6d3a66..aa6271d8aa 100644
	--- a/include/crm_internal.h
	+++ b/include/crm_internal.h
	@@ -1,356 +1,356 @@
	/* crm_internal.h */

	/*
	* Copyright (C) 2006 - 2008
	* Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License as published by
	* the Free Software Foundation; either version 2 of the License, or
	* (at your option) any later version.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#ifndef CRM_INTERNAL__H
	# define CRM_INTERNAL__H

	# include <config.h>
	# include <portability.h>

	# include <glib.h>
	# include <stdbool.h>
	# include <libxml/tree.h>

	# include <crm/lrmd.h>
	# include <crm/common/logging.h>
	# include <crm/common/ipcs.h>

	/* Dynamic loading of libraries */
	void find_library_function(void handle, const char lib, const char *fn, int fatal);
	void convert_const_pointer(const void ptr);

	/* For ACLs */
	char *uid2username(uid_t uid);
	void determine_request_user(const char user, xmlNode request, const char *field);
	const char crm_acl_get_set_user(xmlNode request, const char field, const char peer_user);

	# if ENABLE_ACL
	# include <string.h>
	static inline gboolean
	is_privileged(const char *user)
	{
	if (user == NULL) {
	return FALSE;
	} else if (strcmp(user, CRM_DAEMON_USER) == 0) {
	return TRUE;
	} else if (strcmp(user, "root") == 0) {
	return TRUE;
	}
	return FALSE;
	}
	# endif

	/* CLI option processing*/
	# ifdef HAVE_GETOPT_H
	# include <getopt.h>
	# else
	# define no_argument 0
	# define required_argument 1
	# endif

	# define pcmk_option_default 0x00000
	# define pcmk_option_hidden 0x00001
	# define pcmk_option_paragraph 0x00002
	# define pcmk_option_example 0x00004

	struct crm_option {
	/* Fields from 'struct option' in getopt.h */
	/* name of long option */
	const char *name;
	/*
	* one of no_argument, required_argument, and optional_argument:
	* whether option takes an argument
	*/
	int has_arg;
	/* if not NULL, set flag to val when option found /
	int *flag;
	/* if flag not NULL, value to set flag to; else return value /
	int val;

	/* Custom fields */
	const char *desc;
	long flags;
	};

	void crm_set_options(const char short_options, const char usage, struct crm_option *long_options,
	const char *app_desc);
	int crm_get_option(int argc, char *argv, int index);
	int crm_get_option_long(int argc, char *argv, int index, const char **longname);
	int crm_help(char cmd, int exit_code);

	/* Cluster Option Processing */
	typedef struct pe_cluster_option_s {
	const char *name;
	const char *alt_name;
	const char *type;
	const char *values;
	const char *default_value;

	gboolean(is_valid) (const char );

	const char *description_short;
	const char *description_long;

	} pe_cluster_option;

	const char cluster_option(GHashTable options, gboolean(validate) (const char ),
	const char name, const char old_name, const char *def_value);

	const char get_cluster_pref(GHashTable options, pe_cluster_option * option_list, int len,
	const char *name);

	void config_metadata(const char name, const char version, const char *desc_short,
	const char desc_long, pe_cluster_option option_list, int len);

	void verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len);
	gboolean check_time(const char *value);
	gboolean check_timer(const char *value);
	gboolean check_boolean(const char *value);
	gboolean check_number(const char *value);
	gboolean check_quorum(const char *value);
	gboolean check_utilization(const char *value);

	/* Shared PE/crmd functionality */
	void filter_action_parameters(xmlNode * param_set, const char *version);
	void filter_reload_parameters(xmlNode * param_set, const char *restart_string);

	/* Resource operation updates */
	xmlNode create_operation_update(xmlNode parent, lrmd_event_data_t * event,
	- const char caller_version, int target_rc, const char origin,
	- int level);
	+ const char * caller_version, int target_rc, const char * node,
	+ const char * origin, int level);

	/* char2score */
	extern int node_score_red;
	extern int node_score_green;
	extern int node_score_yellow;
	extern int node_score_infinity;

	/* Assorted convenience functions */
	static inline int
	crm_strlen_zero(const char *s)
	{
	return !s \|\| *s == '\0';
	}

	char add_list_element(char list, const char *value);
	char generate_series_filename(const char directory, const char *series, int sequence,
	gboolean bzip);
	int get_last_sequence(const char directory, const char series);
	void write_last_sequence(const char directory, const char series, int sequence, int max);

	int crm_pid_active(long pid);
	void crm_make_daemon(const char name, gboolean daemonize, const char pidfile);
	gboolean crm_is_writable(const char dir, const char file, const char user, const char group,
	gboolean need_both);

	char generate_op_key(const char rsc_id, const char *op_type, int interval);
	char generate_notify_key(const char rsc_id, const char notify_type, const char op_type);
	char generate_transition_magic_v202(const char transition_key, int op_status);
	char generate_transition_magic(const char transition_key, int op_status, int op_rc);
	char generate_transition_key(int action, int transition_id, int target_rc, const char node);

	static inline long long
	crm_clear_bit(const char function, const char target, long long word, long long bit)
	{
	long long rc = (word & ~bit);

	if (rc == word) {
	/* Unchanged */
	} else if (target) {
	crm_trace("Bit 0x%.8llx for %s cleared by %s", bit, target, function);
	} else {
	crm_trace("Bit 0x%.8llx cleared by %s", bit, function);
	}

	return rc;
	}

	static inline long long
	crm_set_bit(const char function, const char target, long long word, long long bit)
	{
	long long rc = (word \| bit);

	if (rc == word) {
	/* Unchanged */
	} else if (target) {
	crm_trace("Bit 0x%.8llx for %s set by %s", bit, target, function);
	} else {
	crm_trace("Bit 0x%.8llx set by %s", bit, function);
	}

	return rc;
	}

	# define set_bit(word, bit) word = crm_set_bit(__FUNCTION__, NULL, word, bit)
	# define clear_bit(word, bit) word = crm_clear_bit(__FUNCTION__, NULL, word, bit)

	void g_hash_destroy_str(gpointer data);

	long long crm_int_helper(const char text, char *end_text);
	char crm_concat(const char prefix, const char *suffix, char join);
	char generate_hash_key(const char crm_msg_reference, const char *sys);

	bool crm_compress_string(const char data, int length, int max, char *result,
	unsigned int *result_len);

	/! remote tcp/tls helper functions /
	typedef struct crm_remote_s crm_remote_t;

	int crm_remote_send(crm_remote_t * remote, xmlNode * msg);
	int crm_remote_ready(crm_remote_t * remote, int total_timeout /ms / );
	gboolean crm_remote_recv(crm_remote_t * remote, int total_timeout /ms / , int *disconnected);
	xmlNode crm_remote_parse_buffer(crm_remote_t remote);
	int crm_remote_tcp_connect(const char *host, int port);
	int crm_remote_tcp_connect_async(const char host, int port, int timeout, /ms */
	int timer_id, void userdata, void (callback) (void userdata, int sock));

	# ifdef HAVE_GNUTLS_GNUTLS_H
	/*!
	* \internal
	* \brief Initiate the client handshake after establishing the tcp socket.
	* \note This is a blocking function, it will block until the entire handshake
	* is complete or until the timeout period is reached.
	* \retval 0 success
	* \retval negative, failure
	*/
	int crm_initiate_client_tls_handshake(crm_remote_t * remote, int timeout_ms);

	/*!
	* \internal
	* \brief Create client or server session for anon DH encryption credentials
	* \param sock, the socket the session will use for transport
	* \param type, GNUTLS_SERVER or GNUTLS_CLIENT
	* \param credentials, gnutls_anon_server_credentials_t or gnutls_anon_client_credentials_t
	*
	* \retval gnutls_session_t * on success
	* \retval NULL on failure
	*/
	void crm_create_anon_tls_session(int sock, int type, void credentials);

	/*!
	* \internal
	* \brief Create client or server session for PSK credentials
	* \param sock, the socket the session will use for transport
	* \param type, GNUTLS_SERVER or GNUTLS_CLIENT
	* \param credentials, gnutls_psk_server_credentials_t or gnutls_osk_client_credentials_t
	*
	* \retval gnutls_session_t * on success
	* \retval NULL on failure
	*/
	void create_psk_tls_session(int csock, int type, void credentials);
	# endif

	# define REMOTE_MSG_TERMINATOR "\r\n\r\n"

	const char daemon_option(const char option);
	void set_daemon_option(const char option, const char value);
	gboolean daemon_option_enabled(const char daemon, const char option);
	void strip_text_nodes(xmlNode * xml);
	void pcmk_panic(const char *origin);
	void sysrq_init(void);
	pid_t pcmk_locate_sbd(void);
	int crm_pidfile_inuse(const char *filename, long mypid);
	int crm_read_pidfile(const char *filename);

	# define crm_config_err(fmt...) { crm_config_error = TRUE; crm_err(fmt); }
	# define crm_config_warn(fmt...) { crm_config_warning = TRUE; crm_warn(fmt); }

	# define attrd_channel T_ATTRD
	# define F_ATTRD_KEY "attr_key"
	# define F_ATTRD_ATTRIBUTE "attr_name"
	# define F_ATTRD_REGEX "attr_regex"
	# define F_ATTRD_TASK "task"
	# define F_ATTRD_VALUE "attr_value"
	# define F_ATTRD_SET "attr_set"
	# define F_ATTRD_IS_REMOTE "attr_is_remote"
	# define F_ATTRD_SECTION "attr_section"
	# define F_ATTRD_DAMPEN "attr_dampening"
	# define F_ATTRD_IGNORE_LOCALLY "attr_ignore_locally"
	# define F_ATTRD_HOST "attr_host"
	# define F_ATTRD_HOST_ID "attr_host_id"
	# define F_ATTRD_USER "attr_user"
	# define F_ATTRD_WRITER "attr_writer"
	# define F_ATTRD_VERSION "attr_version"

	# if SUPPORT_COROSYNC
	# if CS_USES_LIBQB
	# include <qb/qbipc_common.h>
	# include <corosync/corotypes.h>
	typedef struct qb_ipc_request_header cs_ipc_header_request_t;
	typedef struct qb_ipc_response_header cs_ipc_header_response_t;
	# else
	# include <corosync/corodefs.h>
	# include <corosync/coroipcc.h>
	# include <corosync/coroipc_types.h>
	static inline int
	qb_to_cs_error(int a)
	{
	return a;
	}

	typedef coroipc_request_header_t cs_ipc_header_request_t;
	typedef coroipc_response_header_t cs_ipc_header_response_t;
	# endif
	# else
	typedef struct {
	int size __attribute__ ((aligned(8)));
	int id __attribute__ ((aligned(8)));
	} __attribute__ ((aligned(8))) cs_ipc_header_request_t;

	typedef struct {
	int size __attribute__ ((aligned(8)));
	int id __attribute__ ((aligned(8)));
	int error __attribute__ ((aligned(8)));
	} __attribute__ ((aligned(8))) cs_ipc_header_response_t;

	# endif

	void
	attrd_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb);
	void
	stonith_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb);

	qb_ipcs_service_t *
	crmd_ipc_server_init(struct qb_ipcs_service_handlers *cb);

	void cib_ipc_servers_init(qb_ipcs_service_t **ipcs_ro,
	qb_ipcs_service_t **ipcs_rw,
	qb_ipcs_service_t **ipcs_shm,
	struct qb_ipcs_service_handlers *ro_cb,
	struct qb_ipcs_service_handlers *rw_cb);

	void cib_ipc_servers_destroy(qb_ipcs_service_t *ipcs_ro,
	qb_ipcs_service_t *ipcs_rw,
	qb_ipcs_service_t *ipcs_shm);

	static inline void realloc_safe(void ptr, size_t size)
	{
	void *ret = realloc(ptr, size);

	if(ret == NULL) {
	abort();
	}

	return ret;
	}


	#endif /* CRM_INTERNAL__H */
	diff --git a/lib/common/utils.c b/lib/common/utils.c
	index d70778d67c..a2e39cd8f0 100644
	--- a/lib/common/utils.c
	+++ b/lib/common/utils.c
	@@ -1,2635 +1,2636 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <dlfcn.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <sys/utsname.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <ctype.h>
	#include <pwd.h>
	#include <grp.h>
	#include <time.h>
	#include <libgen.h>
	#include <signal.h>

	#include <qb/qbdefs.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/cib/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/common/ipc.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/mainloop.h>
	#include <crm/attrd.h>
	#include <libxml2/libxml/relaxng.h>

	#ifndef MAXLINE
	# define MAXLINE 512
	#endif

	#ifdef HAVE_GETOPT_H
	# include <getopt.h>
	#endif

	#ifndef PW_BUFFER_LEN
	# define PW_BUFFER_LEN 500
	#endif

	CRM_TRACE_INIT_DATA(common);

	gboolean crm_config_error = FALSE;
	gboolean crm_config_warning = FALSE;
	char *crm_system_name = NULL;

	int node_score_red = 0;
	int node_score_green = 0;
	int node_score_yellow = 0;
	int node_score_infinity = INFINITY;

	static struct crm_option *crm_long_options = NULL;
	static const char *crm_app_description = NULL;
	static char *crm_short_options = NULL;
	static const char *crm_app_usage = NULL;

	int
	crm_exit(int rc)
	{
	mainloop_cleanup();

	#if HAVE_LIBXML2
	crm_trace("cleaning up libxml");
	crm_xml_cleanup();
	#endif

	crm_trace("exit %d", rc);
	qb_log_fini();

	free(crm_short_options);
	free(crm_system_name);

	exit(ABS(rc)); /* Always exit with a positive value so that it can be passed to crm_error
	*
	* Otherwise the system wraps it around and people
	* have to jump through hoops figuring out what the
	* error was
	*/
	return rc; /* Can never happen, but allows return crm_exit(rc)
	* where "return rc" was used previously - which
	* keeps compilers happy.
	*/
	}

	gboolean
	check_time(const char *value)
	{
	if (crm_get_msec(value) < 5000) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_timer(const char *value)
	{
	if (crm_get_msec(value) < 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_boolean(const char *value)
	{
	int tmp = FALSE;

	if (crm_str_to_boolean(value, &tmp) != 1) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_number(const char *value)
	{
	errno = 0;
	if (value == NULL) {
	return FALSE;

	} else if (safe_str_eq(value, MINUS_INFINITY_S)) {

	} else if (safe_str_eq(value, INFINITY_S)) {

	} else {
	crm_int_helper(value, NULL);
	}

	if (errno != 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_quorum(const char *value)
	{
	if (safe_str_eq(value, "stop")) {
	return TRUE;

	} else if (safe_str_eq(value, "freeze")) {
	return TRUE;

	} else if (safe_str_eq(value, "ignore")) {
	return TRUE;

	} else if (safe_str_eq(value, "suicide")) {
	return TRUE;
	}
	return FALSE;
	}

	gboolean
	check_utilization(const char *value)
	{
	char *end = NULL;
	long number = strtol(value, &end, 10);

	if(end && end[0] != '%') {
	return FALSE;
	} else if(number < 0) {
	return FALSE;
	}

	return TRUE;
	}

	int
	char2score(const char *score)
	{
	int score_f = 0;

	if (score == NULL) {

	} else if (safe_str_eq(score, MINUS_INFINITY_S)) {
	score_f = -node_score_infinity;

	} else if (safe_str_eq(score, INFINITY_S)) {
	score_f = node_score_infinity;

	} else if (safe_str_eq(score, "+" INFINITY_S)) {
	score_f = node_score_infinity;

	} else if (safe_str_eq(score, "red")) {
	score_f = node_score_red;

	} else if (safe_str_eq(score, "yellow")) {
	score_f = node_score_yellow;

	} else if (safe_str_eq(score, "green")) {
	score_f = node_score_green;

	} else {
	score_f = crm_parse_int(score, NULL);
	if (score_f > 0 && score_f > node_score_infinity) {
	score_f = node_score_infinity;

	} else if (score_f < 0 && score_f < -node_score_infinity) {
	score_f = -node_score_infinity;
	}
	}

	return score_f;
	}

	char *
	score2char_stack(int score, char *buf, size_t len)
	{
	if (score >= node_score_infinity) {
	strncpy(buf, INFINITY_S, 9);
	} else if (score <= -node_score_infinity) {
	strncpy(buf, MINUS_INFINITY_S , 10);
	} else {
	return crm_itoa_stack(score, buf, len);
	}

	return buf;
	}

	char *
	score2char(int score)
	{
	if (score >= node_score_infinity) {
	return strdup(INFINITY_S);

	} else if (score <= -node_score_infinity) {
	return strdup("-" INFINITY_S);
	}
	return crm_itoa(score);
	}

	const char *
	cluster_option(GHashTable * options, gboolean(validate) (const char ),
	const char name, const char old_name, const char *def_value)
	{
	const char *value = NULL;

	CRM_ASSERT(name != NULL);

	if (options != NULL) {
	value = g_hash_table_lookup(options, name);
	}

	if (value == NULL && old_name && options != NULL) {
	value = g_hash_table_lookup(options, old_name);
	if (value != NULL) {
	crm_config_warn("Using deprecated name '%s' for"
	" cluster option '%s'", old_name, name);
	g_hash_table_insert(options, strdup(name), strdup(value));
	value = g_hash_table_lookup(options, old_name);
	}
	}

	if (value == NULL) {
	crm_trace("Using default value '%s' for cluster option '%s'", def_value, name);

	if (options == NULL) {
	return def_value;

	} else if(def_value == NULL) {
	return def_value;
	}

	g_hash_table_insert(options, strdup(name), strdup(def_value));
	value = g_hash_table_lookup(options, name);
	}

	if (validate && validate(value) == FALSE) {
	crm_config_err("Value '%s' for cluster option '%s' is invalid."
	" Defaulting to %s", value, name, def_value);
	g_hash_table_replace(options, strdup(name), strdup(def_value));
	value = g_hash_table_lookup(options, name);
	}

	return value;
	}

	const char *
	get_cluster_pref(GHashTable * options, pe_cluster_option * option_list, int len, const char *name)
	{
	int lpc = 0;
	const char *value = NULL;
	gboolean found = FALSE;

	for (lpc = 0; lpc < len; lpc++) {
	if (safe_str_eq(name, option_list[lpc].name)) {
	found = TRUE;
	value = cluster_option(options,
	option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name, option_list[lpc].default_value);
	}
	}
	CRM_CHECK(found, crm_err("No option named: %s", name));
	return value;
	}

	void
	config_metadata(const char name, const char version, const char *desc_short,
	const char desc_long, pe_cluster_option option_list, int len)
	{
	int lpc = 0;

	fprintf(stdout, "<?xml version=\"1.0\"?>"
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
	"<resource-agent name=\"%s\">\n"
	" <version>%s</version>\n"
	" <longdesc lang=\"en\">%s</longdesc>\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	" <parameters>\n", name, version, desc_long, desc_short);

	for (lpc = 0; lpc < len; lpc++) {
	if (option_list[lpc].description_long == NULL && option_list[lpc].description_short == NULL) {
	continue;
	}
	fprintf(stdout, " <parameter name=\"%s\" unique=\"0\">\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	" <content type=\"%s\" default=\"%s\"/>\n"
	" <longdesc lang=\"en\">%s%s%s</longdesc>\n"
	" </parameter>\n",
	option_list[lpc].name,
	option_list[lpc].description_short,
	option_list[lpc].type,
	option_list[lpc].default_value,
	option_list[lpc].description_long ? option_list[lpc].
	description_long : option_list[lpc].description_short,
	option_list[lpc].values ? " Allowed values: " : "",
	option_list[lpc].values ? option_list[lpc].values : "");
	}
	fprintf(stdout, " </parameters>\n</resource-agent>\n");
	}

	void
	verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len)
	{
	int lpc = 0;

	for (lpc = 0; lpc < len; lpc++) {
	cluster_option(options,
	option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name, option_list[lpc].default_value);
	}
	}

	char *
	crm_concat(const char prefix, const char suffix, char join)
	{
	int len = 0;
	char *new_str = NULL;

	CRM_ASSERT(prefix != NULL);
	CRM_ASSERT(suffix != NULL);
	len = strlen(prefix) + strlen(suffix) + 2;

	new_str = malloc(len);
	if(new_str) {
	sprintf(new_str, "%s%c%s", prefix, join, suffix);
	new_str[len - 1] = 0;
	}
	return new_str;
	}

	char *
	generate_hash_key(const char crm_msg_reference, const char sys)
	{
	char *hash_key = crm_concat(sys ? sys : "none", crm_msg_reference, '_');

	crm_trace("created hash key: (%s)", hash_key);
	return hash_key;
	}


	char *
	crm_itoa_stack(int an_int, char *buffer, size_t len)
	{
	if (buffer != NULL) {
	snprintf(buffer, len, "%d", an_int);
	}

	return buffer;
	}

	char *
	crm_itoa(int an_int)
	{
	int len = 32;
	char *buffer = NULL;

	buffer = malloc(len + 1);
	if (buffer != NULL) {
	snprintf(buffer, len, "%d", an_int);
	}

	return buffer;
	}

	void
	crm_build_path(const char *path_c, mode_t mode)
	{
	int offset = 1, len = 0;
	char *path = strdup(path_c);

	CRM_CHECK(path != NULL, return);
	for (len = strlen(path); offset < len; offset++) {
	if (path[offset] == '/') {
	path[offset] = 0;
	if (mkdir(path, mode) < 0 && errno != EEXIST) {
	crm_perror(LOG_ERR, "Could not create directory '%s'", path);
	break;
	}
	path[offset] = '/';
	}
	}
	if (mkdir(path, mode) < 0 && errno != EEXIST) {
	crm_perror(LOG_ERR, "Could not create directory '%s'", path);
	}

	free(path);
	}

	int
	crm_user_lookup(const char name, uid_t uid, gid_t * gid)
	{
	int rc = -1;
	char *buffer = NULL;
	struct passwd pwd;
	struct passwd *pwentry = NULL;

	buffer = calloc(1, PW_BUFFER_LEN);
	getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry);
	if (pwentry) {
	rc = 0;
	if (uid) {
	*uid = pwentry->pw_uid;
	}
	if (gid) {
	*gid = pwentry->pw_gid;
	}
	crm_trace("Cluster user %s has uid=%d gid=%d", name, pwentry->pw_uid, pwentry->pw_gid);

	} else {
	crm_err("Cluster user %s does not exist", name);
	}

	free(buffer);
	return rc;
	}

	static int
	crm_version_helper(const char text, char *end_text)
	{
	int atoi_result = -1;

	CRM_ASSERT(end_text != NULL);

	errno = 0;

	if (text != NULL && text[0] != 0) {
	atoi_result = (int)strtol(text, end_text, 10);

	if (errno == EINVAL) {
	crm_err("Conversion of '%s' %c failed", text, text[0]);
	atoi_result = -1;
	}
	}
	return atoi_result;
	}

	/*
	* version1 < version2 : -1
	* version1 = version2 : 0
	* version1 > version2 : 1
	*/
	int
	compare_version(const char version1, const char version2)
	{
	int rc = 0;
	int lpc = 0;
	char ver1_copy = NULL, ver2_copy = NULL;
	char rest1 = NULL, rest2 = NULL;

	if (version1 == NULL && version2 == NULL) {
	return 0;
	} else if (version1 == NULL) {
	return -1;
	} else if (version2 == NULL) {
	return 1;
	}

	ver1_copy = strdup(version1);
	ver2_copy = strdup(version2);
	rest1 = ver1_copy;
	rest2 = ver2_copy;

	while (1) {
	int digit1 = 0;
	int digit2 = 0;

	lpc++;

	if (rest1 == rest2) {
	break;
	}

	if (rest1 != NULL) {
	digit1 = crm_version_helper(rest1, &rest1);
	}

	if (rest2 != NULL) {
	digit2 = crm_version_helper(rest2, &rest2);
	}

	if (digit1 < digit2) {
	rc = -1;
	break;

	} else if (digit1 > digit2) {
	rc = 1;
	break;
	}

	if (rest1 != NULL && rest1[0] == '.') {
	rest1++;
	}
	if (rest1 != NULL && rest1[0] == 0) {
	rest1 = NULL;
	}

	if (rest2 != NULL && rest2[0] == '.') {
	rest2++;
	}
	if (rest2 != NULL && rest2[0] == 0) {
	rest2 = NULL;
	}
	}

	free(ver1_copy);
	free(ver2_copy);

	if (rc == 0) {
	crm_trace("%s == %s (%d)", version1, version2, lpc);
	} else if (rc < 0) {
	crm_trace("%s < %s (%d)", version1, version2, lpc);
	} else if (rc > 0) {
	crm_trace("%s > %s (%d)", version1, version2, lpc);
	}

	return rc;
	}

	gboolean do_stderr = FALSE;

	void
	g_hash_destroy_str(gpointer data)
	{
	free(data);
	}

	#include <sys/types.h>
	/* #include <stdlib.h> */
	/* #include <limits.h> */

	long long
	crm_int_helper(const char text, char *end_text)
	{
	long long result = -1;
	char *local_end_text = NULL;
	int saved_errno = 0;

	errno = 0;

	if (text != NULL) {
	#ifdef ANSI_ONLY
	if (end_text != NULL) {
	result = strtol(text, end_text, 10);
	} else {
	result = strtol(text, &local_end_text, 10);
	}
	#else
	if (end_text != NULL) {
	result = strtoll(text, end_text, 10);
	} else {
	result = strtoll(text, &local_end_text, 10);
	}
	#endif

	saved_errno = errno;
	/* CRM_CHECK(errno != EINVAL); */
	if (errno == EINVAL) {
	crm_err("Conversion of %s failed", text);
	result = -1;

	} else if (errno == ERANGE) {
	crm_err("Conversion of %s was clipped: %lld", text, result);

	} else if (errno != 0) {
	crm_perror(LOG_ERR, "Conversion of %s failed:", text);
	}

	if (local_end_text != NULL && local_end_text[0] != '\0') {
	crm_err("Characters left over after parsing '%s': '%s'", text, local_end_text);
	}

	errno = saved_errno;
	}
	return result;
	}

	int
	crm_parse_int(const char text, const char default_text)
	{
	int atoi_result = -1;

	if (text != NULL) {
	atoi_result = crm_int_helper(text, NULL);
	if (errno == 0) {
	return atoi_result;
	}
	}

	if (default_text != NULL) {
	atoi_result = crm_int_helper(default_text, NULL);
	if (errno == 0) {
	return atoi_result;
	}

	} else {
	crm_err("No default conversion value supplied");
	}

	return -1;
	}

	gboolean
	safe_str_neq(const char a, const char b)
	{
	if (a == b) {
	return FALSE;

	} else if (a == NULL \|\| b == NULL) {
	return TRUE;

	} else if (strcasecmp(a, b) == 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	crm_is_true(const char *s)
	{
	gboolean ret = FALSE;

	if (s != NULL) {
	crm_str_to_boolean(s, &ret);
	}
	return ret;
	}

	int
	crm_str_to_boolean(const char s, int ret)
	{
	if (s == NULL) {
	return -1;

	} else if (strcasecmp(s, "true") == 0
	\|\| strcasecmp(s, "on") == 0
	\|\| strcasecmp(s, "yes") == 0 \|\| strcasecmp(s, "y") == 0 \|\| strcasecmp(s, "1") == 0) {
	*ret = TRUE;
	return 1;

	} else if (strcasecmp(s, "false") == 0
	\|\| strcasecmp(s, "off") == 0
	\|\| strcasecmp(s, "no") == 0 \|\| strcasecmp(s, "n") == 0 \|\| strcasecmp(s, "0") == 0) {
	*ret = FALSE;
	return 1;
	}
	return -1;
	}

	#ifndef NUMCHARS
	# define NUMCHARS "0123456789."
	#endif

	#ifndef WHITESPACE
	# define WHITESPACE " \t\n\r\f"
	#endif

	unsigned long long
	crm_get_interval(const char *input)
	{
	unsigned long long msec = 0;

	if (input == NULL) {
	return msec;

	} else if (input[0] != 'P') {
	long long tmp = crm_get_msec(input);

	if(tmp > 0) {
	msec = tmp;
	}

	} else {
	crm_time_t *interval = crm_time_parse_duration(input);

	msec = 1000 * crm_time_get_seconds(interval);
	crm_time_free(interval);
	}

	return msec;
	}

	long long
	crm_get_msec(const char *input)
	{
	const char *cp = input;
	const char *units;
	long long multiplier = 1000;
	long long divisor = 1;
	long long msec = -1;
	char *end_text = NULL;

	/* double dret; */

	if (input == NULL) {
	return msec;
	}

	cp += strspn(cp, WHITESPACE);
	units = cp + strspn(cp, NUMCHARS);
	units += strspn(units, WHITESPACE);

	if (strchr(NUMCHARS, *cp) == NULL) {
	return msec;
	}

	if (strncasecmp(units, "ms", 2) == 0 \|\| strncasecmp(units, "msec", 4) == 0) {
	multiplier = 1;
	divisor = 1;
	} else if (strncasecmp(units, "us", 2) == 0 \|\| strncasecmp(units, "usec", 4) == 0) {
	multiplier = 1;
	divisor = 1000;
	} else if (strncasecmp(units, "s", 1) == 0 \|\| strncasecmp(units, "sec", 3) == 0) {
	multiplier = 1000;
	divisor = 1;
	} else if (strncasecmp(units, "m", 1) == 0 \|\| strncasecmp(units, "min", 3) == 0) {
	multiplier = 60 * 1000;
	divisor = 1;
	} else if (strncasecmp(units, "h", 1) == 0 \|\| strncasecmp(units, "hr", 2) == 0) {
	multiplier = 60 * 60 * 1000;
	divisor = 1;
	} else if (units != EOS && units != '\n' && *units != '\r') {
	return msec;
	}

	msec = crm_int_helper(cp, &end_text);
	if (msec > LLONG_MAX/multiplier) {
	/* arithmetics overflow while multiplier/divisor mutually exclusive */
	return LLONG_MAX;
	}
	msec *= multiplier;
	msec /= divisor;
	/* dret += 0.5; */
	/* msec = (long long)dret; */
	return msec;
	}

	char *
	generate_op_key(const char rsc_id, const char op_type, int interval)
	{
	int len = 35;
	char *op_id = NULL;

	CRM_CHECK(rsc_id != NULL, return NULL);
	CRM_CHECK(op_type != NULL, return NULL);

	len += strlen(op_type);
	len += strlen(rsc_id);
	op_id = malloc(len);
	CRM_CHECK(op_id != NULL, return NULL);
	sprintf(op_id, "%s_%s_%d", rsc_id, op_type, interval);
	return op_id;
	}

	gboolean
	parse_op_key(const char key, char rsc_id, char op_type, int interval)
	{
	char *notify = NULL;
	char *mutable_key = NULL;
	char *mutable_key_ptr = NULL;
	int len = 0, offset = 0, ch = 0;

	CRM_CHECK(key != NULL, return FALSE);

	*interval = 0;
	len = strlen(key);
	offset = len - 1;

	crm_trace("Source: %s", key);

	while (offset > 0 && isdigit(key[offset])) {
	int digits = len - offset;

	ch = key[offset] - '0';
	CRM_CHECK(ch < 10, return FALSE);
	CRM_CHECK(ch >= 0, return FALSE);
	while (digits > 1) {
	digits--;
	ch = ch * 10;
	}
	*interval += ch;
	offset--;
	}

	crm_trace(" Interval: %d", *interval);
	CRM_CHECK(key[offset] == '_', return FALSE);

	mutable_key = strdup(key);
	mutable_key[offset] = 0;
	offset--;

	while (offset > 0 && key[offset] != '_') {
	offset--;
	}

	CRM_CHECK(key[offset] == '_', free(mutable_key);
	return FALSE);

	mutable_key_ptr = mutable_key + offset + 1;

	crm_trace(" Action: %s", mutable_key_ptr);

	*op_type = strdup(mutable_key_ptr);

	mutable_key[offset] = 0;
	offset--;

	CRM_CHECK(mutable_key != mutable_key_ptr, free(mutable_key);
	return FALSE);

	notify = strstr(mutable_key, "_post_notify");
	if (notify && safe_str_eq(notify, "_post_notify")) {
	notify[0] = 0;
	}

	notify = strstr(mutable_key, "_pre_notify");
	if (notify && safe_str_eq(notify, "_pre_notify")) {
	notify[0] = 0;
	}

	crm_trace(" Resource: %s", mutable_key);
	*rsc_id = mutable_key;

	return TRUE;
	}

	char *
	generate_notify_key(const char rsc_id, const char notify_type, const char *op_type)
	{
	int len = 12;
	char *op_id = NULL;

	CRM_CHECK(rsc_id != NULL, return NULL);
	CRM_CHECK(op_type != NULL, return NULL);
	CRM_CHECK(notify_type != NULL, return NULL);

	len += strlen(op_type);
	len += strlen(rsc_id);
	len += strlen(notify_type);
	if(len > 0) {
	op_id = malloc(len);
	}
	if (op_id != NULL) {
	sprintf(op_id, "%s_%s_notify_%s_0", rsc_id, notify_type, op_type);
	}
	return op_id;
	}

	char *
	generate_transition_magic_v202(const char *transition_key, int op_status)
	{
	int len = 80;
	char *fail_state = NULL;

	CRM_CHECK(transition_key != NULL, return NULL);

	len += strlen(transition_key);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%s", op_status, transition_key);
	}
	return fail_state;
	}

	char *
	generate_transition_magic(const char *transition_key, int op_status, int op_rc)
	{
	int len = 80;
	char *fail_state = NULL;

	CRM_CHECK(transition_key != NULL, return NULL);

	len += strlen(transition_key);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%d;%s", op_status, op_rc, transition_key);
	}
	return fail_state;
	}

	gboolean
	decode_transition_magic(const char magic, char uuid, int transition_id, int *action_id,
	int op_status, int op_rc, int *target_rc)
	{
	int res = 0;
	char *key = NULL;
	gboolean result = TRUE;

	CRM_CHECK(magic != NULL, return FALSE);
	CRM_CHECK(op_rc != NULL, return FALSE);
	CRM_CHECK(op_status != NULL, return FALSE);

	key = calloc(1, strlen(magic) + 1);
	res = sscanf(magic, "%d:%d;%s", op_status, op_rc, key);
	if (res != 3) {
	crm_warn("Only found %d items in: '%s'", res, magic);
	free(key);
	return FALSE;
	}

	CRM_CHECK(decode_transition_key(key, uuid, transition_id, action_id, target_rc), result = FALSE);

	free(key);
	return result;
	}

	char *
	generate_transition_key(int transition_id, int action_id, int target_rc, const char *node)
	{
	int len = 40;
	char *fail_state = NULL;

	CRM_CHECK(node != NULL, return NULL);

	len += strlen(node);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%d:%d:%-*s", action_id, transition_id, target_rc, 36, node);
	}
	return fail_state;
	}

	gboolean
	decode_transition_key(const char key, char uuid, int transition_id, int *action_id,
	int *target_rc)
	{
	int res = 0;
	gboolean done = FALSE;

	CRM_CHECK(uuid != NULL, return FALSE);
	CRM_CHECK(target_rc != NULL, return FALSE);
	CRM_CHECK(action_id != NULL, return FALSE);
	CRM_CHECK(transition_id != NULL, return FALSE);

	*uuid = calloc(1, 37);
	res = sscanf(key, "%d:%d:%d:%36s", action_id, transition_id, target_rc, *uuid);
	switch (res) {
	case 4:
	/* Post Pacemaker 0.6 */
	done = TRUE;
	break;
	case 3:
	case 2:
	/* this can be tricky - the UUID might start with an integer */

	/* Until Pacemaker 0.6 */
	done = TRUE;
	*target_rc = -1;
	res = sscanf(key, "%d:%d:%36s", action_id, transition_id, *uuid);
	if (res == 2) {
	*action_id = -1;
	res = sscanf(key, "%d:%36s", transition_id, *uuid);
	CRM_CHECK(res == 2, done = FALSE);

	} else if (res != 3) {
	CRM_CHECK(res == 3, done = FALSE);
	}
	break;

	case 1:
	/* Prior to Heartbeat 2.0.8 */
	done = TRUE;
	*action_id = -1;
	*target_rc = -1;
	res = sscanf(key, "%d:%36s", transition_id, *uuid);
	CRM_CHECK(res == 2, done = FALSE);
	break;
	default:
	crm_crit("Unhandled sscanf result (%d) for %s", res, key);
	}

	if (strlen(*uuid) != 36) {
	crm_warn("Bad UUID (%s) in sscanf result (%d) for %s", *uuid, res, key);
	}

	if (done == FALSE) {
	crm_err("Cannot decode '%s' rc=%d", key, res);

	free(*uuid);
	*uuid = NULL;
	*target_rc = -1;
	*action_id = -1;
	*transition_id = -1;
	}

	return done;
	}

	void
	filter_action_parameters(xmlNode * param_set, const char *version)
	{
	char *key = NULL;
	char *timeout = NULL;
	char *interval = NULL;

	const char *attr_filter[] = {
	XML_ATTR_ID,
	XML_ATTR_CRM_VERSION,
	XML_LRM_ATTR_OP_DIGEST,
	};

	gboolean do_delete = FALSE;
	int lpc = 0;
	static int meta_len = 0;

	if (meta_len == 0) {
	meta_len = strlen(CRM_META);
	}

	if (param_set == NULL) {
	return;
	}

	for (lpc = 0; lpc < DIMOF(attr_filter); lpc++) {
	xml_remove_prop(param_set, attr_filter[lpc]);
	}

	key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	interval = crm_element_value_copy(param_set, key);
	free(key);

	key = crm_meta_name(XML_ATTR_TIMEOUT);
	timeout = crm_element_value_copy(param_set, key);

	if (param_set) {
	xmlAttrPtr xIter = param_set->properties;

	while (xIter) {
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	do_delete = FALSE;
	if (strncasecmp(prop_name, CRM_META, meta_len) == 0) {
	do_delete = TRUE;
	}

	if (do_delete) {
	xml_remove_prop(param_set, prop_name);
	}
	}
	}

	if (crm_get_msec(interval) > 0 && compare_version(version, "1.0.8") > 0) {
	/* Re-instate the operation's timeout value */
	if (timeout != NULL) {
	crm_xml_add(param_set, key, timeout);
	}
	}

	free(interval);
	free(timeout);
	free(key);
	}

	void
	filter_reload_parameters(xmlNode * param_set, const char *restart_string)
	{
	int len = 0;
	char *name = NULL;
	char *match = NULL;

	if (param_set == NULL) {
	return;
	}

	if (param_set) {
	xmlAttrPtr xIter = param_set->properties;

	while (xIter) {
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	name = NULL;
	len = strlen(prop_name) + 3;

	name = malloc(len);
	if(name) {
	sprintf(name, " %s ", prop_name);
	name[len - 1] = 0;
	match = strstr(restart_string, name);
	}

	if (match == NULL) {
	crm_trace("%s not found in %s", prop_name, restart_string);
	xml_remove_prop(param_set, prop_name);
	}
	free(name);
	}
	}
	}

	extern bool crm_is_daemon;

	/* coverity[+kill] */
	void
	crm_abort(const char file, const char function, int line,
	const char *assert_condition, gboolean do_core, gboolean do_fork)
	{
	int rc = 0;
	int pid = 0;
	int status = 0;

	/* Implied by the parent's error logging below */
	/* crm_write_blackbox(0); */

	if(crm_is_daemon == FALSE) {
	/* This is a command line tool - do not fork */

	/* crm_add_logfile(NULL); * Record it to a file? */
	crm_enable_stderr(TRUE); /* Make sure stderr is enabled so we can tell the caller */
	do_fork = FALSE; /* Just crash if needed */
	}

	if (do_core == FALSE) {
	crm_err("%s: Triggered assert at %s:%d : %s", function, file, line, assert_condition);
	return;

	} else if (do_fork) {
	pid = fork();

	} else {
	crm_err("%s: Triggered fatal assert at %s:%d : %s", function, file, line, assert_condition);
	}

	if (pid == -1) {
	crm_crit("%s: Cannot create core for non-fatal assert at %s:%d : %s",
	function, file, line, assert_condition);
	return;

	} else if(pid == 0) {
	/* Child process */
	abort();
	return;
	}

	/* Parent process */
	crm_err("%s: Forked child %d to record non-fatal assert at %s:%d : %s",
	function, pid, file, line, assert_condition);
	crm_write_blackbox(SIGTRAP, NULL);

	do {
	rc = waitpid(pid, &status, 0);
	if(rc == pid) {
	return; /* Job done */
	}

	} while(errno == EINTR);

	if (errno == ECHILD) {
	/* crm_mon does this */
	crm_trace("Cannot wait on forked child %d - SIGCHLD is probably set to SIG_IGN", pid);
	return;
	}
	crm_perror(LOG_ERR, "Cannot wait on forked child %d", pid);
	}

	char *
	generate_series_filename(const char directory, const char series, int sequence, gboolean bzip)
	{
	int len = 40;
	char *filename = NULL;
	const char *ext = "raw";

	CRM_CHECK(directory != NULL, return NULL);
	CRM_CHECK(series != NULL, return NULL);

	#if !HAVE_BZLIB_H
	bzip = FALSE;
	#endif

	len += strlen(directory);
	len += strlen(series);
	filename = malloc(len);
	CRM_CHECK(filename != NULL, return NULL);

	if (bzip) {
	ext = "bz2";
	}
	sprintf(filename, "%s/%s-%d.%s", directory, series, sequence, ext);

	return filename;
	}

	int
	get_last_sequence(const char directory, const char series)
	{
	FILE *file_strm = NULL;
	int start = 0, length = 0, read_len = 0;
	char *series_file = NULL;
	char *buffer = NULL;
	int seq = 0;
	int len = 36;

	CRM_CHECK(directory != NULL, return 0);
	CRM_CHECK(series != NULL, return 0);

	len += strlen(directory);
	len += strlen(series);
	series_file = malloc(len);
	CRM_CHECK(series_file != NULL, return 0);
	sprintf(series_file, "%s/%s.last", directory, series);

	file_strm = fopen(series_file, "r");
	if (file_strm == NULL) {
	crm_debug("Series file %s does not exist", series_file);
	free(series_file);
	return 0;
	}

	/* see how big the file is */
	start = ftell(file_strm);
	fseek(file_strm, 0L, SEEK_END);
	length = ftell(file_strm);
	fseek(file_strm, 0L, start);

	CRM_ASSERT(length >= 0);
	CRM_ASSERT(start == ftell(file_strm));

	if (length <= 0) {
	crm_info("%s was not valid", series_file);
	free(buffer);
	buffer = NULL;

	} else {
	crm_trace("Reading %d bytes from file", length);
	buffer = calloc(1, (length + 1));
	read_len = fread(buffer, 1, length, file_strm);
	if (read_len != length) {
	crm_err("Calculated and read bytes differ: %d vs. %d", length, read_len);
	free(buffer);
	buffer = NULL;
	}
	}

	seq = crm_parse_int(buffer, "0");
	fclose(file_strm);

	crm_trace("Found %d in %s", seq, series_file);

	free(series_file);
	free(buffer);
	return seq;
	}

	void
	write_last_sequence(const char directory, const char series, int sequence, int max)
	{
	int rc = 0;
	int len = 36;
	FILE *file_strm = NULL;
	char *series_file = NULL;

	CRM_CHECK(directory != NULL, return);
	CRM_CHECK(series != NULL, return);

	if (max == 0) {
	return;
	}
	if (max > 0 && sequence >= max) {
	sequence = 0;
	}

	len += strlen(directory);
	len += strlen(series);
	series_file = malloc(len);

	if(series_file) {
	sprintf(series_file, "%s/%s.last", directory, series);
	file_strm = fopen(series_file, "w");
	}

	if (file_strm != NULL) {
	rc = fprintf(file_strm, "%d", sequence);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Cannot write to series file %s", series_file);
	}

	} else {
	crm_err("Cannot open series file %s for writing", series_file);
	}

	if (file_strm != NULL) {
	fflush(file_strm);
	fclose(file_strm);
	}

	crm_trace("Wrote %d to %s", sequence, series_file);
	free(series_file);
	}

	#define LOCKSTRLEN 11

	int
	crm_pid_active(long pid)
	{
	if (pid <= 0) {
	return -1;

	} else if (kill(pid, 0) < 0 && errno == ESRCH) {
	return 0;
	}
	#ifndef HAVE_PROC_PID
	return 1;
	#else
	{
	int rc = 0;
	int running = 0;
	char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX];

	/* check to make sure pid hasn't been reused by another process */
	snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", pid);

	rc = readlink(proc_path, exe_path, PATH_MAX - 1);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not read from %s", proc_path);
	goto bail;
	}

	exe_path[rc] = 0;
	snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid());
	rc = readlink(proc_path, myexe_path, PATH_MAX - 1);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not read from %s", proc_path);
	goto bail;
	}

	myexe_path[rc] = 0;
	if (strcmp(exe_path, myexe_path) == 0) {
	running = 1;
	}
	}

	bail:
	return running;
	#endif
	}

	int
	crm_read_pidfile(const char *filename)
	{
	int fd;
	long pid = -1;
	char buf[LOCKSTRLEN + 1];

	if ((fd = open(filename, O_RDONLY)) < 0) {
	goto bail;
	}

	if (read(fd, buf, sizeof(buf)) < 1) {
	goto bail;
	}

	if (sscanf(buf, "%lu", &pid) > 0) {
	if (pid <= 0) {
	pid = -ESRCH;
	}
	}

	bail:
	if (fd >= 0) {
	close(fd);
	}
	return pid;
	}

	int
	crm_pidfile_inuse(const char *filename, long mypid)
	{
	long pid = 0;
	struct stat sbuf;
	char buf[LOCKSTRLEN + 1];
	int rc = -ENOENT, fd = 0;

	if ((fd = open(filename, O_RDONLY)) >= 0) {
	if (fstat(fd, &sbuf) >= 0 && sbuf.st_size < LOCKSTRLEN) {
	sleep(2); /* if someone was about to create one,
	* give'm a sec to do so
	*/
	}
	if (read(fd, buf, sizeof(buf)) > 0) {
	if (sscanf(buf, "%lu", &pid) > 0) {
	crm_trace("Got pid %lu from %s\n", pid, filename);
	if (pid <= 1) {
	/* Invalid pid */
	rc = -ENOENT;
	unlink(filename);

	} else if (mypid && pid == mypid) {
	/* In use by us */
	rc = pcmk_ok;

	} else if (crm_pid_active(pid) == FALSE) {
	/* Contains a stale value */
	unlink(filename);
	rc = -ENOENT;

	} else if (mypid && pid != mypid) {
	/* locked by existing process - give up */
	rc = -EEXIST;
	}
	}
	}
	close(fd);
	}
	return rc;
	}

	static int
	crm_lock_pidfile(const char *filename)
	{
	long mypid = 0;
	int fd = 0, rc = 0;
	char buf[LOCKSTRLEN + 1];

	mypid = (unsigned long)getpid();

	rc = crm_pidfile_inuse(filename, 0);
	if (rc == -ENOENT) {
	/* exists but the process is not active */

	} else if (rc != pcmk_ok) {
	/* locked by existing process - give up */
	return rc;
	}

	if ((fd = open(filename, O_CREAT \| O_WRONLY \| O_EXCL, 0644)) < 0) {
	/* Hmmh, why did we fail? Anyway, nothing we can do about it */
	return -errno;
	}

	snprintf(buf, sizeof(buf), "%*lu\n", LOCKSTRLEN - 1, mypid);
	rc = write(fd, buf, LOCKSTRLEN);
	close(fd);

	if (rc != LOCKSTRLEN) {
	crm_perror(LOG_ERR, "Incomplete write to %s", filename);
	return -errno;
	}

	return crm_pidfile_inuse(filename, mypid);
	}

	void
	crm_make_daemon(const char name, gboolean daemonize, const char pidfile)
	{
	int rc;
	long pid;
	const char *devnull = "/dev/null";

	if (daemonize == FALSE) {
	return;
	}

	/* Check before we even try... */
	rc = crm_pidfile_inuse(pidfile, 1);
	if(rc < pcmk_ok && rc != -ENOENT) {
	pid = crm_read_pidfile(pidfile);
	crm_err("%s: already running [pid %ld in %s]", name, pid, pidfile);
	printf("%s: already running [pid %ld in %s]\n", name, pid, pidfile);
	crm_exit(rc);
	}

	pid = fork();
	if (pid < 0) {
	fprintf(stderr, "%s: could not start daemon\n", name);
	crm_perror(LOG_ERR, "fork");
	crm_exit(EINVAL);

	} else if (pid > 0) {
	crm_exit(pcmk_ok);
	}

	rc = crm_lock_pidfile(pidfile);
	if(rc < pcmk_ok) {
	crm_err("Could not lock '%s' for %s: %s (%d)", pidfile, name, pcmk_strerror(rc), rc);
	printf("Could not lock '%s' for %s: %s (%d)\n", pidfile, name, pcmk_strerror(rc), rc);
	crm_exit(rc);
	}

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);

	close(STDIN_FILENO);
	(void)open(devnull, O_RDONLY); /* Stdin: fd 0 */
	close(STDOUT_FILENO);
	(void)open(devnull, O_WRONLY); /* Stdout: fd 1 */
	close(STDERR_FILENO);
	(void)open(devnull, O_WRONLY); /* Stderr: fd 2 */
	}

	gboolean
	crm_is_writable(const char dir, const char file,
	const char user, const char group, gboolean need_both)
	{
	int s_res = -1;
	struct stat buf;
	char *full_file = NULL;
	const char *target = NULL;

	gboolean pass = TRUE;
	gboolean readwritable = FALSE;

	CRM_ASSERT(dir != NULL);
	if (file != NULL) {
	full_file = crm_concat(dir, file, '/');
	target = full_file;
	s_res = stat(full_file, &buf);
	if (s_res == 0 && S_ISREG(buf.st_mode) == FALSE) {
	crm_err("%s must be a regular file", target);
	pass = FALSE;
	goto out;
	}
	}

	if (s_res != 0) {
	target = dir;
	s_res = stat(dir, &buf);
	if (s_res != 0) {
	crm_err("%s must exist and be a directory", dir);
	pass = FALSE;
	goto out;

	} else if (S_ISDIR(buf.st_mode) == FALSE) {
	crm_err("%s must be a directory", dir);
	pass = FALSE;
	}
	}

	if (user) {
	struct passwd *sys_user = NULL;

	sys_user = getpwnam(user);
	readwritable = (sys_user != NULL
	&& buf.st_uid == sys_user->pw_uid && (buf.st_mode & (S_IRUSR \| S_IWUSR)));
	if (readwritable == FALSE) {
	crm_err("%s must be owned and r/w by user %s", target, user);
	if (need_both) {
	pass = FALSE;
	}
	}
	}

	if (group) {
	struct group *sys_grp = getgrnam(group);

	readwritable = (sys_grp != NULL
	&& buf.st_gid == sys_grp->gr_gid && (buf.st_mode & (S_IRGRP \| S_IWGRP)));
	if (readwritable == FALSE) {
	if (need_both \|\| user == NULL) {
	pass = FALSE;
	crm_err("%s must be owned and r/w by group %s", target, group);
	} else {
	crm_warn("%s should be owned and r/w by group %s", target, group);
	}
	}
	}

	out:
	free(full_file);
	return pass;
	}

	char *
	crm_strip_trailing_newline(char *str)
	{
	int len;

	if (str == NULL) {
	return str;
	}

	for (len = strlen(str) - 1; len >= 0 && str[len] == '\n'; len--) {
	str[len] = '\0';
	}

	return str;
	}

	gboolean
	crm_str_eq(const char a, const char b, gboolean use_case)
	{
	if (use_case) {
	return g_strcmp0(a, b) == 0;

	/* TODO - Figure out which calls, if any, really need to be case independant */
	} else if (a == b) {
	return TRUE;

	} else if (a == NULL \|\| b == NULL) {
	/* shouldn't be comparing NULLs */
	return FALSE;

	} else if (strcasecmp(a, b) == 0) {
	return TRUE;
	}
	return FALSE;
	}

	char *
	crm_meta_name(const char *field)
	{
	int lpc = 0;
	int max = 0;
	char *crm_name = NULL;

	CRM_CHECK(field != NULL, return NULL);
	crm_name = crm_concat(CRM_META, field, '_');

	/* Massage the names so they can be used as shell variables */
	max = strlen(crm_name);
	for (; lpc < max; lpc++) {
	switch (crm_name[lpc]) {
	case '-':
	crm_name[lpc] = '_';
	break;
	}
	}
	return crm_name;
	}

	const char *
	crm_meta_value(GHashTable * hash, const char *field)
	{
	char *key = NULL;
	const char *value = NULL;

	key = crm_meta_name(field);
	if (key) {
	value = g_hash_table_lookup(hash, key);
	free(key);
	}

	return value;
	}

	static struct option *
	crm_create_long_opts(struct crm_option *long_options)
	{
	struct option *long_opts = NULL;

	#ifdef HAVE_GETOPT_H
	int index = 0, lpc = 0;

	/*
	* A previous, possibly poor, choice of '?' as the short form of --help
	* means that getopt_long() returns '?' for both --help and for "unknown option"
	*
	* This dummy entry allows us to differentiate between the two in crm_get_option()
	* and exit with the correct error code
	*/
	long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
	long_opts[index].name = "__dummmy__";
	long_opts[index].has_arg = 0;
	long_opts[index].flag = 0;
	long_opts[index].val = '_';
	index++;

	for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
	if (long_options[lpc].name[0] == '-') {
	continue;
	}

	long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
	/*fprintf(stderr, "Creating %d %s = %c\n", index,
	* long_options[lpc].name, long_options[lpc].val); */
	long_opts[index].name = long_options[lpc].name;
	long_opts[index].has_arg = long_options[lpc].has_arg;
	long_opts[index].flag = long_options[lpc].flag;
	long_opts[index].val = long_options[lpc].val;
	index++;
	}

	/* Now create the list terminator */
	long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
	long_opts[index].name = NULL;
	long_opts[index].has_arg = 0;
	long_opts[index].flag = 0;
	long_opts[index].val = 0;
	#endif

	return long_opts;
	}

	void
	crm_set_options(const char short_options, const char app_usage, struct crm_option *long_options,
	const char *app_desc)
	{
	if (short_options) {
	crm_short_options = strdup(short_options);

	} else if (long_options) {
	int lpc = 0;
	int opt_string_len = 0;
	char *local_short_options = NULL;

	for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
	if (long_options[lpc].val && long_options[lpc].val != '-' && long_options[lpc].val < UCHAR_MAX) {
	local_short_options = realloc_safe(local_short_options, opt_string_len + 4);
	local_short_options[opt_string_len++] = long_options[lpc].val;
	/* getopt(3) says: Two colons mean an option takes an optional arg; */
	if (long_options[lpc].has_arg == optional_argument) {
	local_short_options[opt_string_len++] = ':';
	}
	if (long_options[lpc].has_arg >= required_argument) {
	local_short_options[opt_string_len++] = ':';
	}
	local_short_options[opt_string_len] = 0;
	}
	}
	crm_short_options = local_short_options;
	crm_trace("Generated short option string: '%s'", local_short_options);
	}

	if (long_options) {
	crm_long_options = long_options;
	}
	if (app_desc) {
	crm_app_description = app_desc;
	}
	if (app_usage) {
	crm_app_usage = app_usage;
	}
	}

	int
	crm_get_option(int argc, char *argv, int index)
	{
	return crm_get_option_long(argc, argv, index, NULL);
	}

	int
	crm_get_option_long(int argc, char *argv, int index, const char **longname)
	{
	#ifdef HAVE_GETOPT_H
	static struct option *long_opts = NULL;

	if (long_opts == NULL && crm_long_options) {
	long_opts = crm_create_long_opts(crm_long_options);
	}

	if (long_opts) {
	int flag = getopt_long(argc, argv, crm_short_options, long_opts, index);

	switch (flag) {
	case 0:
	if (long_opts[*index].val) {
	return long_opts[*index].val;
	} else if (longname) {
	longname = long_opts[index].name;
	} else {
	crm_notice("Unhandled option --%s", long_opts[*index].name);
	return flag;
	}
	case -1: /* End of option processing */
	break;
	case ':':
	crm_trace("Missing argument");
	crm_help('?', 1);
	break;
	case '?':
	crm_help('?', *index ? 0 : 1);
	break;
	}
	return flag;
	}
	#endif

	if (crm_short_options) {
	return getopt(argc, argv, crm_short_options);
	}

	return -1;
	}

	int
	crm_help(char cmd, int exit_code)
	{
	int i = 0;
	FILE *stream = (exit_code ? stderr : stdout);

	if (cmd == 'v' \|\| cmd == '$') {
	fprintf(stream, "Pacemaker %s\n", VERSION);
	fprintf(stream, "Written by Andrew Beekhof\n");
	goto out;
	}

	if (cmd == '!') {
	fprintf(stream, "Pacemaker %s (Build: %s): %s\n", VERSION, BUILD_VERSION, CRM_FEATURES);
	goto out;
	}

	fprintf(stream, "%s - %s\n", crm_system_name, crm_app_description);

	if (crm_app_usage) {
	fprintf(stream, "Usage: %s %s\n", crm_system_name, crm_app_usage);
	}

	if (crm_long_options) {
	fprintf(stream, "Options:\n");
	for (i = 0; crm_long_options[i].name != NULL; i++) {
	if (crm_long_options[i].flags & pcmk_option_hidden) {

	} else if (crm_long_options[i].flags & pcmk_option_paragraph) {
	fprintf(stream, "%s\n\n", crm_long_options[i].desc);

	} else if (crm_long_options[i].flags & pcmk_option_example) {
	fprintf(stream, "\t#%s\n\n", crm_long_options[i].desc);

	} else if (crm_long_options[i].val == '-' && crm_long_options[i].desc) {
	fprintf(stream, "%s\n", crm_long_options[i].desc);

	} else {
	/* is val printable as char ? */
	if (crm_long_options[i].val && crm_long_options[i].val <= UCHAR_MAX) {
	fprintf(stream, " -%c,", crm_long_options[i].val);
	} else {
	fputs(" ", stream);
	}
	fprintf(stream, " --%s%s\t%s\n", crm_long_options[i].name,
	crm_long_options[i].has_arg == optional_argument ? "[=value]" :
	crm_long_options[i].has_arg == required_argument ? "=value" : "",
	crm_long_options[i].desc ? crm_long_options[i].desc : "");
	}
	}

	} else if (crm_short_options) {
	fprintf(stream, "Usage: %s - %s\n", crm_system_name, crm_app_description);
	for (i = 0; crm_short_options[i] != 0; i++) {
	int has_arg = no_argument /* 0 */;

	if (crm_short_options[i + 1] == ':') {
	if (crm_short_options[i + 2] == ':')
	has_arg = optional_argument /* 2 */;
	else
	has_arg = required_argument /* 1 */;
	}

	fprintf(stream, " -%c %s\n", crm_short_options[i],
	has_arg == optional_argument ? "[value]" :
	has_arg == required_argument ? "{value}" : "");
	i += has_arg;
	}
	}

	fprintf(stream, "\nReport bugs to %s\n", PACKAGE_BUGREPORT);

	out:
	return crm_exit(exit_code);
	}

	void cib_ipc_servers_init(qb_ipcs_service_t **ipcs_ro,
	qb_ipcs_service_t **ipcs_rw,
	qb_ipcs_service_t **ipcs_shm,
	struct qb_ipcs_service_handlers *ro_cb,
	struct qb_ipcs_service_handlers *rw_cb)
	{
	*ipcs_ro = mainloop_add_ipc_server(cib_channel_ro, QB_IPC_NATIVE, ro_cb);
	*ipcs_rw = mainloop_add_ipc_server(cib_channel_rw, QB_IPC_NATIVE, rw_cb);
	*ipcs_shm = mainloop_add_ipc_server(cib_channel_shm, QB_IPC_SHM, rw_cb);

	if (ipcs_ro == NULL \|\| ipcs_rw == NULL \|\| *ipcs_shm == NULL) {
	crm_err("Failed to create cib servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	void cib_ipc_servers_destroy(qb_ipcs_service_t *ipcs_ro,
	qb_ipcs_service_t *ipcs_rw,
	qb_ipcs_service_t *ipcs_shm)
	{
	qb_ipcs_destroy(ipcs_ro);
	qb_ipcs_destroy(ipcs_rw);
	qb_ipcs_destroy(ipcs_shm);
	}

	qb_ipcs_service_t *
	crmd_ipc_server_init(struct qb_ipcs_service_handlers *cb)
	{
	return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb);
	}

	void
	attrd_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb)
	{
	*ipcs = mainloop_add_ipc_server(T_ATTRD, QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Failed to create attrd servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	void
	stonith_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb)
	{
	*ipcs = mainloop_add_ipc_server("stonith-ng", QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Failed to create stonith-ng servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	int
	attrd_update_delegate(crm_ipc_t * ipc, char command, const char host, const char name,
	const char value, const char section, const char set, const char dampen,
	const char *user_name, gboolean is_remote)
	{
	int rc = -ENOTCONN;
	int max = 5;
	enum crm_ipc_flags flags = crm_ipc_flags_none;
	xmlNode *update = create_xml_node(NULL, __FUNCTION__);

	static gboolean connected = TRUE;
	static crm_ipc_t *local_ipc = NULL;

	if (ipc == NULL && local_ipc == NULL) {
	local_ipc = crm_ipc_new(T_ATTRD, 0);
	flags \|= crm_ipc_client_response;
	connected = FALSE;
	}

	if (ipc == NULL) {
	ipc = local_ipc;
	}

	/* remap common aliases */
	if (safe_str_eq(section, "reboot")) {
	section = XML_CIB_TAG_STATUS;

	} else if (safe_str_eq(section, "forever")) {
	section = XML_CIB_TAG_NODES;
	}

	crm_xml_add(update, F_TYPE, T_ATTRD);
	crm_xml_add(update, F_ORIG, crm_system_name);

	if (name == NULL && command == 'U') {
	command = 'R';
	}

	switch (command) {
	case 'u':
	crm_xml_add(update, F_ATTRD_TASK, "update");
	crm_xml_add(update, F_ATTRD_REGEX, name);
	break;
	case 'D':
	case 'U':
	case 'v':
	crm_xml_add(update, F_ATTRD_TASK, "update");
	crm_xml_add(update, F_ATTRD_ATTRIBUTE, name);
	break;
	case 'R':
	crm_xml_add(update, F_ATTRD_TASK, "refresh");
	break;
	case 'q':
	crm_xml_add(update, F_ATTRD_TASK, "query");
	break;
	case 'C':
	crm_xml_add(update, F_ATTRD_TASK, "peer-remove");
	break;
	}

	crm_xml_add(update, F_ATTRD_VALUE, value);
	crm_xml_add(update, F_ATTRD_DAMPEN, dampen);
	crm_xml_add(update, F_ATTRD_SECTION, section);
	crm_xml_add(update, F_ATTRD_HOST, host);
	crm_xml_add(update, F_ATTRD_SET, set);
	crm_xml_add_int(update, F_ATTRD_IS_REMOTE, is_remote);
	#if ENABLE_ACL
	if (user_name) {
	crm_xml_add(update, F_ATTRD_USER, user_name);
	}
	#endif

	while (max > 0) {
	if (connected == FALSE) {
	crm_info("Connecting to cluster... %d retries remaining", max);
	connected = crm_ipc_connect(ipc);
	}

	if (connected) {
	rc = crm_ipc_send(ipc, update, flags, 0, NULL);
	}

	if (ipc != local_ipc) {
	break;

	} else if (rc > 0) {
	break;

	} else if (rc == -EAGAIN \|\| rc == -EALREADY) {
	sleep(5 - max);
	max--;

	} else {
	crm_ipc_close(ipc);
	connected = FALSE;
	sleep(5 - max);
	max--;
	}
	}

	free_xml(update);
	if (rc > 0) {
	crm_debug("Sent update: %s=%s for %s", name, value, host ? host : "localhost");
	rc = pcmk_ok;

	} else {
	crm_debug("Could not send update %s=%s for %s: %s (%d)", name, value,
	host ? host : "localhost", pcmk_strerror(rc), rc);
	}
	return rc;
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
	static void
	append_digest(lrmd_event_data_t * op, xmlNode * update, const char version, const char magic,
	int level)
	{
	/* this will enable us to later determine that the
	* resource's parameters have changed and we should force
	* a restart
	*/
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}

	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	filter_action_parameters(args_xml, version);
	digest = calculate_operation_digest(args_xml, version);

	#if 0
	if (level < get_crm_log_level()
	&& op->interval == 0 && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) {
	char *digest_source = dump_xml_unformatted(args_xml);

	do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
	digest, ID(update), magic, digest_source);
	free(digest_source);
	}
	#endif
	crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);

	free_xml(args_xml);
	free(digest);
	}

	int
	rsc_op_expected_rc(lrmd_event_data_t * op)
	{
	int rc = 0;

	if (op && op->user_data) {
	int dummy = 0;
	char *uuid = NULL;

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &rc);
	free(uuid);
	}
	return rc;
	}

	gboolean
	did_rsc_op_fail(lrmd_event_data_t * op, int target_rc)
	{
	switch (op->op_status) {
	case PCMK_LRM_OP_CANCELLED:
	case PCMK_LRM_OP_PENDING:
	return FALSE;
	break;

	case PCMK_LRM_OP_NOTSUPPORTED:
	case PCMK_LRM_OP_TIMEOUT:
	case PCMK_LRM_OP_ERROR:
	return TRUE;
	break;

	default:
	if (target_rc != op->rc) {
	return TRUE;
	}
	}

	return FALSE;
	}

	xmlNode *
	-create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char *caller_version,
	- int target_rc, const char *origin, int level)
	+create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char * caller_version,
	+ int target_rc, const char * node, const char * origin, int level)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;
	gboolean dc_munges_migrate_ops = (compare_version(caller_version, "3.0.3") < 0);
	gboolean dc_needs_unique_ops = (compare_version(caller_version, "3.0.6") < 0);

	CRM_CHECK(op != NULL, return NULL);
	do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%d)",
	origin, op->rsc_id, op->op_type, services_lrm_status_str(op->op_status),
	op->interval);

	crm_trace("DC version: %s", caller_version);

	task = op->op_type;
	/* remap the task name under various scenarios
	* this makes life easier for the PE when its trying determin the current state
	*/
	if (crm_str_eq(task, "reload", TRUE)) {
	if (op->op_status == PCMK_LRM_OP_DONE) {
	task = CRMD_ACTION_START;
	} else {
	task = CRMD_ACTION_STATUS;
	}

	} else if (dc_munges_migrate_ops && crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) {
	/* if the migrate_from fails it will have enough info to do the right thing */
	if (op->op_status == PCMK_LRM_OP_DONE) {
	task = CRMD_ACTION_STOP;
	} else {
	task = CRMD_ACTION_STATUS;
	}

	} else if (dc_munges_migrate_ops
	&& op->op_status == PCMK_LRM_OP_DONE
	&& crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
	task = CRMD_ACTION_START;
	}

	key = generate_op_key(op->rsc_id, task, op->interval);
	if (dc_needs_unique_ops && op->interval > 0) {
	op_id = strdup(key);

	} else if (crm_str_eq(task, CRMD_ACTION_NOTIFY, TRUE)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = generate_notify_key(op->rsc_id, n_type, n_task);

	/* these are not yet allowed to fail */
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = 0;

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = generate_op_key(op->rsc_id, "last_failure", 0);
	if (op->interval == 0) {
	/* Ensure 'last' gets updated too in case recording-pending="true" */
	op_id_additional = generate_op_key(op->rsc_id, "last", 0);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval > 0) {
	op_id = strdup(key);

	} else {
	op_id = generate_op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = find_entity(parent, XML_LRM_TAG_RSC_OP, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for:"
	" %s_%s_%d %d from %s",
	op->rsc_id, op->op_type, op->interval, op->call_id, origin);
	local_user_data = generate_transition_key(-1, op->call_id, target_rc, FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if(magic == NULL) {
	magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
	}

	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
	crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason);
	+ crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */

	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, op->interval);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (%s_%s_%d): last=%lu change=%lu exec=%lu queue=%lu",
	op->rsc_id, op->op_type, op->interval,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if (op->interval == 0) {
	/* The values are the same for non-recurring ops */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_RUN, op->t_run);
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);

	} else if(op->t_rcchange) {
	/* last-run is not accurate for recurring ops */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_rcchange);

	} else {
	/* ...but is better than nothing otherwise */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);
	}

	crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
	crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
	}
	}

	if (crm_str_eq(op->op_type, CRMD_ACTION_MIGRATE, TRUE)
	\|\| crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) {
	/*
	* Record migrate_source and migrate_target always for migrate ops.
	*/
	const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = XML_LRM_ATTR_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	bool
	pcmk_acl_required(const char *user)
	{
	#if ENABLE_ACL
	if(user == NULL \|\| strlen(user) == 0) {
	crm_trace("no user set");
	return FALSE;

	} else if (strcmp(user, CRM_DAEMON_USER) == 0) {
	return FALSE;

	} else if (strcmp(user, "root") == 0) {
	return FALSE;
	}
	crm_trace("acls required for %s", user);
	return TRUE;
	#else
	crm_trace("acls not supported");
	return FALSE;
	#endif
	}

	#if ENABLE_ACL
	char *
	uid2username(uid_t uid)
	{
	struct passwd *pwent = getpwuid(uid);

	if (pwent == NULL) {
	crm_perror(LOG_ERR, "Cannot get password entry of uid: %d", uid);
	return NULL;

	} else {
	return strdup(pwent->pw_name);
	}
	}

	const char *
	crm_acl_get_set_user(xmlNode * request, const char field, const char peer_user)
	{
	/* field is only checked for backwards compatibility */
	static const char *effective_user = NULL;
	const char *requested_user = NULL;
	const char *user = NULL;

	if(effective_user == NULL) {
	effective_user = uid2username(geteuid());
	}

	requested_user = crm_element_value(request, XML_ACL_TAG_USER);
	if(requested_user == NULL) {
	requested_user = crm_element_value(request, field);
	}

	if (is_privileged(effective_user) == FALSE) {
	/* We're not running as a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
	user = effective_user;

	} else if(peer_user == NULL && requested_user == NULL) {
	/* No user known or requested, use 'effective_user' and make sure one is set for the request */
	user = effective_user;

	} else if(peer_user == NULL) {
	/* No user known, trusting 'requested_user' */
	user = requested_user;

	} else if (is_privileged(peer_user) == FALSE) {
	/* The peer is not a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
	user = peer_user;

	} else if (requested_user == NULL) {
	/* Even if we're privileged, make sure there is always a value set */
	user = peer_user;

	} else {
	/* Legal delegation to 'requested_user' */
	user = requested_user;
	}

	/* Yes, pointer comparision */
	if(user != crm_element_value(request, XML_ACL_TAG_USER)) {
	crm_xml_add(request, XML_ACL_TAG_USER, user);
	}

	if(field != NULL && user != crm_element_value(request, field)) {
	crm_xml_add(request, field, user);
	}

	return requested_user;
	}

	void
	determine_request_user(const char user, xmlNode request, const char *field)
	{
	/* Get our internal validation out of the way first */
	CRM_CHECK(user != NULL && request != NULL && field != NULL, return);

	/* If our peer is a privileged user, we might be doing something on behalf of someone else */
	if (is_privileged(user) == FALSE) {
	/* We're not a privileged user, set or overwrite any existing value for $field */
	crm_xml_replace(request, field, user);

	} else if (crm_element_value(request, field) == NULL) {
	/* Even if we're privileged, make sure there is always a value set */
	crm_xml_replace(request, field, user);

	/* } else { Legal delegation */
	}

	crm_trace("Processing msg as user '%s'", crm_element_value(request, field));
	}
	#endif

	/*
	* This re-implements g_str_hash as it was prior to glib2-2.28:
	*
	* http://git.gnome.org/browse/glib/commit/?id=354d655ba8a54b754cb5a3efb42767327775696c
	*
	* Note that the new g_str_hash is presumably a better hash (it's actually
	* a correct implementation of DJB's hash), but we need to preserve existing
	* behaviour, because the hash key ultimately determines the "sort" order
	* when iterating through GHashTables, which affects allocation of scores to
	* clone instances when iterating through rsc->allowed_nodes. It (somehow)
	* also appears to have some minor impact on the ordering of a few
	* pseudo_event IDs in the transition graph.
	*/
	guint
	g_str_hash_traditional(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + *p;

	return h;
	}

	guint
	crm_strcase_hash(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + g_ascii_tolower(*p);

	return h;
	}

	void *
	find_library_function(void *handle, const char lib, const char *fn, gboolean fatal)
	{
	char *error;
	void *a_function;

	if (*handle == NULL) {
	*handle = dlopen(lib, RTLD_LAZY);
	}

	if (!(*handle)) {
	crm_err("%sCould not open %s: %s", fatal ? "Fatal: " : "", lib, dlerror());
	if (fatal) {
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	return NULL;
	}

	a_function = dlsym(*handle, fn);
	if ((error = dlerror()) != NULL) {
	crm_err("%sCould not find %s in %s: %s", fatal ? "Fatal: " : "", fn, lib, error);
	if (fatal) {
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	return a_function;
	}

	char *
	add_list_element(char list, const char value)
	{
	int len = 0;
	int last = 0;

	if (value == NULL) {
	return list;
	}
	if (list) {
	last = strlen(list);
	}
	len = last + 2; /* +1 space, +1 EOS */
	len += strlen(value);
	list = realloc_safe(list, len);
	sprintf(list + last, " %s", value);
	return list;
	}

	void *
	convert_const_pointer(const void *ptr)
	{
	/* Worst function ever */
	return (void *)ptr;
	}

	#ifdef HAVE_UUID_UUID_H
	# include <uuid/uuid.h>
	#endif

	char *
	crm_generate_uuid(void)
	{
	unsigned char uuid[16];
	char buffer = malloc(37); / Including NUL byte */

	uuid_generate(uuid);
	uuid_unparse(uuid, buffer);
	return buffer;
	}

	#include <md5.h>

	char *
	crm_md5sum(const char *buffer)
	{
	int lpc = 0, len = 0;
	char *digest = NULL;
	unsigned char raw_digest[MD5_DIGEST_SIZE];

	if(buffer != NULL) {
	len = strlen(buffer);
	}

	crm_trace("Beginning digest of %d bytes", len);
	digest = malloc(2 * MD5_DIGEST_SIZE + 1);
	if(digest) {
	md5_buffer(buffer, len, raw_digest);
	for (lpc = 0; lpc < MD5_DIGEST_SIZE; lpc++) {
	sprintf(digest + (2 * lpc), "%02x", raw_digest[lpc]);
	}
	digest[(2 * MD5_DIGEST_SIZE)] = 0;
	crm_trace("Digest %s.", digest);

	} else {
	crm_err("Could not create digest");
	}
	return digest;
	}

	#include <time.h>
	#include <bzlib.h>

	bool
	crm_compress_string(const char data, int length, int max, char result, unsigned int result_len)
	{
	int rc;
	char *compressed = NULL;
	char *uncompressed = strdup(data);
	struct timespec after_t;
	struct timespec before_t;

	if(max == 0) {
	max = (length * 1.1) + 600; /* recomended size */
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &before_t);
	#endif

	/* coverity[returned_null] Ignore */
	compressed = malloc(max);

	*result_len = max;
	rc = BZ2_bzBuffToBuffCompress(compressed, result_len, uncompressed, length, CRM_BZ2_BLOCKS, 0,
	CRM_BZ2_WORK);

	free(uncompressed);

	if (rc != BZ_OK) {
	crm_err("Compression of %d bytes failed: %s (%d)", length, bz2_strerror(rc), rc);
	free(compressed);
	return FALSE;
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &after_t);

	crm_info("Compressed %d bytes into %d (ratio %d:1) in %dms",
	length, result_len, length / (result_len),
	(after_t.tv_sec - before_t.tv_sec) * 1000 + (after_t.tv_nsec -
	before_t.tv_nsec) / 1000000);
	#else
	crm_info("Compressed %d bytes into %d (ratio %d:1)",
	length, result_len, length / (result_len));
	#endif

	*result = compressed;
	return TRUE;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	void
	crm_gnutls_global_init(void)
	{
	signal(SIGPIPE, SIG_IGN);
	gnutls_global_init();
	}
	#endif

	diff --git a/tools/fake_transition.c b/tools/fake_transition.c
	index d6d71eb667..e8c37f7893 100644
	--- a/tools/fake_transition.c
	+++ b/tools/fake_transition.c
	@@ -1,846 +1,846 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>

	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/common/util.h>
	#include <crm/transition.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/status.h>
	#include <allocate.h>
	#include "fake_transition.h"

	static bool fake_quiet = FALSE;
	static cib_t *fake_cib = NULL;
	static GListPtr fake_resource_list = NULL;
	static GListPtr fake_op_fail_list = NULL;
	gboolean bringing_nodes_online = FALSE;

	#define STATUS_PATH_MAX 512

	#define quiet_log(fmt, args...) do { \
	if(fake_quiet) { \
	crm_trace(fmt, ##args); \
	} else { \
	printf(fmt , ##args); \
	} \
	} while(0)

	#define new_node_template "//"XML_CIB_TAG_NODE"[@uname='%s']"
	#define node_template "//"XML_CIB_TAG_STATE"[@uname='%s']"
	#define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
	#define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
	/* #define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']" */


	static void
	inject_transient_attr(xmlNode * cib_node, const char name, const char value)
	{
	xmlNode *attrs = NULL;
	xmlNode *container = NULL;
	xmlNode *nvp = NULL;
	const char *node_uuid = ID(cib_node);
	char *nvp_id = crm_concat(name, node_uuid, '-');

	quiet_log("Injecting attribute %s=%s into %s '%s'", name, value, xmlGetNodePath(cib_node),
	ID(cib_node));

	attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	if (attrs == NULL) {
	attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
	}

	container = first_named_child(attrs, XML_TAG_ATTR_SETS);
	if (container == NULL) {
	container = create_xml_node(attrs, XML_TAG_ATTR_SETS);
	crm_xml_add(container, XML_ATTR_ID, node_uuid);
	}

	nvp = create_xml_node(container, XML_CIB_TAG_NVPAIR);
	crm_xml_add(nvp, XML_ATTR_ID, nvp_id);
	crm_xml_add(nvp, XML_NVPAIR_ATTR_NAME, name);
	crm_xml_add(nvp, XML_NVPAIR_ATTR_VALUE, value);

	free(nvp_id);
	}

	static void
	update_failcounts(xmlNode * cib_node, const char *resource, int interval, int rc)
	{
	if (rc == 0) {
	return;

	} else if (rc == 7 && interval == 0) {
	return;

	} else {
	char *name = NULL;
	char *now = crm_itoa(time(NULL));

	name = crm_concat("fail-count", resource, '-');
	inject_transient_attr(cib_node, name, "value++");

	name = crm_concat("last-failure", resource, '-');
	inject_transient_attr(cib_node, name, now);

	free(name);
	free(now);
	}
	}

	static void
	create_node_entry(cib_t * cib_conn, const char *node)
	{
	int rc = pcmk_ok;
	int max = strlen(new_node_template) + strlen(node) + 1;
	char *xpath = NULL;

	xpath = calloc(1, max);

	snprintf(xpath, max, new_node_template, node);
	rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath \| cib_sync_call \| cib_scope_local);

	if (rc == -ENXIO) {
	xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);

	/* Using node uname as uuid ala corosync/openais */
	crm_xml_add(cib_object, XML_ATTR_ID, node);
	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
	cib_sync_call \| cib_scope_local);
	/* Not bothering with subsequent query to see if it exists,
	we'll bomb out later in the call to query_node_uuid()... */

	free_xml(cib_object);
	}

	free(xpath);
	}

	static lrmd_event_data_t *
	create_op(xmlNode * cib_resource, const char *task, int interval, int outcome)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *xop = NULL;

	op = calloc(1, sizeof(lrmd_event_data_t));

	op->rsc_id = strdup(ID(cib_resource));
	op->interval = interval;
	op->op_type = strdup(task);

	op->rc = outcome;
	op->op_status = 0;
	op->params = NULL; /* TODO: Fill me in */
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	op->call_id = 0;
	for (xop = __xml_first_child(cib_resource); xop != NULL; xop = __xml_next(xop)) {
	int tmp = 0;

	crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
	if (tmp > op->call_id) {
	op->call_id = tmp;
	}
	}
	op->call_id++;

	return op;
	}

	static xmlNode *
	inject_op(xmlNode * cib_resource, lrmd_event_data_t * op, int target_rc)
	{
	- return create_operation_update(cib_resource, op, CRM_FEATURE_SET, target_rc, crm_system_name,
	+ return create_operation_update(cib_resource, op, CRM_FEATURE_SET, target_rc, NULL, crm_system_name,
	LOG_DEBUG_2);
	}

	static xmlNode *
	inject_node_state(cib_t * cib_conn, const char node, const char uuid)
	{
	int rc = pcmk_ok;
	int max = strlen(rsc_template) + strlen(node) + 1;
	char *xpath = NULL;
	xmlNode *cib_object = NULL;

	xpath = calloc(1, max);

	if (bringing_nodes_online) {
	create_node_entry(cib_conn, node);
	}

	snprintf(xpath, max, node_template, node);
	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	if (cib_object && ID(cib_object) == NULL) {
	crm_err("Detected multiple node_state entries for xpath=%s, bailing", xpath);
	crm_log_xml_warn(cib_object, "Duplicates");
	crm_exit(ENOTUNIQ);
	}

	if (rc == -ENXIO) {
	char *found_uuid = NULL;

	if (uuid == NULL) {
	query_node_uuid(cib_conn, node, &found_uuid, NULL);
	} else {
	found_uuid = strdup(uuid);
	}

	cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
	crm_xml_add(cib_object, XML_ATTR_UUID, found_uuid);
	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
	cib_sync_call \| cib_scope_local);
	free_xml(cib_object);
	free(found_uuid);

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath \| cib_sync_call \| cib_scope_local);
	crm_trace("injecting node state for %s. rc is %d", node, rc);
	}

	free(xpath);
	CRM_ASSERT(rc == pcmk_ok);
	return cib_object;
	}

	static xmlNode *
	modify_node(cib_t * cib_conn, char *node, gboolean up)
	{
	xmlNode *cib_node = inject_node_state(cib_conn, node, NULL);

	if (up) {
	crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES);
	crm_xml_add(cib_node, XML_NODE_IS_PEER, ONLINESTATUS);
	crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
	crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);

	} else {
	crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
	crm_xml_add(cib_node, XML_NODE_IS_PEER, OFFLINESTATUS);
	crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
	crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
	}

	crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
	return cib_node;
	}

	static xmlNode *
	find_resource_xml(xmlNode * cib_node, const char *resource)
	{
	char *xpath = NULL;
	xmlNode *match = NULL;
	const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
	int max = strlen(rsc_template) + strlen(resource) + strlen(node) + 1;

	xpath = calloc(1, max);

	snprintf(xpath, max, rsc_template, node, resource);
	match = get_xpath_object(xpath, cib_node, LOG_DEBUG_2);

	free(xpath);
	return match;
	}


	static xmlNode *
	inject_resource(xmlNode * cib_node, const char resource, const char rclass, const char *rtype,
	const char *rprovider)
	{
	xmlNode *lrm = NULL;
	xmlNode *container = NULL;
	xmlNode *cib_resource = NULL;
	char *xpath = NULL;

	cib_resource = find_resource_xml(cib_node, resource);
	if (cib_resource != NULL) {
	return cib_resource;
	}

	/* One day, add query for class, provider, type */

	if (rclass == NULL \|\| rtype == NULL) {
	fprintf(stderr, "Resource %s not found in the status section of %s."
	" Please supply the class and type to continue\n", resource, ID(cib_node));
	return NULL;

	} else if (safe_str_neq(rclass, "ocf")
	&& safe_str_neq(rclass, "stonith")
	&& safe_str_neq(rclass, "heartbeat")
	&& safe_str_neq(rclass, "service")
	&& safe_str_neq(rclass, "upstart")
	&& safe_str_neq(rclass, "systemd")
	&& safe_str_neq(rclass, "lsb")) {
	fprintf(stderr, "Invalid class for %s: %s\n", resource, rclass);
	return NULL;

	} else if (safe_str_eq(rclass, "ocf") && rprovider == NULL) {
	fprintf(stderr, "Please specify the provider for resource %s\n", resource);
	return NULL;
	}

	xpath = (char *)xmlGetNodePath(cib_node);
	crm_info("Injecting new resource %s into %s '%s'", resource, xpath, ID(cib_node));
	free(xpath);

	lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
	if (lrm == NULL) {
	const char *node_uuid = ID(cib_node);

	lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
	crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
	}

	container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
	if (container == NULL) {
	container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
	}

	cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
	crm_xml_add(cib_resource, XML_ATTR_ID, resource);

	crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
	crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
	crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);

	return cib_resource;
	}

	static int
	find_ticket_state(cib_t * the_cib, const char ticket_id, xmlNode * ticket_state_xml)
	{
	int offset = 0;
	static int xpath_max = 1024;
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;

	char *xpath_string = NULL;

	CRM_ASSERT(ticket_state_xml != NULL);
	*ticket_state_xml = NULL;

	xpath_string = calloc(1, xpath_max);
	offset += snprintf(xpath_string + offset, xpath_max - offset, "%s", "/cib/status/tickets");

	if (ticket_id) {
	offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s[@id=\"%s\"]",
	XML_CIB_TAG_TICKET_STATE, ticket_id);
	}
	CRM_LOG_ASSERT(offset > 0);
	rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);

	if (rc != pcmk_ok) {
	goto bail;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_has_children(xml_search)) {
	if (ticket_id) {
	fprintf(stdout, "Multiple ticket_states match ticket_id=%s\n", ticket_id);
	}
	*ticket_state_xml = xml_search;
	} else {
	*ticket_state_xml = xml_search;
	}

	bail:
	free(xpath_string);
	return rc;
	}

	static int
	set_ticket_state_attr(const char ticket_id, const char attr_name,
	const char attr_value, cib_t cib, int cib_options)
	{
	int rc = pcmk_ok;
	xmlNode *xml_top = NULL;
	xmlNode *ticket_state_xml = NULL;

	rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
	if (rc == pcmk_ok) {
	crm_debug("Found a match state for ticket: id=%s", ticket_id);
	xml_top = ticket_state_xml;

	} else if (rc != -ENXIO) {
	return rc;

	} else {
	xmlNode *xml_obj = NULL;

	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
	ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
	crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
	}

	crm_xml_add(ticket_state_xml, attr_name, attr_value);

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);

	free_xml(xml_top);

	return rc;
	}

	void
	modify_configuration(pe_working_set_t * data_set, cib_t *cib,
	const char quorum, const char watchdog, GListPtr node_up, GListPtr node_down, GListPtr node_fail,
	GListPtr op_inject, GListPtr ticket_grant, GListPtr ticket_revoke,
	GListPtr ticket_standby, GListPtr ticket_activate)
	{
	int rc = pcmk_ok;
	GListPtr gIter = NULL;

	xmlNode *cib_op = NULL;
	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;

	lrmd_event_data_t *op = NULL;

	if (quorum) {
	xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);

	quiet_log(" + Setting quorum: %s\n", quorum);
	/* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid); */
	crm_xml_add(top, XML_ATTR_HAVE_QUORUM, quorum);

	rc = cib->cmds->modify(cib, NULL, top, cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	}

	if (watchdog) {
	quiet_log(" + Setting watchdog: %s\n", watchdog);

	rc = update_attr_delegate(cib, cib_sync_call \| cib_scope_local,
	XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	XML_ATTR_HAVE_WATCHDOG, watchdog, FALSE, NULL, NULL);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = node_up; gIter != NULL; gIter = gIter->next) {
	char node = (char )gIter->data;

	quiet_log(" + Bringing node %s online\n", node);
	cib_node = modify_node(cib, node, TRUE);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (gIter = node_down; gIter != NULL; gIter = gIter->next) {
	char xpath[STATUS_PATH_MAX];
	char node = (char )gIter->data;

	quiet_log(" + Taking node %s offline\n", node);
	cib_node = modify_node(cib, node, FALSE);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);

	snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node, XML_CIB_TAG_LRM);
	cib->cmds->delete(cib, xpath, NULL,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node,
	XML_TAG_TRANSIENT_NODEATTRS);
	cib->cmds->delete(cib, xpath, NULL,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	}

	for (gIter = node_fail; gIter != NULL; gIter = gIter->next) {
	char node = (char )gIter->data;

	quiet_log(" + Failing node %s\n", node);
	cib_node = modify_node(cib, node, TRUE);
	crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (gIter = ticket_grant; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	quiet_log(" + Granting ticket %s\n", ticket_id);
	rc = set_ticket_state_attr(ticket_id, "granted", "true",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = ticket_revoke; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	quiet_log(" + Revoking ticket %s\n", ticket_id);
	rc = set_ticket_state_attr(ticket_id, "granted", "false",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = ticket_standby; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	quiet_log(" + Making ticket %s standby\n", ticket_id);
	rc = set_ticket_state_attr(ticket_id, "standby", "true",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = ticket_activate; gIter != NULL; gIter = gIter->next) {
	char ticket_id = (char )gIter->data;

	quiet_log(" + Activating ticket %s\n", ticket_id);
	rc = set_ticket_state_attr(ticket_id, "standby", "false",
	cib, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	}

	for (gIter = op_inject; gIter != NULL; gIter = gIter->next) {
	char spec = (char )gIter->data;

	int rc = 0;
	int outcome = 0;
	int interval = 0;

	char *key = NULL;
	char *node = NULL;
	char *task = NULL;
	char *resource = NULL;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *rprovider = NULL;

	resource_t *rsc = NULL;

	quiet_log(" + Injecting %s into the configuration\n", spec);

	key = calloc(1, strlen(spec) + 1);
	node = calloc(1, strlen(spec) + 1);
	rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
	CRM_CHECK(rc == 3,
	fprintf(stderr, "Invalid operation spec: %s. Only found %d fields\n", spec, rc);
	continue);

	parse_op_key(key, &resource, &task, &interval);

	rsc = pe_find_resource(data_set->resources, resource);
	if (rsc == NULL) {
	fprintf(stderr, " - Invalid resource name: %s\n", resource);
	} else {
	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);

	cib_node = inject_node_state(cib, node, NULL);
	CRM_ASSERT(cib_node != NULL);

	update_failcounts(cib_node, resource, interval, outcome);

	cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
	CRM_ASSERT(cib_resource != NULL);

	op = create_op(cib_resource, task, interval, outcome);
	CRM_ASSERT(op != NULL);

	cib_op = inject_op(cib_resource, op, 0);
	CRM_ASSERT(cib_op != NULL);
	lrmd_free_event(op);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	}
	free(task);
	free(node);
	free(key);
	}
	}

	static gboolean
	exec_pseudo_action(crm_graph_t * graph, crm_action_t * action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	action->confirmed = TRUE;

	quiet_log(" * Pseudo action: %s%s%s\n", task, node ? " on " : "", node ? node : "");
	update_graph(graph, action);
	return TRUE;
	}

	static gboolean
	exec_rsc_action(crm_graph_t * graph, crm_action_t * action)
	{
	int rc = 0;
	GListPtr gIter = NULL;
	lrmd_event_data_t *op = NULL;
	int target_outcome = 0;
	gboolean uname_is_uuid = FALSE;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *resource = NULL;
	const char *rprovider = NULL;
	const char *operation = crm_element_value(action->xml, "operation");
	const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);

	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	char *uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
	const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if (safe_str_eq(operation, CRM_OP_PROBED)
	\|\| safe_str_eq(operation, CRM_OP_REPROBE)) {
	crm_info("Skipping %s op for %s\n", operation, node);
	goto done;
	}

	if (action_rsc == NULL) {
	crm_log_xml_err(action->xml, "Bad");
	free(node); free(uuid);
	return FALSE;
	}

	/* Look for the preferred name
	* If not found, try the expected 'local' name
	* If not found use the preferred name anyway
	*/
	resource = crm_element_value(action_rsc, XML_ATTR_ID);
	if (pe_find_resource(fake_resource_list, resource) == NULL) {
	const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);

	if (pe_find_resource(fake_resource_list, longname)) {
	resource = longname;
	}
	}

	if (safe_str_eq(operation, "delete")) {
	quiet_log(" * Resource action: %-15s delete on %s\n", resource, node);
	goto done;
	}

	rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
	rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
	rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);

	if (target_rc_s != NULL) {
	target_outcome = crm_parse_int(target_rc_s, "0");
	}

	CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call \| cib_scope_local) ==
	pcmk_ok);

	if (router_node) {
	uname_is_uuid = TRUE;
	}

	cib_node = inject_node_state(fake_cib, node, uname_is_uuid ? node : uuid);
	CRM_ASSERT(cib_node != NULL);

	cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
	CRM_ASSERT(cib_resource != NULL);

	op = convert_graph_action(cib_resource, action, 0, target_outcome);
	if (op->interval) {
	quiet_log(" * Resource action: %-15s %s=%d on %s\n", resource, op->op_type, op->interval,
	node);
	} else {
	quiet_log(" * Resource action: %-15s %s on %s\n", resource, op->op_type, node);
	}

	for (gIter = fake_op_fail_list; gIter != NULL; gIter = gIter->next) {
	char spec = (char )gIter->data;
	char *key = NULL;

	key = calloc(1, 1 + strlen(spec));
	snprintf(key, strlen(spec), "%s_%s_%d@%s=", resource, op->op_type, op->interval, node);

	if (strncasecmp(key, spec, strlen(key)) == 0) {
	sscanf(spec, "%[^=]=%d", (int )&op->rc);

	action->failed = TRUE;
	graph->abort_priority = INFINITY;
	printf("\tPretending action %d failed with rc=%d\n", action->id, op->rc);
	update_failcounts(cib_node, resource, op->interval, op->rc);
	free(key);
	break;
	}
	free(key);
	}

	inject_op(cib_resource, op, target_outcome);
	lrmd_free_event(op);

	rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(node); free(uuid);
	free_xml(cib_node);
	action->confirmed = TRUE;
	update_graph(graph, action);
	return TRUE;
	}

	static gboolean
	exec_crmd_action(crm_graph_t * graph, crm_action_t * action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	action->confirmed = TRUE;

	if(rsc) {
	quiet_log(" * Cluster action: %s for %s on %s\n", task, ID(rsc), node);
	} else {
	quiet_log(" * Cluster action: %s on %s\n", task, node);
	}
	update_graph(graph, action);
	return TRUE;
	}

	static gboolean
	exec_stonith_action(crm_graph_t * graph, crm_action_t * action)
	{
	const char *op = crm_meta_value(action->params, "stonith_action");
	char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);

	quiet_log(" * Fencing %s (%s)\n", target, op);
	if(safe_str_neq(op, "on")) {
	int rc = 0;
	char xpath[STATUS_PATH_MAX];
	xmlNode *cib_node = modify_node(fake_cib, target, FALSE);

	crm_xml_add(cib_node, XML_ATTR_ORIGIN, __FUNCTION__);
	CRM_ASSERT(cib_node != NULL);

	rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call \| cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
	fake_cib->cmds->delete(fake_cib, xpath, NULL,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target,
	XML_TAG_TRANSIENT_NODEATTRS);
	fake_cib->cmds->delete(fake_cib, xpath, NULL,
	cib_xpath \| cib_sync_call \| cib_scope_local);

	free_xml(cib_node);
	}

	action->confirmed = TRUE;
	update_graph(graph, action);
	free(target);
	return TRUE;
	}

	int
	run_simulation(pe_working_set_t * data_set, cib_t *cib, GListPtr op_fail_list, bool quiet)
	{
	crm_graph_t *transition = NULL;
	enum transition_status graph_rc = -1;

	crm_graph_functions_t exec_fns = {
	exec_pseudo_action,
	exec_rsc_action,
	exec_crmd_action,
	exec_stonith_action,
	};

	fake_cib = cib;
	fake_quiet = quiet;
	fake_op_fail_list = op_fail_list;

	quiet_log("\nExecuting cluster transition:\n");

	set_graph_functions(&exec_fns);
	transition = unpack_graph(data_set->graph, crm_system_name);
	print_graph(LOG_DEBUG, transition);

	fake_resource_list = data_set->resources;
	do {
	graph_rc = run_graph(transition);

	} while (graph_rc == transition_active);
	fake_resource_list = NULL;

	if (graph_rc != transition_complete) {
	fprintf(stdout, "Transition failed: %s\n", transition_status(graph_rc));
	print_graph(LOG_ERR, transition);
	}
	destroy_graph(transition);
	if (graph_rc != transition_complete) {
	fprintf(stdout, "An invalid transition was produced\n");
	}

	if (quiet == FALSE) {
	xmlNode *cib_object = NULL;
	int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call \| cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	cleanup_alloc_calculations(data_set);
	data_set->input = cib_object;
	}

	if (graph_rc != transition_complete) {
	return graph_rc;
	}
	return 0;
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:03 PM (1 d, 2 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002444
Default Alt Text: (212 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions