No OneTemporary
Actions

Size

277 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/crmd/Makefile.am b/crmd/Makefile.am
	index 2db2410ff0..af4f2f9eef 100644
	--- a/crmd/Makefile.am
	+++ b/crmd/Makefile.am
	@@ -1,76 +1,76 @@
	#
	# Copyright (C) 2004 Andrew Beekhof
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	#
	MAINTAINERCLEANFILES = Makefile.in

	INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \
	-I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl

	halibdir = $(CRM_DAEMON_DIR)

	## binary progs
	halib_PROGRAMS = crmd

	## SOURCES

	noinst_HEADERS = crmd.h crmd_fsa.h crmd_messages.h fsa_defines.h \
	fsa_matrix.h fsa_proto.h crmd_utils.h crmd_callbacks.h \
	crmd_lrm.h te_callbacks.h tengine.h

	crmd_SOURCES = main.c crmd.c corosync.c \
	fsa.c control.c messages.c membership.c callbacks.c \
	election.c join_client.c join_dc.c subsystems.c \
	- cib.c pengine.c tengine.c lrm.c lrm_state.c \
	+ cib.c pengine.c tengine.c lrm.c lrm_state.c remote_lrmd_ra.c \
	utils.c misc.c te_events.c te_actions.c te_utils.c te_callbacks.c

	if BUILD_HEARTBEAT_SUPPORT
	crmd_SOURCES += heartbeat.c
	endif

	crmd_LDADD = $(top_builddir)/lib/fencing/libstonithd.la \
	$(top_builddir)/lib/transition/libtransitioner.la \
	$(top_builddir)/lib/pengine/libpe_rules.la \
	$(top_builddir)/lib/cib/libcib.la \
	$(top_builddir)/lib/cluster/libcrmcluster.la \
	$(top_builddir)/lib/common/libcrmcommon.la \
	$(top_builddir)/lib/services/libcrmservice.la \
	$(top_builddir)/lib/lrmd/liblrmd.la \
	$(CLUSTERLIBS)
	if BUILD_XML_HELP
	man7_MANS = crmd.7

	%.xml: %
	$(top_builddir)/crmd/$< metadata \| $(XSLTPROC) --nonet --novalid --stringparam man.name $< $(top_srcdir)/xml/ocf-meta2man.xsl - > $(top_builddir)/crmd/$@

	%.7: %.xml
	$(XSLTPROC) $(MANPAGE_XSLT) $(top_builddir)/crmd/$<
	endif

	clean-generic:
	rm -f .log .debug .xml ~

	install-exec-local:

	uninstall-local:

	graphs: fsa_inputs.png fsa_inputs_by_action.png fsa_actions_by_state.png

	%.png: %.dot
	dot -Tpng $< > $@

	%.dot : fsa_matrix.h make_dot.pl
	perl $(top_srcdir)/crmd/make_dot.pl $(top_srcdir)/crmd/fsa_matrix.h $(top_builddir)/crmd
	diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
	index 8941fce734..dfe3b324b8 100644
	--- a/crmd/crmd_lrm.h
	+++ b/crmd/crmd_lrm.h
	@@ -1,100 +1,160 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	extern gboolean verify_stopped(enum crmd_fsa_state cur_state, int log_level);
	extern void lrm_clear_last_failure(const char *rsc_id);
	void lrm_op_callback(lrmd_event_data_t * op);

	typedef struct resource_history_s {
	char *id;
	lrmd_rsc_info_t rsc;
	lrmd_event_data_t *last;
	lrmd_event_data_t *failed;
	GList *recurring_op_list;

	/* Resources must be stopped using the same
	* parameters they were started with. This hashtable
	* holds the parameters that should be used for the next stop
	* cmd on this resource. */
	GHashTable *stop_params;
	} rsc_history_t;

	struct recurring_op_s {
	char *rsc_id;
	char *op_type;
	char *op_key;
	int call_id;
	int interval;
	gboolean remove;
	gboolean cancelled;
	};

	typedef struct lrm_state_s {
	const char *node_name;
	- lrmd_t *conn;
	+ void *conn;
	+ void *remote_ra_data;

	GHashTable *resource_history;
	GHashTable *pending_ops;
	GHashTable *deletion_ops;

	int num_lrm_register_fails;
	} lrm_state_t;

	struct pending_deletion_op_s {
	char *rsc;
	ha_msg_input_t *input;
	};

	/*!
	* \brief Clear all state information from a single state entry.
	* \note This does not close the lrmd connection
	*/
	void lrm_state_reset_tables(lrm_state_t *lrm_state);
	GList *lrm_state_get_list(void);

	/*!
	* \brief Initiate internal state tables
	*/
	gboolean lrm_state_init_local(void);

	/*!
	* \brief Destroy all state entries and internal state tables
	*/
	void lrm_state_destroy_all(void);

	/*!
	- * \brief Create remote lrmd connection entry.
	+ * \brief Create lrmd connection entry.
	*/
	-lrm_state_t lrm_state_create_remote(const char node_name, const char *server, int port);
	+lrm_state_t lrm_state_create(const char node_name);

	/*!
	* \brief Destroy lrmd connection keyed of node name
	*/
	void lrm_state_destroy(const char *node_name);

	/*!
	* \brief Find lrm_state data by node name
	*/
	lrm_state_t lrm_state_find(const char node_name);

	/*!
	- * \brief Either find or create a new entry for the local
	- * ipc lrmd connection.
	+ * \brief Either find or create a new entry
	*/
	-lrm_state_t lrm_state_find_or_create_local(const char node_name);
	+lrm_state_t lrm_state_find_or_create(const char node_name);

	+
	+/*!
	+ * The functions below are wrappers for the lrmd api calls the crmd
	+ * uses. These wrapper functions allow us to treat the crmd's remote
	+ * lrmd connection resources the same as regular resources. Internally
	+ * Regular resources go to the lrmd, and remote connection resources are
	+ * handled locally in the crmd.
	+ */
	+void lrm_state_disconnect(lrm_state_t *lrm_state);
	+int lrm_state_ipc_connect(lrm_state_t *lrm_state);
	+int lrm_state_remote_connect_async(lrm_state_t lrm_state, const char server, int port, int timeout);
	+int lrm_state_is_connected(lrm_state_t *lrm_state);
	+int lrm_state_poke_connection(lrm_state_t *lrm_state);
	+
	+int lrm_state_get_metadata (lrm_state_t *lrm_state,
	+ const char *class,
	+ const char *provider,
	+ const char *agent,
	+ char **output,
	+ enum lrmd_call_options options);
	+int lrm_state_cancel(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *action,
	+ int interval);
	+int lrm_state_exec(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *action,
	+ const char *userdata,
	+ int interval, /* ms */
	+ int timeout, /* ms */
	+ int start_delay, /* ms */
	+ lrmd_key_value_t *params);
	+lrmd_rsc_info_t lrm_state_get_rsc_info(lrm_state_t lrm_state,
	+ const char *rsc_id,
	+ enum lrmd_call_options options);
	+int lrm_state_register_rsc(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *class,
	+ const char *provider,
	+ const char *agent,
	+ enum lrmd_call_options options);
	+int lrm_state_unregister_rsc(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ enum lrmd_call_options options);
	+
	+/! These functions are used to manage the remote lrmd connection resources /
	+void remote_lrm_op_callback(lrmd_event_data_t * op);
	+gboolean is_remote_lrmd_ra(const char agent, const char provider, const char *id);
	+int remote_ra_get_metadata(char **output);
	+lrmd_rsc_info_t * remote_ra_get_rsc_info(lrm_state_t lrm_state, const char rsc_id);
	+int remote_ra_cancel(lrm_state_t lrm_state, const char rsc_id, const char *action, int interval);
	+int remote_ra_exec(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *action,
	+ const char *userdata,
	+ int interval, /* ms */
	+ int timeout, /* ms */
	+ int start_delay, /* ms */
	+ lrmd_key_value_t *params);
	+void remote_ra_cleanup(lrm_state_t *lrm_state);
	diff --git a/crmd/lrm.c b/crmd/lrm.c
	index 743a0cc9bd..75c4636b68 100644
	--- a/crmd/lrm.c
	+++ b/crmd/lrm.c
	@@ -1,1997 +1,1996 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/services.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	#define START_DELAY_THRESHOLD 5 * 60 * 1000
	#define MAX_LRM_REG_FAILS 30

	struct delete_event_s {
	int rc;
	const char *rsc;
	lrm_state_t *lrm_state;
	};


	gboolean process_lrm_event(lrm_state_t lrm_state, lrmd_event_data_t op);
	static gboolean is_rsc_active(lrm_state_t lrm_state, const char rsc_id);
	static gboolean build_active_RAs(lrm_state_t lrm_state, xmlNode rsc_list);
	static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
	static int delete_rsc_status(lrm_state_t lrm_state, const char rsc_id, int call_options, const char *user_name);

	static lrmd_event_data_t construct_op(lrm_state_t lrm_state, xmlNode * rsc_op, const char rsc_id, const char operation);
	static void do_lrm_rsc_op(lrm_state_t lrm_state, lrmd_rsc_info_t rsc, const char operation, xmlNode msg, xmlNode * request);

	void send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id);

	static gboolean lrm_state_verify_stopped(lrm_state_t *lrm_state, enum crmd_fsa_state cur_state, int log_level);

	static void
	lrm_connection_destroy(void)
	{
	if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	crm_crit("LRM Connection failed");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	clear_bit(fsa_input_register, R_LRM_CONNECTED);

	} else {
	crm_info("LRM Connection disconnected");
	}

	}

	static char *
	make_stop_id(const char *rsc, int call_id)
	{
	char *op_id = NULL;

	op_id = calloc(1, strlen(rsc) + 34);
	if (op_id != NULL) {
	snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id);
	}
	return op_id;
	}

	static void
	copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") == NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") != NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	update_history_cache(lrm_state_t lrm_state, lrmd_rsc_info_t rsc, lrmd_event_data_t * op)
	{
	int target_rc = 0;
	rsc_history_t *entry = NULL;

	if (op->rsc_deleted) {
	crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
	delete_rsc_status(lrm_state, op->rsc_id, cib_quorum_override, NULL);
	return;
	}

	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	return;
	}

	crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);

	entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
	if (entry == NULL && rsc) {
	entry = calloc(1, sizeof(rsc_history_t));
	entry->id = strdup(op->rsc_id);
	g_hash_table_insert(lrm_state->resource_history, entry->id, entry);

	entry->rsc.id = entry->id;
	entry->rsc.type = strdup(rsc->type);
	entry->rsc.class = strdup(rsc->class);
	if (rsc->provider) {
	entry->rsc.provider = strdup(rsc->provider);
	} else {
	entry->rsc.provider = NULL;
	}

	} else if (entry == NULL) {
	crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
	return;
	}

	target_rc = rsc_op_expected_rc(op);
	if (op->op_status == PCMK_LRM_OP_CANCELLED) {
	if (op->interval > 0) {
	GList gIter, gIterNext;

	crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);

	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIterNext) {
	lrmd_event_data_t *existing = gIter->data;
	gIterNext = gIter->next;

	if (safe_str_eq(op->rsc_id, existing->rsc_id)
	&& safe_str_eq(op->op_type, existing->op_type)
	&& op->interval == existing->interval) {
	lrmd_free_event(existing);
	entry->recurring_op_list = g_list_delete_link(entry->recurring_op_list, gIter);
	}
	}
	return;

	} else {
	crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
	op->rc, op->op_status);
	}

	} else if (did_rsc_op_fail(op, target_rc)) {
	/* We must store failed monitors here
	* - otherwise the block below will cause them to be forgetten them when a stop happens
	*/
	if (entry->failed) {
	lrmd_free_event(entry->failed);
	}
	entry->failed = lrmd_copy_event(op);

	} else if (op->interval == 0) {
	if (entry->last) {
	lrmd_free_event(entry->last);
	}
	entry->last = lrmd_copy_event(op);

	if (op->params &&
	(safe_str_eq(CRMD_ACTION_START, op->op_type) \|\|
	safe_str_eq(CRMD_ACTION_STATUS, op->op_type))) {

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}
	entry->stop_params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
	}
	}

	if (op->interval > 0) {
	crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
	entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));

	} else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
	GList *gIter = entry->recurring_op_list;

	crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
	g_list_length(gIter), op->rsc_id, op->op_type, op->interval);
	for (; gIter != NULL; gIter = gIter->next) {
	lrmd_free_event(gIter->data);
	}
	g_list_free(entry->recurring_op_list);
	entry->recurring_op_list = NULL;
	}
	}

	void
	lrm_op_callback(lrmd_event_data_t * op)
	{
	const char *nodename = NULL;
	lrm_state_t *lrm_state = NULL;

	CRM_CHECK(op != NULL, return);

	- /* determine if this is a callback for a remote or local connection */
	+ /* determine the node name for this connection. */
	nodename = op->remote_nodename ? op->remote_nodename : fsa_our_uname;
	lrm_state = lrm_state_find(nodename);

	CRM_ASSERT(lrm_state != NULL);

	+ if (op->type == lrmd_event_disconnect && (safe_str_eq(lrm_state->node_name, fsa_our_uname))) {
	+ lrm_connection_destroy();
	+ return;
	+ } else if (op->type != lrmd_event_exec_complete) {
	+ return;
	+ }
	+
	process_lrm_event(lrm_state, op);
	}

	/* A_LRM_CONNECT */
	void
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* This only pertains to local lrmd connections. Remote connections are handled as
	* resources within the pengine. Connecting and disconnecting from remote lrmd instances
	* handled differently than the local. */

	lrm_state_t *lrm_state = NULL;

	- lrm_state = lrm_state_find_or_create_local(fsa_our_uname);
	+ lrm_state = lrm_state_find_or_create(fsa_our_uname);
	if (lrm_state == NULL) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	if (action & A_LRM_DISCONNECT) {
	if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
	if(action == A_LRM_DISCONNECT) {
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	clear_bit(fsa_input_register, R_LRM_CONNECTED);
	- lrm_state->conn->cmds->disconnect(lrm_state->conn);
	+ lrm_state_disconnect(lrm_state);
	lrm_state_reset_tables(lrm_state);
	crm_notice("Disconnected from the LRM");
	}

	if (action & A_LRM_CONNECT) {
	int ret = pcmk_ok;

	crm_debug("Connecting to the LRM");
	- ret = lrm_state->conn->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL);
	+ ret = lrm_state_ipc_connect(lrm_state);

	if (ret != pcmk_ok) {
	- if (++lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
	+ if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
	crm_warn("Failed to sign on to the LRM %d"
	" (%d max) times", lrm_state->num_lrm_register_fails, MAX_LRM_REG_FAILS);

	crm_timer_start(wait_timer);
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	if (ret != pcmk_ok) {
	crm_err("Failed to sign on to the LRM %d" " (max) times", lrm_state->num_lrm_register_fails);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	set_bit(fsa_input_register, R_LRM_CONNECTED);
	crm_debug("LRM connection established");
	}

	if (action & ~(A_LRM_CONNECT \| A_LRM_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
	}
	}

	static void
	ghash_print_pending(gpointer key, gpointer value, gpointer user_data)
	{
	const char *stop_id = key;
	int *log_level = user_data;
	struct recurring_op_s *pending = value;

	do_crm_log(*log_level, "Pending action: %s (%s)", stop_id, pending->op_key);
	}

	static void
	ghash_print_pending_for_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	const char *stop_id = key;
	char *rsc = user_data;
	struct recurring_op_s *pending = value;

	if (safe_str_eq(rsc, pending->rsc_id)) {
	crm_notice("%sction %s (%s) incomplete at shutdown",
	pending->interval == 0 ? "A" : "Recurring a", stop_id, pending->op_key);
	}
	}

	static void
	ghash_count_pending(gpointer key, gpointer value, gpointer user_data)
	{
	int *counter = user_data;
	struct recurring_op_s *pending = value;

	if (pending->interval > 0) {
	/* Ignore recurring actions in the shutdown calculations */
	return;
	}

	(*counter)++;
	}

	static gboolean
	lrm_state_verify_stopped(lrm_state_t *lrm_state, enum crmd_fsa_state cur_state, int log_level)
	{
	int counter = 0;
	gboolean rc = TRUE;
	const char *when = "lrm disconnect";

	GHashTableIter gIter;
	rsc_history_t *entry = NULL;

	crm_debug("Checking for active resources before exit");

	if (cur_state == S_TERMINATE) {
	log_level = LOG_ERR;
	when = "shutdown";

	} else if (is_set(fsa_input_register, R_SHUTDOWN)) {
	when = "shutdown... waiting";
	}

	if (lrm_state->pending_ops) {
	- /* TODO check if the state connection is connected, not if fsa bit is set */
	- if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
	+ if (lrm_state_is_connected(lrm_state) == TRUE) {
	/* Only log/complain about non-recurring actions */
	g_hash_table_foreach_remove(lrm_state->pending_ops, stop_recurring_actions, NULL);
	}
	g_hash_table_foreach(lrm_state->pending_ops, ghash_count_pending, &counter);
	}

	if (counter > 0) {
	do_crm_log(log_level, "%d pending LRM operations at %s%s", counter, when);

	if (cur_state == S_TERMINATE \|\| !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
	g_hash_table_foreach(lrm_state->pending_ops, ghash_print_pending, &log_level);
	} else {
	rc = FALSE;
	}
	return rc;
	}

	if (lrm_state->resource_history == NULL) {
	return rc;
	}

	counter = 0;
	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	if (is_rsc_active(lrm_state, entry->id) == FALSE) {
	continue;
	}

	counter++;
	crm_trace("Found %s active", entry->id);
	if (lrm_state->pending_ops) {
	g_hash_table_foreach(lrm_state->pending_ops, ghash_print_pending_for_rsc, entry->id);
	}
	}

	if(counter) {
	crm_err("%d resources were active at %s.", counter, when);
	}

	return rc;
	}

	static char *
	get_rsc_metadata(const char type, const char class, const char *provider)
	{
	char *metadata = NULL;
	/* Always use a local connection for this operation */
	lrm_state_t *lrm_state = lrm_state_find(fsa_our_uname);

	CRM_CHECK(type != NULL, return NULL);
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(lrm_state != NULL, return NULL);

	if (provider == NULL) {
	provider = "heartbeat";
	}

	crm_trace("Retreiving metadata for %s::%s:%s", type, class, provider);
	- lrm_state->conn->cmds->get_metadata(lrm_state->conn, class, provider, type, &metadata, 0);
	+ lrm_state_get_metadata(lrm_state, class, provider, type, &metadata, 0);

	if (metadata) {
	/* copy the metadata because the LRM likes using
	* g_alloc instead of cl_malloc
	*/
	char *m_copy = strdup(metadata);

	g_free(metadata);
	metadata = m_copy;

	} else {
	crm_warn("No metadata found for %s::%s:%s", type, class, provider);
	}

	return metadata;
	}

	typedef struct reload_data_s {
	char *key;
	char *metadata;
	time_t last_query;
	gboolean can_reload;
	GListPtr restart_list;
	} reload_data_t;

	static void
	g_hash_destroy_reload(gpointer data)
	{
	reload_data_t *reload = data;

	free(reload->key);
	free(reload->metadata);
	g_list_free_full(reload->restart_list, free);
	free(reload);
	}

	GHashTable *reload_hash = NULL;
	static GListPtr
	get_rsc_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int len = 0;
	char *key = NULL;
	char *copy = NULL;
	const char *value = NULL;
	const char *provider = NULL;

	xmlNode *param = NULL;
	xmlNode *params = NULL;
	xmlNode *actions = NULL;
	xmlNode *metadata = NULL;

	time_t now = time(NULL);
	reload_data_t *reload = NULL;

	if (reload_hash == NULL) {
	reload_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_reload);
	}

	provider = rsc->provider;
	if (provider == NULL) {
	provider = "heartbeat";
	}

	len = strlen(rsc->type) + strlen(rsc->class) + strlen(provider) + 4;
	/* coverity[returned_null] Ignore */
	key = malloc( len);
	snprintf(key, len, "%s::%s:%s", rsc->type, rsc->class, provider);

	reload = g_hash_table_lookup(reload_hash, key);

	if (reload && ((now - 9) > reload->last_query)
	&& safe_str_eq(op->op_type, RSC_START)) {
	reload = NULL; /* re-query */
	}

	if (reload == NULL) {
	xmlNode *action = NULL;

	reload = calloc(1, sizeof(reload_data_t));
	g_hash_table_replace(reload_hash, key, reload);

	reload->last_query = now;
	reload->key = key;
	key = NULL;
	reload->metadata = get_rsc_metadata(rsc->type, rsc->class, provider);

	metadata = string2xml(reload->metadata);
	if (metadata == NULL) {
	crm_err("Metadata for %s::%s:%s is not valid XML",
	rsc->provider, rsc->class, rsc->type);
	goto cleanup;
	}

	actions = find_xml_node(metadata, "actions", TRUE);

	for (action = __xml_first_child(actions); action != NULL; action = __xml_next(action)) {
	if (crm_str_eq((const char *)action->name, "action", TRUE)) {
	value = crm_element_value(action, "name");
	if (safe_str_eq("reload", value)) {
	reload->can_reload = TRUE;
	break;
	}
	}
	}

	if (reload->can_reload == FALSE) {
	goto cleanup;
	}

	params = find_xml_node(metadata, "parameters", TRUE);
	for (param = __xml_first_child(params); param != NULL; param = __xml_next(param)) {
	if (crm_str_eq((const char *)param->name, "parameter", TRUE)) {
	value = crm_element_value(param, "unique");
	if (crm_is_true(value)) {
	value = crm_element_value(param, "name");
	if (value == NULL) {
	crm_err("%s: NULL param", key);
	continue;
	}
	crm_debug("Attr %s is not reloadable", value);
	copy = strdup(value);
	CRM_CHECK(copy != NULL, continue);
	reload->restart_list = g_list_append(reload->restart_list, copy);
	}
	}
	}
	}

	cleanup:
	free(key);
	free_xml(metadata);
	return reload->restart_list;
	}

	static void
	append_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, xmlNode * update, const char *version)
	{
	int len = 0;
	char *list = NULL;
	char *digest = NULL;
	const char *value = NULL;
	xmlNode *restart = NULL;
	GListPtr restart_list = NULL;
	GListPtr lpc = NULL;

	if (op->interval > 0) {
	/* monitors are not reloadable */
	return;

	} else if (op->params == NULL) {
	crm_debug("%s has no parameters", ID(update));
	return;

	} else if (rsc == NULL) {
	return;

	} else if (crm_str_eq(CRMD_ACTION_STOP, op->op_type, TRUE)) {
	/* Stopped resources don't need to be reloaded */
	return;

	} else if (compare_version("1.0.8", version) > 0) {
	/* Caller version does not support reloads */
	return;
	}

	restart_list = get_rsc_restart_list(rsc, op);
	if (restart_list == NULL) {
	/* Resource does not support reloads */
	return;
	}

	restart = create_xml_node(NULL, XML_TAG_PARAMS);
	for (lpc = restart_list; lpc != NULL; lpc = lpc->next) {
	const char param = (const char )lpc->data;

	int start = len;

	CRM_CHECK(param != NULL, continue);
	value = g_hash_table_lookup(op->params, param);
	if (value != NULL) {
	crm_xml_add(restart, param, value);
	}
	len += strlen(param) + 2;
	list = realloc(list, len + 1);
	sprintf(list + start, " %s ", param);
	}

	digest = calculate_operation_digest(restart, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list);
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	crm_trace("%s: %s, %s", rsc->id, digest, list);
	crm_log_xml_trace(restart, "restart digest source");

	free_xml(restart);
	free(digest);
	free(list);
	}

	static gboolean
	build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *src)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	const char *caller_version = CRM_FEATURE_SET;

	if (op == NULL) {
	return FALSE;

	} else if (AM_I_DC) {

	} else if (fsa_our_dc_version != NULL) {
	caller_version = fsa_our_dc_version;
	} else if (op->params == NULL) {
	caller_version = fsa_our_dc_version;
	} else {
	/* there is a small risk in formerly mixed clusters that
	* it will be sub-optimal.
	* however with our upgrade policy, the update we send
	* should still be completely supported anyway
	*/
	caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
	crm_debug("Falling back to operation originator version: %s", caller_version);
	}

	target_rc = rsc_op_expected_rc(op);
	xml_op = create_operation_update(parent, op, caller_version, target_rc, src, LOG_DEBUG);

	if (xml_op) {
	append_restart_list(rsc, op, xml_op, caller_version);
	}
	return TRUE;
	}

	static gboolean
	is_rsc_active(lrm_state_t lrm_state, const char rsc_id)
	{
	rsc_history_t *entry = NULL;

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	if (entry == NULL \|\| entry->last == NULL) {
	return FALSE;
	}

	crm_trace("Processing %s: %s.%d=%d",
	rsc_id, entry->last->op_type, entry->last->interval, entry->last->rc);
	if (entry->last->rc == PCMK_EXECRA_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) {
	return FALSE;

	} else if (entry->last->rc == PCMK_EXECRA_OK
	&& safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) {
	/* a stricter check is too complex...
	* leave that to the PE
	*/
	return FALSE;

	} else if (entry->last->rc == PCMK_EXECRA_NOT_RUNNING) {
	return FALSE;

	} else if (entry->last->interval == 0 && entry->last->rc == PCMK_EXECRA_NOT_CONFIGURED) {
	/* Badly configured resources can't be reliably stopped */
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	build_active_RAs(lrm_state_t lrm_state, xmlNode rsc_list)
	{
	GHashTableIter iter;
	rsc_history_t *entry = NULL;

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {

	GList *gIter = NULL;
	xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);

	crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.class);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);

	build_operation_update(xml_rsc, &(entry->rsc), entry->last, __FUNCTION__);
	build_operation_update(xml_rsc, &(entry->rsc), entry->failed, __FUNCTION__);
	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
	build_operation_update(xml_rsc, &(entry->rsc), gIter->data, __FUNCTION__);
	}
	}

	return FALSE;
	}

	static xmlNode *
	do_lrm_query_internal(lrm_state_t *lrm_state, gboolean is_replace)
	{
	xmlNode *xml_result = NULL;
	xmlNode *xml_state = NULL;
	xmlNode *xml_data = NULL;
	xmlNode *rsc_list = NULL;

	crm_node_t *peer = crm_get_peer(0, lrm_state->node_name);
	xml_state = do_update_node_cib(peer, node_update_cluster\|node_update_peer, NULL, __FUNCTION__);

	/* The next two lines shouldn't be necessary for newer DCs */
	crm_xml_add(xml_state, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
	crm_xml_add(xml_state, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);

	xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
	crm_xml_add(xml_data, XML_ATTR_ID, fsa_our_uuid);
	rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);

	/* Build a list of active (not always running) resources */
	build_active_RAs(lrm_state, rsc_list);

	xml_result = create_cib_fragment(xml_state, XML_CIB_TAG_STATUS);
	crm_log_xml_trace(xml_state, "Current state of the LRM");
	free_xml(xml_state);

	return xml_result;
	}

	xmlNode *
	do_lrm_query(gboolean is_replace, const char *node_name)
	{
	lrm_state_t *lrm_state = lrm_state_find(node_name);
	if (!lrm_state) {
	crm_err("Could not query lrm state for lrmd node %s", node_name);
	return NULL;
	}
	return do_lrm_query_internal(lrm_state, is_replace);
	}

	static void
	notify_deleted(lrm_state_t lrm_state, ha_msg_input_t input, const char *rsc_id, int rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, from_host, rsc_id, rc == pcmk_ok ? "" : " not");

	op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
	CRM_ASSERT(op != NULL);

	if (rc == pcmk_ok) {
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_OK;
	} else {
	op->op_status = PCMK_LRM_OP_ERROR;
	op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	}

	send_direct_ack(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);

	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	/* this isn't expected - trigger a new transition */
	time_t now = time(NULL);
	char *now_s = crm_itoa(now);

	crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id);

	update_attr_delegate(
	fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"last-lrm-refresh", now_s, FALSE, NULL);

	free(now_s);
	}
	}

	static gboolean
	lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	struct delete_event_s *event = user_data;
	struct pending_deletion_op_s *op = value;

	if (safe_str_eq(event->rsc, op->rsc)) {
	notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
	return TRUE;
	}
	return FALSE;
	}

	static gboolean
	lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
	{
	const char *rsc = user_data;
	struct recurring_op_s *pending = value;

	if (safe_str_eq(rsc, pending->rsc_id)) {
	crm_info("Removing op %s:%d for deleted resource %s",
	pending->op_key, pending->call_id, rsc);
	return TRUE;
	}
	return FALSE;
	}

	/*
	* Remove the rsc from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/
	#define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"

	static int
	delete_rsc_status(lrm_state_t lrm_state, const char rsc_id, int call_options, const char *user_name)
	{
	char *rsc_xpath = NULL;
	int max = 0;
	int rc = pcmk_ok;

	CRM_CHECK(rsc_id != NULL, return -ENXIO);

	max = strlen(rsc_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + 1;
	rsc_xpath = calloc(1, max);
	snprintf(rsc_xpath, max, rsc_template, lrm_state->node_name, rsc_id);

	rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
	NULL, NULL, call_options \| cib_xpath, user_name);

	free(rsc_xpath);
	return rc;
	}

	static void
	delete_rsc_entry(lrm_state_t lrm_state, ha_msg_input_t input, const char rsc_id, GHashTableIter rsc_gIter, int rc, const char *user_name)
	{
	struct delete_event_s event;

	CRM_CHECK(rsc_id != NULL, return);

	if (rc == pcmk_ok) {
	char *rsc_id_copy = strdup(rsc_id);

	if (rsc_gIter)
	g_hash_table_iter_remove(rsc_gIter);
	else
	g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
	crm_debug("sync: Sending delete op for %s", rsc_id_copy);
	delete_rsc_status(lrm_state, rsc_id_copy, cib_quorum_override, user_name);

	g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);

	free(rsc_id_copy);
	}

	if (input) {
	notify_deleted(lrm_state, input, rsc_id, rc);
	}

	event.rc = rc;
	event.rsc = rsc_id;
	event.lrm_state = lrm_state;
	g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
	}

	/*
	* Remove the op from the CIB
	*
	* Avoids refreshing the entire LRM section of this host
	*/

	#define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
	#define op_call_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']"

	static void
	delete_op_entry(lrm_state_t lrm_state, lrmd_event_data_t op, const char rsc_id, const char key, int call_id)
	{
	xmlNode *xml_top = NULL;

	if (op != NULL) {
	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);

	if(op->interval > 0) {
	char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval);
	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, XML_ATTR_ID, op_id);
	free(op_id);
	}

	crm_debug("async: Sending delete op for %s_%s_%d (call=%d)",
	op->rsc_id, op->op_type, op->interval, op->call_id);

	fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, cib_quorum_override);

	} else if (rsc_id != NULL && key != NULL) {
	int max = 0;
	char *op_xpath = NULL;

	if (call_id > 0) {
	max =
	strlen(op_call_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + strlen(key) +
	10;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_call_template, lrm_state->node_name, rsc_id, key, call_id);

	} else {
	max = strlen(op_template) + strlen(rsc_id) + strlen(lrm_state->node_name) + strlen(key) + 1;
	op_xpath = calloc(1, max);
	snprintf(op_xpath, max, op_template, lrm_state->node_name, rsc_id, key);
	}

	crm_debug("sync: Sending delete op for %s (call=%d)", rsc_id, call_id);
	fsa_cib_conn->cmds->delete(fsa_cib_conn, op_xpath, NULL, cib_quorum_override \| cib_xpath);

	free(op_xpath);

	} else {
	crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key);
	return;
	}

	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	void lrm_clear_last_failure(const char *rsc_id)
	{
	char *attr = NULL;
	GHashTableIter iter;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;
	rsc_history_t *entry = NULL;

	attr = generate_op_key(rsc_id, "last_failure", 0);

	/* This clears last failure for every lrm state that has this rsc.*/
	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	delete_op_entry(lrm_state, NULL, rsc_id, attr, 0);

	if (!lrm_state->resource_history) {
	continue;
	}

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
	if (safe_str_eq(rsc_id, entry->id)) {
	lrmd_free_event(entry->failed);
	entry->failed = NULL;
	}
	}
	}
	free(attr);

	}

	static gboolean
	cancel_op(lrm_state_t lrm_state, const char rsc_id, const char *key, int op, gboolean remove)
	{
	int rc = pcmk_ok;
	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != 0, return FALSE);
	CRM_CHECK(rsc_id != NULL, return FALSE);
	if (key == NULL) {
	key = make_stop_id(rsc_id, op);
	}
	pending = g_hash_table_lookup(lrm_state->pending_ops, key);

	if (pending) {
	if (remove && pending->remove == FALSE) {
	pending->remove = TRUE;
	crm_debug("Scheduling %s for removal", key);
	}

	if (pending->cancelled) {
	crm_debug("Operation %s already cancelled", key);
	return TRUE;
	}

	pending->cancelled = TRUE;

	} else {
	crm_info("No pending op found for %s", key);
	return TRUE;
	}

	crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);

	- rc = lrm_state->conn->cmds->cancel(lrm_state->conn,
	+ rc = lrm_state_cancel(lrm_state,
	pending->rsc_id,
	pending->op_type,
	pending->interval);

	if (rc == pcmk_ok) {
	crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
	} else {
	crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
	/* The caller needs to make sure the entry is
	* removed from the pending_ops list
	*
	* Usually by returning TRUE inside the worker function
	* supplied to g_hash_table_foreach_remove()
	*
	* Not removing the entry from pending_ops will block
	* the node from shutting down
	*/
	return FALSE;
	}

	return TRUE;
	}

	struct cancel_data {
	gboolean done;
	gboolean remove;
	const char *key;
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
	{
	struct cancel_data *data = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (safe_str_eq(op->op_key, data->key)) {
	data->done = TRUE;
	if (cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove) == FALSE) {
	return TRUE;
	}
	}
	return FALSE;
	}

	static gboolean
	cancel_op_key(lrm_state_t lrm_state, lrmd_rsc_info_t rsc, const char *key, gboolean remove)
	{
	struct cancel_data data;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(key != NULL, return FALSE);

	data.key = key;
	data.rsc = rsc;
	data.done = FALSE;
	data.remove = remove;
	data.lrm_state = lrm_state;

	g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
	return data.done;
	}

	static lrmd_rsc_info_t *
	get_lrm_resource(lrm_state_t lrm_state, xmlNode resource, xmlNode * op_msg, gboolean do_create)
	{
	lrmd_rsc_info_t *rsc = NULL;
	const char *id = ID(resource);
	const char *type = crm_element_value(resource, XML_ATTR_TYPE);
	const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER);
	const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG);

	crm_trace("Retrieving %s from the LRM.", id);
	CRM_CHECK(id != NULL, return NULL);

	- rsc = lrm_state->conn->cmds->get_rsc_info(lrm_state->conn, id, 0);
	+ rsc = lrm_state_get_rsc_info(lrm_state, id, 0);
	+
	if (!rsc && long_id) {
	- rsc = lrm_state->conn->cmds->get_rsc_info(lrm_state->conn, long_id, 0);
	+ rsc = lrm_state_get_rsc_info(lrm_state, long_id, 0);
	}

	if (!rsc && do_create) {
	CRM_CHECK(class != NULL, return NULL);
	CRM_CHECK(type != NULL, return NULL);

	crm_trace("Adding rsc %s before operation", id);

	- lrm_state->conn->cmds->register_rsc(lrm_state->conn,
	+ lrm_state_register_rsc(lrm_state,
	id, class, provider, type, lrmd_opt_drop_recurring);

	- rsc = lrm_state->conn->cmds->get_rsc_info(lrm_state->conn, id, 0);
	+ rsc = lrm_state_get_rsc_info(lrm_state, id, 0);

	if (!rsc) {
	fsa_data_t *msg_data = NULL;

	crm_err("Could not add resource %s to LRM", id);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}

	return rsc;
	}

	static void
	delete_resource(lrm_state_t *lrm_state,
	const char *id,
	lrmd_rsc_info_t * rsc,
	GHashTableIter *gIter,
	const char *sys,
	const char *host,
	const char *user,
	ha_msg_input_t * request)
	{
	int rc = pcmk_ok;

	crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host);

	if (rsc) {
	- rc = lrm_state->conn->cmds->unregister_rsc(lrm_state->conn, id, 0);
	+ rc = lrm_state_unregister_rsc(lrm_state, id, 0);
	}

	if (rc == pcmk_ok) {
	crm_trace("Resource '%s' deleted", id);
	} else if (rc == -EINPROGRESS) {
	crm_info("Deletion of resource '%s' pending", id);
	if (request) {
	struct pending_deletion_op_s *op = NULL;
	char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);

	op = calloc(1, sizeof(struct pending_deletion_op_s));
	op->rsc = strdup(rsc->id);
	op->input = copy_ha_msg_input(request);
	g_hash_table_insert(lrm_state->deletion_ops, ref, op);
	}
	return;
	} else {
	crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d",
	id, sys, user ? user : "internal", host, rc);
	}

	delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
	}

	/* A_LRM_INVOKE */
	void
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean done = FALSE;
	gboolean create_rsc = TRUE;
	lrm_state_t *lrm_state = NULL;
	const char *crm_op = NULL;
	const char *from_sys = NULL;
	const char *from_host = NULL;
	const char *operation = NULL;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *user_name = NULL;

	/* TODO lrm_state. Once lrmd connection resources are enabled and controlled by the
	* pengine, the input xml should tell us what lrm_state object we should be using here.
	* Until that work is done, we assume the local connection for now. */
	lrm_state = lrm_state_find(fsa_our_uname);

	#if ENABLE_ACL
	user_name = crm_element_value(input->msg, F_CRM_USER);
	crm_trace("LRM command from user '%s'", user_name);
	#endif

	crm_op = crm_element_value(input->msg, F_CRM_TASK);
	from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
	from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
	}

	crm_trace("LRM command from: %s", from_sys);

	if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
	operation = CRMD_ACTION_DELETE;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	operation = CRM_OP_LRM_REFRESH;

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
	rsc_history_t *entry = NULL;
	lrmd_event_data_t *op = NULL;
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* The lrmd can not fail a resource, it does not understand the
	* concept of success or failure in relation to a resource, it simply
	* executes operations and reports the results. We determine what a failure is.
	* Becaues of this, if we want to fail a resource we have to fake what we
	* understand a failure to look like.
	*
	* To do this we create a fake lrmd operation event for the resource
	* we want to fail. We then pass that event to the lrmd client callback
	* so it will be processed as if it actually came from the lrmd. */
	op = construct_op(lrm_state, input->xml, ID(xml_rsc), "asyncmon");
	free((char *) op->user_data);
	op->user_data = NULL;
	entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
	/* Make sure the call id is greater than the last successful operation,
	* otherwise the failure will not result in a possible recovery of the resource
	* as it could appear the failure occurred before the successful start */
	if (entry && entry->last) {
	op->call_id = entry->last->call_id + 1;
	if (op->call_id < 0) {
	op->call_id = 1;
	}
	}
	op->interval = 0;
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	CRM_ASSERT(op != NULL);

	# if ENABLE_ACL
	if (user_name && is_privileged(user_name) == FALSE) {
	crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;
	}
	# endif

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);
	if (rsc) {
	crm_info("Failing resource %s...", rsc->id);
	process_lrm_event(lrm_state, op);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_OK;
	lrmd_free_rsc_info(rsc);
	} else {
	crm_info("Cannot find/create resource in order to fail it...");
	crm_log_xml_warn(input->msg, "bad input");
	}

	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;

	} else if (input->xml != NULL) {
	operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
	}

	if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
	int rc = pcmk_ok;
	xmlNode *fragment = do_lrm_query_internal(lrm_state, TRUE);

	fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
	crm_info("Forced a local LRM refresh: call=%d", rc);
	free_xml(fragment);

	} else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
	xmlNode *data = do_lrm_query_internal(lrm_state, FALSE);
	xmlNode *reply = create_reply(input->msg, data);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_err("Unable to route reply");
	crm_log_xml_err(reply, "reply");
	}
	free_xml(reply);
	free_xml(data);

	} else if (safe_str_eq(operation, CRM_OP_PROBED)) {
	update_attrd(NULL, CRM_OP_PROBED, XML_BOOLEAN_TRUE, user_name);

	} else if (safe_str_eq(crm_op, CRM_OP_REPROBE)) {
	GHashTableIter gIter;
	rsc_history_t *entry = NULL;

	crm_notice("Forcing the status of all resources to be redetected");

	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, from_host, user_name, NULL);
	}

	/* Now delete the copy in the CIB */
	erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);

	/* And finally, _delete_ the value in attrd
	* Setting it to FALSE results in the PE sending us back here again
	*/
	update_attrd(NULL, CRM_OP_PROBED, NULL, user_name);

	} else if (operation != NULL) {
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *params = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* only the first 16 chars are used by the LRM */
	params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);

	if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	create_rsc = FALSE;
	}

	rsc = get_lrm_resource(lrm_state, xml_rsc, input->xml, create_rsc);

	if (rsc == NULL && create_rsc) {
	crm_err("Invalid resource definition");
	crm_log_xml_warn(input->msg, "bad input");

	} else if (rsc == NULL) {
	lrmd_event_data_t *op = NULL;

	crm_notice("Not creating resource for a %s event: %s", operation, ID(input->xml));
	delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok, user_name);

	op = construct_op(lrm_state, input->xml, ID(xml_rsc), operation);
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = PCMK_EXECRA_OK;
	CRM_ASSERT(op != NULL);
	send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);

	} else if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
	lrmd_event_data_t *op = NULL;
	char *op_key = NULL;
	char *meta_key = NULL;
	int call = 0;
	const char *call_id = NULL;
	const char *op_task = NULL;
	const char *op_interval = NULL;

	CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	op_interval = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
	op_task = crm_element_value(params, meta_key);
	free(meta_key);

	meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
	call_id = crm_element_value(params, meta_key);
	free(meta_key);

	CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);
	CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command");
	return);

	op = construct_op(lrm_state, input->xml, rsc->id, op_task);
	CRM_ASSERT(op != NULL);
	op_key = generate_op_key(rsc->id, op_task, crm_parse_int(op_interval, "0"));

	crm_debug("PE requested op %s (call=%s) be cancelled",
	op_key, call_id ? call_id : "NA");
	call = crm_parse_int(call_id, "0");
	if (call == 0) {
	/* the normal case when the PE cancels a recurring op */
	done = cancel_op_key(lrm_state, rsc, op_key, TRUE);

	} else {
	/* the normal case when the PE cancels an orphan op */
	done = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
	}

	if (done == FALSE) {
	crm_debug("Nothing known about operation %d for %s", call, op_key);
	delete_op_entry(lrm_state, NULL, rsc->id, op_key, call);

	/* needed?? surely not otherwise the cancel_op_(_key) wouldn't
	* have failed in the first place
	*/
	g_hash_table_remove(lrm_state->pending_ops, op_key);
	}

	op->rc = PCMK_EXECRA_OK;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(from_host, from_sys, rsc, op, rsc->id);

	free(op_key);
	lrmd_free_event(op);

	} else if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
	int cib_rc = pcmk_ok;

	CRM_ASSERT(rsc != NULL);

	cib_rc = delete_rsc_status(lrm_state, rsc->id, cib_dryrun \| cib_sync_call, user_name);
	if (cib_rc != pcmk_ok) {
	lrmd_event_data_t *op = NULL;

	crm_err
	("Attempt of deleting resource status '%s' from CIB for %s (user=%s) on %s failed: (rc=%d) %s",
	rsc->id, from_sys, user_name ? user_name : "unknown", from_host, cib_rc,
	pcmk_strerror(cib_rc));

	op = construct_op(lrm_state, input->xml, rsc->id, operation);
	op->op_status = PCMK_LRM_OP_ERROR;

	if (cib_rc == -EACCES) {
	op->rc = PCMK_EXECRA_INSUFFICIENT_PRIV;
	} else {
	op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	}
	send_direct_ack(from_host, from_sys, NULL, op, rsc->id);
	lrmd_free_event(op);
	return;
	}

	delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys, from_host, user_name, input);

	} else if (rsc != NULL) {
	do_lrm_rsc_op(lrm_state, rsc, operation, input->xml, input->msg);
	}

	lrmd_free_rsc_info(rsc);

	} else {
	crm_err("Operation was neither a lrm_query, nor a rsc op. %s", crm_str(crm_op));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	static lrmd_event_data_t *
	construct_op(lrm_state_t lrm_state, xmlNode rsc_op, const char rsc_id, const char operation)
	{
	lrmd_event_data_t *op = NULL;
	const char *op_delay = NULL;
	const char *op_timeout = NULL;
	const char *op_interval = NULL;
	GHashTable *params = NULL;

	const char *transition = NULL;

	CRM_LOG_ASSERT(rsc_id != NULL);

	op = calloc(1, sizeof(lrmd_event_data_t));
	op->type = lrmd_event_exec_complete;
	op->op_type = strdup(operation);
	op->op_status = PCMK_LRM_OP_PENDING;
	op->rc = -1;
	op->rsc_id = strdup(rsc_id);
	op->interval = 0;
	op->timeout = 0;
	op->start_delay = 0;

	if (rsc_op == NULL) {
	CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
	op->user_data = NULL;
	/* the stop_all_resources() case
	* by definition there is no DC (or they'd be shutting
	* us down).
	* So we should put our version here.
	*/
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_insert(op->params,
	strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));

	crm_trace("Constructed %s op for %s", operation, rsc_id);
	return op;
	}

	params = xml2list(rsc_op);
	g_hash_table_remove(params, CRM_META "_op_target_rc");

	op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
	op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
	op_interval = crm_meta_value(params, XML_LRM_ATTR_INTERVAL);

	op->interval = crm_parse_int(op_interval, "0");
	op->timeout = crm_parse_int(op_timeout, "0");
	op->start_delay = crm_parse_int(op_delay, "0");

	if (safe_str_neq(operation, RSC_STOP)) {
	op->params = params;

	} else {
	rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);

	/* If we do not have stop parameters cached, use
	* whatever we are given */
	if (!entry \|\| !entry->stop_params) {
	op->params = params;
	} else {
	/* Copy the cached parameter list so that we stop the resource
	* with the old attributes, not the new ones */
	op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_foreach(params, copy_meta_keys, op->params);
	g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
	g_hash_table_destroy(params);
	params = NULL;
	}
	}

	/* sanity */
	if (op->interval < 0) {
	op->interval = 0;
	}
	if (op->timeout <= 0) {
	op->timeout = op->interval;
	}
	if (op->start_delay < 0) {
	op->start_delay = 0;
	}

	transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
	CRM_CHECK(transition != NULL, return op);

	op->user_data = strdup(transition);

	if (op->interval != 0) {
	if (safe_str_eq(operation, CRMD_ACTION_START)
	\|\| safe_str_eq(operation, CRMD_ACTION_STOP)) {
	crm_err("Start and Stop actions cannot have an interval: %d", op->interval);
	op->interval = 0;
	}
	}

	crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval);

	return op;
	}

	void
	send_direct_ack(const char to_host, const char to_sys,
	lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id)
	{
	xmlNode *reply = NULL;
	xmlNode update, iter;
	xmlNode *fragment;
	crm_node_t *peer = NULL;

	CRM_CHECK(op != NULL, return);
	if (op->rsc_id == NULL) {
	CRM_LOG_ASSERT(rsc_id != NULL);
	op->rsc_id = strdup(rsc_id);
	}
	if (to_sys == NULL) {
	to_sys = CRM_SYSTEM_TENGINE;
	}

	peer = crm_get_peer(0, fsa_our_uname);
	update = do_update_node_cib(peer, node_update_none, NULL, __FUNCTION__);

	iter = create_xml_node(update, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);

	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);
	fragment = create_cib_fragment(update, XML_CIB_TAG_STATUS);

	reply = create_request(CRM_OP_INVOKE_LRM, fragment, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);

	crm_log_xml_trace(update, "ACK Update");

	crm_debug("ACK'ing resource op %s_%s_%d from %s: %s",
	op->rsc_id, op->op_type, op->interval, op->user_data,
	crm_element_value(reply, XML_ATTR_REFERENCE));

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}

	free_xml(fragment);
	free_xml(update);
	free_xml(reply);
	}

	gboolean
	verify_stopped(enum crmd_fsa_state cur_state, int log_level)
	{
	gboolean res = TRUE;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;

	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
	/* keep iterating through all even when false is returned */
	res = FALSE;
	}
	}

	set_bit(fsa_input_register, R_SENT_RSC_STOP);
	return res;
	}

	struct stop_recurring_action_s {
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	struct stop_recurring_action_s *event = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	if (op->interval != 0 && safe_str_eq(op->rsc_id, event->rsc->id)) {
	if (cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE) == FALSE) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static gboolean
	stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
	{
	lrm_state_t *lrm_state = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;
	gboolean remove = FALSE;

	if (op->interval != 0) {
	remove = cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static void
	do_lrm_rsc_op(lrm_state_t lrm_state, lrmd_rsc_info_t rsc, const char operation, xmlNode msg, xmlNode * request)
	{
	int call_id = 0;
	char *op_id = NULL;
	lrmd_event_data_t *op = NULL;
	lrmd_key_value_t *params = NULL;
	fsa_data_t *msg_data = NULL;
	const char *transition = NULL;

	CRM_CHECK(rsc != NULL, return);

	if (msg != NULL) {
	transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
	if (transition == NULL) {
	crm_log_xml_err(msg, "Missing transition number");
	}
	}

	op = construct_op(lrm_state, msg, rsc->id, operation);

	/* stop the monitor before stopping the resource */
	if (crm_str_eq(operation, CRMD_ACTION_STOP, TRUE)
	\|\| crm_str_eq(operation, CRMD_ACTION_DEMOTE, TRUE)
	\|\| crm_str_eq(operation, CRMD_ACTION_PROMOTE, TRUE)
	\|\| crm_str_eq(operation, CRMD_ACTION_MIGRATE, TRUE)) {


	struct stop_recurring_action_s data;
	data.rsc = rsc;
	data.lrm_state = lrm_state;
	g_hash_table_foreach_remove(lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);
	}

	/* now do the op */
	crm_debug("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval);

	if (fsa_state != S_NOT_DC && fsa_state != S_POLICY_ENGINE && fsa_state != S_TRANSITION_ENGINE) {
	if (safe_str_neq(operation, "fail")
	&& safe_str_neq(operation, CRMD_ACTION_STOP)) {
	crm_info("Discarding attempt to perform action %s on %s"
	" in state %s", operation, rsc->id, fsa_state2string(fsa_state));
	op->rc = 99;
	op->op_status = PCMK_LRM_OP_ERROR;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	lrmd_free_event(op);
	free(op_id);
	return;
	}
	}

	op_id = generate_op_key(rsc->id, op->op_type, op->interval);

	if (op->interval > 0) {
	/* cancel it so we can then restart it without conflict */
	cancel_op_key(lrm_state, rsc, op_id, FALSE);
	}

	if (op->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, op->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	params = lrmd_key_value_add(params, key, value);
	}
	}

	- call_id = lrm_state->conn->cmds->exec(lrm_state->conn,
	+ call_id = lrm_state_exec(lrm_state,
	rsc->id,
	op->op_type,
	op->user_data,
	op->interval,
	op->timeout,
	op->start_delay,
	- lrmd_opt_notify_changes_only,
	params);

	if (call_id <= 0) {
	crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);

	} else {
	/* record all operations so we can wait
	* for them to complete during shutdown
	*/
	char *call_id_s = make_stop_id(rsc->id, call_id);
	struct recurring_op_s *pending = NULL;

	pending = calloc(1, sizeof(struct recurring_op_s));
	crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);

	pending->call_id = call_id;
	pending->interval = op->interval;
	pending->op_type = strdup(operation);
	pending->op_key = strdup(op_id);
	pending->rsc_id = strdup(rsc->id);
	g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);

	if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) {
	char *uuid = NULL;
	int dummy = 0, target_rc = 0;

	crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc);
	free(uuid);

	op->rc = target_rc;
	op->op_status = PCMK_LRM_OP_DONE;
	send_direct_ack(NULL, NULL, rsc, op, rsc->id);
	}
	}

	free(op_id);
	lrmd_free_event(op);
	return;
	}

	int last_resource_update = 0;

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource update %d complete: rc=%d", call_id, rc);
	break;
	default:
	crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
	}

	if(call_id == last_resource_update) {
	last_resource_update = 0;
	trigger_fsa(fsa_source);
	}
	}

	static int
	do_update_resource(lrm_state_t lrm_state, lrmd_rsc_info_t rsc, lrmd_event_data_t * op)
	{
	/*
	<status>
	<nodes_status id=uname>
	<lrm>
	<lrm_resources>
	<lrm_resource id=...>
	</...>
	*/
	int rc = pcmk_ok;
	xmlNode update, iter = NULL;
	int call_opt = cib_quorum_override;

	CRM_CHECK(op != NULL, return 0);

	if (fsa_state == S_ELECTION \|\| fsa_state == S_PENDING) {
	crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state));
	call_opt \|= cib_scope_local;
	}

	iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
	update = iter;
	iter = create_xml_node(iter, XML_CIB_TAG_STATE);

	set_uuid(iter, XML_ATTR_UUID, lrm_state->node_name);
	crm_xml_add(iter, XML_ATTR_UNAME, lrm_state->node_name);
	crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);

	iter = create_xml_node(iter, XML_CIB_TAG_LRM);
	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);

	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);

	build_operation_update(iter, rsc, op, __FUNCTION__);

	if (rsc) {
	crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
	crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class);
	crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);

	CRM_CHECK(rsc->type != NULL, crm_err("Resource %s has no value for type", op->rsc_id));
	CRM_CHECK(rsc->class != NULL, crm_err("Resource %s has no value for class", op->rsc_id));

	} else {
	crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id);
	goto cleanup;
	}

	/* make it an asyncronous call and be done with it
	*
	* Best case:
	* the resource state will be discovered during
	* the next signup or election.
	*
	* Bad case:
	* we are shutting down and there is no DC at the time,
	* but then why were we shutting down then anyway?
	* (probably because of an internal error)
	*
	* Worst case:
	* we get shot for having resources "running" when the really weren't
	*
	* the alternative however means blocking here for too long, which
	* isnt acceptable
	*/
	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);

	if(rc > 0) {
	last_resource_update = rc;
	}

	/* the return code is a call number, not an error code */
	crm_trace("Sent resource state update message: %d for %s=%d on %s", rc,
	op->op_type, op->interval, op->rsc_id);
	fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);

	cleanup:
	free_xml(update);
	return rc;
	}

	void
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
	{
	CRM_CHECK(FALSE, return);
	}

	gboolean
	process_lrm_event(lrm_state_t lrm_state, lrmd_event_data_t op)
	{
	char *op_id = NULL;
	char *op_key = NULL;

	int update_id = 0;
	int log_level = LOG_ERR;
	gboolean removed = FALSE;
	lrmd_rsc_info_t *rsc = NULL;

	struct recurring_op_s *pending = NULL;

	CRM_CHECK(op != NULL, return FALSE);

	- if (op->type == lrmd_event_disconnect) {
	- lrm_connection_destroy();
	- return TRUE;
	- } else if (op->type != lrmd_event_exec_complete) {
	- return TRUE;
	- }
	-
	CRM_CHECK(op->rsc_id != NULL, return FALSE);
	op_id = make_stop_id(op->rsc_id, op->call_id);
	pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
	op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
	- rsc = lrm_state->conn->cmds->get_rsc_info(lrm_state->conn, op->rsc_id, 0);
	+ rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);

	switch (op->op_status) {
	case PCMK_LRM_OP_ERROR:
	case PCMK_LRM_OP_PENDING:
	case PCMK_LRM_OP_NOTSUPPORTED:
	break;
	case PCMK_LRM_OP_CANCELLED:
	log_level = LOG_INFO;
	break;
	case PCMK_LRM_OP_DONE:
	log_level = LOG_NOTICE;
	break;
	case PCMK_LRM_OP_TIMEOUT:
	log_level = LOG_DEBUG_3;
	crm_err("LRM operation %s (%d) %s (timeout=%dms)",
	op_key, op->call_id, services_lrm_status_str(op->op_status), op->timeout);
	break;
	default:
	crm_err("Mapping unknown status (%d) to ERROR", op->op_status);
	op->op_status = PCMK_LRM_OP_ERROR;
	}

	if (op->op_status == PCMK_LRM_OP_ERROR
	&& (op->rc == PCMK_EXECRA_RUNNING_MASTER \|\| op->rc == PCMK_EXECRA_NOT_RUNNING)) {
	/* Leave it up to the TE/PE to decide if this is an error */
	op->op_status = PCMK_LRM_OP_DONE;
	log_level = LOG_INFO;
	}

	if (op->op_status != PCMK_LRM_OP_CANCELLED) {
	if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
	/* Keep notify ops out of the CIB */
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
	} else {
	update_id = do_update_resource(lrm_state, rsc, op);
	}
	} else if (op->interval == 0) {
	/* This will occur when "crm resource cleanup" is called while actions are in-flight */
	crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
	send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);

	} else if (pending == NULL) {
	/* Operations that are cancelled may safely be removed
	* from the pending op list before the lrmd completion event
	* is received. Only report non-cancelled ops here. */
	if (op->op_status != PCMK_LRM_OP_CANCELLED) {
	crm_err("Op %s (call=%d): No 'pending' entry", op_key, op->call_id);
	}
	} else if (op->user_data == NULL) {
	crm_err("Op %s (call=%d): No user data", op_key, op->call_id);

	} else if (pending->remove) {
	delete_op_entry(lrm_state, op, op->rsc_id, op_key, op->call_id);

	} else {
	/* Before a stop is called, no need to direct ack */
	crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id);
	}

	if ((op->interval == 0) && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
	removed = TRUE;
	crm_trace("Op %s (call=%d, stop-id=%s): Confirmed", op_key, op->call_id, op_id);
	}

	if (op->op_status == PCMK_LRM_OP_DONE) {
	do_crm_log(log_level,
	"LRM operation %s (call=%d, rc=%d, cib-update=%d, confirmed=%s) %s",
	op_key, op->call_id, op->rc, update_id, removed ? "true" : "false",
	lrmd_event_rc2str(op->rc));
	} else {
	do_crm_log(log_level,
	"LRM operation %s (call=%d, status=%d, cib-update=%d, confirmed=%s) %s",
	op_key, op->call_id, op->op_status, update_id, removed ? "true" : "false",
	services_lrm_status_str(op->op_status));
	}

	if (op->output) {
	char *prefix = g_strdup_printf("%s_%s_%d:%d", op->rsc_id, op->op_type, op->interval, op->call_id);

	if (op->rc) {
	crm_log_output(LOG_NOTICE, prefix, op->output);
	} else {
	crm_log_output(LOG_DEBUG, prefix, op->output);
	}
	g_free(prefix);
	}

	if (op->rsc_deleted) {
	crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
	delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
	}

	/* If a shutdown was escalated while operations were pending,
	* then the FSA will be stalled right now... allow it to continue
	*/
	mainloop_set_trigger(fsa_source);
	update_history_cache(lrm_state, rsc, op);

	lrmd_free_rsc_info(rsc);
	free(op_key);
	free(op_id);

	return TRUE;
	}

	diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
	index e28904d5a0..6b7a325e26 100644
	--- a/crmd/lrm_state.c
	+++ b/crmd/lrm_state.c
	@@ -1,220 +1,396 @@
	/*
	* Copyright (C) 2012 David Vossel <dvossel@redhat.com>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	GHashTable *lrm_state_table = NULL;

	static void
	history_cache_destroy(gpointer data)
	{
	rsc_history_t *entry = data;

	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}

	free(entry->rsc.type);
	free(entry->rsc.class);
	free(entry->rsc.provider);

	lrmd_free_event(entry->failed);
	lrmd_free_event(entry->last);
	free(entry->id);
	free(entry);
	}

	static void
	free_deletion_op(gpointer value)
	{
	struct pending_deletion_op_s *op = value;

	free(op->rsc);
	delete_ha_msg_input(op->input);
	free(op);
	}

	static void
	free_recurring_op(gpointer value)
	{
	struct recurring_op_s op = (struct recurring_op_s )value;

	free(op->rsc_id);
	free(op->op_type);
	free(op->op_key);
	free(op);
	}


	-static lrm_state_t *
	-internal_state_create(const char *node_name)
	+lrm_state_t *
	+lrm_state_create(const char *node_name)
	{

	lrm_state_t *state = calloc(1, sizeof(lrm_state_t));

	if (!state) {
	return NULL;
	} else if (!node_name) {
	crm_err("No node name given for lrm state object");
	return NULL;
	}

	state->node_name = strdup(node_name);

	state->deletion_ops = g_hash_table_new_full(crm_str_hash,
	g_str_equal,
	g_hash_destroy_str,
	free_deletion_op);

	state->pending_ops = g_hash_table_new_full(crm_str_hash,
	g_str_equal,
	g_hash_destroy_str,
	free_recurring_op);

	state->resource_history = g_hash_table_new_full(crm_str_hash,
	g_str_equal,
	NULL,
	history_cache_destroy);

	g_hash_table_insert(lrm_state_table, (char *) state->node_name, state);
	return state;

	}
	-static lrm_state_t *
	-lrm_state_create_local(const char *node_name)
	-{
	-
	- lrm_state_t *state = internal_state_create(node_name);
	-
	- if (state) {
	- state->conn = lrmd_api_new();
	- state->conn->cmds->set_callback(state->conn, lrm_op_callback);
	- }
	- return state;
	-}
	-
	-lrm_state_t *
	-lrm_state_create_remote(const char node_name, const char server, int port)
	-{
	- lrm_state_t *state;
	-
	- if (!server \|\| !port) {
	- return NULL;
	- }
	-
	- state = internal_state_create(node_name);
	-
	- if (state) {
	- state->conn = lrmd_remote_api_new(node_name, server, port);
	- state->conn->cmds->set_callback(state->conn, lrm_op_callback);
	- }
	- return state;
	-}

	void lrm_state_destroy(const char *node_name)
	{
	g_hash_table_remove(lrm_state_table, node_name);
	}

	static void
	internal_lrm_state_destroy(gpointer data)
	{
	lrm_state_t *lrm_state = data;
	if (!lrm_state) {
	return;
	}

	+ remote_ra_cleanup(lrm_state);
	lrmd_api_delete(lrm_state->conn);

	if (lrm_state->resource_history) {
	g_hash_table_destroy(lrm_state->resource_history);
	}
	if (lrm_state->deletion_ops) {
	g_hash_table_destroy(lrm_state->deletion_ops);
	}
	if (lrm_state->pending_ops) {
	g_hash_table_destroy(lrm_state->pending_ops);
	}

	free((char *) lrm_state->node_name);
	free(lrm_state);
	}

	void lrm_state_reset_tables(lrm_state_t *lrm_state)
	{

	if (lrm_state->resource_history) {
	g_hash_table_remove_all(lrm_state->resource_history);
	}
	if (lrm_state->deletion_ops) {
	g_hash_table_remove_all(lrm_state->deletion_ops);
	}
	if (lrm_state->pending_ops) {
	g_hash_table_remove_all(lrm_state->pending_ops);
	}
	}

	gboolean lrm_state_init_local(void)
	{
	if (lrm_state_table) {
	return TRUE;
	}

	lrm_state_table = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, internal_lrm_state_destroy);
	if (!lrm_state_table) {
	return FALSE;
	}

	return TRUE;
	}

	void lrm_state_destroy_all(void)
	{
	if (lrm_state_table) {
	g_hash_table_destroy(lrm_state_table);
	}
	}

	lrm_state_t lrm_state_find(const char node_name)
	{
	return g_hash_table_lookup(lrm_state_table, node_name);
	}

	-lrm_state_t lrm_state_find_or_create_local(const char node_name)
	+lrm_state_t lrm_state_find_or_create(const char node_name)
	{
	lrm_state_t *lrm_state;

	lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
	if (!lrm_state) {
	- lrm_state = lrm_state_create_local(node_name);
	+ lrm_state = lrm_state_create(node_name);
	}

	return lrm_state;
	}

	GList *lrm_state_get_list(void)
	{
	return g_hash_table_get_values(lrm_state_table);
	}
	+
	+void
	+lrm_state_disconnect(lrm_state_t *lrm_state)
	+{
	+ if (!lrm_state->conn) {
	+ return;
	+ }
	+ ((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);
	+ lrmd_api_delete(lrm_state->conn);
	+ lrm_state->conn = NULL;
	+}
	+
	+int
	+lrm_state_is_connected(lrm_state_t *lrm_state)
	+{
	+ if (!lrm_state->conn) {
	+ return FALSE;
	+ }
	+ return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
	+}
	+
	+int lrm_state_poke_connection(lrm_state_t *lrm_state)
	+{
	+
	+ if (!lrm_state->conn) {
	+ return -1;
	+ }
	+ return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
	+}
	+
	+int
	+lrm_state_ipc_connect(lrm_state_t *lrm_state)
	+{
	+ int ret;
	+
	+ if (!lrm_state->conn) {
	+ lrm_state->conn = lrmd_api_new();
	+ ((lrmd_t *)lrm_state->conn)->cmds->set_callback(lrm_state->conn, lrm_op_callback);
	+ }
	+
	+ ret = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL);
	+
	+ if (ret != pcmk_ok) {
	+ lrm_state->num_lrm_register_fails++;
	+ } else {
	+ lrm_state->num_lrm_register_fails = 0;
	+ }
	+
	+ return ret;
	+}
	+
	+int
	+lrm_state_remote_connect_async(lrm_state_t lrm_state, const char server, int port, int timeout_ms)
	+{
	+ int ret;
	+
	+ if (!lrm_state->conn) {
	+ lrm_state->conn = lrmd_remote_api_new(lrm_state->node_name, server, port);
	+ if (!lrm_state->conn) {
	+ return -1;
	+ }
	+ ((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, remote_lrm_op_callback);
	+ }
	+
	+ crm_trace("initiating remote connection to %s at %d with timeout %d", server, port, timeout_ms);
	+ ret = ((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name, timeout_ms);
	+
	+ if (ret != pcmk_ok) {
	+ lrm_state->num_lrm_register_fails++;
	+ } else {
	+ lrm_state->num_lrm_register_fails = 0;
	+ }
	+
	+ return ret;
	+}
	+
	+int lrm_state_get_metadata (lrm_state_t *lrm_state,
	+ const char *class,
	+ const char *provider,
	+ const char *agent,
	+ char **output,
	+ enum lrmd_call_options options)
	+{
	+ if (!lrm_state->conn) {
	+ return -ENOTCONN;
	+ }
	+ if (is_remote_lrmd_ra(agent, provider, NULL)) {
	+ return remote_ra_get_metadata(output);
	+ }
	+
	+ /* Optimize this... only retrieve metadata from local lrmd connection. Perhaps consider
	+ * caching result. */
	+ return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata(lrm_state->conn, class, provider, agent, output, options);
	+}
	+
	+int lrm_state_cancel(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *action,
	+ int interval)
	+{
	+ if (!lrm_state->conn) {
	+ return -ENOTCONN;
	+ }
	+
	+ /* Optimize this, cancel requires a synced request/response to the server.
	+ * Figure out a way to make this async. */
	+ if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	+ return remote_ra_cancel(lrm_state, rsc_id, action, interval);
	+ }
	+ return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id, action, interval);
	+}
	+
	+lrmd_rsc_info_t lrm_state_get_rsc_info(lrm_state_t lrm_state,
	+ const char *rsc_id,
	+ enum lrmd_call_options options)
	+{
	+ if (!lrm_state->conn) {
	+ return NULL;
	+ }
	+ /* optimize this... this function is a synced round trip from client to daemon.
	+ * It should be possible to cache the resource info in the lrmd client to prevent this. */
	+ if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	+ return remote_ra_get_rsc_info(lrm_state, rsc_id);
	+ }
	+
	+ return ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
	+}
	+
	+int lrm_state_exec(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *action,
	+ const char *userdata,
	+ int interval, /* ms */
	+ int timeout, /* ms */
	+ int start_delay, /* ms */
	+ lrmd_key_value_t *params)
	+{
	+
	+ if (!lrm_state->conn) {
	+ lrmd_key_value_freeall(params);
	+ return -ENOTCONN;
	+ }
	+
	+ if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	+ return remote_ra_exec(lrm_state,
	+ rsc_id,
	+ action,
	+ userdata,
	+ interval,
	+ timeout,
	+ start_delay,
	+ params);
	+ }
	+
	+ return ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn,
	+ rsc_id,
	+ action,
	+ userdata,
	+ interval,
	+ timeout,
	+ start_delay,
	+ lrmd_opt_notify_changes_only,
	+ params);
	+}
	+
	+int lrm_state_register_rsc(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *class,
	+ const char *provider,
	+ const char *agent,
	+ enum lrmd_call_options options)
	+{
	+ if (!lrm_state->conn) {
	+ return -ENOTCONN;
	+ }
	+
	+ /* optimize this... this function is a synced round trip from client to daemon.
	+ * The crmd/lrm.c code path should be re-factored to allow the register of resources
	+ * to be performed async. The lrmd client api needs to make an async version
	+ * of register available. */
	+ if (is_remote_lrmd_ra(agent, provider, NULL)) {
	+ return lrm_state_find_or_create(rsc_id) ? pcmk_ok : -1;
	+ }
	+
	+ return ((lrmd_t *) lrm_state->conn)->cmds->register_rsc(lrm_state->conn, rsc_id, class, provider, agent, options);
	+}
	+
	+int lrm_state_unregister_rsc(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ enum lrmd_call_options options)
	+{
	+ if (!lrm_state->conn) {
	+ return -ENOTCONN;
	+ }
	+
	+ /* optimize this... this function is a synced round trip from client to daemon.
	+ * The crmd/lrm.c code path that uses this function should always treat it as an
	+ * async operation. The lrmd client api needs to make an async version unreg available. */
	+ if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	+ lrm_state_destroy(rsc_id);
	+ return pcmk_ok;
	+ }
	+
	+ return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
	+}
	diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
	new file mode 100644
	index 0000000000..022141f0c2
	--- /dev/null
	+++ b/crmd/remote_lrmd_ra.c
	@@ -0,0 +1,602 @@
	+/*
	+ * Copyright (C) 2013 David Vossel <dvossel@redhat.com>
	+ *
	+ * This program is free software; you can redistribute it and/or
	+ * modify it under the terms of the GNU General Public
	+ * License as published by the Free Software Foundation; either
	+ * version 2 of the License, or (at your option) any later version.
	+ *
	+ * This software is distributed in the hope that it will be useful,
	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	+ * General Public License for more details.
	+ *
	+ * You should have received a copy of the GNU General Public
	+ * License along with this library; if not, write to the Free Software
	+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	+ */
	+
	+#include <crm_internal.h>
	+#include <crm/crm.h>
	+
	+#include <crmd.h>
	+#include <crmd_fsa.h>
	+#include <crmd_messages.h>
	+#include <crmd_callbacks.h>
	+#include <crmd_lrm.h>
	+#include <crm/lrmd.h>
	+#include <crm/services.h>
	+
	+#define REMOTE_LRMD_RA "remote"
	+
	+int remote_ra_callid = 0;
	+
	+typedef struct remote_ra_cmd_s {
	+ /! the local node the cmd is issued from /
	+ char *owner;
	+ /! the remote node the cmd is executed on /
	+ char *rsc_id;
	+ /! the action to execute /
	+ char *action;
	+ /! some string the client wants us to give it back /
	+ char *userdata;
	+ /! start delay in ms /
	+ int start_delay;
	+ /! timer id used for start delay. /
	+ int delay_id;
	+ /! timeout in ms for cmd /
	+ int timeout;
	+ /! recurring interval in ms /
	+ int interval;
	+ /! interval timer id /
	+ int interval_id;
	+ int reported_success;
	+ int monitor_timeout_id;
	+ /! action parameters /
	+ lrmd_key_value_t *params;
	+ /! executed rc /
	+ int rc;
	+ int op_status;
	+ int call_id;
	+ gboolean cancel;
	+} remote_ra_cmd_t;
	+
	+typedef struct remote_ra_data_s {
	+ crm_trigger_t *work;
	+ remote_ra_cmd_t *cur_cmd;
	+ GList *cmds;
	+ GList *recurring_cmds;
	+
	+} remote_ra_data_t;
	+
	+static void
	+free_cmd(gpointer user_data)
	+{
	+ remote_ra_cmd_t *cmd = user_data;
	+
	+ if (!cmd) {
	+ return;
	+ }
	+ if (cmd->delay_id) {
	+ g_source_remove(cmd->delay_id);
	+ }
	+ if (cmd->interval_id) {
	+ g_source_remove(cmd->interval_id);
	+ }
	+ if (cmd->monitor_timeout_id) {
	+ g_source_remove(cmd->monitor_timeout_id);
	+ }
	+ free(cmd->owner);
	+ free(cmd->rsc_id);
	+ free(cmd->action);
	+ free(cmd->userdata);
	+ lrmd_key_value_freeall(cmd->params);
	+ free(cmd);
	+}
	+
	+static gboolean
	+recurring_helper(gpointer data)
	+{
	+ remote_ra_cmd_t *cmd = data;
	+ lrm_state_t *connection_rsc = NULL;
	+
	+ cmd->interval_id = 0;
	+ connection_rsc = lrm_state_find(cmd->rsc_id);
	+ if (connection_rsc && connection_rsc->remote_ra_data) {
	+ remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
	+
	+ ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);
	+ ra_data->cmds = g_list_append(ra_data->cmds, cmd);
	+ mainloop_set_trigger(ra_data->work);
	+ }
	+ return FALSE;
	+}
	+
	+static gboolean
	+start_delay_helper(gpointer data)
	+{
	+ remote_ra_cmd_t *cmd = data;
	+ lrm_state_t *connection_rsc = NULL;
	+
	+ cmd->delay_id = 0;
	+ connection_rsc = lrm_state_find(cmd->rsc_id);
	+ if (connection_rsc && connection_rsc->remote_ra_data) {
	+ remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
	+ mainloop_set_trigger(ra_data->work);
	+ }
	+ return FALSE;
	+}
	+
	+static void
	+report_remote_ra_result(remote_ra_cmd_t *cmd)
	+{
	+ lrmd_event_data_t op = { 0, };
	+
	+ op.type = lrmd_event_exec_complete;
	+ op.rsc_id = cmd->rsc_id;
	+ op.op_type = cmd->action;
	+ op.user_data = cmd->userdata;
	+ op.timeout = cmd->timeout;
	+ op.rc = cmd->rc;
	+ op.op_status = cmd->op_status;
	+ if (cmd->params) {
	+ lrmd_key_value_t *tmp;
	+
	+ op.params = g_hash_table_new_full(crm_str_hash,
	+ g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	+ for (tmp = cmd->params; tmp; tmp = tmp->next) {
	+ g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
	+ }
	+
	+ }
	+ op.call_id = cmd->call_id;
	+ op.remote_nodename = cmd->owner;
	+
	+ lrm_op_callback(&op);
	+
	+ if (op.params) {
	+ g_hash_table_destroy(op.params);
	+ }
	+}
	+
	+static gboolean
	+monitor_timeout_cb(gpointer data)
	+{
	+ lrm_state_t *lrm_state = NULL;
	+ remote_ra_cmd_t *cmd = data;
	+
	+ crm_debug("Poke async response timed out for node %s", cmd->rsc_id);
	+ cmd->monitor_timeout_id = 0;
	+ cmd->op_status = PCMK_LRM_OP_TIMEOUT;
	+ cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	+
	+ lrm_state = lrm_state_find(cmd->rsc_id);
	+ if (lrm_state && lrm_state->remote_ra_data) {
	+ remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	+
	+ if (ra_data->cur_cmd == cmd) {
	+ ra_data->cur_cmd = NULL;
	+ }
	+ if (ra_data->cmds) {
	+ mainloop_set_trigger(ra_data->work);
	+ }
	+ }
	+
	+ report_remote_ra_result(cmd);
	+ free_cmd(cmd);
	+ return FALSE;
	+}
	+
	+void
	+remote_lrm_op_callback(lrmd_event_data_t * op)
	+{
	+ lrm_state_t *lrm_state = NULL;
	+ remote_ra_data_t *ra_data = NULL;
	+ remote_ra_cmd_t *cmd = NULL;
	+
	+ crm_debug("remote connection event - event_type:%s node:%s action:%s rc:%s op_status:%s",
	+ lrmd_event_type2str(op->type),
	+ op->remote_nodename,
	+ op->op_type ? op->op_type : "none",
	+ lrmd_event_rc2str(op->rc),
	+ services_lrm_status_str(op->op_status));
	+
	+ /* we filter connection events and map them to start events for the remote agent */
	+ if ((op->type != lrmd_event_connect) && (op->type != lrmd_event_poke)) {
	+ crm_debug("not a connection or poke event, pass it up to crmd lrm callback");
	+ lrm_op_callback(op);
	+ return;
	+ } else if (!op->remote_nodename) {
	+ crm_debug("no node name, dropping event.");
	+ return;
	+ }
	+
	+ lrm_state = lrm_state_find(op->remote_nodename);
	+ if (!lrm_state \|\| !lrm_state->remote_ra_data) {
	+ crm_debug("lrm_state info not found for remote lrmd connection event");
	+ lrm_op_callback(op);
	+ return;
	+ }
	+
	+ ra_data = lrm_state->remote_ra_data;
	+
	+ if (!ra_data->cur_cmd) {
	+ crm_debug("no event to match");
	+ lrm_op_callback(op);
	+ return;
	+ }
	+
	+ cmd = ra_data->cur_cmd;
	+
	+ /* Start actions and migrate from actions complete after connection
	+ * comes back to us. */
	+ if (op->type == lrmd_event_connect && (
	+ safe_str_eq(cmd->action, "start") \|\|
	+ safe_str_eq(cmd->action, "migrate_from"))) {
	+
	+ if (op->connection_rc == -ETIMEDOUT) {
	+ cmd->op_status = PCMK_LRM_OP_TIMEOUT;
	+ cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	+ } else if (op->connection_rc < 0) {
	+ cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	+ cmd->op_status = PCMK_LRM_OP_ERROR;
	+ } else {
	+ cmd->rc = PCMK_EXECRA_OK;
	+ cmd->op_status = PCMK_LRM_OP_DONE;
	+ }
	+
	+ crm_debug("remote lrmd connect event matched %s action. ", cmd->action);
	+ report_remote_ra_result(cmd);
	+ if (ra_data->cmds) {
	+ mainloop_set_trigger(ra_data->work);
	+ }
	+ ra_data->cur_cmd = NULL;
	+ free_cmd(cmd);
	+ return;
	+ } else if (op->type == lrmd_event_poke && safe_str_eq(cmd->action, "monitor")) {
	+
	+ if (cmd->monitor_timeout_id) {
	+ g_source_remove(cmd->monitor_timeout_id);
	+ cmd->monitor_timeout_id = 0;
	+ }
	+
	+ /* Only report success the first time, after that only worry about failures.
	+ * For this function, if we get the poke pack, it is always a success. Pokes
	+ * only fail if the send fails, or the response times out. */
	+ if (!cmd->reported_success) {
	+ cmd->rc = PCMK_EXECRA_OK;
	+ cmd->op_status = PCMK_LRM_OP_DONE;
	+ report_remote_ra_result(cmd);
	+ cmd->reported_success = 1;
	+ }
	+
	+ crm_debug("remote lrmd poke event matched %s action. ", cmd->action);
	+
	+ /* success, keep rescheduling if interval is present. */
	+ if (cmd->interval && (cmd->cancel == FALSE)) {
	+ ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
	+ cmd->interval_id = g_timeout_add(cmd->interval, recurring_helper, cmd);
	+ cmd = NULL; /* prevent free */
	+ }
	+
	+ if (ra_data->cmds) {
	+ mainloop_set_trigger(ra_data->work);
	+ }
	+ ra_data->cur_cmd = NULL;
	+ free_cmd(cmd);
	+ return;
	+ }
	+
	+ crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
	+}
	+
	+static gboolean
	+handle_remote_ra_exec(gpointer user_data)
	+{
	+ int rc = 0;
	+ lrm_state_t *lrm_state = user_data;
	+ remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	+ remote_ra_cmd_t *cmd;
	+ GList *first = NULL;
	+ if (ra_data->cur_cmd) {
	+ /* still waiting on previous cmd*/
	+ return TRUE;
	+ }
	+
	+ while (ra_data->cmds) {
	+ first = ra_data->cmds;
	+ cmd = first->data;
	+ if (cmd->delay_id) {
	+ /* still waiting for start delay timer to trip */
	+ return TRUE;
	+ }
	+
	+ ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
	+ g_list_free_1(first);
	+
	+ if (!strcmp(cmd->action, "start") \|\| !strcmp(cmd->action, "migrate_from")) {
	+ const char *server = NULL;
	+ lrmd_key_value_t *tmp = NULL;
	+ int port = 0;
	+
	+ for (tmp = cmd->params; tmp; tmp = tmp->next) {
	+ if (safe_str_eq(tmp->key, "server")) {
	+ server = tmp->value;
	+ }
	+ if (safe_str_eq(tmp->key, "port")) {
	+ port = atoi(tmp->value);
	+ }
	+ }
	+
	+ rc = lrm_state_remote_connect_async(lrm_state, server, port, cmd->timeout);
	+
	+ if (rc == 0) {
	+ /* take care of this later when we get async connection result */
	+ crm_debug("began remote lrmd connect, waiting for connect event.");
	+ ra_data->cur_cmd = cmd;
	+ return TRUE;
	+ } else {
	+ crm_debug("connect failed, not expecting to match any connection event later");
	+ cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	+ cmd->op_status = PCMK_LRM_OP_ERROR;
	+ }
	+ report_remote_ra_result(cmd);
	+
	+ } else if (!strcmp(cmd->action, "monitor")) {
	+
	+ if (lrm_state_is_connected(lrm_state) == TRUE) {
	+ rc = lrm_state_poke_connection(lrm_state);
	+ if (rc < 0) {
	+ cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
	+ cmd->op_status = PCMK_LRM_OP_ERROR;
	+ }
	+ } else {
	+ rc = -1;
	+ cmd->op_status = PCMK_LRM_OP_DONE;
	+ cmd->rc = PCMK_EXECRA_NOT_RUNNING;
	+ }
	+
	+ if (rc == 0) {
	+ crm_debug("poked remote lrmd at node %s, waiting for async response.", cmd->rsc_id);
	+ ra_data->cur_cmd = cmd;
	+ cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
	+ return TRUE;
	+ }
	+ report_remote_ra_result(cmd);
	+
	+ } else if (!strcmp(cmd->action, "stop")) {
	+ lrm_state_disconnect(lrm_state);
	+ cmd->rc = PCMK_EXECRA_OK;
	+ cmd->op_status = PCMK_LRM_OP_DONE;
	+ report_remote_ra_result(cmd);
	+
	+ } else if (!strcmp(cmd->action, "migrate_to")) {
	+ /* no-op. */
	+ cmd->rc = PCMK_EXECRA_OK;
	+ cmd->op_status = PCMK_LRM_OP_DONE;
	+ report_remote_ra_result(cmd);
	+ }
	+
	+ free_cmd(cmd);
	+ }
	+
	+ return TRUE;
	+}
	+
	+static void
	+remote_ra_data_init(lrm_state_t *lrm_state)
	+{
	+ remote_ra_data_t *ra_data = NULL;
	+
	+ if (lrm_state->remote_ra_data) {
	+ return;
	+ }
	+
	+ ra_data = calloc(1, sizeof(remote_ra_data_t));
	+ ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
	+ lrm_state->remote_ra_data = ra_data;
	+}
	+
	+void
	+remote_ra_cleanup(lrm_state_t *lrm_state)
	+{
	+ remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	+
	+ if (!ra_data) {
	+ return;
	+ }
	+
	+ if (ra_data->cmds) {
	+ g_list_free_full(ra_data->cmds, free_cmd);
	+ }
	+
	+ if (ra_data->cmds) {
	+ g_list_free_full(ra_data->recurring_cmds, free_cmd);
	+ }
	+ mainloop_destroy_trigger(ra_data->work);
	+ free(ra_data);
	+ lrm_state->remote_ra_data = NULL;
	+}
	+
	+gboolean
	+is_remote_lrmd_ra(const char agent, const char provider, const char *id)
	+{
	+ if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
	+ return TRUE;
	+ }
	+ if (id && lrm_state_find(id)) {
	+ return TRUE;
	+ }
	+
	+ return FALSE;
	+}
	+
	+int
	+remote_ra_get_metadata(char **output)
	+{
	+#define remote_metadata \
	+"<?xml version=\"1.0\"?>\n"\
	+"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"\
	+"<resource-agent name=\"remote\" version=\"0.1\">\n"\
	+" <version>1.0</version>\n"\
	+" <parameters>\n"\
	+" <parameter name=\"server\" unique=\"1\">\n"\
	+" <longdesc lang=\"en\">\n"\
	+" Server location to connect to. This can be an ip address or hostname.\n"\
	+" </longdesc>\n"\
	+" <shortdesc lang=\"en\">Server location</shortdesc>\n"\
	+" <content type=\"string\"/>\n"\
	+" </parameter>\n"\
	+" <parameter name=\"port\" unique=\"1\">\n"\
	+" <longdesc lang=\"en\">\n"\
	+" tcp port to connect to.\n"\
	+" </longdesc>\n"\
	+" <shortdesc lang=\"en\">tcp port</shortdesc>\n"\
	+" <content type=\"string\" default=\"1984\"/>\n"\
	+" </parameter>\n"\
	+" </parameters>\n"\
	+" <actions>\n"\
	+" <action name=\"start\" timeout=\"15\" />\n"\
	+" <action name=\"stop\" timeout=\"15\" />\n"\
	+" <action name=\"monitor\" timeout=\"15\" />\n"\
	+" <action name=\"migrate_to\" timeout=\"15\" />\n"\
	+" <action name=\"migrate_from\" timeout=\"15\" />\n"\
	+" <action name=\"meta-data\" timeout=\"5\" />\n"\
	+" </actions>\n"\
	+"</resource-agent>\n"
	+
	+ *output = strdup(remote_metadata);
	+ return 0;
	+}
	+
	+lrmd_rsc_info_t *
	+remote_ra_get_rsc_info(lrm_state_t lrm_state, const char rsc_id)
	+{
	+ lrmd_rsc_info_t *info = NULL;
	+ if ((lrm_state_find(rsc_id))) {
	+ info = calloc(1, sizeof(lrmd_rsc_info_t));
	+
	+ info->id = strdup(rsc_id);
	+ info->type = strdup(REMOTE_LRMD_RA);
	+ info->class = strdup("ocf");
	+ info->provider = strdup("pacemaker");
	+ }
	+
	+ return info;
	+}
	+
	+static gboolean
	+is_remote_ra_supported_action(const char *action)
	+{
	+ if (!action) {
	+ return FALSE;
	+ } else if (strcmp(action,"start") &&
	+ strcmp(action,"stop") &&
	+ strcmp(action,"migrate_to") &&
	+ strcmp(action,"migrate_from") &&
	+ strcmp(action,"monitor")) {
	+ return FALSE;
	+ }
	+
	+ return TRUE;
	+}
	+
	+static GList *
	+remove_cmd(GList list, const char action, int interval)
	+{
	+ remote_ra_cmd_t *cmd = NULL;
	+ GListPtr gIter = NULL;
	+ for (gIter = list; gIter != NULL; gIter = gIter->next) {
	+ cmd = gIter->data;
	+ if (cmd->interval == interval && safe_str_eq(cmd->action, action)) {
	+ break;
	+ }
	+ cmd = NULL;
	+ }
	+ if (cmd) {
	+ list = g_list_remove(list, cmd);
	+ free_cmd(cmd);
	+ }
	+ return list;
	+}
	+
	+int remote_ra_cancel(lrm_state_t lrm_state, const char rsc_id, const char *action, int interval)
	+{
	+ lrm_state_t *connection_rsc = NULL;
	+ remote_ra_data_t *ra_data = NULL;
	+
	+ connection_rsc = lrm_state_find(rsc_id);
	+ if (!connection_rsc \|\| !connection_rsc->remote_ra_data) {
	+ return -EINVAL;
	+ }
	+
	+ ra_data = connection_rsc->remote_ra_data;
	+ ra_data->cmds = remove_cmd(ra_data->cmds, action, interval);
	+ ra_data->recurring_cmds = remove_cmd(ra_data->cmds, action, interval);
	+ if (ra_data->cur_cmd &&
	+ (ra_data->cur_cmd->interval == interval) &&
	+ (safe_str_eq(ra_data->cur_cmd->action, action))) {
	+
	+ ra_data->cur_cmd->cancel = TRUE;
	+ }
	+
	+ return 0;
	+}
	+
	+int remote_ra_exec(lrm_state_t *lrm_state,
	+ const char *rsc_id,
	+ const char *action,
	+ const char *userdata,
	+ int interval, /* ms */
	+ int timeout, /* ms */
	+ int start_delay, /* ms */
	+ lrmd_key_value_t *params)
	+{
	+ int rc = 0;
	+ lrm_state_t *connection_rsc = NULL;
	+ remote_ra_cmd_t *cmd = NULL;
	+ remote_ra_data_t *ra_data = NULL;
	+
	+ if (is_remote_ra_supported_action(action) == FALSE) {
	+ rc = -EINVAL;
	+ goto exec_done;
	+ }
	+
	+ connection_rsc = lrm_state_find(rsc_id);
	+ if (!connection_rsc) {
	+ rc = -EINVAL;
	+ goto exec_done;
	+ }
	+
	+ remote_ra_data_init(connection_rsc);
	+
	+ cmd = calloc(1, sizeof(remote_ra_cmd_t));
	+ cmd->owner = strdup(lrm_state->node_name);
	+ cmd->rsc_id = strdup(rsc_id);
	+ cmd->action = strdup(action);
	+ cmd->userdata = strdup(userdata);
	+ cmd->interval = interval;
	+ cmd->timeout = timeout;
	+ cmd->start_delay = start_delay;
	+ cmd->params = params;
	+
	+ remote_ra_callid++;
	+ if (remote_ra_callid <=0) {
	+ remote_ra_callid = 1;
	+ }
	+ cmd->call_id = remote_ra_callid;
	+
	+ if (cmd->start_delay) {
	+ cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
	+ }
	+ ra_data = connection_rsc->remote_ra_data;
	+
	+ ra_data->cmds = g_list_append(ra_data->cmds, cmd);
	+ mainloop_set_trigger(ra_data->work);
	+
	+ return cmd->call_id;
	+exec_done:
	+
	+ lrmd_key_value_freeall(params);
	+ return rc;
	+}
	diff --git a/include/crm/lrmd.h b/include/crm/lrmd.h
	index a07c9757eb..c5e4fe05c6 100644
	--- a/include/crm/lrmd.h
	+++ b/include/crm/lrmd.h
	@@ -1,466 +1,508 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	/**
	* \file
	* \brief Local Resource Manager
	* \ingroup lrm
	*/

	#ifndef LRMD__H
	#define LRMD__H

	typedef struct lrmd_s lrmd_t;
	typedef struct lrmd_key_value_s {
	char *key;
	char *value;
	struct lrmd_key_value_s *next;
	} lrmd_key_value_t;


	/* INDENT-OFF */
	#define DEFAULT_REMOTE_KEY_LOCATION "/etc/pacemaker/authkey"
	#define ALT_REMOTE_KEY_LOCATION "/etc/corosync/authkey"
	#define DEFAULT_REMOTE_PORT 1984
	#define DEFAULT_REMOTE_USERNAME "lrmd"

	#define F_LRMD_OPERATION "lrmd_op"
	#define F_LRMD_CLIENTNAME "lrmd_clientname"
	#define F_LRMD_CLIENTID "lrmd_clientid"
	#define F_LRMD_REMOTE_MSG_TYPE "lrmd_remote_msg_type"
	#define F_LRMD_REMOTE_MSG_ID "lrmd_remote_msg_id"
	#define F_LRMD_CALLBACK_TOKEN "lrmd_async_id"
	#define F_LRMD_CALLID "lrmd_callid"
	#define F_LRMD_CANCEL_CALLID "lrmd_cancel_callid"
	#define F_LRMD_CALLOPTS "lrmd_callopt"
	#define F_LRMD_CALLDATA "lrmd_calldata"
	#define F_LRMD_RC "lrmd_rc"
	#define F_LRMD_EXEC_RC "lrmd_exec_rc"
	#define F_LRMD_OP_STATUS "lrmd_exec_op_status"
	#define F_LRMD_TIMEOUT "lrmd_timeout"
	#define F_LRMD_CLASS "lrmd_class"
	#define F_LRMD_PROVIDER "lrmd_provider"
	#define F_LRMD_TYPE "lrmd_type"
	#define F_LRMD_ORIGIN "lrmd_origin"

	#define F_LRMD_RSC_RUN_TIME "lrmd_run_time"
	#define F_LRMD_RSC_RCCHANGE_TIME "lrmd_rcchange_time"
	#define F_LRMD_RSC_EXEC_TIME "lrmd_exec_time"
	#define F_LRMD_RSC_QUEUE_TIME "lrmd_queue_time"

	#define F_LRMD_RSC_ID "lrmd_rsc_id"
	#define F_LRMD_RSC_ACTION "lrmd_rsc_action"
	#define F_LRMD_RSC_USERDATA_STR "lrmd_rsc_userdata_str"
	#define F_LRMD_RSC_OUTPUT "lrmd_rsc_output"
	#define F_LRMD_RSC_START_DELAY "lrmd_rsc_start_delay"
	#define F_LRMD_RSC_INTERVAL "lrmd_rsc_interval"
	#define F_LRMD_RSC_METADATA "lrmd_rsc_metadata_res"
	#define F_LRMD_RSC_DELETED "lrmd_rsc_deleted"
	#define F_LRMD_RSC "lrmd_rsc"

	#define LRMD_OP_RSC_CHK_REG "lrmd_rsc_check_register"
	#define LRMD_OP_RSC_REG "lrmd_rsc_register"
	#define LRMD_OP_RSC_EXEC "lrmd_rsc_exec"
	#define LRMD_OP_RSC_CANCEL "lrmd_rsc_cancel"
	#define LRMD_OP_RSC_UNREG "lrmd_rsc_unregister"
	#define LRMD_OP_RSC_INFO "lrmd_rsc_info"
	#define LRMD_OP_RSC_METADATA "lrmd_rsc_metadata"
	+#define LRMD_OP_POKE "lrmd_rsc_poke"

	#define T_LRMD "lrmd"
	#define T_LRMD_REPLY "lrmd_reply"
	#define T_LRMD_NOTIFY "lrmd_notify"
	/* INDENT-ON */

	/*!
	* \brief Create a new local lrmd connection
	*/
	lrmd_t *lrmd_api_new(void);

	/*!
	* \brief Create a new remote lrmd connection using tls backend
	*
	* \note nodename and server may be the same value.
	*
	* \param nodename, the remote node name identified with this connection.
	* \param server, the server to connect to.
	* \param port, the port to connect to.
	*/
	lrmd_t lrmd_remote_api_new(const char nodename, const char *server, int port);

	/*!
	* \brief Use after lrmd_poll returns 1.
	*
	* \param fd to poll on
	* \param timeout in ms
	*
	* \retval true - connection is still up
	* \retval false - disconnected
	*/
	bool lrmd_dispatch(lrmd_t *lrmd);

	/*!
	* \brief Poll for a specified timeout period to determine if a message
	* is ready for dispatch.
	* \retval 1 msg is ready
	* \retval 0 timeout occured
	* \retval negative error code
	*/
	int lrmd_poll(lrmd_t *lrmd, int timeout);

	/*!
	* \brief Destroy lrmd object
	*/
	void lrmd_api_delete(lrmd_t * lrmd);
	lrmd_key_value_t lrmd_key_value_add(lrmd_key_value_t kvp,
	const char *key,
	const char *value);

	/* INDENT-OFF */
	/* Reserved for future use */
	enum lrmd_call_options {
	lrmd_opt_none = 0x00000000,
	/* lrmd_opt_sync_call = 0x00000001, //Not implemented, patches welcome. */
	/! Only notify the client originating a exec() the results /
	lrmd_opt_notify_orig_only = 0x00000002,
	/*! Drop recurring operations initiated by a client when client disconnects.
	* This call_option is only valid when registering a resource. */
	lrmd_opt_drop_recurring = 0x00000003,
	/! Only send out notifications for recurring operations whenthe result changes /
	lrmd_opt_notify_changes_only = 0x00000004,
	};

	enum lrmd_callback_event {
	lrmd_event_register,
	lrmd_event_unregister,
	lrmd_event_exec_complete,
	lrmd_event_disconnect,
	+ lrmd_event_connect,
	+ lrmd_event_poke,
	};

	enum lrmd_exec_rc {
	PCMK_EXECRA_OK = 0,
	PCMK_EXECRA_UNKNOWN_ERROR = 1,
	PCMK_EXECRA_INVALID_PARAM = 2,
	PCMK_EXECRA_UNIMPLEMENT_FEATURE = 3,
	PCMK_EXECRA_INSUFFICIENT_PRIV = 4,
	PCMK_EXECRA_NOT_INSTALLED = 5,
	PCMK_EXECRA_NOT_CONFIGURED = 6,
	PCMK_EXECRA_NOT_RUNNING = 7,
	PCMK_EXECRA_RUNNING_MASTER = 8,
	PCMK_EXECRA_FAILED_MASTER = 9,

	/* For status command only */
	PCMK_EXECRA_STATUS_UNKNOWN = 14,
	};
	/* INDENT-ON */

	typedef struct lrmd_event_data_s {
	/! Type of event, register, unregister, call_completed... /
	enum lrmd_callback_event type;

	/! The resource this event occurred on. /
	const char *rsc_id;
	/! The action performed, start, stop, monitor... /
	const char *op_type;
	/! The userdata string given do exec() api function /
	const char *user_data;

	/! The client api call id associated with this event /
	int call_id;
	/! The operation's timeout period in ms. /
	int timeout;
	/! The operation's recurring interval in ms. /
	int interval;
	/! The operation's start delay value in ms. /
	int start_delay;
	/! This operation that just completed is on a deleted rsc. /
	int rsc_deleted;

	/! The executed ra return code /
	enum lrmd_exec_rc rc;
	/! The lrmd status returned for exec_complete events /
	int op_status;
	/! stdout from resource agent operation /
	const char *output;
	/! Timestamp of when op ran /
	unsigned int t_run;
	/! Timestamp of last rc change /
	unsigned int t_rcchange;
	/! Time in length op took to execute /
	unsigned int exec_time;
	/! Time in length spent in queue /
	unsigned int queue_time;

	+ /! int connection result. Used for connection and poke events/
	+ int connection_rc;
	+
	/* This is a GHashTable containing the
	* parameters given to the operation */
	void *params;

	- /* If this is a remote connection, this is the remote node name
	- * registered for the connection. */
	+ /* client node name associated with this conneciton.
	+ * This is useful if multiple clients are being utilized by
	+ * a single process. This name allows the actions to be matched
	+ * to the proper client. */
	const char *remote_nodename;

	} lrmd_event_data_t;

	lrmd_event_data_t lrmd_copy_event(lrmd_event_data_t event);
	void lrmd_free_event(lrmd_event_data_t *event);

	typedef struct lrmd_rsc_info_s {
	char *id;
	char *type;
	char *class;
	char *provider;
	} lrmd_rsc_info_t;

	lrmd_rsc_info_t lrmd_copy_rsc_info(lrmd_rsc_info_t rsc_info);
	void lrmd_free_rsc_info(lrmd_rsc_info_t *rsc_info);

	typedef void (lrmd_event_callback)(lrmd_event_data_t event);

	typedef struct lrmd_list_s {
	const char *val;
	struct lrmd_list_s *next;
	} lrmd_list_t;

	void lrmd_list_freeall(lrmd_list_t *head);
	+void lrmd_key_value_freeall(lrmd_key_value_t * head);

	typedef struct lrmd_api_operations_s
	{
	/*!
	* \brief Connect from the lrmd.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (connect) (lrmd_t lrmd, const char client_name, int fd);

	+ /*!
	+ * \brief Establish an connection to lrmd, don't block while connecting.
	+ * \note this function requires the use of mainloop.
	+ *
	+ * \note The is returned using the event callback.
	+ * \note When this function returns 0, the callback will be invoked
	+ * to report the final result of the connect.
	+ * \retval 0, connect in progress, wait for event callback
	+ * \retval -1, failure.
	+ */
	+ int (connect_async) (lrmd_t lrmd, const char client_name, int timeout /ms */);
	+
	+ /*!
	+ * \brief Is connected to lrmd daemon?
	+ *
	+ * \retval 0, false
	+ * \retval 1, true
	+ */
	+ int (is_connected) (lrmd_t lrmd);
	+
	+ /*!
	+ * \brief Poke lrmd connection to verify it is still capable of serving requests
	+ * \note The response comes in the form of a poke event to the callback.
	+ *
	+ * \retval 0, wait for response in callback
	+ * \retval -1, connection failure, callback may not be invoked
	+ */
	+ int (poke_connection) (lrmd_t lrmd);
	+
	/*!
	* \brief Disconnect from the lrmd.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (disconnect)(lrmd_t lrmd);

	/*!
	* \brief Register a resource with the lrmd.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (register_rsc) (lrmd_t lrmd,
	const char *rsc_id,
	const char *class,
	const char *provider,
	const char *agent,
	enum lrmd_call_options options);

	/*!
	* \brief Retrieve registration info for a rsc
	*
	* \retval info on success
	* \retval NULL on failure
	*/
	lrmd_rsc_info_t (get_rsc_info) (lrmd_t *lrmd,
	const char *rsc_id,
	enum lrmd_call_options options);

	/*!
	* \brief Unregister a resource from the lrmd.
	*
	* \note All pending and recurring operations will be cancelled
	* automatically.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \retval 0, success
	* \retval -1, success, but operations are currently executing on the rsc which will
	* return once they are completed.
	* \retval negative error code on failure
	*
	*/
	int (unregister_rsc) (lrmd_t lrmd,
	const char *rsc_id,
	enum lrmd_call_options options);

	/*!
	* \brief Sets the callback to receive lrmd events on.
	*/
	void (set_callback) (lrmd_t lrmd,
	lrmd_event_callback callback);

	/*!
	* \brief Issue a command on a resource
	*
	* \note Asynchronous, command is queued in daemon on function return, but
	* execution of command is not synced.
	*
	* \note Operations on individual resources are guaranteed to occur
	* in the order the client api calls them in.
	*
	* \note Operations between different resources are not guaranteed
	* to occur in any specific order in relation to one another
	* regardless of what order the client api is called in.
	* \retval call_id to track async event result on success
	* \retval negative error code on failure
	*/
	int (exec)(lrmd_t lrmd,
	const char *rsc_id,
	const char *action,
	const char userdata, / userdata string given back in event notification */
	int interval, /* ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options,
	lrmd_key_value_t params); / ownership of params is given up to api here */

	/*!
	* \brief Cancel a recurring command.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \note The cancel is completed async from this call.
	* We can be guaranteed the cancel has completed once
	* the callback receives an exec_complete event with
	* the lrmd_op_status signifying that the operation is
	* cancelled.
	* \note For each resource, cancel operations and exec operations
	* are processed in the order they are received.
	* It is safe to assume that for a single resource, a cancel
	* will occur in the lrmd before an exec if the client's cancel
	* api call occurs before the exec api call.
	*
	* It is not however safe to assume any operation on one resource will
	* occur before an operation on another resource regardless of
	* the order the client api is called in.
	*
	* \retval 0, cancel command sent.
	* \retval negative error code on failure
	*/
	int (cancel)(lrmd_t lrmd,
	const char *rsc_id,
	const char *action,
	int interval);

	/*!
	* \brief Get the metadata documentation for a resource.
	*
	* \note Value is returned in output. Output must be freed when set
	*
	* \retval lrmd_ok success
	* \retval negative error code on failure
	*/
	int (get_metadata) (lrmd_t lrmd,
	const char *class,
	const char *provider,
	const char *agent,
	char **output,
	enum lrmd_call_options options);

	/*!
	* \brief Retrieve a list of installed resource agents.
	*
	* \note if class is not provided, all known agents will be returned
	* \note list must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_agents)(lrmd_t lrmd, lrmd_list_t *agents, const char class, const char *provider);

	/*!
	* \brief Retrieve a list of resource agent providers
	*
	* \note When the agent is provided, only the agent's provider will be returned
	* \note When no agent is supplied, all providers will be returned.
	* \note List must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_ocf_providers)(lrmd_t lrmd,
	const char *agent,
	lrmd_list_t **providers);

	/*!
	* \brief Retrieve a list of standards supported by this machine/installation
	*
	* \note List must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_standards)(lrmd_t lrmd, lrmd_list_t **standards);

	} lrmd_api_operations_t;

	struct lrmd_s {
	lrmd_api_operations_t *cmds;
	void *private;
	};

	static inline const char *
	lrmd_event_rc2str(enum lrmd_exec_rc rc)
	{
	switch(rc) {
	case PCMK_EXECRA_OK:
	return "ok";
	case PCMK_EXECRA_UNKNOWN_ERROR:
	return "unknown error";
	case PCMK_EXECRA_INVALID_PARAM:
	return "invalid parameter";
	case PCMK_EXECRA_UNIMPLEMENT_FEATURE:
	return "unimplemented feature";
	case PCMK_EXECRA_INSUFFICIENT_PRIV:
	return "insufficient privileges";
	case PCMK_EXECRA_NOT_INSTALLED:
	return "not installed";
	case PCMK_EXECRA_NOT_CONFIGURED:
	return "not configured";
	case PCMK_EXECRA_NOT_RUNNING:
	return "not running";
	case PCMK_EXECRA_RUNNING_MASTER:
	return "master";
	case PCMK_EXECRA_FAILED_MASTER:
	return "master (failed)";
	case PCMK_EXECRA_STATUS_UNKNOWN:
	return "status: unknown";
	default:
	break;
	}
	return "<unknown>";
	}

	static inline const char *
	lrmd_event_type2str(enum lrmd_callback_event type)
	{
	switch (type) {
	case lrmd_event_register:
	return "register";
	case lrmd_event_unregister:
	return "unregister";
	case lrmd_event_exec_complete:
	return "exec_complete";
	case lrmd_event_disconnect:
	return "disconnect";
	+ case lrmd_event_connect:
	+ return "connect";
	+ case lrmd_event_poke:
	+ return "poke";
	}
	return "unknown";
	}

	#endif
	diff --git a/include/crm_internal.h b/include/crm_internal.h
	index ca963c6d5b..a555aab3d5 100644
	--- a/include/crm_internal.h
	+++ b/include/crm_internal.h
	@@ -1,305 +1,310 @@
	/* crm_internal.h */

	/*
	* Copyright (C) 2006 - 2008
	* Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License as published by
	* the Free Software Foundation; either version 2 of the License, or
	* (at your option) any later version.
	*
	* This program is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#ifndef CRM_INTERNAL__H
	# define CRM_INTERNAL__H

	# include <config.h>
	# include <portability.h>

	# include <glib.h>
	# include <stdbool.h>
	# include <libxml/tree.h>

	# include <crm/lrmd.h>
	# include <crm/common/logging.h>

	/* Dynamic loading of libraries */
	void find_library_function(void handle, const char lib, const char *fn, int fatal);
	void convert_const_pointer(const void ptr);

	/* For ACLs */
	char *uid2username(uid_t uid);
	void determine_request_user(char user, xmlNode request, const char *field);

	# if ENABLE_ACL
	# include <string.h>
	static inline gboolean
	is_privileged(const char *user)
	{
	if (user == NULL) {
	return FALSE;
	} else if (strcmp(user, CRM_DAEMON_USER) == 0) {
	return TRUE;
	} else if (strcmp(user, "root") == 0) {
	return TRUE;
	}
	return FALSE;
	}
	# endif

	/* CLI option processing*/
	# ifdef HAVE_GETOPT_H
	# include <getopt.h>
	# else
	# define no_argument 0
	# define required_argument 1
	# endif

	# define pcmk_option_default 0x00000
	# define pcmk_option_hidden 0x00001
	# define pcmk_option_paragraph 0x00002
	# define pcmk_option_example 0x00004

	struct crm_option {
	/* Fields from 'struct option' in getopt.h */
	/* name of long option */
	const char *name;
	/*
	* one of no_argument, required_argument, and optional_argument:
	* whether option takes an argument
	*/
	int has_arg;
	/* if not NULL, set flag to val when option found /
	int *flag;
	/* if flag not NULL, value to set flag to; else return value /
	int val;

	/* Custom fields */
	const char *desc;
	long flags;
	};

	void crm_set_options(const char short_options, const char usage, struct crm_option long_options, const char app_desc);
	int crm_get_option(int argc, char *argv, int index);
	int crm_get_option_long(int argc, char *argv, int index, const char **longname);
	void crm_help(char cmd, int exit_code);

	/* Cluster Option Processing */
	typedef struct pe_cluster_option_s {
	const char *name;
	const char *alt_name;
	const char *type;
	const char *values;
	const char *default_value;

	gboolean(is_valid) (const char );

	const char *description_short;
	const char *description_long;

	} pe_cluster_option;

	const char cluster_option(GHashTable options, gboolean(validate) (const char ),
	const char name, const char old_name, const char *def_value);

	const char get_cluster_pref(GHashTable options, pe_cluster_option * option_list, int len, const char *name);

	void config_metadata(const char name, const char version, const char *desc_short,
	const char desc_long, pe_cluster_option option_list, int len);

	void verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len);
	gboolean check_time(const char *value);
	gboolean check_timer(const char *value);
	gboolean check_boolean(const char *value);
	gboolean check_number(const char *value);

	/* Shared PE/crmd functionality */
	void filter_action_parameters(xmlNode * param_set, const char *version);
	void filter_reload_parameters(xmlNode * param_set, const char *restart_string);

	/* Resource operation updates */
	xmlNode create_operation_update(xmlNode parent, lrmd_event_data_t event, const char caller_version,
	int target_rc, const char *origin, int level);

	/* char2score */
	extern int node_score_red;
	extern int node_score_green;
	extern int node_score_yellow;
	extern int node_score_infinity;


	/* Assorted convenience functions */
	static inline int
	crm_strlen_zero(const char *s)
	{
	return !s \|\| *s == '\0';
	}

	char add_list_element(char list, const char *value);
	char generate_series_filename(const char directory, const char *series, int sequence, gboolean bzip);
	int get_last_sequence(const char directory, const char series);
	void write_last_sequence(const char directory, const char series, int sequence, int max);

	int crm_pid_active(long pid);
	void crm_make_daemon(const char name, gboolean daemonize, const char pidfile);
	gboolean crm_is_writable(const char dir, const char file, const char user, const char group, gboolean need_both);

	char generate_op_key(const char rsc_id, const char *op_type, int interval);
	char generate_notify_key(const char rsc_id, const char notify_type, const char op_type);
	char generate_transition_magic_v202(const char transition_key, int op_status);
	char generate_transition_magic(const char transition_key, int op_status, int op_rc);
	char generate_transition_key(int action, int transition_id, int target_rc, const char node);

	static inline long long
	crm_clear_bit(const char function, const char target, long long word, long long bit)
	{
	long long rc = (word & ~bit);

	if(rc == word) {
	/* Unchanged */
	} else if (target) {
	crm_trace("Bit 0x%.8llx for %s cleared by %s", bit, target, function);
	} else {
	crm_trace("Bit 0x%.8llx cleared by %s", bit, function);
	}

	return rc;
	}

	static inline long long
	crm_set_bit(const char function, const char target, long long word, long long bit)
	{
	long long rc = (word\|bit);

	if(rc == word) {
	/* Unchanged */
	} else if (target) {
	crm_trace("Bit 0x%.8llx for %s set by %s", bit, target, function);
	} else {
	crm_trace("Bit 0x%.8llx set by %s", bit, function);
	}

	return rc;
	}

	# define set_bit(word, bit) word = crm_set_bit(__PRETTY_FUNCTION__, NULL, word, bit)
	# define clear_bit(word, bit) word = crm_clear_bit(__PRETTY_FUNCTION__, NULL, word, bit)

	void g_hash_destroy_str(gpointer data);

	long long crm_int_helper(const char text, char *end_text);
	char crm_concat(const char prefix, const char *suffix, char join);
	char generate_hash_key(const char crm_msg_reference, const char *sys);


	bool crm_compress_string(
	const char data, int length, int max, char result, unsigned int result_len);

	/! remote tcp/tls helper functions /
	typedef struct crm_remote_s crm_remote_t;

	int crm_remote_send(crm_remote_t remote, xmlNode msg);
	int crm_remote_ready(crm_remote_t remote, int total_timeout /ms */);
	gboolean crm_remote_recv(crm_remote_t remote, int total_timeout /ms /, int disconnected);
	xmlNode crm_remote_parse_buffer(crm_remote_t remote);
	int crm_remote_tcp_connect(const char *host, int port);
	+int crm_remote_tcp_connect_async(const char *host,
	+ int port,
	+ int timeout, /ms/
	+ void *userdata,
	+ void (callback)(void userdata, int sock));

	#ifdef HAVE_GNUTLS_GNUTLS_H
	/*!
	* \internal
	* \brief Initiate the client handshake after establishing the tcp socket.
	* \note This is a blocking function, it will block until the entire handshake
	* is complete or until the timeout period is reached.
	* \retval 0 success
	* \retval negative, failure
	*/
	int crm_initiate_client_tls_handshake(crm_remote_t *remote, int timeout_ms);
	/*!
	* \internal
	* \brief Create client or server session for anon DH encryption credentials
	* \param sock, the socket the session will use for transport
	* \param type, GNUTLS_SERVER or GNUTLS_CLIENT
	* \param credentials, gnutls_anon_server_credentials_t or gnutls_anon_client_credentials_t
	*
	* \retval gnutls_session * on success
	* \retval NULL on failure
	*/
	void crm_create_anon_tls_session(int sock, int type, void credentials);
	/*!
	* \internal
	* \brief Create client or server session for PSK credentials
	* \param sock, the socket the session will use for transport
	* \param type, GNUTLS_SERVER or GNUTLS_CLIENT
	* \param credentials, gnutls_psk_server_credentials_t or gnutls_osk_client_credentials_t
	*
	* \retval gnutls_session * on success
	* \retval NULL on failure
	*/
	void create_psk_tls_session(int csock, int type, void credentials);
	#endif

	#define REMOTE_MSG_TERMINATOR "\r\n\r\n"

	const char daemon_option(const char option);
	void set_daemon_option(const char option, const char value);
	gboolean daemon_option_enabled(const char daemon, const char option);
	void strip_text_nodes(xmlNode *xml);

	# define crm_config_err(fmt...) { crm_config_error = TRUE; crm_err(fmt); }
	# define crm_config_warn(fmt...) { crm_config_warning = TRUE; crm_warn(fmt); }

	# define attrd_channel T_ATTRD
	# define F_ATTRD_KEY "attr_key"
	# define F_ATTRD_ATTRIBUTE "attr_name"
	# define F_ATTRD_TASK "task"
	# define F_ATTRD_VALUE "attr_value"
	# define F_ATTRD_SET "attr_set"
	# define F_ATTRD_SECTION "attr_section"
	# define F_ATTRD_DAMPEN "attr_dampening"
	# define F_ATTRD_IGNORE_LOCALLY "attr_ignore_locally"
	# define F_ATTRD_HOST "attr_host"
	# define F_ATTRD_USER "attr_user"

	# if SUPPORT_COROSYNC
	# if CS_USES_LIBQB
	# include <qb/qbipc_common.h>
	# include <corosync/corotypes.h>
	typedef struct qb_ipc_request_header cs_ipc_header_request_t;
	typedef struct qb_ipc_response_header cs_ipc_header_response_t;
	# else
	# include <corosync/corodefs.h>
	# include <corosync/coroipcc.h>
	# include <corosync/coroipc_types.h>
	static inline int
	qb_to_cs_error(int a)
	{
	return a;
	}

	typedef coroipc_request_header_t cs_ipc_header_request_t;
	typedef coroipc_response_header_t cs_ipc_header_response_t;
	# endif
	# else
	typedef struct {
	int size __attribute__ ((aligned(8)));
	int id __attribute__ ((aligned(8)));
	} __attribute__ ((aligned(8))) cs_ipc_header_request_t;

	typedef struct {
	int size __attribute__ ((aligned(8)));
	int id __attribute__ ((aligned(8)));
	int error __attribute__ ((aligned(8)));
	} __attribute__ ((aligned(8))) cs_ipc_header_response_t;

	# endif

	#endif /* CRM_INTERNAL__H */
	diff --git a/lib/common/remote.c b/lib/common/remote.c
	index ff5361ce4b..fe59b1cefb 100644
	--- a/lib/common/remote.c
	+++ b/lib/common/remote.c
	@@ -1,718 +1,898 @@
	/*
	* Copyright (c) 2008 Andrew Beekhof
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/socket.h>
	#include <arpa/inet.h>
	#include <netinet/ip.h>
	#include <netdb.h>


	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <glib.h>

	#include <crm/common/ipcs.h>
	#include <crm/common/xml.h>
	+#include <crm/common/mainloop.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#ifdef HAVE_GNUTLS_GNUTLS_H
	const int psk_tls_kx_order[] = {
	GNUTLS_KX_DHE_PSK,
	GNUTLS_KX_PSK,
	};
	const int anon_tls_kx_order[] = {
	GNUTLS_KX_ANON_DH,
	GNUTLS_KX_DHE_RSA,
	GNUTLS_KX_DHE_DSS,
	GNUTLS_KX_RSA,
	0
	};

	int
	crm_initiate_client_tls_handshake(crm_remote_t *remote, int timeout_ms)
	{
	int rc = 0;
	int pollrc = 0;
	time_t start = time(NULL);

	do {
	rc = gnutls_handshake(*remote->tls_session);
	if (rc == GNUTLS_E_INTERRUPTED \|\| rc == GNUTLS_E_AGAIN) {
	pollrc = crm_remote_ready(remote, 1000);
	if (pollrc < 0) {
	/* poll returned error, there is no hope */
	rc = -1;
	}
	}

	} while (((time(NULL) - start) < (timeout_ms/1000)) &&
	(rc == GNUTLS_E_INTERRUPTED \|\| rc == GNUTLS_E_AGAIN));

	if(rc < 0) {
	crm_trace("gnutls_handshake() failed with %d", rc);
	}
	return rc;
	}

	void *
	crm_create_anon_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT /, void credentials)
	{
	gnutls_session *session = gnutls_malloc(sizeof(gnutls_session));

	gnutls_init(session, type);
	# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT
	/* http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication */
	gnutls_priority_set_direct(*session, "NORMAL:+ANON-DH", NULL);
	/* gnutls_priority_set_direct (session, "NONE:+VERS-TLS-ALL:+CIPHER-ALL:+MAC-ALL:+SIGN-ALL:+COMP-ALL:+ANON-DH", NULL); /
	# else
	gnutls_set_default_priority(*session);
	gnutls_kx_set_priority(*session, anon_tls_kx_order);
	# endif
	gnutls_transport_set_ptr(*session, (gnutls_transport_ptr) GINT_TO_POINTER(csock));
	switch (type) {
	case GNUTLS_SERVER:
	gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_server_credentials_t) credentials);
	break;
	case GNUTLS_CLIENT:
	gnutls_credentials_set(*session, GNUTLS_CRD_ANON, (gnutls_anon_client_credentials_t) credentials);
	break;
	}

	return session;
	}

	void *
	create_psk_tls_session(int csock, int type /* GNUTLS_SERVER, GNUTLS_CLIENT /, void credentials)
	{
	gnutls_session *session = gnutls_malloc(sizeof(gnutls_session));

	gnutls_init(session, type);
	# ifdef HAVE_GNUTLS_PRIORITY_SET_DIRECT
	gnutls_priority_set_direct(*session, "NORMAL:+DHE-PSK:+PSK", NULL);
	# else
	gnutls_set_default_priority(*session);
	gnutls_kx_set_priority(*session, psk_tls_kx_order);
	# endif
	gnutls_transport_set_ptr(*session, (gnutls_transport_ptr) GINT_TO_POINTER(csock));
	switch (type) {
	case GNUTLS_SERVER:
	gnutls_credentials_set(*session, GNUTLS_CRD_PSK, (gnutls_psk_server_credentials_t) credentials);
	break;
	case GNUTLS_CLIENT:
	gnutls_credentials_set(*session, GNUTLS_CRD_PSK, (gnutls_psk_client_credentials_t) credentials);
	break;
	}

	return session;
	}


	static int
	crm_send_tls(gnutls_session * session, const char *buf, size_t len)
	{
	const char *unsent = buf;
	int rc = 0;
	int total_send;

	if (buf == NULL) {
	return -1;
	}

	total_send = len;
	crm_trace("Message size: %d", len);

	while (TRUE) {
	rc = gnutls_record_send(*session, unsent, len);

	if (rc == GNUTLS_E_INTERRUPTED \|\| rc == GNUTLS_E_AGAIN) {
	crm_debug("Retry");

	} else if (rc < 0) {
	crm_err("Connection terminated rc = %d", rc);
	break;

	} else if (rc < len) {
	crm_debug("Only sent %d of %d bytes", rc, len);
	len -= rc;
	unsent += rc;
	} else {
	crm_debug("Sent %d bytes", rc);
	break;
	}
	}

	return rc < 0 ? rc : total_send;
	}


	/*!
	* \internal
	* \brief Read bytes off non blocking tls session.
	*
	* \param session - tls session to read
	* \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
	*
	* \note only use with NON-Blocking sockets. Should only be used after polling socket.
	* This function will return once max_size is met, the socket read buffer
	* is empty, or an error is encountered.
	*
	* \retval '\0' terminated buffer on success
	*/
	static char *
	crm_recv_tls(gnutls_session * session, size_t max_size, size_t recv_len, int disconnected)
	{
	char *buf = NULL;
	int rc = 0;
	size_t len = 0;
	size_t chunk_size = max_size ? max_size : 1024;
	size_t buf_size = 0;
	size_t read_size = 0;

	if (session == NULL) {
	if (disconnected) {
	*disconnected = 1;
	}
	goto done;
	}

	buf = calloc(1, chunk_size + 1);
	buf_size = chunk_size;

	while (TRUE) {
	read_size = buf_size - len;

	/* automatically grow the buffer when needed if max_size is not set.*/
	if (!max_size && (read_size < (chunk_size / 2))) {
	buf_size += chunk_size;
	crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
	buf = realloc(buf, buf_size + 1);
	CRM_ASSERT(buf != NULL);

	read_size = buf_size - len;
	}

	rc = gnutls_record_recv(*session, buf + len, read_size);

	if (rc > 0) {
	crm_trace("Got %d more bytes.", rc);
	len += rc;
	/* always null terminate buffer, the +1 to alloc always allows for this.*/
	buf[len] = '\0';
	}
	if (max_size && (max_size == read_size)) {
	crm_trace("Buffer max read size %d met" , max_size);
	goto done;
	}

	/* process any errors. */
	if (rc == GNUTLS_E_INTERRUPTED) {
	crm_trace("EINTR encoutered, retry tls read");
	} else if (rc == GNUTLS_E_AGAIN) {
	crm_trace("non-blocking, exiting read on rc = %d", rc);
	goto done;
	} else if (rc <= 0) {
	if (rc == 0) {
	crm_debug("EOF encoutered during TLS read");
	} else {
	crm_debug("Error receiving message: %s (%d)", gnutls_strerror(rc), rc);
	}
	if (disconnected) {
	*disconnected = 1;
	}
	goto done;
	}
	}

	done:
	if (recv_len) {
	*recv_len = len;
	}
	if (!len) {
	free(buf);
	buf = NULL;
	}
	return buf;

	}
	#endif

	static int
	crm_send_plaintext(int sock, const char *buf, size_t len)
	{

	int rc = 0;
	const char *unsent = buf;
	int total_send;

	if (buf == NULL) {
	return -1;
	}
	total_send = len;

	crm_trace("Message on socket %d: size=%d", sock, len);
	retry:
	rc = write(sock, unsent, len);
	if (rc < 0) {
	switch (errno) {
	case EINTR:
	case EAGAIN:
	crm_trace("Retry");
	goto retry;
	default:
	crm_perror(LOG_ERR, "Could only write %d of the remaining %d bytes", rc, (int) len);
	break;
	}

	} else if (rc < len) {
	crm_trace("Only sent %d of %d remaining bytes", rc, len);
	len -= rc;
	unsent += rc;
	goto retry;

	} else {
	crm_trace("Sent %d bytes: %.100s", rc, buf);
	}

	return rc < 0 ? rc : total_send;

	}

	/*!
	* \internal
	* \brief Read bytes off non blocking socket.
	*
	* \param session - tls session to read
	* \param max_size - max bytes allowed to read for buffer. 0 assumes no limit
	*
	* \note only use with NON-Blocking sockets. Should only be used after polling socket.
	* This function will return once max_size is met, the socket read buffer
	* is empty, or an error is encountered.
	*
	* \retval '\0' terminated buffer on success
	*/
	static char *
	crm_recv_plaintext(int sock, size_t max_size, size_t recv_len, int disconnected)
	{
	char *buf = NULL;
	ssize_t rc = 0;
	ssize_t len = 0;
	ssize_t chunk_size = max_size ? max_size : 1024;
	size_t buf_size = 0;
	size_t read_size = 0;

	if (sock <= 0) {
	if (disconnected) {
	*disconnected = 1;
	}
	goto done;
	}

	buf = calloc(1, chunk_size + 1);
	buf_size = chunk_size;

	while (TRUE) {
	errno = 0;
	read_size = buf_size - len;

	/* automatically grow the buffer when needed if max_size is not set.*/
	if (!max_size && (read_size < (chunk_size / 2))) {
	buf_size += chunk_size;
	crm_trace("Grow buffer by %d more bytes. buf is now %d bytes", (int)chunk_size, buf_size);
	buf = realloc(buf, buf_size + 1);
	CRM_ASSERT(buf != NULL);

	read_size = buf_size - len;
	}

	rc = read(sock, buf + len, chunk_size);

	if (rc > 0) {
	crm_trace("Got %d more bytes. errno=%d", (int)rc, errno);
	len += rc;
	/* always null terminate buffer, the +1 to alloc always allows for this.*/
	buf[len] = '\0';
	}
	if (max_size && (max_size == read_size)) {
	crm_trace("Buffer max read size %d met" , max_size);
	goto done;
	}

	if (rc > 0) {
	continue;
	} else if (rc == 0) {
	if (disconnected) {
	*disconnected = 1;
	}
	crm_trace("EOF encoutered during read");
	goto done;
	}

	/* process errors */
	if (errno == EINTR) {
	crm_trace("EINTER encoutered, retry socket read.");
	} else if (errno == EAGAIN) {
	crm_trace("non-blocking, exiting read on rc = %d", rc);
	goto done;
	} else if (errno <= 0) {
	if (disconnected) {
	*disconnected = 1;
	}
	crm_debug("Error receiving message: %d", (int)rc);
	goto done;
	}
	}

	done:
	if (recv_len) {
	*recv_len = len;
	}
	if (!len) {
	free(buf);
	buf = NULL;
	}
	return buf;
	}

	static int
	crm_remote_send_raw(crm_remote_t remote, const char buf, size_t len)
	{
	int rc = -ESOCKTNOSUPPORT;

	if(remote->tcp_socket) {
	rc = crm_send_plaintext(remote->tcp_socket, buf, len);
	#ifdef HAVE_GNUTLS_GNUTLS_H

	} else if(remote->tls_session) {
	rc = crm_send_tls(remote->tls_session, buf, len);
	#endif
	} else {
	crm_err("Unsupported connection type");
	}
	return rc;
	}

	int
	crm_remote_send(crm_remote_t remote, xmlNode msg)
	{
	int rc = -1;
	char *xml_text = NULL;
	int len = 0;

	xml_text = dump_xml_unformatted(msg);
	if (xml_text) {
	len = strlen(xml_text);
	} else {
	crm_err("Invalid XML, can not send msg");
	return -1;
	}

	rc = crm_remote_send_raw(remote, xml_text, len);
	if (rc >= 0) {
	rc = crm_remote_send_raw(remote, REMOTE_MSG_TERMINATOR, strlen(REMOTE_MSG_TERMINATOR));
	}

	if (rc < 0) {
	crm_err("Failed to send remote msg, rc = %d", rc);
	}

	free(xml_text);
	return rc;
	}

	/*!
	* \internal
	* \brief handles the recv buffer and parsing out msgs.
	* \note new_data is owned by this function once it is passed in.
	*/
	xmlNode *
	crm_remote_parse_buffer(crm_remote_t *remote)
	{
	char *buf = NULL;
	char *start = NULL;
	char *end = NULL;
	xmlNode *xml = NULL;

	if (remote->buffer == NULL) {
	return NULL;
	}

	/* take ownership of the buffer */
	buf = remote->buffer;
	remote->buffer = NULL;

	/* MSGS are separated by a '\r\n\r\n'. Split a message off the buffer and return it. */
	start = buf;
	end = strstr(start, REMOTE_MSG_TERMINATOR);

	while (!xml && end) {

	/* grab the message */
	end[0] = '\0';
	end += strlen(REMOTE_MSG_TERMINATOR);

	xml = string2xml(start);
	if (xml == NULL) {
	crm_err("Couldn't parse: '%.120s'", start);
	}
	start = end;
	end = strstr(start, REMOTE_MSG_TERMINATOR);
	}

	if (xml && start) {
	/* we have msgs left over, save it until next time */
	remote->buffer = strdup(start);
	free(buf);
	} else if (!xml) {
	/* no msg present */
	remote->buffer = buf;
	}

	return xml;
	}

	/*!
	* \internal
	* \brief Determine if a remote session has data to read
	*
	* \retval 0, timeout occured.
	* \retval positive, data is ready to be read
	* \retval negative, session has ended
	*/
	int
	crm_remote_ready(crm_remote_t remote, int timeout / ms */)
	{
	struct pollfd fds = { 0, };
	int sock = 0;
	int rc = 0;
	time_t start;

	if (remote->tcp_socket) {
	sock = remote->tcp_socket;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	} else if(remote->tls_session) {
	void sock_ptr = gnutls_transport_get_ptr(remote->tls_session);
	sock = GPOINTER_TO_INT(sock_ptr);
	#endif
	} else {
	crm_err("Unsupported connection type");
	}

	if (sock <= 0) {
	crm_trace("No longer connected");
	return -ENOTCONN;
	}

	start = time(NULL);
	errno = 0;
	do {
	fds.fd = sock;
	fds.events = POLLIN;

	/* If we got an EINTR while polling, and we have a
	* specific timeout we are trying to honor, attempt
	* to adjust the timeout to the closest second. */
	if (errno == EINTR && (timeout > 0)) {
	timeout = timeout - ((time(NULL) - start) * 1000);
	if (timeout < 1000) {
	timeout = 1000;
	}
	}

	rc = poll(&fds, 1, timeout);
	} while (rc < 0 && errno == EINTR);

	return rc;
	}

	/*!
	* \internal
	* \brief Read data off the socket until at least one full message is present or timeout occures.
	* \retval TRUE message read
	* \retval FALSE full message not read
	*/

	gboolean
	crm_remote_recv(crm_remote_t remote, int total_timeout /ms /, int disconnected)
	{
	int ret;
	size_t request_len = 0;
	time_t start = time(NULL);
	char *raw_request = NULL;
	int remaining_timeout = 0;

	if (total_timeout == 0) {
	total_timeout = 10000;
	} else if (total_timeout < 0) {
	total_timeout = 60000;
	}
	*disconnected = 0;

	remaining_timeout = total_timeout;
	while ((remaining_timeout > 0) && !(*disconnected)) {

	/* read some more off the tls buffer if we still have time left. */
	crm_trace("waiting to receive remote msg, starting timeout %d, remaining_timeout %d", total_timeout, remaining_timeout);
	ret = crm_remote_ready(remote, remaining_timeout);
	raw_request = NULL;

	if (ret == 0) {
	crm_err("poll timed out (%d ms) while waiting to receive msg", remaining_timeout);
	return FALSE;

	} else if (ret < 0) {
	if (errno != EINTR) {
	crm_debug("poll returned error while waiting for msg, rc: %d, errno: %d", ret, errno);
	*disconnected = 1;
	return FALSE;
	}
	crm_debug("poll EINTR encountered during poll, retrying");

	} else if (remote->tcp_socket) {
	raw_request = crm_recv_plaintext(remote->tcp_socket, 0, &request_len, disconnected);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	} else if(remote->tls_session) {
	raw_request = crm_recv_tls(remote->tls_session, 0, &request_len, disconnected);
	#endif
	} else {
	crm_err("Unsupported connection type");
	}

	remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000);

	if (!raw_request) {
	crm_debug("Empty msg received after poll");
	continue;
	}

	if (remote->buffer) {
	int old_len = strlen(remote->buffer);

	crm_trace("Expanding recv buffer from %d to %d", old_len, old_len+request_len);

	remote->buffer = realloc(remote->buffer, old_len + request_len + 1);
	memcpy(remote->buffer + old_len, raw_request, request_len);
	*(remote->buffer+old_len+request_len) = '\0';
	free(raw_request);

	} else {
	remote->buffer = raw_request;
	}

	if (strstr(remote->buffer, REMOTE_MSG_TERMINATOR)) {
	return TRUE;
	}
	}

	return FALSE;
	}

	+struct tcp_async_cb_data {
	+ gboolean success;
	+ int sock;
	+ void *userdata;
	+ void (callback)(void userdata, int sock);
	+ int timeout; /ms/
	+ time_t start;
	+};
	+
	+static gboolean
	+check_connect_finished(gpointer userdata)
	+{
	+ struct tcp_async_cb_data *cb_data = userdata;
	+ int rc = 0;
	+ int sock = cb_data->sock;
	+ int error = 0;
	+
	+ fd_set rset, wset;
	+ socklen_t len = sizeof(error);
	+ struct timeval ts = { 0, };
	+
	+ if (cb_data->success == TRUE) {
	+ goto dispatch_done;
	+ }
	+
	+ FD_ZERO(&rset);
	+ FD_SET(sock, &rset);
	+ wset = rset;
	+
	+ crm_trace("fd %d: checking to see if connect finished", sock);
	+ rc = select(sock + 1, &rset, &wset, NULL, &ts);
	+
	+ if (rc < 0) {
	+ rc = errno;
	+ if ((errno == EINPROGRESS) \|\| (errno == EAGAIN)) {
	+ /* reschedule if there is still time left */
	+ if ((time(NULL) - cb_data->start) < (cb_data->timeout / 1000)) {
	+ goto reschedule;
	+ } else {
	+ rc = -ETIMEDOUT;
	+ }
	+ }
	+ crm_trace("fd %d: select failed %d connect dispatch ", rc);
	+ goto dispatch_done;
	+ } else if (rc == 0) {
	+ if ((time(NULL) - cb_data->start) < (cb_data->timeout / 1000)) {
	+ goto reschedule;
	+ }
	+ crm_err("fd %d: timeout during select", sock);
	+ rc = -ETIMEDOUT;
	+ goto dispatch_done;
	+ } else {
	+ crm_trace("fd %d: select returned success", sock);
	+ rc = 0;
	+ }
	+
	+ /* can we read or write to the socket now? */
	+ if (FD_ISSET(sock, &rset) \|\| FD_ISSET(sock, &wset)) {
	+ if (getsockopt(sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) {
	+ crm_trace("fd %d: call to getsockopt failed", sock);
	+ rc = -1;
	+ goto dispatch_done;
	+ }
	+
	+ if (error) {
	+ crm_trace("fd %d: error returned from getsockopt: %d", sock, error);
	+ rc = -1;
	+ goto dispatch_done;
	+ }
	+ } else {
	+ crm_trace("neither read nor write set after select");
	+ rc = -1;
	+ goto dispatch_done;
	+ }
	+
	+dispatch_done:
	+ if (!rc) {
	+ crm_trace("fd %d: connected", sock);
	+ /* Success, set the return code to the sock to report to the callback */
	+ rc = cb_data->sock;
	+ cb_data->sock = 0;
	+ } else {
	+ close(sock);
	+ }
	+ free(cb_data);
	+
	+ if (cb_data->callback) {
	+ cb_data->callback(cb_data->userdata, rc);
	+ }
	+ return FALSE;
	+
	+reschedule:
	+
	+ /* will check again next interval */
	+ return TRUE;
	+}
	+
	+static int
	+internal_tcp_connect_async(int sock,
	+ const struct sockaddr *addr,
	+ socklen_t addrlen,
	+ int timeout /* ms */,
	+ void *userdata,
	+ void (callback)(void userdata, int sock))
	+{
	+ int rc = 0;
	+ int flag = 0;
	+ int interval;
	+ struct tcp_async_cb_data *cb_data = NULL;
	+
	+ if ((flag = fcntl(sock, F_GETFL)) >= 0) {
	+ if (fcntl(sock, F_SETFL, flag \| O_NONBLOCK) < 0) {
	+ crm_err( "fcntl() write failed");
	+ return -1;
	+ }
	+ }
	+
	+ rc = connect(sock, addr, addrlen);
	+
	+ if (rc < 0 && (errno != EINPROGRESS) && (errno != EAGAIN)) {
	+ return -1;
	+ }
	+
	+ cb_data = calloc(1, sizeof(struct tcp_async_cb_data));
	+ cb_data->userdata = userdata;
	+ cb_data->callback = callback;
	+ cb_data->sock = sock;
	+ cb_data->timeout = timeout;
	+ cb_data->start = time(NULL);
	+
	+ if (rc == 0) {
	+ /* The connect was successful immediately, we still return to mainloop
	+ * and let this callback get called later. This avoids the user of this api
	+ * to have to account for the fact the callback could be invoked within this
	+ * function before returning. */
	+ cb_data->success = TRUE;
	+ interval = 1;
	+ }
	+
	+ /* Check connect finished is mostly doing a non-block poll on the socket
	+ * to see if we can read/write to it. Once we can, the connect has completed.
	+ * This method allows us to connect to the server without blocking mainloop.
	+ *
	+ * This is a poor man's way of polling to see when the connection finished.
	+ * At some point we should figure out a way to use a mainloop fd callback for this.
	+ * Something about the way mainloop is currently polling prevents this from working at the
	+ * moment though. */
	+ crm_trace("fd %d: scheduling to check if connect finished in %dms second", sock, interval);
	+ g_timeout_add(interval, check_connect_finished, cb_data);
	+
	+ return 0;
	+}
	+
	+static int
	+internal_tcp_connect(int sock,
	+ const struct sockaddr *addr,
	+ socklen_t addrlen)
	+{
	+ int flag = 0;
	+ int rc = connect(sock, addr, addrlen);
	+
	+ if (rc == 0) {
	+ if ((flag = fcntl(sock, F_GETFL)) >= 0) {
	+ if (fcntl(sock, F_SETFL, flag \| O_NONBLOCK) < 0) {
	+ crm_err( "fcntl() write failed");
	+ return -1;
	+ }
	+ }
	+ }
	+
	+ return rc;
	+}
	+
	/*!
	* \internal
	* \brief tcp connection to server at specified port
	- * \retval positive, socket fd.
	* \retval negative, failed to connect.
	*/
	int
	-crm_remote_tcp_connect(const char *host, int port)
	+crm_remote_tcp_connect_async(const char *host,
	+ int port,
	+ int timeout, /ms/
	+ void *userdata,
	+ void (callback)(void userdata, int sock))
	{
	struct addrinfo *res;
	struct addrinfo *rp;
	struct addrinfo hints;
	const char *server = host;
	int ret_ga;
	int sock;

	/* getaddrinfo */
	memset(&hints, 0, sizeof(struct addrinfo));
	hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_flags = AI_CANONNAME;

	crm_debug("Looking up %s", server);
	ret_ga = getaddrinfo(server, NULL, &hints, &res);
	if (ret_ga) {
	crm_err("getaddrinfo: %s", gai_strerror(ret_ga));
	return -1;
	}

	if (!res \|\| !res->ai_addr) {
	crm_err("getaddrinfo failed");
	return -1;
	}

	for (rp = res; rp != NULL; rp = rp->ai_next) {
	struct sockaddr *addr = rp->ai_addr;
	- int flag = 0;
	if (!addr) {
	continue;
	}

	if (rp->ai_canonname) {
	server = res->ai_canonname;
	}
	crm_debug("Got address %s for %s", server, host);

	/* create socket */
	sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
	if (sock == -1) {
	crm_err("Socket creation failed for remote client connection.");
	continue;
	}
	if (addr->sa_family == AF_INET6) {
	struct sockaddr_in6 addr_in = (struct sockaddr_in6 ) addr;
	addr_in->sin6_port = htons(port);
	} else {
	struct sockaddr_in addr_in = (struct sockaddr_in ) addr;
	addr_in->sin_port = htons(port);
	crm_info("Attempting to connect to remote server at %s:%d", inet_ntoa(addr_in->sin_addr), port);
	}

	- if (connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) {
	- if ((flag = fcntl(sock, F_GETFL)) >= 0) {
	- if (fcntl(sock, F_SETFL, flag \| O_NONBLOCK) < 0) {
	- crm_err( "fcntl() write failed");
	- close(sock);
	- sock = -1;
	- continue;
	- }
	+ if (callback) {
	+ if (internal_tcp_connect_async(sock, rp->ai_addr, rp->ai_addrlen, timeout, userdata, callback) == 0) {
	+ return 0; /* Success for now, we'll hear back later in the callback */
	+ }
	+
	+ } else {
	+ if (internal_tcp_connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) {
	+ break; /* Success */
	}
	- break; /* Success */
	}

	close(sock);
	sock = -1;
	}
	freeaddrinfo(res);

	return sock;
	}

	+int
	+crm_remote_tcp_connect(const char *host, int port)
	+{
	+ return crm_remote_tcp_connect_async(host, port, -1, NULL, NULL);
	+}
	diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
	index c9cec65be1..5ad2d11d62 100644
	--- a/lib/lrmd/lrmd_client.c
	+++ b/lib/lrmd/lrmd_client.c
	@@ -1,1661 +1,1846 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/msg_xml.h>

	#include <crm/stonith-ng.h>


	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#include <sys/socket.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>
	#include <netdb.h>

	CRM_TRACE_INIT_DATA(lrmd);

	static stonith_t *stonith_api = NULL;

	static int lrmd_api_disconnect(lrmd_t *lrmd);
	-static int lrmd_connected(lrmd_t *lrmd);
	+static int lrmd_api_is_connected(lrmd_t *lrmd);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	-
	#define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
	gnutls_psk_client_credentials_t psk_cred_s;
	int lrmd_tls_set_key(gnutls_datum_t key, const char location);
	static void lrmd_tls_disconnect(lrmd_t *lrmd);
	static int global_remote_msg_id = 0;
	int lrmd_tls_send_msg(crm_remote_t session, xmlNode msg, uint32_t id, const char *msg_type);
	static void lrmd_tls_connection_destroy(gpointer userdata);
	#endif

	typedef struct lrmd_private_s {
	enum client_type type;
	char *token;
	mainloop_io_t *source;

	/* IPC parameters */
	crm_ipc_t *ipc;

	crm_remote_t *remote;

	/* Extra TLS parameters */
	char *remote_nodename;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	char *server;
	int port;
	gnutls_psk_client_credentials_t psk_cred_c;

	int sock;
	GList *pending_notify;
	crm_trigger_t *process_notify;
	#endif

	lrmd_event_callback callback;

	} lrmd_private_t;

	static lrmd_list_t *
	lrmd_list_add(lrmd_list_t * head, const char *value)
	{
	lrmd_list_t p, end;

	p = calloc(1, sizeof(lrmd_list_t));
	p->val = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_list_freeall(lrmd_list_t * head)
	{
	lrmd_list_t *p;

	while (head) {
	char val = (char )head->val;

	p = head->next;
	free(val);
	free(head);
	head = p;
	}
	}

	lrmd_key_value_t *
	lrmd_key_value_add(lrmd_key_value_t * head, const char key, const char value)
	{
	lrmd_key_value_t p, end;

	p = calloc(1, sizeof(lrmd_key_value_t));
	p->key = strdup(key);
	p->value = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	-static void
	+void
	lrmd_key_value_freeall(lrmd_key_value_t * head)
	{
	lrmd_key_value_t *p;

	while (head) {
	p = head->next;
	free(head->key);
	free(head->value);
	free(head);
	head = p;
	}
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	g_hash_table_replace(user_data, strdup(key), strdup(value));
	}

	lrmd_event_data_t *
	lrmd_copy_event(lrmd_event_data_t * event)
	{
	lrmd_event_data_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_event_data_t));

	/* This will get all the int values.
	* we just have to be careful not to leave any
	* dangling pointers to strings. */
	memcpy(copy, event, sizeof(lrmd_event_data_t));

	copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
	copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
	copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
	copy->output = event->output ? strdup(event->output) : NULL;
	copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;

	if (event->params) {
	copy->params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (copy->params != NULL) {
	g_hash_table_foreach(event->params, dup_attr, copy->params);
	}
	}

	return copy;
	}

	void
	lrmd_free_event(lrmd_event_data_t * event)
	{
	if (!event) {
	return;
	}

	/* free gives me grief if i try to cast */
	free((char *)event->rsc_id);
	free((char *)event->op_type);
	free((char *)event->user_data);
	free((char *)event->output);
	free((char *)event->remote_nodename);
	if (event->params) {
	g_hash_table_destroy(event->params);
	}
	free(event);
	}

	static int
	lrmd_dispatch_internal(lrmd_t lrmd, xmlNode msg)
	{
	const char *type;
	lrmd_private_t *native = lrmd->private;
	lrmd_event_data_t event = { 0, };

	if (!native->callback) {
	/* no callback set */
	crm_trace("notify event received but client has not set callback");
	return 1;
	}

	event.remote_nodename = native->remote_nodename;
	type = crm_element_value(msg, F_LRMD_OPERATION);
	crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
	event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);

	if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) {
	event.type = lrmd_event_register;
	} else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) {
	event.type = lrmd_event_unregister;
	} else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) {
	crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
	crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval);
	crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
	crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
	crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
	crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);

	crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run);
	crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange);
	crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
	crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);

	event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
	event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
	event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
	event.type = lrmd_event_exec_complete;

	event.params = xml2list(msg);
	+ } else if (crm_str_eq(type, LRMD_OP_POKE, TRUE)) {
	+ event.type = lrmd_event_poke;
	} else {
	return 1;
	}

	crm_trace("op %s notify event received", type);
	native->callback(&event);

	if (event.params) {
	g_hash_table_destroy(event.params);
	}
	return 1;
	}

	static int
	lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;
	xmlNode *msg;
	int rc;

	if (!native->callback) {
	/* no callback set */
	return 1;
	}

	msg = string2xml(buffer);
	rc = lrmd_dispatch_internal(lrmd, msg);
	free_xml(msg);
	return rc;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_free_xml(gpointer userdata)
	{
	free_xml((xmlNode *) userdata);
	}

	static int
	lrmd_tls_connected(lrmd_t *lrmd)
	{
	lrmd_private_t *native = lrmd->private;
	if (native->remote->tls_session) {
	return TRUE;
	}

	return FALSE;
	}

	static int
	lrmd_tls_dispatch(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;
	xmlNode *xml = NULL;
	int rc = 0;
	int disconnected = 0;

	if (lrmd_tls_connected(lrmd) == FALSE) {
	crm_trace("tls dispatch triggered after disconnect");
	return 0;
	}

	crm_trace("tls_dispatch triggered");

	/* First check if there are any pending notifies to process that came
	* while we were waiting for replies earlier. */
	if (native->pending_notify) {
	GList *iter = NULL;
	crm_trace("Processing pending notifies");
	for (iter = native->pending_notify; iter; iter = iter->next) {
	lrmd_dispatch_internal(lrmd, iter->data);
	}
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	/* Next read the current buffer and see if there are any messages to handle. */
	rc = crm_remote_ready(native->remote, 0);
	if (rc == 0) {
	/* nothing to read, see if any full messages are already in buffer. */
	xml = crm_remote_parse_buffer(native->remote);
	} else if (rc < 0) {
	disconnected = 1;
	} else {
	crm_remote_recv(native->remote, -1, &disconnected);
	xml = crm_remote_parse_buffer(native->remote);
	}
	while (xml) {
	lrmd_dispatch_internal(lrmd, xml);
	free_xml(xml);
	xml = crm_remote_parse_buffer(native->remote);
	}

	if (disconnected) {
	crm_info("Server disconnected while reading remote server msg.");
	lrmd_tls_disconnect(lrmd);
	return 0;
	}
	return 1;
	}
	#endif

	/* Not used with mainloop */
	int lrmd_poll(lrmd_t *lrmd, int timeout)
	{
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	return crm_ipc_ready(native->ipc);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	if (native->pending_notify) {
	return 1;
	} else if (native->remote->buffer
	&& strstr(native->remote->buffer, REMOTE_MSG_TERMINATOR)) {
	return 1;
	}

	return crm_remote_ready(native->remote, 0);
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return 0;
	}

	/* Not used with mainloop */
	bool
	lrmd_dispatch(lrmd_t * lrmd)
	{
	lrmd_private_t *private = NULL;

	CRM_ASSERT(lrmd != NULL);

	private = lrmd->private;
	switch (private->type) {
	case CRM_CLIENT_IPC:
	while (crm_ipc_ready(private->ipc)) {
	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);
	lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
	}
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	lrmd_tls_dispatch(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", private->type);
	}

	- if (lrmd_connected(lrmd) == FALSE) {
	+ if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("Connection closed");
	return FALSE;
	}

	return TRUE;
	}

	static xmlNode *
	lrmd_create_op(const char token, const char op, xmlNode * data, enum lrmd_call_options options)
	{
	xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");

	crm_xml_add(op_msg, F_TYPE, T_LRMD);
	crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_LRMD_OPERATION, op);
	crm_trace("Sending call options: %.8lx, %d", (long)options, options);
	crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);

	if (data != NULL) {
	add_message_xml(op_msg, F_LRMD_CALLDATA, data);
	}

	return op_msg;
	}

	static void
	lrmd_ipc_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;

	crm_info("IPC connection destroyed");

	/* Prevent these from being cleaned up in lrmd_api_disconnect() */
	native->ipc = NULL;
	native->source = NULL;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_disconnect;
	+ event.remote_nodename = native->remote_nodename;
	native->callback(&event);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->private;

	crm_info("TLS connection destroyed");

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	}
	if (native->psk_cred_c) {
	gnutls_psk_free_client_credentials(native->psk_cred_c);
	}
	if (native->sock) {
	close(native->sock);
	}
	if (native->process_notify) {
	mainloop_destroy_trigger(native->process_notify);
	native->process_notify = NULL;
	}
	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	free(native->remote->buffer);
	native->remote->buffer = NULL;
	native->source = 0;
	native->sock = 0;
	native->psk_cred_c = NULL;
	native->remote->tls_session = NULL;
	native->sock = 0;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.remote_nodename = native->remote_nodename;
	event.type = lrmd_event_disconnect;
	native->callback(&event);
	}
	return;
	}

	int
	lrmd_tls_send_msg(crm_remote_t session, xmlNode msg, uint32_t id, const char *msg_type)
	{
	int rc = -1;

	crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
	crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);

	rc = crm_remote_send(session, msg);

	if (rc < 0) {
	crm_err("Failed to send remote lrmd tls msg, rc = %d" , rc);
	return rc;
	}

	return rc;
	}

	static xmlNode *
	lrmd_tls_recv_reply(lrmd_t lrmd, int total_timeout, int expected_reply_id, int disconnected)
	{
	lrmd_private_t *native = lrmd->private;
	xmlNode *xml = NULL;
	time_t start = time(NULL);
	const char *msg_type = NULL;
	int reply_id = 0;
	int remaining_timeout = 0;

	- if (total_timeout == 0) {
	+ /* A timeout of 0 here makes no sense. We have to wait a period of time
	+ * for the response to come back. If -1 or 0, default to 10 seconds. */
	+ if (total_timeout <= 0) {
	total_timeout = 10000;
	- } else if (total_timeout == -1) {
	- total_timeout = 30000;
	}

	while (!xml) {

	xml = crm_remote_parse_buffer(native->remote);
	if (!xml) {
	/* read some more off the tls buffer if we still have time left. */
	if (remaining_timeout) {
	remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000);
	} else {
	remaining_timeout = total_timeout;
	}
	if (remaining_timeout <= 0) {
	return NULL;
	}

	crm_remote_recv(native->remote, remaining_timeout, disconnected);
	xml = crm_remote_parse_buffer(native->remote);
	if (!xml \|\| *disconnected) {
	return NULL;
	}
	}

	CRM_ASSERT(xml != NULL);

	crm_element_value_int(xml, F_LRMD_REMOTE_MSG_ID, &reply_id);
	msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);

	if (!msg_type) {
	crm_err("Empty msg type received while waiting for reply");
	free_xml(xml);
	xml = NULL;
	} else if (safe_str_eq(msg_type, "notify")) {
	/* got a notify while waiting for reply, trigger the notify to be processed later */
	crm_info("queueing notify");
	native->pending_notify = g_list_append(native->pending_notify, xml);
	if (native->process_notify) {
	crm_info("notify trigger set.");
	mainloop_set_trigger(native->process_notify);
	}
	xml = NULL;
	} else if (safe_str_neq(msg_type, "reply")) {
	/* msg isn't a reply, make some noise */
	crm_err("Expected a reply, got %s", msg_type);
	free_xml(xml);
	xml = NULL;
	} else if (reply_id != expected_reply_id) {
	crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
	free_xml(xml);
	xml = NULL;
	}
	}

	if (native->remote->buffer && native->process_notify) {
	mainloop_set_trigger(native->process_notify);
	}

	return xml;
	}

	static int
	-lrmd_tls_send_recv(lrmd_t lrmd, xmlNode msg, int timeout, xmlNode **reply)
	+lrmd_tls_send(lrmd_t lrmd, xmlNode msg)
	{
	int rc = 0;
	- int disconnected = 0;
	- xmlNode *xml = NULL;
	lrmd_private_t *native = lrmd->private;
	-
	- if (lrmd_tls_connected(lrmd) == FALSE) {
	- return -1;
	- }
	-
	global_remote_msg_id++;
	if (global_remote_msg_id <= 0) {
	global_remote_msg_id = 1;
	}

	rc = lrmd_tls_send_msg(native->remote, msg, global_remote_msg_id, "request");
	if (rc <= 0) {
	crm_err("Remote lrmd send failed, disconnecting");
	lrmd_tls_disconnect(lrmd);
	return -ENOTCONN;
	}
	+ return pcmk_ok;
	+}
	+
	+static int
	+lrmd_tls_send_recv(lrmd_t lrmd, xmlNode msg, int timeout, xmlNode **reply)
	+{
	+ int rc = 0;
	+ int disconnected = 0;
	+ xmlNode *xml = NULL;
	+
	+ if (lrmd_tls_connected(lrmd) == FALSE) {
	+ return -1;
	+ }
	+
	+ rc = lrmd_tls_send(lrmd, msg);
	+ if (rc < 0) {
	+ return rc;
	+ }

	xml = lrmd_tls_recv_reply(lrmd, timeout, global_remote_msg_id, &disconnected);

	if (disconnected) {
	crm_err("Remote lrmd server disconnected while waiting for reply with id %d. ", global_remote_msg_id);
	lrmd_tls_disconnect(lrmd);
	rc = -ENOTCONN;
	} else if (!xml) {
	crm_err("Remote lrmd never received reply for request id %d. timeout: %dms ", global_remote_msg_id, timeout);
	rc = -ECOMM;
	}

	if (reply) {
	*reply = xml;
	} else {
	free_xml(xml);
	}

	return rc;
	}
	#endif

	static int
	lrmd_send_xml(lrmd_t lrmd, xmlNode msg, int timeout, xmlNode **reply)
	{
	int rc = -1;
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return rc;
	}

	static int
	-lrmd_connected(lrmd_t *lrmd)
	+lrmd_send_xml_no_reply(lrmd_t lrmd, xmlNode msg)
	+{
	+ int rc = -1;
	+ lrmd_private_t *native = lrmd->private;
	+
	+ switch (native->type) {
	+ case CRM_CLIENT_IPC:
	+ rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_none, 0, NULL);
	+ break;
	+ case CRM_CLIENT_TLS:
	+#ifdef HAVE_GNUTLS_GNUTLS_H
	+ rc = lrmd_tls_send(lrmd, msg);
	+#endif
	+ break;
	+ default:
	+ crm_err("Unsupported connection type: %d", native->type);
	+ }
	+
	+ return rc;
	+}
	+
	+static int
	+lrmd_api_is_connected(lrmd_t *lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	return crm_ipc_connected(native->ipc);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	return lrmd_tls_connected(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	return 0;
	}

	static int
	-lrmd_send_command(lrmd_t * lrmd, const char op, xmlNode data, xmlNode ** output_data, int timeout, /* ms. defaults to 1000 if set to 0 */
	- enum lrmd_call_options options)
	+lrmd_send_command(lrmd_t * lrmd,
	+ const char *op,
	+ xmlNode * data,
	+ xmlNode ** output_data,
	+ int timeout, /* ms. defaults to 1000 if set to 0 */
	+ enum lrmd_call_options options,
	+ gboolean expect_reply) /* TODO we need to reduce usage of this boolean */
	{
	int rc = pcmk_ok;
	int reply_id = -1;
	lrmd_private_t *native = lrmd->private;
	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	- if (!lrmd_connected(lrmd)) {
	+ if (!lrmd_api_is_connected(lrmd)) {
	return -ENOTCONN;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	CRM_CHECK(native->token != NULL,;);
	crm_trace("sending %s op to lrmd", op);

	op_msg = lrmd_create_op(native->token, op, data, options);

	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);

	- rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
	- free_xml(op_msg);
	+ if (expect_reply) {
	+ rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
	+ } else {
	+ rc = lrmd_send_xml_no_reply(lrmd, op_msg);
	+ goto done;
	+ }

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, F_LRMD_CALLID, &reply_id);
	crm_trace("%s op reply received", op);
	if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
	rc = -ENOMSG;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (output_data) {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	done:
	- if (lrmd_connected(lrmd) == FALSE) {
	+ if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("LRMD disconnected");
	}

	+ free_xml(op_msg);
	free_xml(op_reply);
	return rc;
	}

	+static int
	+lrmd_api_poke_connection(lrmd_t *lrmd)
	+{
	+ int rc;
	+ xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	+
	+ crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	+ rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0, FALSE);
	+ free_xml(data);
	+
	+ return rc;
	+}
	+
	static int
	lrmd_handshake(lrmd_t lrmd, const char name)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->private;
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, "lrmd_command");

	crm_xml_add(hello, F_TYPE, T_LRMD);
	crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(hello, F_LRMD_CLIENTNAME, name);

	rc = lrmd_send_xml(lrmd, hello, -1, &reply);

	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc);
	rc = -ECOMM;
	} else if (reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;
	} else {
	const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
	const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);

	if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);

	+ if (rc != pcmk_ok) {
	+ lrmd_api_disconnect(lrmd);
	+ }
	return rc;
	}

	static int
	lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->private;

	static struct ipc_client_callbacks lrmd_callbacks = {
	.dispatch = lrmd_ipc_dispatch,
	.destroy = lrmd_ipc_connection_destroy
	};

	crm_info("Connecting to lrmd");

	if (fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new("lrmd", 0);
	if (native->ipc && crm_ipc_connect(native->ipc)) {
	*fd = crm_ipc_get_fd(native->ipc);
	} else if (native->ipc) {
	rc = -ENOTCONN;
	}
	} else {
	native->source = mainloop_add_ipc_client("lrmd", G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the LRMD API");
	rc = -ENOTCONN;
	}

	return rc;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	int lrmd_tls_set_key(gnutls_datum_t key, const char location)
	{
	FILE *stream;
	int read_len = 256;
	int cur_len = 0;
	int buf_len = read_len;
	static char *key_cache = NULL;
	static size_t key_cache_len = 0;
	static time_t key_cache_updated;

	if (key_cache) {
	time_t now = time(NULL);

	if ((now - key_cache_updated) < 60) {
	key->data = gnutls_malloc(key_cache_len + 1);
	key->size = key_cache_len;
	memcpy(key->data, key_cache, key_cache_len);

	crm_debug("using cached LRMD key");
	return 0;
	} else {
	key_cache_len = 0;
	key_cache_updated = 0;
	free(key_cache);
	key_cache = NULL;
	crm_debug("clearing lrmd key cache");
	}
	}

	stream = fopen(location, "r");
	if (!stream) {
	return -1;
	}

	key->data = gnutls_malloc(read_len);
	while (!feof(stream)) {
	char next;
	if (cur_len == buf_len) {
	buf_len = cur_len + read_len;
	key->data = gnutls_realloc(key->data, buf_len);
	}
	next = fgetc(stream);
	if (next == EOF && feof(stream)) {
	break;
	}

	key->data[cur_len] = next;
	cur_len++;
	}
	fclose(stream);

	key->size = cur_len;
	if (!cur_len) {
	gnutls_free(key->data);
	key->data = 0;
	return -1;
	}

	if (!key_cache) {
	key_cache = calloc(1, key->size+1);
	memcpy(key_cache, key->data, key->size);

	key_cache_len = key->size;
	key_cache_updated = time(NULL);
	}

	return 0;
	}

	static int
	lrmd_tls_key_cb(gnutls_session_t session, char *username, gnutls_datum_t key)
	{
	int rc = 0;

	if (lrmd_tls_set_key(key, DEFAULT_REMOTE_KEY_LOCATION)) {
	rc = lrmd_tls_set_key(key, ALT_REMOTE_KEY_LOCATION);
	}
	if (rc) {
	crm_err("No lrmd remote key found");
	return -1;
	}

	*username = gnutls_malloc(strlen(DEFAULT_REMOTE_USERNAME) + 1);
	strcpy(*username, DEFAULT_REMOTE_USERNAME);

	return rc;
	}
	#endif

	+static void
	+lrmd_gnutls_global_init(void)
	+{
	+ static int gnutls_init = 0;
	+
	+ if (!gnutls_init) {
	+ gnutls_global_init();
	+ }
	+ gnutls_init = 1;
	+}
	+
	+static void
	+report_async_connection_result(lrmd_t *lrmd, int rc)
	+{
	+ lrmd_private_t *native = lrmd->private;
	+ if (native->callback) {
	+ lrmd_event_data_t event = { 0, };
	+ event.type = lrmd_event_connect;
	+ event.remote_nodename = native->remote_nodename;
	+ event.connection_rc = rc;
	+ native->callback(&event);
	+ }
	+}
	+
	+#ifdef HAVE_GNUTLS_GNUTLS_H
	+static void
	+lrmd_tcp_connect_cb(void *userdata, int sock)
	+{
	+ lrmd_t *lrmd = userdata;
	+ lrmd_private_t *native = lrmd->private;
	+ char name[256] = { 0, };
	+ static struct mainloop_fd_callbacks lrmd_tls_callbacks =
	+ {
	+ .dispatch = lrmd_tls_dispatch,
	+ .destroy = lrmd_tls_connection_destroy,
	+ };
	+ int rc = sock;
	+
	+ if (rc < 0) {
	+ lrmd_tls_connection_destroy(lrmd);
	+ crm_info("remote lrmd connect to %s at port %d failed", native->server, native->port);
	+ report_async_connection_result(lrmd, rc);
	+ return;
	+ }
	+
	+ /* TODO continue with tls stuff now that tcp connect passed. make this async as well soon
	+ * to avoid all blocking code in the client. */
	+ native->sock = sock;
	+ gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	+ gnutls_psk_set_client_credentials_function(native->psk_cred_c, lrmd_tls_key_cb);
	+ native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);
	+
	+ if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
	+ crm_warn("Client tls handshake failed for server %s:%d. Disconnecting", native->server, native->port);
	+ gnutls_deinit(*native->remote->tls_session);
	+ gnutls_free(native->remote->tls_session);
	+ native->remote->tls_session = NULL;
	+ lrmd_tls_connection_destroy(lrmd);
	+ report_async_connection_result(lrmd, -1);
	+ return;
	+ }
	+
	+ crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server, native->port);
	+
	+ snprintf(name, 128, "remote-lrmd-%s:%d", native->server, native->port);
	+
	+ native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
	+ native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
	+
	+ rc = lrmd_handshake(lrmd, name);
	+ report_async_connection_result(lrmd, rc);
	+
	+ return;
	+}
	+#endif
	+
	+static int
	+lrmd_tls_connect_async(lrmd_t lrmd, int timeout /ms*/)
	+{
	+#ifdef HAVE_GNUTLS_GNUTLS_H
	+ int rc = 0;
	+ lrmd_private_t *native = lrmd->private;
	+
	+ lrmd_gnutls_global_init();
	+
	+ rc = crm_remote_tcp_connect_async(native->server, native->port, timeout, lrmd, lrmd_tcp_connect_cb);
	+
	+ return rc;
	+#else
	+ crm_err("TLS not enabled for this build.");
	+ return -ENOTCONN;
	+#endif
	+}
	+
	static int
	lrmd_tls_connect(lrmd_t lrmd, int fd)
	{
	#ifdef HAVE_GNUTLS_GNUTLS_H
	static struct mainloop_fd_callbacks lrmd_tls_callbacks =
	{
	.dispatch = lrmd_tls_dispatch,
	.destroy = lrmd_tls_connection_destroy,
	};

	lrmd_private_t *native = lrmd->private;
	- static int gnutls_init = 0;
	int sock;

	- if (!gnutls_init) {
	- gnutls_global_init();
	- }
	+ lrmd_gnutls_global_init();

	- gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	- gnutls_psk_set_client_credentials_function(native->psk_cred_c, lrmd_tls_key_cb);
	sock = crm_remote_tcp_connect(native->server, native->port);
	if (sock <= 0) {
	crm_warn("Could not establish remote lrmd connection to %s", native->server);
	lrmd_tls_connection_destroy(lrmd);
	return -ENOTCONN;
	}

	native->sock = sock;
	+ gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	+ gnutls_psk_set_client_credentials_function(native->psk_cred_c, lrmd_tls_key_cb);
	native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);

	if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
	crm_err("Session creation for %s:%d failed", native->server, native->port);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = NULL;
	lrmd_tls_connection_destroy(lrmd);
	return -1;
	}

	crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server, native->port);

	if (fd) {
	*fd = sock;
	} else {
	char name[256] = { 0, };
	snprintf(name, 128, "remote-lrmd-%s:%d", native->server, native->port);

	native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
	native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
	}
	return pcmk_ok;
	#else
	crm_err("TLS not enabled for this build.");
	return -ENOTCONN;
	#endif
	}

	static int
	lrmd_api_connect(lrmd_t * lrmd, const char name, int fd)
	{
	int rc = -ENOTCONN;
	lrmd_private_t *native = lrmd->private;

	switch (native->type) {
	case CRM_CLIENT_IPC:
	rc = lrmd_ipc_connect(lrmd, fd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	rc = lrmd_tls_connect(lrmd, fd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	if (rc == pcmk_ok) {
	rc = lrmd_handshake(lrmd, name);
	}

	- if (rc != pcmk_ok) {
	- lrmd_api_disconnect(lrmd);
	+ return rc;
	+}
	+
	+static int
	+lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
	+{
	+ int rc = 0;
	+ lrmd_private_t *native = lrmd->private;
	+
	+ if (!native->callback) {
	+ crm_err("Async connect not possible, no lrmd client callback set.");
	+ return -1;
	+ }
	+
	+ switch (native->type) {
	+ case CRM_CLIENT_IPC:
	+ /* fake async connection with ipc. it should be fast
	+ * enough that we gain very little from async */
	+ rc = lrmd_api_connect(lrmd, name, NULL);
	+ if (!rc) {
	+ report_async_connection_result(lrmd, rc);
	+ }
	+ break;
	+ case CRM_CLIENT_TLS:
	+ rc = lrmd_tls_connect_async(lrmd, timeout);
	+ if (rc) {
	+ /* connection failed, report rc now */
	+ report_async_connection_result(lrmd, rc);
	+ }
	+ break;
	+ default:
	+ crm_err("Unsupported connection type: %d", native->type);
	}

	return rc;
	}

	static void
	lrmd_ipc_disconnect(lrmd_t *lrmd)
	{
	lrmd_private_t *native = lrmd->private;
	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;

	} else if(native->ipc) {
	/* Not attached to mainloop */
	crm_ipc_t *ipc = native->ipc;
	native->ipc = NULL;
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_disconnect(lrmd_t *lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = 0;
	}

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;

	} else if(native->sock) {
	close(native->sock);
	}

	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}
	}
	#endif

	static int
	lrmd_api_disconnect(lrmd_t *lrmd)
	{
	lrmd_private_t *native = lrmd->private;

	crm_info("Disconnecting from lrmd service");
	switch (native->type) {
	case CRM_CLIENT_IPC:
	lrmd_ipc_disconnect(lrmd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	lrmd_tls_disconnect(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported connection type: %d", native->type);
	}

	free(native->token);
	native->token = NULL;
	return 0;
	}

	static int
	lrmd_api_register_rsc(lrmd_t * lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char type, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = NULL;

	if (!class \|\| !type \|\| !rsc_id) {
	return -EINVAL;
	}
	if (safe_str_eq(class, "ocf") && !provider) {
	return -EINVAL;
	}

	data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_CLASS, class);
	crm_xml_add(data, F_LRMD_PROVIDER, provider);
	crm_xml_add(data, F_LRMD_TYPE, type);
	- rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options);
	+ rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	static int
	lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	- rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options);
	+ rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	lrmd_rsc_info_t *
	lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	lrmd_rsc_info_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_rsc_info_t));

	copy->id = strdup(rsc_info->id);
	copy->type = strdup(rsc_info->type);
	copy->class = strdup(rsc_info->class);
	if (rsc_info->provider) {
	copy->provider = strdup(rsc_info->provider);
	}

	return copy;
	}

	void
	lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	if (!rsc_info) {
	return;
	}
	free(rsc_info->id);
	free(rsc_info->type);
	free(rsc_info->class);
	free(rsc_info->provider);
	free(rsc_info);
	}

	static lrmd_rsc_info_t *
	lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	lrmd_rsc_info_t *rsc_info = NULL;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *output = NULL;
	const char *class = NULL;
	const char *provider = NULL;
	const char *type = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	- lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options);
	+ lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
	free_xml(data);

	if (!output) {
	return NULL;
	}

	class = crm_element_value(output, F_LRMD_CLASS);
	provider = crm_element_value(output, F_LRMD_PROVIDER);
	type = crm_element_value(output, F_LRMD_TYPE);

	if (!class \|\| !type) {
	free_xml(output);
	return NULL;
	} else if (safe_str_eq(class, "ocf") && !provider) {
	free_xml(output);
	return NULL;
	}

	rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
	rsc_info->id = strdup(rsc_id);
	rsc_info->class = strdup(class);
	if (provider) {
	rsc_info->provider = strdup(provider);
	}
	rsc_info->type = strdup(type);

	free_xml(output);
	return rsc_info;
	}

	static void
	lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
	{
	lrmd_private_t *native = lrmd->private;

	native->callback = callback;
	}

	static int
	stonith_get_metadata(const char provider, const char type, char **output)
	{
	int rc = pcmk_ok;

	stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0);
	if (*output == NULL) {
	rc = -EIO;
	}
	return rc;
	}

	static int
	lsb_get_metadata(const char type, char *output)
	{

	#define lsb_metadata_template \
	"<?xml version=\"1.0\"?>\n"\
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"\
	"<resource-agent name=\"%s\" version=\"0.1\">\n"\
	" <version>1.0</version>\n"\
	" <longdesc lang=\"en\">\n"\
	" %s"\
	" </longdesc>\n"\
	" <shortdesc lang=\"en\">%s</shortdesc>\n"\
	" <parameters>\n"\
	" </parameters>\n"\
	" <actions>\n"\
	" <action name=\"start\" timeout=\"15\" />\n"\
	" <action name=\"stop\" timeout=\"15\" />\n"\
	" <action name=\"status\" timeout=\"15\" />\n"\
	" <action name=\"restart\" timeout=\"15\" />\n"\
	" <action name=\"force-reload\" timeout=\"15\" />\n"\
	" <action name=\"monitor\" timeout=\"15\" interval=\"15\" />\n"\
	" <action name=\"meta-data\" timeout=\"5\" />\n"\
	" </actions>\n"\
	" <special tag=\"LSB\">\n"\
	" <Provides>%s</Provides>\n"\
	" <Required-Start>%s</Required-Start>\n"\
	" <Required-Stop>%s</Required-Stop>\n"\
	" <Should-Start>%s</Should-Start>\n"\
	" <Should-Stop>%s</Should-Stop>\n"\
	" <Default-Start>%s</Default-Start>\n"\
	" <Default-Stop>%s</Default-Stop>\n"\
	" </special>\n"\
	"</resource-agent>\n"

	#define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
	#define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
	#define PROVIDES "# Provides:"
	#define REQ_START "# Required-Start:"
	#define REQ_STOP "# Required-Stop:"
	#define SHLD_START "# Should-Start:"
	#define SHLD_STOP "# Should-Stop:"
	#define DFLT_START "# Default-Start:"
	#define DFLT_STOP "# Default-Stop:"
	#define SHORT_DSCR "# Short-Description:"
	#define DESCRIPTION "# Description:"

	#define lsb_meta_helper_free_value(m) \
	if ((m) != NULL) { \
	xmlFree(m); \
	(m) = NULL; \
	}

	#define lsb_meta_helper_get_value(buffer, ptr, keyword) \
	if (!ptr && !strncasecmp(buffer, keyword, strlen(keyword))) { \
	(ptr) = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer+strlen(keyword)); \
	continue; \
	}

	char ra_pathname[PATH_MAX] = { 0, };
	FILE *fp;
	GString *meta_data = NULL;
	char buffer[1024];
	char *provides = NULL;
	char *req_start = NULL;
	char *req_stop = NULL;
	char *shld_start = NULL;
	char *shld_stop = NULL;
	char *dflt_start = NULL;
	char *dflt_stop = NULL;
	char *s_dscrpt = NULL;
	char *xml_l_dscrpt = NULL;
	GString *l_dscrpt = NULL;

	snprintf(ra_pathname, sizeof(ra_pathname), "%s%s%s",
	type[0] == '/' ? "" : LSB_ROOT_DIR, type[0] == '/' ? "" : "/", type);

	if (!(fp = fopen(ra_pathname, "r"))) {
	return -EIO;
	}

	/* Enter into the lsb-compliant comment block */
	while (fgets(buffer, sizeof(buffer), fp)) {
	/* Now suppose each of the following eight arguments contain only one line */
	lsb_meta_helper_get_value(buffer, provides, PROVIDES)
	lsb_meta_helper_get_value(buffer, req_start, REQ_START)
	lsb_meta_helper_get_value(buffer, req_stop, REQ_STOP)
	lsb_meta_helper_get_value(buffer, shld_start, SHLD_START)
	lsb_meta_helper_get_value(buffer, shld_stop, SHLD_STOP)
	lsb_meta_helper_get_value(buffer, dflt_start, DFLT_START)
	lsb_meta_helper_get_value(buffer, dflt_stop, DFLT_STOP)
	lsb_meta_helper_get_value(buffer, s_dscrpt, SHORT_DSCR)

	/* Long description may cross multiple lines */
	if ((l_dscrpt == NULL) && (0 == strncasecmp(buffer, DESCRIPTION, strlen(DESCRIPTION)))) {
	l_dscrpt = g_string_new(buffer + strlen(DESCRIPTION));
	/* Between # and keyword, more than one space, or a tab character,
	* indicates the continuation line. Extracted from LSB init script standard */
	while (fgets(buffer, sizeof(buffer), fp)) {
	if (!strncmp(buffer, "# ", 3) \|\| !strncmp(buffer, "#\t", 2)) {
	buffer[0] = ' ';
	l_dscrpt = g_string_append(l_dscrpt, buffer);
	} else {
	fputs(buffer, fp);
	break; /* Long description ends */
	}
	}
	continue;
	}
	if (l_dscrpt) {
	xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(l_dscrpt->str));
	}
	if (!strncasecmp(buffer, LSB_INITSCRIPT_INFOEND_TAG, strlen(LSB_INITSCRIPT_INFOEND_TAG))) {
	/* Get to the out border of LSB comment block */
	break;
	}
	if (buffer[0] != '#') {
	break; /* Out of comment block in the beginning */
	}
	}
	fclose(fp);

	meta_data = g_string_new("");
	g_string_sprintf(meta_data, lsb_metadata_template, type,
	(xml_l_dscrpt == NULL) ? type : xml_l_dscrpt,
	(s_dscrpt == NULL) ? type : s_dscrpt, (provides == NULL) ? "" : provides,
	(req_start == NULL) ? "" : req_start, (req_stop == NULL) ? "" : req_stop,
	(shld_start == NULL) ? "" : shld_start, (shld_stop == NULL) ? "" : shld_stop,
	(dflt_start == NULL) ? "" : dflt_start, (dflt_stop == NULL) ? "" : dflt_stop);

	lsb_meta_helper_free_value(xml_l_dscrpt);
	lsb_meta_helper_free_value(s_dscrpt);
	lsb_meta_helper_free_value(provides);
	lsb_meta_helper_free_value(req_start);
	lsb_meta_helper_free_value(req_stop);
	lsb_meta_helper_free_value(shld_start);
	lsb_meta_helper_free_value(shld_stop);
	lsb_meta_helper_free_value(dflt_start);
	lsb_meta_helper_free_value(dflt_stop);

	if (l_dscrpt) {
	g_string_free(l_dscrpt, TRUE);
	}

	*output = strdup(meta_data->str);
	g_string_free(meta_data, TRUE);

	return pcmk_ok;
	}

	static int
	generic_get_metadata(const char standard, const char provider, const char type, char *output)
	{
	svc_action_t *action = resources_action_create(type,
	standard,
	provider,
	type,
	"meta-data",
	0,
	5000,
	NULL);

	if (!(services_action_sync(action))) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	if (!action->stdout_data) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	*output = strdup(action->stdout_data);
	services_action_free(action);

	return pcmk_ok;
	}

	static int
	lrmd_api_get_metadata(lrmd_t * lrmd,
	const char *class,
	const char *provider,
	const char type, char *output, enum lrmd_call_options options)
	{
	if (!class \|\| !type) {
	return -EINVAL;
	}

	if (safe_str_eq(class, "stonith")) {
	return stonith_get_metadata(provider, type, output);
	} else if (safe_str_eq(class, "lsb")) {
	return lsb_get_metadata(type, output);
	}
	return generic_get_metadata(class, provider, type, output);
	}

	static int
	lrmd_api_exec(lrmd_t * lrmd, const char rsc_id, const char action, const char userdata, int interval, / ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
	xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
	lrmd_key_value_t *tmp = NULL;

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);

	for (tmp = params; tmp; tmp = tmp->next) {
	hash2field((gpointer) tmp->key, (gpointer) tmp->value, args);
	}

	- rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options);
	+ rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	static int
	lrmd_api_cancel(lrmd_t * lrmd, const char rsc_id, const char action, int interval)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);

	crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add(data, F_LRMD_RSC_ACTION, action);
	crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
	crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
	- rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0);
	+ rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
	free_xml(data);
	return rc;
	}

	static int
	list_stonith_agents(lrmd_list_t ** resources)
	{
	int rc = 0;
	stonith_key_value_t *stonith_resources = NULL;
	stonith_key_value_t *dIter = NULL;

	stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);

	for (dIter = stonith_resources; dIter; dIter = dIter->next) {
	rc++;
	if(resources) {
	resources = lrmd_list_add(resources, dIter->value);
	}
	}

	stonith_key_value_freeall(stonith_resources, 1, 0);
	return rc;
	}

	static int
	lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
	const char *provider)
	{
	int rc = 0;

	if (safe_str_eq(class, "stonith")) {
	rc += list_stonith_agents(resources);

	} else {
	GListPtr gIter = NULL;
	GList *agents = resources_list_agents(class, provider);

	for (gIter = agents; gIter != NULL; gIter = gIter->next) {
	resources = lrmd_list_add(resources, (const char *)gIter->data);
	rc++;
	}
	g_list_free_full(agents, free);

	if (!class) {
	rc += list_stonith_agents(resources);
	}
	}

	if(rc == 0) {
	crm_notice("No agents found for class %s", class);
	rc = -EPROTONOSUPPORT;
	}
	return rc;
	}

	static int
	does_provider_have_agent(const char agent, const char provider, const char *class)
	{
	int found = 0;
	GList *agents = NULL;
	GListPtr gIter2 = NULL;

	agents = resources_list_agents(class, provider);
	for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
	if (safe_str_eq(agent, gIter2->data)) {
	found = 1;
	}
	}
	g_list_free_full(agents, free);

	return found;
	}

	static int
	lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char agent, lrmd_list_t * providers)
	{
	int rc = pcmk_ok;
	char *provider = NULL;
	GList *ocf_providers = NULL;
	GListPtr gIter = NULL;

	ocf_providers = resources_list_providers("ocf");

	for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
	provider = gIter->data;
	if (!agent \|\| does_provider_have_agent(agent, provider, "ocf")) {
	providers = lrmd_list_add(providers, (const char *)gIter->data);
	rc++;
	}
	}

	g_list_free_full(ocf_providers, free);
	return rc;
	}

	static int
	lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
	{
	int rc = 0;
	GList *standards = NULL;
	GListPtr gIter = NULL;

	standards = resources_list_standards();

	for (gIter = standards; gIter != NULL; gIter = gIter->next) {
	supported = lrmd_list_add(supported, (const char *)gIter->data);
	rc++;
	}

	if(list_stonith_agents(NULL) > 0) {
	supported = lrmd_list_add(supported, "stonith");
	rc++;
	}

	g_list_free_full(standards, free);
	return rc;
	}

	lrmd_t *
	lrmd_api_new(void)
	{
	lrmd_t *new_lrmd = NULL;
	lrmd_private_t *pvt = NULL;

	new_lrmd = calloc(1, sizeof(lrmd_t));
	pvt = calloc(1, sizeof(lrmd_private_t));
	pvt->remote = calloc(1, sizeof(crm_remote_t));
	new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));

	pvt->type = CRM_CLIENT_IPC;
	new_lrmd->private = pvt;

	new_lrmd->cmds->connect = lrmd_api_connect;
	+ new_lrmd->cmds->connect_async = lrmd_api_connect_async;
	+ new_lrmd->cmds->is_connected = lrmd_api_is_connected;
	+ new_lrmd->cmds->poke_connection = lrmd_api_poke_connection;
	new_lrmd->cmds->disconnect = lrmd_api_disconnect;
	new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
	new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
	new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
	new_lrmd->cmds->set_callback = lrmd_api_set_callback;
	new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
	new_lrmd->cmds->exec = lrmd_api_exec;
	new_lrmd->cmds->cancel = lrmd_api_cancel;
	new_lrmd->cmds->list_agents = lrmd_api_list_agents;
	new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
	new_lrmd->cmds->list_standards = lrmd_api_list_standards;

	if (!stonith_api) {
	stonith_api = stonith_api_new();
	}

	return new_lrmd;
	}

	lrmd_t *
	lrmd_remote_api_new(const char nodename, const char server, int port)
	{
	#ifdef HAVE_GNUTLS_GNUTLS_H
	lrmd_t *new_lrmd = lrmd_api_new();
	lrmd_private_t *native = new_lrmd->private;

	if (!nodename && !server) {
	return NULL;
	}

	native->type = CRM_CLIENT_TLS;
	native->remote_nodename = nodename ? strdup(nodename) : strdup(server);
	native->server = server ? strdup(server) : strdup(nodename);
	native->port = port ? port : DEFAULT_REMOTE_PORT;
	return new_lrmd;
	#else
	crm_err("GNUTLS is not enabled for this build, remote LRMD client can not be created");
	return NULL;
	#endif

	}

	void
	lrmd_api_delete(lrmd_t * lrmd)
	{
	if (!lrmd) {
	return;
	}
	lrmd->cmds->disconnect(lrmd); /* no-op if already disconnected */
	free(lrmd->cmds);
	if (lrmd->private) {
	lrmd_private_t *native = lrmd->private;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	free(native->server);
	#endif
	free(native->remote_nodename);
	free(native->remote);
	}
	free(lrmd->private);
	free(lrmd);
	}
	diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c
	index 92693a68b8..d2ed7d2ebd 100644
	--- a/lrmd/lrmd.c
	+++ b/lrmd/lrmd.c
	@@ -1,1222 +1,1224 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipcs.h>
	#include <crm/msg_xml.h>

	#include <lrmd_private.h>

	#ifdef HAVE_SYS_TIMEB_H
	# include <sys/timeb.h>
	#endif

	GHashTable *rsc_list = NULL;

	typedef struct lrmd_cmd_s {
	int timeout;
	int interval;
	int start_delay;

	int call_id;
	int exec_rc;
	int lrmd_op_status;

	int call_opts;
	/* Timer ids, must be removed on cmd destruction. */
	int delay_id;
	int stonith_recurring_id;

	int rsc_deleted;

	char *client_id;
	char *origin;
	char *rsc_id;
	char *action;
	char *output;
	char *userdata_str;

	#ifdef HAVE_SYS_TIMEB_H
	/* Timestamp of when op ran */
	struct timeb t_run;
	/* Timestamp of when op was queued */
	struct timeb t_queue;
	/* Timestamp of last rc change */
	struct timeb t_rcchange;
	#endif

	int first_notify_sent;
	int last_notify_rc;
	int last_notify_op_status;
	int last_pid;

	GHashTable *params;
	} lrmd_cmd_t;

	static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc);
	static gboolean lrmd_rsc_dispatch(gpointer user_data);
	static void cancel_all_recurring(lrmd_rsc_t rsc, const char client_id);

	static void
	log_finished(lrmd_cmd_t *cmd, int exec_time, int queue_time)
	{
	char pid_str[32] = { 0, };
	int log_level = LOG_INFO;

	if (cmd->last_pid) {
	snprintf(pid_str, 32, "%d", cmd->last_pid);
	}

	if (safe_str_eq(cmd->action, "monitor")) {
	log_level = LOG_DEBUG;
	}

	#ifdef HAVE_SYS_TIMEB_H
	do_crm_log(log_level, "finished - rsc:%s action:%s call_id:%d %s%s exit-code:%d exec-time:%dms queue-time:%dms",
	cmd->rsc_id,
	cmd->action,
	cmd->call_id,
	cmd->last_pid ? "pid:" : "",
	pid_str,
	cmd->exec_rc,
	exec_time,
	queue_time);
	#else
	do_crm_log(log_level, "finished - rsc:%s action:%s call_id:%d %s%s exit-code:%d",
	cmd->rsc_id,
	cmd->action,
	cmd->call_id,
	cmd->last_pid ? "pid:" : "",
	pid_str,
	cmd->exec_rc);
	#endif
	}

	static void
	log_execute(lrmd_cmd_t *cmd)
	{
	int log_level = LOG_INFO;

	if (safe_str_eq(cmd->action, "monitor")) {
	log_level = LOG_DEBUG;
	}

	do_crm_log(log_level, "executing - rsc:%s action:%s call_id:%d",
	cmd->rsc_id, cmd->action, cmd->call_id);
	}

	static lrmd_rsc_t *
	build_rsc_from_xml(xmlNode * msg)
	{
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
	lrmd_rsc_t *rsc = NULL;

	rsc = calloc(1, sizeof(lrmd_rsc_t));

	crm_element_value_int(msg, F_LRMD_CALLOPTS, &rsc->call_opts);

	rsc->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
	rsc->class = crm_element_value_copy(rsc_xml, F_LRMD_CLASS);
	rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER);
	rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE);
	rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_rsc_dispatch, rsc);
	return rsc;
	}

	static lrmd_cmd_t *
	create_lrmd_cmd(xmlNode * msg, crm_client_t * client)
	{
	int call_options = 0;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
	lrmd_cmd_t *cmd = NULL;

	cmd = calloc(1, sizeof(lrmd_cmd_t));

	crm_element_value_int(msg, F_LRMD_CALLOPTS, &call_options);
	cmd->call_opts = call_options;
	cmd->client_id = strdup(client->id);

	crm_element_value_int(msg, F_LRMD_CALLID, &cmd->call_id);
	crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &cmd->interval);
	crm_element_value_int(rsc_xml, F_LRMD_TIMEOUT, &cmd->timeout);
	crm_element_value_int(rsc_xml, F_LRMD_RSC_START_DELAY, &cmd->start_delay);

	cmd->origin = crm_element_value_copy(rsc_xml, F_LRMD_ORIGIN);
	cmd->action = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ACTION);
	cmd->userdata_str = crm_element_value_copy(rsc_xml, F_LRMD_RSC_USERDATA_STR);
	cmd->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);

	cmd->params = xml2list(rsc_xml);

	return cmd;
	}

	static void
	free_lrmd_cmd(lrmd_cmd_t * cmd)
	{
	if (cmd->stonith_recurring_id) {
	g_source_remove(cmd->stonith_recurring_id);
	}
	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}
	if (cmd->params) {
	g_hash_table_destroy(cmd->params);
	}
	free(cmd->origin);
	free(cmd->action);
	free(cmd->userdata_str);
	free(cmd->rsc_id);
	free(cmd->output);
	free(cmd->client_id);
	free(cmd);
	}

	static gboolean
	stonith_recurring_op_helper(gpointer data)
	{
	lrmd_cmd_t *cmd = data;
	lrmd_rsc_t *rsc;

	cmd->stonith_recurring_id = 0;

	if (!cmd->rsc_id) {
	return FALSE;
	}

	rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);

	CRM_ASSERT(rsc != NULL);
	/* take it out of recurring_ops list, and put it in the pending ops
	* to be executed */
	rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
	rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
	#ifdef HAVE_SYS_TIMEB_H
	ftime(&cmd->t_queue);
	#endif
	mainloop_set_trigger(rsc->work);

	return FALSE;
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	lrmd_cmd_t *cmd = data;
	lrmd_rsc_t *rsc = NULL;

	cmd->delay_id = 0;
	rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;

	if (rsc) {
	mainloop_set_trigger(rsc->work);
	}

	return FALSE;
	}

	static void
	schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	CRM_CHECK(cmd != NULL, return);
	CRM_CHECK(rsc != NULL, return);

	crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id);
	rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
	#ifdef HAVE_SYS_TIMEB_H
	ftime(&cmd->t_queue);
	#endif
	mainloop_set_trigger(rsc->work);

	if (cmd->start_delay) {
	cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
	}

	}

	static void
	send_reply(crm_client_t * client, int rc, uint32_t id, int call_id)
	{
	int send_rc = 0;
	xmlNode *reply = NULL;

	reply = create_xml_node(NULL, T_LRMD_REPLY);
	crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(reply, F_LRMD_RC, rc);
	crm_xml_add_int(reply, F_LRMD_CALLID, call_id);

	send_rc = lrmd_server_send_reply(client, id, reply);

	free_xml(reply);
	if (send_rc < 0) {
	crm_warn("LRMD reply to %s failed: %d", client->name, send_rc);
	}
	}

	static void
	send_client_notify(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *update_msg = user_data;
	crm_client_t *client = value;

	if (client == NULL) {
	crm_err("Asked to send event to NULL client");
	return;
	} else if (client->name == NULL) {
	crm_trace("Asked to send event to client with no name");
	return;
	}

	if (lrmd_server_send_notify(client, update_msg) <= 0) {
	crm_warn("Notification of client %s/%s failed", client->name, client->id);
	}
	}

	#ifdef HAVE_SYS_TIMEB_H
	static int
	time_diff_ms(struct timeb now, struct timeb old)
	{
	int sec = difftime(now->time, old->time);
	int ms = now->millitm - old->millitm;

	if (old->time == 0) {
	return 0;
	}

	return (sec * 1000) + ms;
	}
	#endif

	static void
	send_cmd_complete_notify(lrmd_cmd_t * cmd)
	{
	int exec_time = 0;
	int queue_time = 0;
	xmlNode *notify = NULL;
	#ifdef HAVE_SYS_TIMEB_H
	struct timeb now = { 0, };

	ftime(&now);
	exec_time = time_diff_ms(&now, &cmd->t_run);
	queue_time = time_diff_ms(&cmd->t_run, &cmd->t_queue);
	#endif

	log_finished(cmd, exec_time, queue_time);

	/* if the first notify result for a cmd has already been sent earlier, and the
	* the option to only send notifies on result changes is set. Check to see
	* if the last result is the same as the new one. If so, suppress this update */
	if (cmd->first_notify_sent && (cmd->call_opts & lrmd_opt_notify_changes_only)) {
	if (cmd->last_notify_rc == cmd->exec_rc &&
	cmd->last_notify_op_status == cmd->lrmd_op_status) {

	/* only send changes */
	return;
	}

	}

	cmd->first_notify_sent = 1;
	cmd->last_notify_rc = cmd->exec_rc;
	cmd->last_notify_op_status = cmd->lrmd_op_status;

	notify = create_xml_node(NULL, T_LRMD_NOTIFY);

	crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(notify, F_LRMD_TIMEOUT, cmd->timeout);
	crm_xml_add_int(notify, F_LRMD_RSC_INTERVAL, cmd->interval);
	crm_xml_add_int(notify, F_LRMD_RSC_START_DELAY, cmd->start_delay);
	crm_xml_add_int(notify, F_LRMD_EXEC_RC, cmd->exec_rc);
	crm_xml_add_int(notify, F_LRMD_OP_STATUS, cmd->lrmd_op_status);
	crm_xml_add_int(notify, F_LRMD_CALLID, cmd->call_id);
	crm_xml_add_int(notify, F_LRMD_RSC_DELETED, cmd->rsc_deleted);

	#ifdef HAVE_SYS_TIMEB_H
	crm_xml_add_int(notify, F_LRMD_RSC_RUN_TIME, cmd->t_run.time);
	crm_xml_add_int(notify, F_LRMD_RSC_RCCHANGE_TIME, cmd->t_rcchange.time);
	crm_xml_add_int(notify, F_LRMD_RSC_EXEC_TIME, exec_time);
	crm_xml_add_int(notify, F_LRMD_RSC_QUEUE_TIME, queue_time);
	#endif

	crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_RSC_EXEC);
	crm_xml_add(notify, F_LRMD_RSC_ID, cmd->rsc_id);
	crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->action);
	crm_xml_add(notify, F_LRMD_RSC_USERDATA_STR, cmd->userdata_str);
	crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->output);

	if (cmd->params) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter iter;

	xmlNode *args = create_xml_node(notify, XML_TAG_ATTRS);

	g_hash_table_iter_init(&iter, cmd->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	hash2field((gpointer) key, (gpointer) value, args);
	}
	}

	if (cmd->client_id && (cmd->call_opts & lrmd_opt_notify_orig_only)) {
	crm_client_t *client = crm_client_get_by_id(cmd->client_id);

	if (client) {
	send_client_notify(client->id, client, notify);
	}
	} else {
	g_hash_table_foreach(client_connections, send_client_notify, notify);
	}

	free_xml(notify);
	}

	static void
	send_generic_notify(int rc, xmlNode * request)
	{
	int call_id = 0;
	xmlNode *notify = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
	const char *op = crm_element_value(request, F_LRMD_OPERATION);

	crm_element_value_int(request, F_LRMD_CALLID, &call_id);

	notify = create_xml_node(NULL, T_LRMD_NOTIFY);
	crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(notify, F_LRMD_RC, rc);
	crm_xml_add_int(notify, F_LRMD_CALLID, call_id);
	crm_xml_add(notify, F_LRMD_OPERATION, op);
	crm_xml_add(notify, F_LRMD_RSC_ID, rsc_id);

	g_hash_table_foreach(client_connections, send_client_notify, notify);

	free_xml(notify);
	}

	static void
	cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)
	{
	crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action, rsc?rsc->active:NULL, cmd);

	if (rsc && (rsc->active == cmd)) {
	rsc->active = NULL;
	mainloop_set_trigger(rsc->work);
	}

	if (!rsc) {
	cmd->rsc_deleted = 1;
	}

	send_cmd_complete_notify(cmd);

	/* crmd expects lrmd to automatically cancel recurring ops after rsc stops */
	if (rsc && safe_str_eq(cmd->action, "stop")) {
	cancel_all_recurring(rsc, NULL);
	}

	if (cmd->interval && (cmd->lrmd_op_status == PCMK_LRM_OP_CANCELLED)) {
	if (rsc) {
	rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
	rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
	}
	free_lrmd_cmd(cmd);
	} else if (cmd->interval == 0) {
	if (rsc) {
	rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
	}
	free_lrmd_cmd(cmd);
	} else {
	/* Clear all the values pertaining just to the last iteration of a recurring op. */
	cmd->lrmd_op_status = 0;
	cmd->last_pid = 0;
	memset(&cmd->t_run, 0, sizeof(cmd->t_run));
	memset(&cmd->t_queue, 0, sizeof(cmd->t_queue));
	free(cmd->output);
	cmd->output = NULL;
	}
	}

	static int
	lsb2uniform_rc(const char *action, int rc)
	{
	if (rc < 0) {
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	/* status has different return codes that everything else. */
	if (!safe_str_eq(action, "status") && !safe_str_eq(action, "monitor")) {
	if (rc > PCMK_LSB_NOT_RUNNING) {
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}
	return rc;
	}

	switch (rc) {
	case PCMK_LSB_STATUS_OK:
	return PCMK_EXECRA_OK;
	case PCMK_LSB_STATUS_NOT_INSTALLED:
	return PCMK_EXECRA_NOT_INSTALLED;
	case PCMK_LSB_STATUS_VAR_PID:
	case PCMK_LSB_STATUS_VAR_LOCK:
	case PCMK_LSB_STATUS_NOT_RUNNING:
	return PCMK_EXECRA_NOT_RUNNING;
	default:
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	static int
	ocf2uniform_rc(int rc)
	{
	if (rc < 0 \|\| rc > PCMK_OCF_FAILED_MASTER) {
	return PCMK_EXECRA_UNKNOWN_ERROR;
	}

	return rc;
	}

	static int
	stonith2uniform_rc(const char *action, int rc)
	{
	if (rc == -ENODEV) {
	if (safe_str_eq(action, "stop")) {
	rc = PCMK_EXECRA_OK;
	} else if (safe_str_eq(action, "start")) {
	rc = PCMK_EXECRA_NOT_INSTALLED;
	} else {
	rc = PCMK_EXECRA_NOT_RUNNING;
	}
	} else if (rc != 0) {
	rc = PCMK_EXECRA_UNKNOWN_ERROR;
	}
	return rc;
	}

	static int
	get_uniform_rc(const char standard, const char action, int rc)
	{
	if (safe_str_eq(standard, "ocf")) {
	return ocf2uniform_rc(rc);
	} else if (safe_str_eq(standard, "stonith")) {
	return stonith2uniform_rc(action, rc);
	} else if (safe_str_eq(standard, "systemd")) {
	return rc;
	} else if (safe_str_eq(standard, "upstart")) {
	return rc;
	} else {
	return lsb2uniform_rc(action, rc);
	}
	}

	void
	client_disconnect_cleanup(const char *client_id)
	{
	GHashTableIter iter;
	lrmd_rsc_t *rsc = NULL;
	char *key = NULL;

	g_hash_table_iter_init(&iter, rsc_list);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & rsc)) {
	if (rsc->call_opts & lrmd_opt_drop_recurring) {
	/* This client is disconnecting, drop any recurring operations
	* it may have initiated on the resource */
	cancel_all_recurring(rsc, client_id);
	}
	}
	}

	static void
	action_complete(svc_action_t * action)
	{
	lrmd_rsc_t *rsc;
	lrmd_cmd_t *cmd = action->cb_data;

	if (!cmd) {
	crm_err("LRMD action (%s) completed does not match any known operations.", action->id);
	return;
	}
	#ifdef HAVE_SYS_TIMEB_H
	if (cmd->exec_rc != action->rc) {
	ftime(&cmd->t_rcchange);
	}
	#endif

	cmd->last_pid = action->pid;
	cmd->exec_rc = get_uniform_rc(action->standard, cmd->action, action->rc);
	cmd->lrmd_op_status = action->status;
	rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;

	if (action->stdout_data) {
	cmd->output = strdup(action->stdout_data);
	}

	cmd_finalize(cmd, rsc);
	}


	static void
	stonith_action_complete(lrmd_cmd_t *cmd, int rc)
	{
	int recurring = cmd->interval;
	lrmd_rsc_t *rsc = NULL;
	cmd->exec_rc = get_uniform_rc("stonith", cmd->action, rc);

	rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);

	if (cmd->lrmd_op_status == PCMK_LRM_OP_CANCELLED) {
	recurring = 0;
	/* do nothing */
	} else if (rc) {
	/* Attempt to map return codes to op status if possible */
	switch (rc) {
	case -EPROTONOSUPPORT:
	cmd->lrmd_op_status = PCMK_LRM_OP_NOTSUPPORTED;
	break;
	case -ETIME:
	cmd->lrmd_op_status = PCMK_LRM_OP_TIMEOUT;
	break;
	default:
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	}
	} else {
	/* command successful */
	cmd->lrmd_op_status = PCMK_LRM_OP_DONE;
	if (safe_str_eq(cmd->action, "start") && rsc) {
	rsc->stonith_started = 1;
	}
	}

	if (recurring && rsc) {
	if (cmd->stonith_recurring_id) {
	g_source_remove(cmd->stonith_recurring_id);
	}
	cmd->stonith_recurring_id = g_timeout_add(cmd->interval, stonith_recurring_op_helper, cmd);
	}

	cmd_finalize(cmd, rsc);
	}

	static void
	lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t *data)
	{
	stonith_action_complete(data->userdata, data->rc);
	}

	void
	stonith_connection_failed(void)
	{
	GHashTableIter iter;
	GList *cmd_list = NULL;
	GList *cmd_iter = NULL;
	lrmd_rsc_t *rsc = NULL;
	char *key = NULL;

	g_hash_table_iter_init(&iter, rsc_list);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & rsc)) {
	if (safe_str_eq(rsc->class, "stonith")) {
	if (rsc->recurring_ops) {
	cmd_list = g_list_concat(cmd_list, rsc->recurring_ops);
	}
	if (rsc->pending_ops) {
	cmd_list = g_list_concat(cmd_list, rsc->pending_ops);
	}
	rsc->pending_ops = rsc->recurring_ops = NULL;
	}
	}

	if (!cmd_list) {
	return;
	}

	crm_err("STONITH connection failed, finalizing %d pending operations.", g_list_length(cmd_list));
	for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
	stonith_action_complete(cmd_iter->data, -ENOTCONN);
	}
	g_list_free(cmd_list);
	}

	static int
	lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	int rc = 0;
	int do_monitor = 0;

	stonith_t *stonith_api = get_stonith_connection();

	if (!stonith_api) {
	cmd->exec_rc = get_uniform_rc("stonith", cmd->action, -ENOTCONN);
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	cmd_finalize(cmd, rsc);
	return -EUNATCH;
	}

	if (safe_str_eq(cmd->action, "start")) {
	char *key = NULL;
	char *value = NULL;
	stonith_key_value_t *device_params = NULL;

	if (cmd->params) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, cmd->params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	device_params = stonith_key_value_add(device_params, key, value);
	}
	}

	/* Stonith automatically registers devices from the CIB when changes occur,
	* but to avoid a possible race condition between stonith receiving the CIB update
	* and the lrmd requesting that resource, the lrmd still registers the device as well.
	* Stonith knows how to handle duplicate device registrations correctly. */
	rc = stonith_api->cmds->register_device(stonith_api,
	st_opt_sync_call,
	cmd->rsc_id,
	rsc->provider, rsc->type, device_params);

	stonith_key_value_freeall(device_params, 1, 1);
	if (rc == 0) {
	do_monitor = 1;
	}
	} else if (safe_str_eq(cmd->action, "stop")) {
	rc = stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call, cmd->rsc_id);
	rsc->stonith_started = 0;
	} else if (safe_str_eq(cmd->action, "monitor")) {
	if (cmd->interval) {
	do_monitor = 1;
	} else {
	rc = rsc->stonith_started ? 0 : -ENODEV;
	}
	}

	if (!do_monitor) {
	goto cleanup_stonith_exec;
	}

	rc = stonith_api->cmds->monitor(stonith_api,
	0, cmd->rsc_id, cmd->timeout / 1000 );

	rc = stonith_api->cmds->register_callback(
	stonith_api,
	rc,
	0,
	0,
	cmd,
	"lrmd_stonith_callback",
	lrmd_stonith_callback);

	/* don't cleanup yet, we will find out the result of the monitor later */
	if (rc > 0) {
	rsc->active = cmd;
	return rc;
	} else if (rc == 0) {
	rc = -1;
	}

	cleanup_stonith_exec:
	stonith_action_complete(cmd, rc);
	return rc;
	}

	static const char *
	normalize_action_name(lrmd_rsc_t * rsc, const char *action)
	{
	if (safe_str_eq(action, "monitor") &&
	(safe_str_eq(rsc->class, "lsb") \|\|
	safe_str_eq(rsc->class, "service") \|\| safe_str_eq(rsc->class, "systemd"))) {
	return "status";
	}
	return action;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	g_hash_table_replace(user_data, strdup(key), strdup(value));
	}

	static int
	lrmd_rsc_execute_service_lib(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
	{
	svc_action_t *action = NULL;
	GHashTable *params_copy = NULL;

	CRM_ASSERT(rsc);
	CRM_ASSERT(cmd);

	crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s",
	rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type);

	if (cmd->params) {
	params_copy = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (params_copy != NULL) {
	g_hash_table_foreach(cmd->params, dup_attr, params_copy);
	}
	}

	action = resources_action_create(rsc->rsc_id,
	rsc->class,
	rsc->provider,
	rsc->type,
	normalize_action_name(rsc, cmd->action),
	cmd->interval, cmd->timeout, params_copy);

	if (!action) {
	crm_err("Failed to create action, action:%s on resource %s", cmd->action, rsc->rsc_id);
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	goto exec_done;
	}

	action->cb_data = cmd;

	/* 'cmd' may not be valid after this point
	*
	* Upstart and systemd both synchronously determine monitor/status
	* results and call action_complete (which may free 'cmd') if necessary
	*/
	if (services_action_async(action, action_complete)) {
	return TRUE;
	}

	cmd->exec_rc = action->rc;
	cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
	services_action_free(action);
	action = NULL;

	exec_done:
	cmd_finalize(cmd, rsc);
	return TRUE;
	}

	static gboolean
	lrmd_rsc_execute(lrmd_rsc_t * rsc)
	{
	lrmd_cmd_t *cmd = NULL;

	CRM_CHECK(rsc != NULL, return FALSE);

	if (rsc->active) {
	crm_trace("%s is still active", rsc->rsc_id);
	return TRUE;
	}

	if (rsc->pending_ops) {
	GList *first = rsc->pending_ops;

	cmd = first->data;
	if (cmd->delay_id) {
	crm_trace
	("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms",
	cmd->rsc_id, cmd->action, cmd->start_delay);
	return TRUE;
	}
	rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first);
	g_list_free_1(first);

	#ifdef HAVE_SYS_TIMEB_H
	ftime(&cmd->t_run);
	}
	#endif

	if (!cmd) {
	crm_trace("Nothing further to do for %s", rsc->rsc_id);
	return TRUE;
	}

	rsc->active = cmd; /* only one op at a time for a rsc */
	if (cmd->interval) {
	rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
	}

	log_execute(cmd);

	if (safe_str_eq(rsc->class, "stonith")) {
	lrmd_rsc_execute_stonith(rsc, cmd);
	} else {
	lrmd_rsc_execute_service_lib(rsc, cmd);
	}

	return TRUE;
	}

	static gboolean
	lrmd_rsc_dispatch(gpointer user_data)
	{
	return lrmd_rsc_execute(user_data);
	}

	void
	free_rsc(gpointer data)
	{
	GListPtr gIter = NULL;
	lrmd_rsc_t *rsc = data;
	int is_stonith = safe_str_eq(rsc->class, "stonith");

	for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	/* command was never executed */
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	cmd_finalize(cmd, NULL);
	}
	/* frees list, but not list elements. */
	g_list_free(rsc->pending_ops);

	for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (is_stonith) {
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	cmd_finalize(cmd, NULL);
	} else {
	/* This command is already handed off to service library,
	* let service library cancel it and tell us via the callback
	* when it is cancelled. The rsc can be safely destroyed
	* even if we are waiting for the cancel result */
	services_action_cancel(rsc->rsc_id, cmd->action, cmd->interval);
	}
	}
	/* frees list, but not list elements. */
	g_list_free(rsc->recurring_ops);

	free(rsc->rsc_id);
	free(rsc->class);
	free(rsc->provider);
	free(rsc->type);
	mainloop_destroy_trigger(rsc->work);

	free(rsc);
	}

	static int
	process_lrmd_signon(crm_client_t * client, uint32_t id, xmlNode * request)
	{
	xmlNode *reply = create_xml_node(NULL, "reply");

	crm_xml_add(reply, F_LRMD_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(reply, F_LRMD_CLIENTID, client->id);
	lrmd_server_send_reply(client, id, reply);

	free_xml(reply);
	return pcmk_ok;
	}

	static int
	process_lrmd_rsc_register(crm_client_t * client, uint32_t id, xmlNode * request)
	{
	int rc = pcmk_ok;
	lrmd_rsc_t *rsc = build_rsc_from_xml(request);
	lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id);

	if (dup &&
	safe_str_eq(rsc->class, dup->class) &&
	safe_str_eq(rsc->provider, dup->provider) && safe_str_eq(rsc->type, dup->type)) {

	crm_warn("Can't add, RSC '%s' already present in the rsc list (%d active resources)",
	rsc->rsc_id, g_hash_table_size(rsc_list));

	free_rsc(rsc);
	return rc;
	}

	g_hash_table_replace(rsc_list, rsc->rsc_id, rsc);
	crm_info("Added '%s' to the rsc list (%d active resources)",
	rsc->rsc_id, g_hash_table_size(rsc_list));

	return rc;
	}

	static void
	process_lrmd_get_rsc_info(crm_client_t * client, uint32_t id, xmlNode * request)
	{
	int rc = pcmk_ok;
	int send_rc = 0;
	int call_id = 0;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
	xmlNode *reply = NULL;
	lrmd_rsc_t *rsc = NULL;

	crm_element_value_int(request, F_LRMD_CALLID, &call_id);

	if (!rsc_id) {
	rc = -ENODEV;
	goto get_rsc_done;
	}

	if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	rc = -ENODEV;
	goto get_rsc_done;
	}

	get_rsc_done:

	reply = create_xml_node(NULL, T_LRMD_REPLY);
	crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__);
	crm_xml_add_int(reply, F_LRMD_RC, rc);
	crm_xml_add_int(reply, F_LRMD_CALLID, call_id);

	if (rsc) {
	crm_xml_add(reply, F_LRMD_RSC_ID, rsc->rsc_id);
	crm_xml_add(reply, F_LRMD_CLASS, rsc->class);
	crm_xml_add(reply, F_LRMD_PROVIDER, rsc->provider);
	crm_xml_add(reply, F_LRMD_TYPE, rsc->type);
	}

	send_rc = lrmd_server_send_reply(client, id, reply);

	if (send_rc < 0) {
	crm_warn("LRMD reply to %s failed: %d", client->name, send_rc);
	}

	free_xml(reply);
	}

	static int
	process_lrmd_rsc_unregister(crm_client_t * client, uint32_t id, xmlNode * request)
	{
	int rc = pcmk_ok;
	lrmd_rsc_t *rsc = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);

	if (!rsc_id) {
	return -ENODEV;
	}

	if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	return pcmk_ok;
	}

	if (rsc->active) {
	/* let the caller know there are still active ops on this rsc to watch for */
	crm_trace("Operation still in progress: %p", rsc->active);
	rc = -EINPROGRESS;
	}

	g_hash_table_remove(rsc_list, rsc_id);

	return rc;
	}

	static int
	process_lrmd_rsc_exec(crm_client_t * client, uint32_t id, xmlNode * request)
	{
	lrmd_rsc_t *rsc = NULL;
	lrmd_cmd_t *cmd = NULL;
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);

	if (!rsc_id) {
	return -EINVAL;
	}
	if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
	crm_info("Resource '%s' not found (%d active resources)",
	rsc_id, g_hash_table_size(rsc_list));
	return -ENODEV;
	}

	cmd = create_lrmd_cmd(request, client);
	schedule_lrmd_cmd(rsc, cmd);

	return cmd->call_id;
	}

	static int
	cancel_op(const char rsc_id, const char action, int interval)
	{
	GListPtr gIter = NULL;
	lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id);

	/* How to cancel an action.
	* 1. Check pending ops list, if it hasn't been handed off
	* to the service library or stonith recurring list remove
	* it there and that will stop it.
	* 2. If it isn't in the pending ops list, then its either a
	* recurring op in the stonith recurring list, or the service
	* library's recurring list. Stop it there
	* 3. If not found in any lists, then this operation has either
	* been executed already and is not a recurring operation, or
	* never existed.
	*/
	if (!rsc) {
	return -ENODEV;
	}

	for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (safe_str_eq(cmd->action, action) && cmd->interval == interval) {
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	cmd_finalize(cmd, rsc);
	return pcmk_ok;
	}
	}

	if (safe_str_eq(rsc->class, "stonith")) {
	/* The service library does not handle stonith operations.
	* We have to handle recurring stonith opereations ourselves. */
	for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
	lrmd_cmd_t *cmd = gIter->data;

	if (safe_str_eq(cmd->action, action) && cmd->interval == interval) {
	cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
	if (rsc->active != cmd) {
	cmd_finalize(cmd, rsc);
	}
	return pcmk_ok;
	}
	}
	} else if (services_action_cancel(rsc_id, normalize_action_name(rsc, action), interval) == TRUE) {
	/* The service library will tell the action_complete callback function
	* this action was cancelled, which will destroy the cmd and remove
	* it from the recurring_op list. Do not do that in this function
	* if the service library says it cancelled it. */
	return pcmk_ok;
	}

	return -EOPNOTSUPP;
	}

	static void
	cancel_all_recurring(lrmd_rsc_t rsc, const char client_id)
	{
	GList *cmd_list = NULL;
	GList *cmd_iter = NULL;

	/* Notice a copy of each list is created when concat is called.
	* This prevents odd behavior from occurring when the cmd_list
	* is iterated through later on. It is possible the cancel_op
	* function may end up modifying the recurring_ops and pending_ops
	* lists. If we did not copy those lists, our cmd_list iteration
	* could get messed up.*/
	if (rsc->recurring_ops) {
	cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->recurring_ops));
	}
	if (rsc->pending_ops) {
	cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->pending_ops));
	}
	if (!cmd_list) {
	return;
	}

	for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
	lrmd_cmd_t *cmd = cmd_iter->data;
	if (cmd->interval == 0) {
	continue;
	}

	if (client_id && safe_str_neq(cmd->client_id, client_id)) {
	continue;
	}

	cancel_op(rsc->rsc_id, cmd->action, cmd->interval);
	}
	/* frees only the copied list data, not the cmds */
	g_list_free(cmd_list);
	}

	static int
	process_lrmd_rsc_cancel(crm_client_t * client, uint32_t id, xmlNode * request)
	{
	xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
	const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
	const char *action = crm_element_value(rsc_xml, F_LRMD_RSC_ACTION);
	int interval = 0;

	crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &interval);

	if (!rsc_id \|\| !action) {
	return -EINVAL;
	}

	return cancel_op(rsc_id, action, interval);
	}

	void
	process_lrmd_message(crm_client_t * client, uint32_t id, xmlNode * request)
	{
	int rc = pcmk_ok;
	int call_id = 0;
	const char *op = crm_element_value(request, F_LRMD_OPERATION);
	int do_reply = 0;
	int do_notify = 0;
	int exit = 0;

	crm_trace("Processing %s operation from %s", op, client->id);
	crm_element_value_int(request, F_LRMD_CALLID, &call_id);

	if (crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
	rc = process_lrmd_signon(client, id, request);
	} else if (crm_str_eq(op, LRMD_OP_RSC_REG, TRUE)) {
	rc = process_lrmd_rsc_register(client, id, request);
	do_notify = 1;
	do_reply = 1;
	} else if (crm_str_eq(op, LRMD_OP_RSC_INFO, TRUE)) {
	process_lrmd_get_rsc_info(client, id, request);
	} else if (crm_str_eq(op, LRMD_OP_RSC_UNREG, TRUE)) {
	rc = process_lrmd_rsc_unregister(client, id, request);
	/* don't notify anyone about failed un-registers */
	if (rc == pcmk_ok \|\| rc == -EINPROGRESS) {
	do_notify = 1;
	}
	do_reply = 1;
	} else if (crm_str_eq(op, LRMD_OP_RSC_EXEC, TRUE)) {
	rc = process_lrmd_rsc_exec(client, id, request);
	do_reply = 1;
	} else if (crm_str_eq(op, LRMD_OP_RSC_CANCEL, TRUE)) {
	rc = process_lrmd_rsc_cancel(client, id, request);
	do_reply = 1;
	+ } else if (crm_str_eq(op, LRMD_OP_POKE, TRUE)) {
	+ do_notify = 1;
	} else {
	rc = -EOPNOTSUPP;
	do_reply = 1;
	crm_err("Unknown %s from %s", op, client->name);
	crm_log_xml_warn(request, "UnknownOp");
	}

	crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d, exit=%d",
	op, client->id, rc, do_reply, do_notify, exit);

	if (do_reply) {
	send_reply(client, rc, id, call_id);
	}

	if (do_notify) {
	send_generic_notify(rc, request);
	}

	if (exit) {
	lrmd_shutdown(0);
	}
	}
	diff --git a/lrmd/test.c b/lrmd/test.c
	index f720ade6a3..874e70b524 100644
	--- a/lrmd/test.c
	+++ b/lrmd/test.c
	@@ -1,590 +1,615 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>

	#include <crm/pengine/status.h>
	#include <crm/cib.h>
	#include <crm/lrmd.h>

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"help", 0, 0, '?'},
	{"verbose", 0, 0, 'V', "\t\tPrint out logs and events to screen"},
	{"quiet", 0, 0, 'Q', "\t\tSuppress all output to screen"},
	{"tls", 0, 0, 'S', "\t\tUse tls backend for local connection"},
	{"listen", 1, 0, 'l', "\tListen for a specific event string"},
	{"api-call", 1, 0, 'c', "\tDirectly relates to lrmd api functions"},
	{"no-wait", 0, 0, 'w', "\tMake api call and do not wait for result."},
	{"is-running", 0, 0, 'R', "\tDetermine if a resource is registered and running."},
	{"notify-orig", 0, 0, 'n', "\tOnly notify this client the results of an api action."},
	{"notify-changes", 0, 0, 'o', "\tOnly notify client changes to recurring operations."},
	{"-spacer-", 1, 0, '-', "\nParameters for api-call option"},
	{"action", 1, 0, 'a'},
	{"rsc-id", 1, 0, 'r'},
	{"cancel-call-id", 1, 0, 'x'},
	{"provider", 1, 0, 'P'},
	{"class", 1, 0, 'C'},
	{"type", 1, 0, 'T'},
	{"interval", 1, 0, 'i'},
	{"timeout", 1, 0, 't'},
	{"start-delay", 1, 0, 's'},
	{"param-key", 1, 0, 'k'},
	{"param-val", 1, 0, 'v'},

	{"-spacer-", 1, 0, '-'},
	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	cib_t *cib_conn = NULL;
	static int exec_call_id = 0;
	static int exec_call_opts = 0;
	extern void cleanup_alloc_calculations(pe_working_set_t * data_set);
	+static gboolean start_test(gpointer user_data);
	+static void try_connect(void);

	static struct {
	int verbose;
	int quiet;
	int print;
	int interval;
	int timeout;
	int start_delay;
	int cancel_call_id;
	int no_wait;
	int is_running;
	int no_connect;
	const char *api_call;
	const char *rsc_id;
	const char *provider;
	const char *class;
	const char *type;
	const char *action;
	const char *listen;
	lrmd_key_value_t *params;
	} options;

	GMainLoop *mainloop = NULL;
	lrmd_t *lrmd_conn = NULL;

	static char event_buf_v0[1024];

	static void
	test_exit(int rc)
	{
	lrmd_api_delete(lrmd_conn);
	crm_exit(rc);
	}

	#define print_result(result) \
	if (!options.quiet) { \
	result; \
	} \

	#define report_event(event) \
	snprintf(event_buf_v0, sizeof(event_buf_v0), "NEW_EVENT event_type:%s rsc_id:%s action:%s rc:%s op_status:%s", \
	lrmd_event_type2str(event->type), \
	event->rsc_id, \
	event->op_type ? event->op_type : "none", \
	lrmd_event_rc2str(event->rc), \
	services_lrm_status_str(event->op_status)); \
	crm_info("%s", event_buf_v0);;

	static void
	test_shutdown(int nsig)
	{
	lrmd_api_delete(lrmd_conn);
	lrmd_conn = NULL;
	}

	static void
	read_events(lrmd_event_data_t * event)
	{
	report_event(event);
	if (options.listen) {
	if (safe_str_eq(options.listen, event_buf_v0)) {
	print_result(printf("LISTEN EVENT SUCCESSFUL\n"));
	test_exit(0);
	}
	}

	if (exec_call_id && (event->call_id == exec_call_id)) {
	if (event->op_status == 0 && event->rc == 0) {
	print_result(printf("API-CALL SUCCESSFUL for 'exec'\n"));
	} else {
	print_result(printf("API-CALL FAILURE for 'exec', rc:%d lrmd_op_status:%s\n",
	event->rc, services_lrm_status_str(event->op_status)));
	test_exit(-1);
	}

	if (!options.listen) {
	test_exit(0);
	}
	}
	}

	static gboolean
	timeout_err(gpointer data)
	{
	print_result(printf("LISTEN EVENT FAILURE - timeout occurred, never found.\n"));
	test_exit(-1);

	return FALSE;
	}

	+static void
	+connection_events(lrmd_event_data_t * event)
	+{
	+ int rc = event->connection_rc;
	+
	+ if (event->type != lrmd_event_connect) {
	+ /* ignore */
	+ return;
	+ }
	+
	+ if (!rc) {
	+ crm_info("lrmd client connection established");
	+ start_test(NULL);
	+ return;
	+ } else {
	+ sleep(1);
	+ try_connect();
	+ crm_notice("lrmd client connection failed");
	+ }
	+}
	+
	static void
	try_connect(void)
	{
	int tries = 10;
	- int i = 0;
	+ static int num_tries = 0;
	int rc = 0;

	- for (i = 0; i < tries; i++) {
	- rc = lrmd_conn->cmds->connect(lrmd_conn, "lrmd", NULL);
	+ lrmd_conn->cmds->set_callback(lrmd_conn, connection_events);
	+ for (; num_tries < tries; num_tries++) {
	+ rc = lrmd_conn->cmds->connect_async(lrmd_conn, "lrmd", 3000);

	if (!rc) {
	- crm_info("lrmd client connection established");
	- return;
	- } else {
	- crm_info("lrmd client connection failed");
	+ return; /* we'll hear back in async callback */
	}
	sleep(1);
	}

	print_result(printf("API CONNECTION FAILURE\n"));
	test_exit(-1);
	}

	static gboolean
	start_test(gpointer user_data)
	{
	int rc = 0;

	if (!options.no_connect) {
	- try_connect();
	+ if (!lrmd_conn->cmds->is_connected(lrmd_conn)) {
	+ try_connect();
	+ /* async connect, this funciton will get called back into. */
	+ return 0;
	+ }
	}
	lrmd_conn->cmds->set_callback(lrmd_conn, read_events);

	if (options.timeout) {
	g_timeout_add(options.timeout, timeout_err, NULL);
	}

	if (!options.api_call) {
	return 0;
	}

	if (safe_str_eq(options.api_call, "exec")) {
	rc = lrmd_conn->cmds->exec(lrmd_conn,
	options.rsc_id,
	options.action,
	NULL,
	options.interval,
	options.timeout,
	options.start_delay, exec_call_opts, options.params);

	if (rc > 0) {
	exec_call_id = rc;
	print_result(printf("API-CALL 'exec' action pending, waiting on response\n"));
	}

	} else if (safe_str_eq(options.api_call, "register_rsc")) {
	rc = lrmd_conn->cmds->register_rsc(lrmd_conn,
	options.rsc_id,
	options.class, options.provider, options.type, 0);
	} else if (safe_str_eq(options.api_call, "get_rsc_info")) {
	lrmd_rsc_info_t *rsc_info;

	rsc_info = lrmd_conn->cmds->get_rsc_info(lrmd_conn, options.rsc_id, 0);

	if (rsc_info) {
	print_result(printf("RSC_INFO: id:%s class:%s provider:%s type:%s\n",
	rsc_info->id, rsc_info->class,
	rsc_info->provider ? rsc_info->provider : "<none>",
	rsc_info->type));
	lrmd_free_rsc_info(rsc_info);
	rc = pcmk_ok;
	} else {
	rc = -1;
	}
	} else if (safe_str_eq(options.api_call, "unregister_rsc")) {
	rc = lrmd_conn->cmds->unregister_rsc(lrmd_conn, options.rsc_id, 0);
	} else if (safe_str_eq(options.api_call, "cancel")) {
	rc = lrmd_conn->cmds->cancel(lrmd_conn, options.rsc_id, options.action, options.interval);
	} else if (safe_str_eq(options.api_call, "metadata")) {
	char *output = NULL;

	rc = lrmd_conn->cmds->get_metadata(lrmd_conn,
	options.class,
	options.provider, options.type, &output, 0);
	if (rc == pcmk_ok) {
	print_result(printf("%s", output));
	free(output);
	}
	} else if (safe_str_eq(options.api_call, "list_agents")) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;

	rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, options.class, options.provider);

	if (rc > 0) {
	print_result(printf("%d agents found\n", rc));
	for (iter = list; iter != NULL; iter = iter->next) {
	print_result(printf("%s\n", iter->val));
	}
	lrmd_list_freeall(list);
	rc = 0;
	} else {
	print_result(printf("API_CALL FAILURE - no agents found\n"));
	rc = -1;
	}
	} else if (safe_str_eq(options.api_call, "list_ocf_providers")) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;

	rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, options.type, &list);

	if (rc > 0) {
	print_result(printf("%d providers found\n", rc));
	for (iter = list; iter != NULL; iter = iter->next) {
	print_result(printf("%s\n", iter->val));
	}
	lrmd_list_freeall(list);
	rc = 0;
	} else {
	print_result(printf("API_CALL FAILURE - no providers found\n"));
	rc = -1;
	}

	} else if (safe_str_eq(options.api_call, "list_standards")) {
	lrmd_list_t *list = NULL;
	lrmd_list_t *iter = NULL;

	rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);

	if (rc > 0) {
	print_result(printf("%d standards found\n", rc));
	for (iter = list; iter != NULL; iter = iter->next) {
	print_result(printf("%s\n", iter->val));
	}
	lrmd_list_freeall(list);
	rc = 0;
	} else {
	print_result(printf("API_CALL FAILURE - no providers found\n"));
	rc = -1;
	}

	} else if (options.api_call) {
	print_result(printf("API-CALL FAILURE unknown action '%s'\n", options.action));
	test_exit(-1);
	}

	if (rc < 0) {
	print_result(printf("API-CALL FAILURE for '%s' api_rc:%d\n", options.api_call, rc));
	test_exit(-1);
	}

	if (options.api_call && rc == pcmk_ok) {
	print_result(printf("API-CALL SUCCESSFUL for '%s'\n", options.api_call));
	if (!options.listen) {
	test_exit(0);
	}
	}

	if (options.no_wait) {
	/* just make the call and exit regardless of anything else. */
	test_exit(0);
	}

	return 0;
	}

	static resource_t *
	find_rsc_or_clone(const char rsc, pe_working_set_t data_set)
	{
	resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);

	if (the_rsc == NULL) {
	char *as_clone = crm_concat(rsc, "0", ':');

	the_rsc = pe_find_resource(data_set->resources, as_clone);
	free(as_clone);
	}
	return the_rsc;
	}

	static int
	generate_params(void)
	{
	int rc = 0;
	pe_working_set_t data_set;
	xmlNode *cib_xml_copy = NULL;
	resource_t *rsc = NULL;
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTableIter iter;

	if (options.params) {
	return 0;
	}

	set_working_set_defaults(&data_set);

	cib_conn = cib_new();
	rc = cib_conn->cmds->signon(cib_conn, "lrmd_test", cib_query);
	if (rc != pcmk_ok) {
	crm_err("Error signing on to the CIB service: %s\n", pcmk_strerror(rc));
	rc = -1;
	goto param_gen_bail;
	}

	cib_xml_copy = get_cib_copy(cib_conn);

	if (!cib_xml_copy) {
	crm_err("Error retrieving cib copy.");
	rc = -1;
	goto param_gen_bail;
	}

	if (cli_config_update(&cib_xml_copy, NULL, FALSE) == FALSE) {
	crm_err("Error updating cib configuration");
	rc = -1;
	goto param_gen_bail;
	}

	data_set.input = cib_xml_copy;
	data_set.now = crm_time_new(NULL);

	cluster_status(&data_set);
	if (options.rsc_id) {
	rsc = find_rsc_or_clone(options.rsc_id, &data_set);
	}

	if (!rsc) {
	crm_err("Resource does not exist in config");
	rc = -1;
	goto param_gen_bail;
	}

	params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	meta = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	get_rsc_attributes(params, rsc, NULL, &data_set);
	get_meta_attributes(meta, rsc, NULL, &data_set);

	if (params) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	options.params = lrmd_key_value_add(options.params, key, value);
	}
	g_hash_table_destroy(params);
	}

	if (meta) {
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	options.params = lrmd_key_value_add(options.params, crm_name, value);
	free(crm_name);
	}
	g_hash_table_destroy(meta);
	}

	param_gen_bail:

	cleanup_alloc_calculations(&data_set);
	return rc;
	}

	int
	main(int argc, char **argv)
	{
	int option_index = 0;
	int argerr = 0;
	int flag;
	char *key = NULL;
	char *val = NULL;
	gboolean use_tls = FALSE;
	crm_trigger_t *trig;

	crm_set_options(NULL, "mode [options]", long_options,
	"Inject commands into the lrmd and watch for events\n");

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch (flag) {
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'V':
	options.verbose = 1;
	break;
	case 'Q':
	options.quiet = 1;
	options.verbose = 0;
	break;
	case 'l':
	options.listen = optarg;
	break;
	case 'w':
	options.no_wait = 1;
	break;
	case 'R':
	options.is_running = 1;
	break;
	case 'n':
	exec_call_opts = lrmd_opt_notify_orig_only;
	break;
	case 'o':
	exec_call_opts = lrmd_opt_notify_changes_only;
	break;
	case 'c':
	options.api_call = optarg;
	break;
	case 'a':
	options.action = optarg;
	break;
	case 'r':
	options.rsc_id = optarg;
	break;
	case 'x':
	options.cancel_call_id = atoi(optarg);
	break;
	case 'P':
	options.provider = optarg;
	break;
	case 'C':
	options.class = optarg;
	break;
	case 'T':
	options.type = optarg;
	break;
	case 'i':
	options.interval = atoi(optarg);
	break;
	case 't':
	options.timeout = atoi(optarg);
	break;
	case 's':
	options.start_delay = atoi(optarg);
	break;
	case 'k':
	key = optarg;
	if (key && val) {
	options.params = lrmd_key_value_add(options.params, key, val);
	key = val = NULL;
	}
	break;
	case 'v':
	val = optarg;
	if (key && val) {
	options.params = lrmd_key_value_add(options.params, key, val);
	key = val = NULL;
	}
	break;
	case 'S':
	use_tls = TRUE;
	break;
	default:
	++argerr;
	break;
	}
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}
	if (optind > argc) {
	++argerr;
	}

	if (!options.listen &&
	(safe_str_eq(options.api_call, "metadata") \|\|
	safe_str_eq(options.api_call, "list_agents") \|\|
	safe_str_eq(options.api_call, "list_standards") \|\|
	safe_str_eq(options.api_call, "list_ocf_providers"))) {
	options.no_connect = 1;
	}

	crm_log_init("lrmd_ctest", LOG_INFO, TRUE, options.verbose ? TRUE : FALSE, argc, argv, FALSE);

	if (options.is_running) {
	if (!options.timeout) {
	options.timeout = 30000;
	}
	options.interval = 0;
	if (!options.rsc_id) {
	crm_err("rsc-id must be given when is-running is used");
	test_exit(-1);
	}

	if (generate_params()) {
	print_result(printf
	("Failed to retrieve rsc parameters from cib, can not determine if rsc is running.\n"));
	test_exit(-1);
	}
	options.api_call = "exec";
	options.action = "monitor";
	exec_call_opts = lrmd_opt_notify_orig_only;
	}

	/* if we can't perform an api_call or listen for events,
	* there is nothing to do */
	if (!options.api_call && !options.listen) {
	crm_err("Nothing to be done. Please specify 'api-call' and/or 'listen'");
	return 0;
	}

	if (use_tls) {
	lrmd_conn = lrmd_remote_api_new(NULL, "localhost", 0);
	} else {
	lrmd_conn = lrmd_api_new();
	}
	trig = mainloop_add_trigger(G_PRIORITY_HIGH, start_test, NULL);
	mainloop_set_trigger(trig);
	mainloop_add_signal(SIGTERM, test_shutdown);

	crm_info("Starting");
	mainloop = g_main_new(FALSE);
	g_main_run(mainloop);

	if (cib_conn != NULL) {
	cib_conn->cmds->signoff(cib_conn);
	cib_delete(cib_conn);
	}

	test_exit(0);
	return 0;
	}
	diff --git a/lrmd/tls_backend.c b/lrmd/tls_backend.c
	index 8689660d4d..8c9f99e729 100644
	--- a/lrmd/tls_backend.c
	+++ b/lrmd/tls_backend.c
	@@ -1,341 +1,341 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/mainloop.h>

	#include <lrmd_private.h>

	#include <sys/socket.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	-#define LRMD_REMOTE_AUTH_TIMEOUT 3000
	+#define LRMD_REMOTE_AUTH_TIMEOUT 10000
	gnutls_psk_server_credentials_t psk_cred_s;
	gnutls_dh_params_t dh_params;
	static int ssock = 0;
	extern int lrmd_call_id;

	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}

	static int
	lrmd_remote_client_msg(gpointer data)
	{
	int id = 0;
	int rc = 0;
	int disconnected = 0;
	xmlNode *request = NULL;
	crm_client_t *client = data;

	if (client->remote->tls_handshake_complete == FALSE) {
	int rc = 0;

	/* Muliple calls to handshake will be required, this callback
	* will be invoked once the client sends more handshake data. */
	do {
	rc = gnutls_handshake(*client->remote->tls_session);

	if (rc < 0 && rc != GNUTLS_E_AGAIN) {
	crm_err("Remote lrmd tls handshake failed");
	return -1;
	}
	} while (rc == GNUTLS_E_INTERRUPTED);

	if (rc == 0) {
	crm_debug("Remote lrmd tls handshake completed");
	client->remote->tls_handshake_complete = TRUE;
	if (client->remote->auth_timeout) {
	g_source_remove(client->remote->auth_timeout);
	}
	client->remote->auth_timeout = 0;
	}
	return 0;
	}

	rc = crm_remote_ready(client->remote, 0);
	if (rc == 0) {
	/* no msg to read */
	return 0;
	} else if (rc < 0) {
	crm_info("Client disconnected during remote client read");
	return -1;
	}

	crm_remote_recv(client->remote, -1, &disconnected);

	request = crm_remote_parse_buffer(client->remote);
	while (request) {
	crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
	crm_trace("processing request from remote client with remote msg id %d", id);
	if (!client->name) {
	const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
	if (value) {
	client->name = strdup(value);
	}
	}

	lrmd_call_id++;
	if (lrmd_call_id < 1) {
	lrmd_call_id = 1;
	}

	crm_xml_add(request, F_LRMD_CLIENTID, client->id);
	crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
	crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);

	process_lrmd_message(client, id, request);
	free_xml(request);

	/* process all the messages in the current buffer */
	request = crm_remote_parse_buffer(client->remote);
	}

	if (disconnected) {
	crm_info("Client disconnect detected in tls msg dispatcher.");
	return -1;
	}

	return 0;
	}

	static void
	lrmd_remote_client_destroy(gpointer user_data)
	{
	crm_client_t *client = user_data;

	if (client == NULL) {
	return;
	}

	client_disconnect_cleanup(client->id);

	crm_notice("LRMD client disconnecting remote client - name: %s id: %s",
	client->name ? client->name : "<unknown>",
	client->id);

	if (client->remote->tls_session) {
	void *sock_ptr;
	int csock;
	sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
	csock = GPOINTER_TO_INT(sock_ptr);

	gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*client->remote->tls_session);
	gnutls_free(client->remote->tls_session);
	close(csock);
	}

	crm_client_destroy(client);

	return;
	}

	static gboolean
	lrmd_auth_timeout_cb(gpointer data)
	{
	crm_client_t *client = data;

	client->remote->auth_timeout = 0;

	if (client->remote->tls_handshake_complete == TRUE) {
	return FALSE;
	}

	mainloop_del_fd(client->remote->source);
	client->remote->source = NULL;
	crm_err("Remote client authentication timed out");

	return FALSE;
	}

	static int
	lrmd_remote_listen(gpointer data)
	{
	int csock = 0;
	int flag = 0;
	unsigned laddr;
	struct sockaddr_in addr;
	gnutls_session *session = NULL;
	crm_client_t *new_client = NULL;

	static struct mainloop_fd_callbacks lrmd_remote_fd_cb =
	{
	.dispatch = lrmd_remote_client_msg,
	.destroy = lrmd_remote_client_destroy,
	};

	/* accept the connection */
	laddr = sizeof(addr);
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	crm_debug("New remote connection from %s", inet_ntoa(addr.sin_addr));

	if (csock == -1) {
	crm_err("accept socket failed");
	return TRUE;
	}

	if ((flag = fcntl(csock, F_GETFL)) >= 0) {
	if (fcntl(csock, F_SETFL, flag \| O_NONBLOCK) < 0) {
	crm_err( "fcntl() write failed");
	close(csock);
	return TRUE;
	}
	} else {
	crm_err( "fcntl() read failed");
	close(csock);
	return TRUE;
	}

	session = create_psk_tls_session(csock, GNUTLS_SERVER, psk_cred_s);
	if (session == NULL) {
	crm_err("TLS session creation failed");
	close(csock);
	return TRUE;
	}

	new_client = calloc(1, sizeof(crm_client_t));
	new_client->remote = calloc(1, sizeof(crm_remote_t));
	new_client->kind = CRM_CLIENT_TLS;
	new_client->remote->tls_session = session;
	new_client->id = crm_generate_uuid();
	new_client->remote->auth_timeout = g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client);
	crm_notice("LRMD client connection established. %p id: %s", new_client, new_client->id);

	new_client->remote->source = mainloop_add_fd(
	"lrmd-remote-client", G_PRIORITY_DEFAULT, csock, new_client, &lrmd_remote_fd_cb);
	g_hash_table_insert(client_connections, new_client->id, new_client);

	return TRUE;
	}

	static void
	lrmd_remote_connection_destroy(gpointer user_data)
	{
	crm_notice("Remote tls server disconnected");
	return;
	}

	static int
	lrmd_tls_server_key_cb(gnutls_session_t session, const char username, gnutls_datum_t key)
	{
	int rc = 0;

	if (lrmd_tls_set_key(key, DEFAULT_REMOTE_KEY_LOCATION)) {
	rc = lrmd_tls_set_key(key, ALT_REMOTE_KEY_LOCATION);
	}
	if (rc) {
	crm_err("No lrmd remote key found");
	return -1;
	}

	return rc;
	}

	int
	lrmd_init_remote_tls_server(int port)
	{
	int rc;
	struct sockaddr_in saddr;
	int optval;
	static struct mainloop_fd_callbacks remote_listen_fd_callbacks =
	{
	.dispatch = lrmd_remote_listen,
	.destroy = lrmd_remote_connection_destroy,
	};

	crm_notice("Starting a tls listener on port %d.", port);
	gnutls_global_init();
	gnutls_global_set_log_function(debug_log);

	gnutls_dh_params_init(&dh_params);
	gnutls_dh_params_generate2(dh_params, 1024);
	gnutls_psk_allocate_server_credentials(&psk_cred_s);
	gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
	gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);

	/* create server socket */
	ssock = socket(AF_INET, SOCK_STREAM, 0);
	if (ssock == -1) {
	crm_err("Can not create server socket.");
	return -1;
	}

	/* reuse address */
	optval = 1;
	rc = setsockopt(ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
	if(rc < 0) {
	crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener");
	}

	rc = -1;

	/* bind server socket */
	memset(&saddr, '\0', sizeof(saddr));
	saddr.sin_family = AF_INET;
	saddr.sin_addr.s_addr = INADDR_ANY;
	saddr.sin_port = htons(port);
	if (bind(ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) {
	crm_err("Can not bind server socket.");
	goto init_remote_cleanup;
	}

	if (listen(ssock, 10) == -1) {
	crm_err("Can not start listen.");
	goto init_remote_cleanup;
	}

	mainloop_add_fd("lrmd-remote", G_PRIORITY_DEFAULT, ssock, NULL, &remote_listen_fd_callbacks);

	rc = ssock;
	init_remote_cleanup:
	if (rc < 0) {
	close(ssock);
	ssock = 0;
	}
	return rc;

	}

	void
	lrmd_tls_server_destroy(void)
	{
	if (psk_cred_s) {
	gnutls_psk_free_server_credentials(psk_cred_s);
	psk_cred_s = 0;
	}

	if (ssock > 0) {
	close(ssock);
	ssock = 0;
	}
	}
	#endif

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 21, 7:41 PM (11 h, 2 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1665596
Default Alt Text: (277 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions