No OneTemporary
Actions

Size

74 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
	index 0e52ff6b1c..497d3f936f 100644
	--- a/crmd/lrm_state.c
	+++ b/crmd/lrm_state.c
	@@ -1,747 +1,747 @@
	/*
	* Copyright (C) 2012 David Vossel <davidvossel@gmail.com>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>

	GHashTable *lrm_state_table = NULL;
	extern GHashTable *proxy_table;
	int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
	void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg));

	static void
	free_rsc_info(gpointer value)
	{
	lrmd_rsc_info_t *rsc_info = value;

	lrmd_free_rsc_info(rsc_info);
	}

	static void
	free_deletion_op(gpointer value)
	{
	struct pending_deletion_op_s *op = value;

	free(op->rsc);
	delete_ha_msg_input(op->input);
	free(op);
	}

	static void
	free_recurring_op(gpointer value)
	{
	struct recurring_op_s op = (struct recurring_op_s )value;

	free(op->user_data);
	free(op->rsc_id);
	free(op->op_type);
	free(op->op_key);
	if (op->params) {
	g_hash_table_destroy(op->params);
	}
	free(op);
	}

	static gboolean
	fail_pending_op(gpointer key, gpointer value, gpointer user_data)
	{
	lrmd_event_data_t event = { 0, };
	lrm_state_t *lrm_state = user_data;
	struct recurring_op_s op = (struct recurring_op_s )value;

	crm_trace("Pre-emptively failing %s_%s_%d on %s (call=%s, %s)",
	op->rsc_id, op->op_type, op->interval,
	lrm_state->node_name, key, op->user_data);

	event.type = lrmd_event_exec_complete;
	event.rsc_id = op->rsc_id;
	event.op_type = op->op_type;
	event.user_data = op->user_data;
	event.timeout = 0;
	event.interval = op->interval;
	event.rc = PCMK_OCF_CONNECTION_DIED;
	event.op_status = PCMK_LRM_OP_ERROR;
	event.t_run = op->start_time;
	event.t_rcchange = op->start_time;

	event.call_id = op->call_id;
	event.remote_nodename = lrm_state->node_name;
	event.params = op->params;

	process_lrm_event(lrm_state, &event, op);
	return TRUE;
	}

	gboolean
	lrm_state_is_local(lrm_state_t *lrm_state)
	{
	if (lrm_state == NULL \|\| fsa_our_uname == NULL) {
	return FALSE;
	}

	if (strcmp(lrm_state->node_name, fsa_our_uname) != 0) {
	return FALSE;
	}

	return TRUE;

	}

	lrm_state_t *
	lrm_state_create(const char *node_name)
	{
	lrm_state_t *state = NULL;

	if (!node_name) {
	crm_err("No node name given for lrm state object");
	return NULL;
	}

	state = calloc(1, sizeof(lrm_state_t));
	if (!state) {
	return NULL;
	}

	state->node_name = strdup(node_name);

	state->rsc_info_cache = g_hash_table_new_full(crm_str_hash,
	g_str_equal, NULL, free_rsc_info);

	state->deletion_ops = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, free_deletion_op);

	state->pending_ops = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, free_recurring_op);

	state->resource_history = g_hash_table_new_full(crm_str_hash,
	g_str_equal, NULL, history_free);

	g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
	return state;

	}

	void
	lrm_state_destroy(const char *node_name)
	{
	g_hash_table_remove(lrm_state_table, node_name);
	}

	static gboolean
	remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data)
	{
	remote_proxy_t *proxy = value;
	const char *node_name = user_data;

	if (safe_str_eq(node_name, proxy->node_name)) {
	return TRUE;
	}

	return FALSE;
	}

	static void
	internal_lrm_state_destroy(gpointer data)
	{
	lrm_state_t *lrm_state = data;

	if (!lrm_state) {
	return;
	}

	crm_trace("Destroying proxy table %s with %d members", lrm_state->node_name, g_hash_table_size(proxy_table));
	g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name);
	remote_ra_cleanup(lrm_state);
	lrmd_api_delete(lrm_state->conn);

	if (lrm_state->rsc_info_cache) {
	crm_trace("Destroying rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache));
	g_hash_table_destroy(lrm_state->rsc_info_cache);
	}
	if (lrm_state->resource_history) {
	crm_trace("Destroying history op cache with %d members", g_hash_table_size(lrm_state->resource_history));
	g_hash_table_destroy(lrm_state->resource_history);
	}
	if (lrm_state->deletion_ops) {
	crm_trace("Destroying deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops));
	g_hash_table_destroy(lrm_state->deletion_ops);
	}
	if (lrm_state->pending_ops) {
	crm_trace("Destroying pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops));
	g_hash_table_destroy(lrm_state->pending_ops);
	}

	free((char *)lrm_state->node_name);
	free(lrm_state);
	}

	void
	lrm_state_reset_tables(lrm_state_t * lrm_state)
	{
	if (lrm_state->resource_history) {
	crm_trace("Re-setting history op cache with %d members",
	g_hash_table_size(lrm_state->resource_history));
	g_hash_table_remove_all(lrm_state->resource_history);
	}
	if (lrm_state->deletion_ops) {
	crm_trace("Re-setting deletion op cache with %d members",
	g_hash_table_size(lrm_state->deletion_ops));
	g_hash_table_remove_all(lrm_state->deletion_ops);
	}
	if (lrm_state->pending_ops) {
	crm_trace("Re-setting pending op cache with %d members",
	g_hash_table_size(lrm_state->pending_ops));
	g_hash_table_remove_all(lrm_state->pending_ops);
	}
	if (lrm_state->rsc_info_cache) {
	crm_trace("Re-setting rsc info cache with %d members",
	g_hash_table_size(lrm_state->rsc_info_cache));
	g_hash_table_remove_all(lrm_state->rsc_info_cache);
	}
	}

	gboolean
	lrm_state_init_local(void)
	{
	if (lrm_state_table) {
	return TRUE;
	}

	lrm_state_table =
	g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, internal_lrm_state_destroy);
	if (!lrm_state_table) {
	return FALSE;
	}

	proxy_table =
	g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, remote_proxy_free);
	if (!proxy_table) {
	g_hash_table_destroy(lrm_state_table);
	return FALSE;
	}

	return TRUE;
	}

	void
	lrm_state_destroy_all(void)
	{
	if (lrm_state_table) {
	crm_trace("Destroying state table with %d members", g_hash_table_size(lrm_state_table));
	g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL;
	}
	if(proxy_table) {
	crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table));
	g_hash_table_destroy(proxy_table); proxy_table = NULL;
	}
	}

	lrm_state_t *
	lrm_state_find(const char *node_name)
	{
	if (!node_name) {
	return NULL;
	}
	return g_hash_table_lookup(lrm_state_table, node_name);
	}

	lrm_state_t *
	lrm_state_find_or_create(const char *node_name)
	{
	lrm_state_t *lrm_state;

	lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
	if (!lrm_state) {
	lrm_state = lrm_state_create(node_name);
	}

	return lrm_state;
	}

	GList *
	lrm_state_get_list(void)
	{
	return g_hash_table_get_values(lrm_state_table);
	}

	void
	lrm_state_disconnect(lrm_state_t * lrm_state)
	{
	int removed = 0;

	if (!lrm_state->conn) {
	return;
	}
	crm_trace("Disconnecting %s", lrm_state->node_name);
	((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);

	if (is_not_set(fsa_input_register, R_SHUTDOWN)) {
	removed = g_hash_table_foreach_remove(lrm_state->pending_ops, fail_pending_op, lrm_state);
	crm_trace("Synthesized %d operation failures for %s", removed, lrm_state->node_name);
	}

	lrmd_api_delete(lrm_state->conn);
	lrm_state->conn = NULL;
	}

	int
	lrm_state_is_connected(lrm_state_t * lrm_state)
	{
	if (!lrm_state->conn) {
	return FALSE;
	}
	return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
	}

	int
	lrm_state_poke_connection(lrm_state_t * lrm_state)
	{

	if (!lrm_state->conn) {
	return -1;
	}
	return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
	}

	int
	lrm_state_ipc_connect(lrm_state_t * lrm_state)
	{
	int ret;

	if (!lrm_state->conn) {
	lrm_state->conn = lrmd_api_new();
	((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, lrm_op_callback);
	}

	ret = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL);

	if (ret != pcmk_ok) {
	lrm_state->num_lrm_register_fails++;
	} else {
	lrm_state->num_lrm_register_fails = 0;
	}

	return ret;
	}

	static int
	remote_proxy_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	/* Async responses from cib and friends back to clients via pacemaker_remoted */
	xmlNode *xml = NULL;
	remote_proxy_t *proxy = userdata;
	lrm_state_t *lrm_state = lrm_state_find(proxy->node_name);
	uint32_t flags;

	if (lrm_state == NULL) {
	return 0;
	}

	xml = string2xml(buffer);
	if (xml == NULL) {
	crm_warn("Received a NULL msg from IPC service.");
	return 1;
	}

	flags = crm_ipc_buffer_flags(proxy->ipc);
	if (flags & crm_ipc_proxied_relay_response) {
	crm_trace("Passing response back to %.8s on %s: %.200s - request id: %d", proxy->session_id, proxy->node_name, buffer, proxy->last_request_id);
	remote_proxy_relay_response(lrm_state->conn, proxy->session_id, xml, proxy->last_request_id);
	proxy->last_request_id = 0;

	} else {
	crm_trace("Passing event back to %.8s on %s: %.200s", proxy->session_id, proxy->node_name, buffer);
	remote_proxy_relay_event(lrm_state->conn, proxy->session_id, xml);
	}
	free_xml(xml);
	return 1;
	}

	static void
	remote_proxy_disconnected(void *userdata)
	{
	remote_proxy_t *proxy = userdata;
	lrm_state_t *lrm_state = lrm_state_find(proxy->node_name);

	crm_trace("Destroying %s (%p)", lrm_state->node_name, userdata);

	proxy->source = NULL;
	proxy->ipc = NULL;

	if (lrm_state && lrm_state->conn) {
	remote_proxy_notify_destroy(lrm_state->conn, proxy->session_id);
	}
	g_hash_table_remove(proxy_table, proxy->session_id);
	}

	static remote_proxy_t *
	remote_proxy_new(const char node_name, const char session_id, const char *channel)
	{
	static struct ipc_client_callbacks proxy_callbacks = {
	.dispatch = remote_proxy_dispatch_internal,
	.destroy = remote_proxy_disconnected
	};
	remote_proxy_t *proxy = calloc(1, sizeof(remote_proxy_t));

	proxy->node_name = strdup(node_name);
	proxy->session_id = strdup(session_id);

	if (safe_str_eq(channel, CRM_SYSTEM_CRMD)) {
	proxy->is_local = TRUE;
	} else {
	proxy->source = mainloop_add_ipc_client(channel, G_PRIORITY_LOW, 0, proxy, &proxy_callbacks);
	proxy->ipc = mainloop_get_ipc_client(proxy->source);

	if (proxy->source == NULL) {
	remote_proxy_free(proxy);
	return NULL;
	}
	}

	crm_trace("created proxy session ID %s", proxy->session_id);
	g_hash_table_insert(proxy_table, proxy->session_id, proxy);

	return proxy;
	}

	gboolean
	crmd_is_proxy_session(const char *session)
	{
	return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE;
	}

	void
	crmd_proxy_send(const char session, xmlNode msg)
	{
	remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
	lrm_state_t *lrm_state = NULL;

	if (!proxy) {
	return;
	}
	crm_log_xml_trace(msg, "to-proxy");
	lrm_state = lrm_state_find(proxy->node_name);
	if (lrm_state) {
	crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name);
	remote_proxy_relay_event(lrm_state->conn, session, msg);
	}
	}

	static void
	crmd_proxy_dispatch(const char session, xmlNode msg)
	{

	crm_log_xml_trace(msg, "CRMd-PROXY[inbound]");

	crm_xml_add(msg, F_CRM_SYS_FROM, session);
	if (crmd_authorize_message(msg, NULL, session)) {
	route_message(C_IPC_MESSAGE, msg);
	}

	trigger_fsa(fsa_source);
	}

	static void
	remote_proxy_cb(lrmd_t lrmd, void userdata, xmlNode *msg)
	{
	lrm_state_t *lrm_state = userdata;
	const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
	const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
	int msg_id = 0;

	/* sessions are raw ipc connections to IPC,
	* all we do is proxy requests/responses exactly
	* like they are given to us at the ipc level. */

	CRM_CHECK(op != NULL, return);
	CRM_CHECK(session != NULL, return);

	crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);

	/* This is msg from remote ipc client going to real ipc server */
	- if (safe_str_eq(op, "new")) {
	+ if (safe_str_eq(op, LRMD_IPC_OP_NEW)) {
	const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);

	CRM_CHECK(channel != NULL, return);

	if (remote_proxy_new(lrm_state->node_name, session, channel) == NULL) {
	remote_proxy_notify_destroy(lrmd, session);
	}
	crm_trace("new remote proxy client established to %s, session id %s", channel, session);
	- } else if (safe_str_eq(op, "destroy")) {
	+ } else if (safe_str_eq(op, LRMD_IPC_OP_DESTROY)) {
	remote_proxy_end_session(session);

	- } else if (safe_str_eq(op, "request")) {
	+ } else if (safe_str_eq(op, LRMD_IPC_OP_REQUEST)) {
	int flags = 0;
	xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
	const char *name = crm_element_value(msg, F_LRMD_IPC_CLIENT);
	remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);

	CRM_CHECK(request != NULL, return);

	if (proxy == NULL) {
	/* proxy connection no longer exists */
	remote_proxy_notify_destroy(lrmd, session);
	return;
	} else if ((proxy->is_local == FALSE) && (crm_ipc_connected(proxy->ipc) == FALSE)) {
	remote_proxy_end_session(session);
	return;
	}
	proxy->last_request_id = 0;
	crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
	crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");

	#if ENABLE_ACL
	CRM_ASSERT(lrm_state->node_name);
	crm_acl_get_set_user(request, F_LRMD_IPC_USER, lrm_state->node_name);
	#endif

	if (proxy->is_local) {
	/* this is for the crmd, which we are, so don't try
	* and connect/send to ourselves over ipc. instead
	* do it directly. */
	crmd_proxy_dispatch(session, request);
	if (flags & crm_ipc_client_response) {
	xmlNode *op_reply = create_xml_node(NULL, "ack");

	crm_xml_add(op_reply, "function", __FUNCTION__);
	crm_xml_add_int(op_reply, "line", __LINE__);
	remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
	free_xml(op_reply);
	}

	} else if(is_set(flags, crm_ipc_proxied)) {
	const char *type = crm_element_value(request, F_TYPE);
	int rc = 0;

	if (safe_str_eq(type, T_ATTRD)
	&& crm_element_value(request, F_ATTRD_HOST) == NULL) {
	crm_xml_add(request, F_ATTRD_HOST, proxy->node_name);
	}

	rc = crm_ipc_send(proxy->ipc, request, flags, 5000, NULL);

	if(rc < 0) {
	xmlNode *op_reply = create_xml_node(NULL, "nack");

	crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);

	/* Send a n'ack so the caller doesn't block */
	crm_xml_add(op_reply, "function", __FUNCTION__);
	crm_xml_add_int(op_reply, "line", __LINE__);
	crm_xml_add_int(op_reply, "rc", rc);
	remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
	free_xml(op_reply);

	} else {
	crm_trace("Relayed %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
	proxy->last_request_id = msg_id;
	}

	} else {
	int rc = pcmk_ok;
	xmlNode *op_reply = NULL;
	/* For backwards compatibility with pacemaker_remoted <= 1.1.10 */

	crm_trace("Relaying %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);

	rc = crm_ipc_send(proxy->ipc, request, flags, 10000, &op_reply);
	if(rc < 0) {
	crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);
	} else {
	crm_trace("Relayed %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
	}

	if(op_reply) {
	remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
	free_xml(op_reply);
	}
	}
	} else {
	crm_err("Unknown proxy operation: %s", op);
	}
	}

	int
	lrm_state_remote_connect_async(lrm_state_t * lrm_state, const char *server, int port,
	int timeout_ms)
	{
	int ret;

	if (!lrm_state->conn) {
	lrm_state->conn = lrmd_remote_api_new(lrm_state->node_name, server, port);
	if (!lrm_state->conn) {
	return -1;
	}
	((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, remote_lrm_op_callback);
	lrmd_internal_set_proxy_callback(lrm_state->conn, lrm_state, remote_proxy_cb);
	}

	crm_trace("initiating remote connection to %s at %d with timeout %d", server, port, timeout_ms);
	ret =
	((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name,
	timeout_ms);

	if (ret != pcmk_ok) {
	lrm_state->num_lrm_register_fails++;
	} else {
	lrm_state->num_lrm_register_fails = 0;
	}

	return ret;
	}

	int
	lrm_state_get_metadata(lrm_state_t * lrm_state,
	const char *class,
	const char *provider,
	const char agent, char *output, enum lrmd_call_options options)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* Optimize this... only retrieve metadata from local lrmd connection. Perhaps consider
	* caching result. */
	return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata(lrm_state->conn, class, provider, agent,
	output, options);
	}

	int
	lrm_state_cancel(lrm_state_t * lrm_state, const char rsc_id, const char action, int interval)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* Optimize this, cancel requires a synced request/response to the server.
	* Figure out a way to make this async. */
	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	return remote_ra_cancel(lrm_state, rsc_id, action, interval);
	}
	return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id, action, interval);
	}

	lrmd_rsc_info_t *
	lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options)
	{
	lrmd_rsc_info_t *rsc = NULL;

	if (!lrm_state->conn) {
	return NULL;
	}
	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	return remote_ra_get_rsc_info(lrm_state, rsc_id);
	}

	rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id);
	if (rsc == NULL) {
	/* only contact the lrmd if we don't already have a cached rsc info */
	rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
	if (rsc == NULL) {
	return NULL;
	}
	/* cache the result */
	g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc);
	}

	return lrmd_copy_rsc_info(rsc);

	}

	int
	lrm_state_exec(lrm_state_t * lrm_state, const char rsc_id, const char action, const char userdata, int interval, / ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	lrmd_key_value_t * params)
	{

	if (!lrm_state->conn) {
	lrmd_key_value_freeall(params);
	return -ENOTCONN;
	}

	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	return remote_ra_exec(lrm_state,
	rsc_id, action, userdata, interval, timeout, start_delay, params);
	}

	return ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn,
	rsc_id,
	action,
	userdata,
	interval,
	timeout,
	start_delay,
	lrmd_opt_notify_changes_only, params);
	}

	int
	lrm_state_register_rsc(lrm_state_t * lrm_state,
	const char *rsc_id,
	const char *class,
	const char provider, const char agent, enum lrmd_call_options options)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* optimize this... this function is a synced round trip from client to daemon.
	* The crmd/lrm.c code path should be re-factored to allow the register of resources
	* to be performed async. The lrmd client api needs to make an async version
	* of register available. */
	if (is_remote_lrmd_ra(agent, provider, NULL)) {
	return lrm_state_find_or_create(rsc_id) ? pcmk_ok : -1;
	}

	return ((lrmd_t *) lrm_state->conn)->cmds->register_rsc(lrm_state->conn, rsc_id, class,
	provider, agent, options);
	}

	int
	lrm_state_unregister_rsc(lrm_state_t * lrm_state,
	const char *rsc_id, enum lrmd_call_options options)
	{
	if (!lrm_state->conn) {
	return -ENOTCONN;
	}

	/* optimize this... this function is a synced round trip from client to daemon.
	* The crmd/lrm.c code path that uses this function should always treat it as an
	* async operation. The lrmd client api needs to make an async version unreg available. */
	if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
	lrm_state_destroy(rsc_id);
	return pcmk_ok;
	}

	g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id);

	return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
	}
	diff --git a/include/crm/lrmd.h b/include/crm/lrmd.h
	index 5a3c6ce3c5..5c74798cd7 100644
	--- a/include/crm/lrmd.h
	+++ b/include/crm/lrmd.h
	@@ -1,466 +1,472 @@
	/*
	* Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	/**
	* \file
	* \brief Local Resource Manager
	* \ingroup lrmd
	*/
	#include <stdbool.h>
	#include <crm/services.h>

	#ifndef LRMD__H
	# define LRMD__H

	typedef struct lrmd_s lrmd_t;
	typedef struct lrmd_key_value_s {
	char *key;
	char *value;
	struct lrmd_key_value_s *next;
	} lrmd_key_value_t;

	#define LRMD_PROTOCOL_VERSION "1.0"

	/* INDENT-OFF */
	#define DEFAULT_REMOTE_KEY_LOCATION "/etc/pacemaker/authkey"
	#define ALT_REMOTE_KEY_LOCATION "/etc/corosync/authkey"
	#define DEFAULT_REMOTE_PORT 3121
	#define DEFAULT_REMOTE_USERNAME "lrmd"

	#define F_LRMD_OPERATION "lrmd_op"
	#define F_LRMD_CLIENTNAME "lrmd_clientname"
	#define F_LRMD_IS_IPC_PROVIDER "lrmd_is_ipc_provider"
	#define F_LRMD_CLIENTID "lrmd_clientid"
	#define F_LRMD_PROTOCOL_VERSION "lrmd_protocol_version"
	#define F_LRMD_REMOTE_MSG_TYPE "lrmd_remote_msg_type"
	#define F_LRMD_REMOTE_MSG_ID "lrmd_remote_msg_id"
	#define F_LRMD_CALLBACK_TOKEN "lrmd_async_id"
	#define F_LRMD_CALLID "lrmd_callid"
	#define F_LRMD_CANCEL_CALLID "lrmd_cancel_callid"
	#define F_LRMD_CALLOPTS "lrmd_callopt"
	#define F_LRMD_CALLDATA "lrmd_calldata"
	#define F_LRMD_RC "lrmd_rc"
	#define F_LRMD_EXEC_RC "lrmd_exec_rc"
	#define F_LRMD_OP_STATUS "lrmd_exec_op_status"
	#define F_LRMD_TIMEOUT "lrmd_timeout"
	#define F_LRMD_CLASS "lrmd_class"
	#define F_LRMD_PROVIDER "lrmd_provider"
	#define F_LRMD_TYPE "lrmd_type"
	#define F_LRMD_ORIGIN "lrmd_origin"

	#define F_LRMD_RSC_RUN_TIME "lrmd_run_time"
	#define F_LRMD_RSC_RCCHANGE_TIME "lrmd_rcchange_time"
	#define F_LRMD_RSC_EXEC_TIME "lrmd_exec_time"
	#define F_LRMD_RSC_QUEUE_TIME "lrmd_queue_time"

	#define F_LRMD_RSC_ID "lrmd_rsc_id"
	#define F_LRMD_RSC_ACTION "lrmd_rsc_action"
	#define F_LRMD_RSC_USERDATA_STR "lrmd_rsc_userdata_str"
	#define F_LRMD_RSC_OUTPUT "lrmd_rsc_output"
	#define F_LRMD_RSC_EXIT_REASON "lrmd_rsc_exit_reason"
	#define F_LRMD_RSC_START_DELAY "lrmd_rsc_start_delay"
	#define F_LRMD_RSC_INTERVAL "lrmd_rsc_interval"
	#define F_LRMD_RSC_METADATA "lrmd_rsc_metadata_res"
	#define F_LRMD_RSC_DELETED "lrmd_rsc_deleted"
	#define F_LRMD_RSC "lrmd_rsc"

	#define LRMD_OP_RSC_CHK_REG "lrmd_rsc_check_register"
	#define LRMD_OP_RSC_REG "lrmd_rsc_register"
	#define LRMD_OP_RSC_EXEC "lrmd_rsc_exec"
	#define LRMD_OP_RSC_CANCEL "lrmd_rsc_cancel"
	#define LRMD_OP_RSC_UNREG "lrmd_rsc_unregister"
	#define LRMD_OP_RSC_INFO "lrmd_rsc_info"
	#define LRMD_OP_RSC_METADATA "lrmd_rsc_metadata"
	#define LRMD_OP_POKE "lrmd_rsc_poke"
	#define LRMD_OP_NEW_CLIENT "lrmd_rsc_new_client"

	+#define LRMD_IPC_OP_NEW "new"
	+#define LRMD_IPC_OP_DESTROY "destroy"
	+#define LRMD_IPC_OP_EVENT "event"
	+#define LRMD_IPC_OP_REQUEST "request"
	+#define LRMD_IPC_OP_RESPONSE "response"
	+
	#define F_LRMD_IPC_OP "lrmd_ipc_op"
	#define F_LRMD_IPC_IPC_SERVER "lrmd_ipc_server"
	#define F_LRMD_IPC_SESSION "lrmd_ipc_session"
	#define F_LRMD_IPC_CLIENT "lrmd_ipc_client"
	#define F_LRMD_IPC_PROXY_NODE "lrmd_ipc_proxy_node"
	#define F_LRMD_IPC_USER "lrmd_ipc_user"
	#define F_LRMD_IPC_MSG "lrmd_ipc_msg"
	#define F_LRMD_IPC_MSG_ID "lrmd_ipc_msg_id"
	#define F_LRMD_IPC_MSG_FLAGS "lrmd_ipc_msg_flags"

	#define T_LRMD "lrmd"
	#define T_LRMD_REPLY "lrmd_reply"
	#define T_LRMD_NOTIFY "lrmd_notify"
	#define T_LRMD_IPC_PROXY "lrmd_ipc_proxy"
	/* INDENT-ON */

	/*!
	* \brief Create a new local lrmd connection
	*/
	lrmd_t *lrmd_api_new(void);

	/*!
	* \brief Create a new remote lrmd connection using tls backend
	*
	* \param nodename name of remote node identified with this connection
	* \param server name of server to connect to
	* \param port port number to connect to
	*
	* \note nodename and server may be the same value.
	*/
	lrmd_t lrmd_remote_api_new(const char nodename, const char *server, int port);

	/*!
	* \brief Use after lrmd_poll returns 1 to read and dispatch a message
	*
	* \param[in,out] lrmd lrmd connection object
	*
	* \return TRUE if connection is still up, FALSE if disconnected
	*/
	bool lrmd_dispatch(lrmd_t * lrmd);

	/*!
	* \brief Poll for a specified timeout period to determine if a message
	* is ready for dispatch.
	* \retval 1 msg is ready
	* \retval 0 timeout occured
	* \retval negative error code
	*/
	int lrmd_poll(lrmd_t * lrmd, int timeout);

	/*!
	* \brief Destroy lrmd object
	*/
	void lrmd_api_delete(lrmd_t * lrmd);
	lrmd_key_value_t lrmd_key_value_add(lrmd_key_value_t kvp, const char key, const char value);

	/* INDENT-OFF */
	/* Reserved for future use */
	enum lrmd_call_options {
	lrmd_opt_none = 0x00000000,
	/* lrmd_opt_sync_call = 0x00000001, //Not implemented, patches welcome. */
	/! Only notify the client originating a exec() the results /
	lrmd_opt_notify_orig_only = 0x00000002,
	/*! Drop recurring operations initiated by a client when client disconnects.
	* This call_option is only valid when registering a resource. When used
	* remotely with the pacemaker_remote daemon, this option means that recurring
	* operations will be dropped once all the remote connections disconnect. */
	lrmd_opt_drop_recurring = 0x00000003,
	/! Only send out notifications for recurring operations whenthe result changes /
	lrmd_opt_notify_changes_only = 0x00000004,
	};

	enum lrmd_callback_event {
	lrmd_event_register,
	lrmd_event_unregister,
	lrmd_event_exec_complete,
	lrmd_event_disconnect,
	lrmd_event_connect,
	lrmd_event_poke,
	lrmd_event_new_client,
	};

	/* INDENT-ON */

	typedef struct lrmd_event_data_s {
	/! Type of event, register, unregister, call_completed... /
	enum lrmd_callback_event type;

	/! The resource this event occurred on. /
	const char *rsc_id;
	/! The action performed, start, stop, monitor... /
	const char *op_type;
	/! The userdata string given do exec() api function /
	const char *user_data;

	/! The client api call id associated with this event /
	int call_id;
	/! The operation's timeout period in ms. /
	int timeout;
	/! The operation's recurring interval in ms. /
	int interval;
	/! The operation's start delay value in ms. /
	int start_delay;
	/! This operation that just completed is on a deleted rsc. /
	int rsc_deleted;

	/! The executed ra return code mapped to OCF /
	enum ocf_exitcode rc;
	/! The lrmd status returned for exec_complete events /
	int op_status;
	/! stdout from resource agent operation /
	const char *output;
	/! Timestamp of when op ran /
	unsigned int t_run;
	/! Timestamp of last rc change /
	unsigned int t_rcchange;
	/! Time in length op took to execute /
	unsigned int exec_time;
	/! Time in length spent in queue /
	unsigned int queue_time;

	/! int connection result. Used for connection and poke events /
	int connection_rc;

	/* This is a GHashTable containing the
	* parameters given to the operation */
	void *params;

	/* client node name associated with this conneciton.
	* This is useful if multiple clients are being utilized by
	* a single process. This name allows the actions to be matched
	* to the proper client. */
	const char *remote_nodename;

	/! exit failure reason string from resource agent operation /
	const char *exit_reason;

	} lrmd_event_data_t;

	lrmd_event_data_t lrmd_copy_event(lrmd_event_data_t event);
	void lrmd_free_event(lrmd_event_data_t * event);

	typedef struct lrmd_rsc_info_s {
	char *id;
	char *type;
	char *class;
	char *provider;
	} lrmd_rsc_info_t;

	lrmd_rsc_info_t lrmd_copy_rsc_info(lrmd_rsc_info_t rsc_info);
	void lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info);

	typedef void (lrmd_event_callback) (lrmd_event_data_t event);

	typedef struct lrmd_list_s {
	const char *val;
	struct lrmd_list_s *next;
	} lrmd_list_t;

	void lrmd_list_freeall(lrmd_list_t * head);
	void lrmd_key_value_freeall(lrmd_key_value_t * head);

	typedef struct lrmd_api_operations_s {
	/*!
	* \brief Connect from the lrmd.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (connect) (lrmd_t lrmd, const char client_name, int fd);

	/*!
	* \brief Establish an connection to lrmd, don't block while connecting.
	* \note this function requires the use of mainloop.
	*
	* \note The is returned using the event callback.
	* \note When this function returns 0, the callback will be invoked
	* to report the final result of the connect.
	* \retval 0, connect in progress, wait for event callback
	* \retval -1, failure.
	*/
	int (connect_async) (lrmd_t lrmd, const char client_name, int timeout /ms */ );

	/*!
	* \brief Is connected to lrmd daemon?
	*
	* \retval 0, false
	* \retval 1, true
	*/
	int (is_connected) (lrmd_t lrmd);

	/*!
	* \brief Poke lrmd connection to verify it is still capable of serving requests
	* \note The response comes in the form of a poke event to the callback.
	*
	* \retval 0, wait for response in callback
	* \retval -1, connection failure, callback may not be invoked
	*/
	int (poke_connection) (lrmd_t lrmd);

	/*!
	* \brief Disconnect from the lrmd.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (disconnect) (lrmd_t lrmd);

	/*!
	* \brief Register a resource with the lrmd.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (register_rsc) (lrmd_t lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char agent, enum lrmd_call_options options);

	/*!
	* \brief Retrieve registration info for a rsc
	*
	* \retval info on success
	* \retval NULL on failure
	*/
	lrmd_rsc_info_t (get_rsc_info) (lrmd_t * lrmd,
	const char *rsc_id, enum lrmd_call_options options);

	/*!
	* \brief Unregister a resource from the lrmd.
	*
	* \note All pending and recurring operations will be cancelled
	* automatically.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \retval 0, success
	* \retval -1, success, but operations are currently executing on the rsc which will
	* return once they are completed.
	* \retval negative error code on failure
	*
	*/
	int (unregister_rsc) (lrmd_t lrmd, const char *rsc_id, enum lrmd_call_options options);

	/*!
	* \brief Sets the callback to receive lrmd events on.
	*/
	void (set_callback) (lrmd_t lrmd, lrmd_event_callback callback);

	/*!
	* \brief Issue a command on a resource
	*
	* \note Asynchronous, command is queued in daemon on function return, but
	* execution of command is not synced.
	*
	* \note Operations on individual resources are guaranteed to occur
	* in the order the client api calls them in.
	*
	* \note Operations between different resources are not guaranteed
	* to occur in any specific order in relation to one another
	* regardless of what order the client api is called in.
	* \retval call_id to track async event result on success
	* \retval negative error code on failure
	*/
	int (exec) (lrmd_t lrmd, const char rsc_id, const char action, const char userdata, / userdata string given back in event notification */
	int interval, /* ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params); /* ownership of params is given up to api here */

	/*!
	* \brief Cancel a recurring command.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \note The cancel is completed async from this call.
	* We can be guaranteed the cancel has completed once
	* the callback receives an exec_complete event with
	* the lrmd_op_status signifying that the operation is
	* cancelled.
	* \note For each resource, cancel operations and exec operations
	* are processed in the order they are received.
	* It is safe to assume that for a single resource, a cancel
	* will occur in the lrmd before an exec if the client's cancel
	* api call occurs before the exec api call.
	*
	* It is not however safe to assume any operation on one resource will
	* occur before an operation on another resource regardless of
	* the order the client api is called in.
	*
	* \retval 0, cancel command sent.
	* \retval negative error code on failure
	*/
	int (cancel) (lrmd_t lrmd, const char rsc_id, const char action, int interval);

	/*!
	* \brief Get the metadata documentation for a resource.
	*
	* \note Value is returned in output. Output must be freed when set
	*
	* \retval lrmd_ok success
	* \retval negative error code on failure
	*/
	int (get_metadata) (lrmd_t lrmd,
	const char *class,
	const char *provider,
	const char agent, char *output, enum lrmd_call_options options);

	/*!
	* \brief Retrieve a list of installed resource agents.
	*
	* \note if class is not provided, all known agents will be returned
	* \note list must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_agents) (lrmd_t lrmd, lrmd_list_t ** agents, const char *class,
	const char *provider);

	/*!
	* \brief Retrieve a list of resource agent providers
	*
	* \note When the agent is provided, only the agent's provider will be returned
	* \note When no agent is supplied, all providers will be returned.
	* \note List must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_ocf_providers) (lrmd_t lrmd, const char agent, lrmd_list_t * providers);

	/*!
	* \brief Retrieve a list of standards supported by this machine/installation
	*
	* \note List must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_standards) (lrmd_t lrmd, lrmd_list_t ** standards);

	} lrmd_api_operations_t;

	struct lrmd_s {
	lrmd_api_operations_t *cmds;
	void *private;
	};

	static inline const char *
	lrmd_event_type2str(enum lrmd_callback_event type)
	{
	switch (type) {
	case lrmd_event_register:
	return "register";
	case lrmd_event_unregister:
	return "unregister";
	case lrmd_event_exec_complete:
	return "exec_complete";
	case lrmd_event_disconnect:
	return "disconnect";
	case lrmd_event_connect:
	return "connect";
	case lrmd_event_poke:
	return "poke";
	case lrmd_event_new_client:
	return "new_client";
	}
	return "unknown";
	}

	#endif
	diff --git a/lib/lrmd/proxy_common.c b/lib/lrmd/proxy_common.c
	index 50c59c32a6..a0f5e6293b 100644
	--- a/lib/lrmd/proxy_common.c
	+++ b/lib/lrmd/proxy_common.c
	@@ -1,100 +1,100 @@
	/*
	* Copyright (c) 2015 David Vossel <davidvossel@gmail.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>

	#include <crm/pengine/status.h>
	#include <crm/cib.h>
	#include <crm/lrmd.h>

	int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
	GHashTable *proxy_table = NULL;

	void
	remote_proxy_notify_destroy(lrmd_t lrmd, const char session_id)
	{
	/* sending to the remote node that an ipc connection has been destroyed */
	xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	- crm_xml_add(msg, F_LRMD_IPC_OP, "destroy");
	+ crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_DESTROY);
	crm_xml_add(msg, F_LRMD_IPC_SESSION, session_id);
	lrmd_internal_proxy_send(lrmd, msg);
	free_xml(msg);
	}

	void
	remote_proxy_relay_event(lrmd_t lrmd, const char session_id, xmlNode *msg)
	{
	/* sending to the remote node an event msg. */
	xmlNode *event = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	- crm_xml_add(event, F_LRMD_IPC_OP, "event");
	+ crm_xml_add(event, F_LRMD_IPC_OP, LRMD_IPC_OP_EVENT);
	crm_xml_add(event, F_LRMD_IPC_SESSION, session_id);
	add_message_xml(event, F_LRMD_IPC_MSG, msg);
	crm_log_xml_explicit(event, "EventForProxy");
	lrmd_internal_proxy_send(lrmd, event);
	free_xml(event);
	}

	void
	remote_proxy_relay_response(lrmd_t lrmd, const char session_id, xmlNode *msg, int msg_id)
	{
	/* sending to the remote node a response msg. */
	xmlNode *response = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	- crm_xml_add(response, F_LRMD_IPC_OP, "response");
	+ crm_xml_add(response, F_LRMD_IPC_OP, LRMD_IPC_OP_RESPONSE);
	crm_xml_add(response, F_LRMD_IPC_SESSION, session_id);
	crm_xml_add_int(response, F_LRMD_IPC_MSG_ID, msg_id);
	add_message_xml(response, F_LRMD_IPC_MSG, msg);
	lrmd_internal_proxy_send(lrmd, response);
	free_xml(response);
	}

	void
	remote_proxy_end_session(const char *session)
	{
	remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);

	if (proxy == NULL) {
	return;
	}
	crm_trace("ending session ID %s", proxy->session_id);

	if (proxy->source) {
	mainloop_del_ipc_client(proxy->source);
	}
	}

	void
	remote_proxy_free(gpointer data)
	{
	remote_proxy_t *proxy = data;

	crm_trace("freed proxy session ID %s", proxy->session_id);
	free(proxy->node_name);
	free(proxy->session_id);
	free(proxy);
	}


	diff --git a/lrmd/ipc_proxy.c b/lrmd/ipc_proxy.c
	index d95a396b53..164a9ff269 100644
	--- a/lrmd/ipc_proxy.c
	+++ b/lrmd/ipc_proxy.c
	@@ -1,424 +1,424 @@
	/*
	* Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/


	#include <crm_internal.h>
	#include <lrmd_private.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipcs.h>
	#include <crm/cib/internal.h>
	#include <crm/fencing/internal.h>

	static qb_ipcs_service_t *cib_ro = NULL;
	static qb_ipcs_service_t *cib_rw = NULL;
	static qb_ipcs_service_t *cib_shm = NULL;

	static qb_ipcs_service_t *attrd_ipcs = NULL;
	static qb_ipcs_service_t *crmd_ipcs = NULL;
	static qb_ipcs_service_t *stonith_ipcs = NULL;

	/* ipc providers == crmd clients connecting from cluster nodes */
	static GHashTable *ipc_providers = NULL;
	/* ipc clients == things like cibadmin, crm_resource, connecting locally */
	static GHashTable *ipc_clients = NULL;

	/*!
	* \internal
	* \brief Get an IPC proxy provider
	*
	* \return Pointer to a provider if one exists, NULL otherwise
	*
	* \note Grab the first provider available; any provider will work, and usually
	* there will be only one. These are client connections originating from a
	* cluster node's crmd.
	*/
	crm_client_t *
	ipc_proxy_get_provider()
	{
	if (ipc_providers) {
	GHashTableIter iter;
	gpointer key = NULL;
	gpointer value = NULL;

	g_hash_table_iter_init(&iter, ipc_providers);
	if (g_hash_table_iter_next(&iter, &key, &value)) {
	return (crm_client_t*)value;
	}
	}
	return NULL;
	}

	static int32_t
	ipc_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid, const char *ipc_channel)
	{
	crm_client_t *client;
	crm_client_t *ipc_proxy = ipc_proxy_get_provider();
	xmlNode *msg;

	crm_trace("Connection %p on channel %s", c, ipc_channel);

	if (ipc_proxy == NULL) {
	crm_err("No ipc providers available for uid %d gid %d", uid, gid);
	return -EREMOTEIO;
	}

	/* this new client is a local ipc client on a remote
	* guest wanting to access the ipc on any available cluster nodes */
	client = crm_client_new(c, uid, gid);
	if (client == NULL) {
	return -EREMOTEIO;
	}

	/* This ipc client is bound to a single ipc provider. If the
	* provider goes away, this client is disconnected */
	client->userdata = strdup(ipc_proxy->id);
	client->name = crm_strdup_printf("proxy-%s-%d-%.8s", ipc_channel, client->pid, client->id);

	g_hash_table_insert(ipc_clients, client->id, client);

	msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	- crm_xml_add(msg, F_LRMD_IPC_OP, "new");
	+ crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_NEW);
	crm_xml_add(msg, F_LRMD_IPC_IPC_SERVER, ipc_channel);
	crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
	lrmd_server_send_notify(ipc_proxy, msg);
	free_xml(msg);
	crm_debug("created new ipc proxy with session id %s", client->id);
	return 0;
	}

	static int32_t
	crmd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	return ipc_proxy_accept(c, uid, gid, CRM_SYSTEM_CRMD);
	}

	static int32_t
	attrd_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	return ipc_proxy_accept(c, uid, gid, T_ATTRD);
	}

	static int32_t
	stonith_proxy_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	return ipc_proxy_accept(c, uid, gid, "stonith-ng");
	}

	static int32_t
	cib_proxy_accept_rw(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	return ipc_proxy_accept(c, uid, gid, cib_channel_rw);
	}

	static int32_t
	cib_proxy_accept_ro(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	return ipc_proxy_accept(c, uid, gid, cib_channel_ro);
	}

	static void
	ipc_proxy_created(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	}

	void
	ipc_proxy_forward_client(crm_client_t ipc_proxy, xmlNode xml)
	{
	const char *session = crm_element_value(xml, F_LRMD_IPC_SESSION);
	const char *msg_type = crm_element_value(xml, F_LRMD_IPC_OP);
	xmlNode *msg = get_message_xml(xml, F_LRMD_IPC_MSG);
	crm_client_t *ipc_client = crm_client_get_by_id(session);
	int rc = 0;

	if (ipc_client == NULL) {
	xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	- crm_xml_add(msg, F_LRMD_IPC_OP, "destroy");
	+ crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_DESTROY);
	crm_xml_add(msg, F_LRMD_IPC_SESSION, session);
	lrmd_server_send_notify(ipc_proxy, msg);
	free_xml(msg);
	return;
	}

	/* This is an event or response from the ipc provider
	* going to the local ipc client.
	*
	* Looking at the chain of events.
	*
	* -----remote node----------------\|---- cluster node ------
	* ipc_client <--1--> this code <--2--> crmd:remote_proxy_cb/remote_proxy_relay_event() <----3----> ipc server
	*
	* This function is receiving a msg from connection 2
	* and forwarding it to connection 1.
	*/

	- if (safe_str_eq(msg_type, "event")) {
	+ if (safe_str_eq(msg_type, LRMD_IPC_OP_EVENT)) {
	crm_trace("Sending event to %s", ipc_client->id);
	rc = crm_ipcs_send(ipc_client, 0, msg, crm_ipc_server_event);

	- } else if (safe_str_eq(msg_type, "response")) {
	+ } else if (safe_str_eq(msg_type, LRMD_IPC_OP_RESPONSE)) {
	int msg_id = 0;

	crm_element_value_int(xml, F_LRMD_IPC_MSG_ID, &msg_id);
	crm_trace("Sending response to %d - %s", ipc_client->request_id, ipc_client->id);
	rc = crm_ipcs_send(ipc_client, msg_id, msg, FALSE);

	CRM_LOG_ASSERT(msg_id == ipc_client->request_id);
	ipc_client->request_id = 0;

	- } else if (safe_str_eq(msg_type, "destroy")) {
	+ } else if (safe_str_eq(msg_type, LRMD_IPC_OP_DESTROY)) {
	qb_ipcs_disconnect(ipc_client->ipcs);

	} else {
	crm_err("Unknown ipc proxy msg type %s" , msg_type);
	}

	if (rc < 0) {
	crm_warn("IPC Proxy send to ipc client %s failed, rc = %d", ipc_client->id, rc);
	}
	}

	static int32_t
	ipc_proxy_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	crm_client_t *client = crm_client_get(c);
	crm_client_t *ipc_proxy = crm_client_get_by_id(client->userdata);
	xmlNode *request = NULL;
	xmlNode *msg = NULL;

	if (!ipc_proxy) {
	qb_ipcs_disconnect(client->ipcs);
	return 0;
	}

	/* This is a request from the local ipc client going
	* to the ipc provider.
	*
	* Looking at the chain of events.
	*
	* -----remote node----------------\|---- cluster node ------
	* ipc_client <--1--> this code <--2--> crmd:remote_proxy_dispatch_internal() <----3----> ipc server
	*
	* This function is receiving a request from connection
	* 1 and forwarding it to connection 2.
	*/
	request = crm_ipcs_recv(client, data, size, &id, &flags);

	if (!request) {
	return 0;
	}

	CRM_CHECK(client != NULL, crm_err("Invalid client");
	free_xml(request); return FALSE);
	CRM_CHECK(client->id != NULL, crm_err("Invalid client: %p", client);
	free_xml(request); return FALSE);

	/* this ensures that synced request/responses happen over the event channel
	* in the crmd, allowing the crmd to process the messages async */
	set_bit(flags, crm_ipc_proxied);
	client->request_id = id;

	msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	- crm_xml_add(msg, F_LRMD_IPC_OP, "request");
	+ crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_REQUEST);
	crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
	crm_xml_add(msg, F_LRMD_IPC_CLIENT, crm_client_name(client));
	crm_xml_add(msg, F_LRMD_IPC_USER, client->user);
	crm_xml_add_int(msg, F_LRMD_IPC_MSG_ID, id);
	crm_xml_add_int(msg, F_LRMD_IPC_MSG_FLAGS, flags);
	add_message_xml(msg, F_LRMD_IPC_MSG, request);
	lrmd_server_send_notify(ipc_proxy, msg);
	free_xml(request);
	free_xml(msg);

	return 0;
	}

	static int32_t
	ipc_proxy_closed(qb_ipcs_connection_t * c)
	{
	crm_client_t *client = crm_client_get(c);
	crm_client_t *ipc_proxy;

	if (client == NULL) {
	return 0;
	}

	ipc_proxy = crm_client_get_by_id(client->userdata);

	crm_trace("Connection %p", c);

	if (ipc_proxy) {
	xmlNode *msg = create_xml_node(NULL, T_LRMD_IPC_PROXY);
	- crm_xml_add(msg, F_LRMD_IPC_OP, "destroy");
	+ crm_xml_add(msg, F_LRMD_IPC_OP, LRMD_IPC_OP_DESTROY);
	crm_xml_add(msg, F_LRMD_IPC_SESSION, client->id);
	lrmd_server_send_notify(ipc_proxy, msg);
	free_xml(msg);
	}

	g_hash_table_remove(ipc_clients, client->id);

	free(client->userdata);
	client->userdata = NULL;
	crm_client_destroy(client);
	return 0;
	}

	static void
	ipc_proxy_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	ipc_proxy_closed(c);
	}

	static struct qb_ipcs_service_handlers crmd_proxy_callbacks = {
	.connection_accept = crmd_proxy_accept,
	.connection_created = ipc_proxy_created,
	.msg_process = ipc_proxy_dispatch,
	.connection_closed = ipc_proxy_closed,
	.connection_destroyed = ipc_proxy_destroy
	};

	static struct qb_ipcs_service_handlers attrd_proxy_callbacks = {
	.connection_accept = attrd_proxy_accept,
	.connection_created = ipc_proxy_created,
	.msg_process = ipc_proxy_dispatch,
	.connection_closed = ipc_proxy_closed,
	.connection_destroyed = ipc_proxy_destroy
	};

	static struct qb_ipcs_service_handlers stonith_proxy_callbacks = {
	.connection_accept = stonith_proxy_accept,
	.connection_created = ipc_proxy_created,
	.msg_process = ipc_proxy_dispatch,
	.connection_closed = ipc_proxy_closed,
	.connection_destroyed = ipc_proxy_destroy
	};

	static struct qb_ipcs_service_handlers cib_proxy_callbacks_ro = {
	.connection_accept = cib_proxy_accept_ro,
	.connection_created = ipc_proxy_created,
	.msg_process = ipc_proxy_dispatch,
	.connection_closed = ipc_proxy_closed,
	.connection_destroyed = ipc_proxy_destroy
	};

	static struct qb_ipcs_service_handlers cib_proxy_callbacks_rw = {
	.connection_accept = cib_proxy_accept_rw,
	.connection_created = ipc_proxy_created,
	.msg_process = ipc_proxy_dispatch,
	.connection_closed = ipc_proxy_closed,
	.connection_destroyed = ipc_proxy_destroy
	};

	void
	ipc_proxy_add_provider(crm_client_t *ipc_proxy)
	{
	if (ipc_providers == NULL) {
	return;
	}
	g_hash_table_insert(ipc_providers, ipc_proxy->id, ipc_proxy);
	}

	void
	ipc_proxy_remove_provider(crm_client_t *ipc_proxy)
	{
	GHashTableIter iter;
	crm_client_t *ipc_client = NULL;
	char *key = NULL;
	GList *remove_these = NULL;
	GListPtr gIter = NULL;

	if (ipc_providers == NULL) {
	return;
	}

	g_hash_table_remove(ipc_providers, ipc_proxy->id);

	g_hash_table_iter_init(&iter, ipc_clients);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & ipc_client)) {
	const char *proxy_id = ipc_client->userdata;
	if (safe_str_eq(proxy_id, ipc_proxy->id)) {
	crm_info("ipc proxy connection for client %s pid %d destroyed because cluster node disconnected.",
	ipc_client->id, ipc_client->pid);
	/* we can't remove during the iteration, so copy items
	* to a list we can destroy later */
	remove_these = g_list_append(remove_these, ipc_client);
	}
	}

	for (gIter = remove_these; gIter != NULL; gIter = gIter->next) {
	ipc_client = gIter->data;
	qb_ipcs_disconnect(ipc_client->ipcs);
	}

	/* just frees the list, not the elements in the list */
	g_list_free(remove_these);
	}

	void
	ipc_proxy_init(void)
	{
	ipc_clients = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
	ipc_providers = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);

	cib_ipc_servers_init(&cib_ro,
	&cib_rw,
	&cib_shm,
	&cib_proxy_callbacks_ro,
	&cib_proxy_callbacks_rw);

	attrd_ipc_server_init(&attrd_ipcs, &attrd_proxy_callbacks);
	stonith_ipc_server_init(&stonith_ipcs, &stonith_proxy_callbacks);
	crmd_ipcs = crmd_ipc_server_init(&crmd_proxy_callbacks);
	if (crmd_ipcs == NULL) {
	crm_err("Failed to create crmd server: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	void
	ipc_proxy_cleanup(void)
	{
	if (ipc_providers) {
	g_hash_table_destroy(ipc_providers);
	}
	if (ipc_clients) {
	g_hash_table_destroy(ipc_clients);
	}
	cib_ipc_servers_destroy(cib_ro, cib_rw, cib_shm);
	qb_ipcs_destroy(attrd_ipcs);
	qb_ipcs_destroy(stonith_ipcs);
	qb_ipcs_destroy(crmd_ipcs);
	cib_ro = NULL;
	cib_rw = NULL;
	cib_shm = NULL;
	ipc_providers = NULL;
	ipc_clients = NULL;
	}
	diff --git a/lrmd/remote_ctl.c b/lrmd/remote_ctl.c
	index ad859541cb..1983c880de 100644
	--- a/lrmd/remote_ctl.c
	+++ b/lrmd/remote_ctl.c
	@@ -1,526 +1,526 @@
	/*
	* Copyright (c) 2015 David Vossel <davidvossel@gmail.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>

	#include <crm/pengine/status.h>
	#include <crm/cib.h>
	#include <crm/lrmd.h>

	extern GHashTable *proxy_table;
	void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg));

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"help", 0, 0, '?'},
	{"verbose", 0, 0, 'V', "\t\tPrint out logs and events to screen"},
	{"quiet", 0, 0, 'Q', "\t\tSuppress all output to screen"},
	{"tls", 1, 0, 'S', "\t\tSet tls host to contact"},
	{"tls-port", 1, 0, 'p', "\t\tUse custom tls port"},
	{"node", 1, 0, 'n', "\tNode name to use for ipc proxy"},
	{"api-call", 1, 0, 'c', "\tDirectly relates to lrmd api functions"},
	{"-spacer-", 1, 0, '-', "\nParameters for api-call option"},
	{"action", 1, 0, 'a'},
	{"rsc-id", 1, 0, 'r'},
	{"provider", 1, 0, 'P'},
	{"class", 1, 0, 'C'},
	{"type", 1, 0, 'T'},
	{"timeout", 1, 0, 't'},
	{"param-key", 1, 0, 'k'},
	{"param-val", 1, 0, 'v'},

	{"-spacer-", 1, 0, '-'},
	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	static int wait_poke = 0;
	static int exec_call_id = 0;
	static gboolean client_start(gpointer user_data);
	static void try_connect(void);

	static struct {
	int verbose;
	int quiet;
	int print;
	int interval;
	int timeout;
	int port;
	const char *node_name;
	const char *api_call;
	const char *rsc_id;
	const char *provider;
	const char *class;
	const char *type;
	const char *action;
	const char *listen;
	const char *tls_host;
	lrmd_key_value_t *params;
	} options;

	GMainLoop *mainloop = NULL;
	lrmd_t *lrmd_conn = NULL;

	static void
	client_exit(int rc)
	{
	lrmd_api_delete(lrmd_conn);
	if (proxy_table) {
	g_hash_table_destroy(proxy_table); proxy_table = NULL;
	}
	exit(rc);
	}

	static void
	client_shutdown(int nsig)
	{
	lrmd_api_delete(lrmd_conn);
	lrmd_conn = NULL;
	}

	static void
	read_events(lrmd_event_data_t * event)
	{
	if (wait_poke && event->type == lrmd_event_poke) {
	client_exit(PCMK_OCF_OK);
	}
	if ((event->call_id == exec_call_id) && (event->type == lrmd_event_exec_complete)) {
	if (event->output) {
	crm_info("%s", event->output);
	}
	if (event->exit_reason) {
	fprintf(stderr, "%s%s\n", PCMK_OCF_REASON_PREFIX, event->exit_reason);
	}
	client_exit(event->rc);
	}
	}

	static gboolean
	timeout_err(gpointer data)
	{
	crm_err("timed out in remote_client\n");
	client_exit(PCMK_OCF_TIMEOUT);

	return FALSE;
	}

	static void
	connection_events(lrmd_event_data_t * event)
	{
	int rc = event->connection_rc;

	if (event->type != lrmd_event_connect) {
	/* ignore */
	return;
	}

	if (!rc) {
	client_start(NULL);
	return;
	} else {
	sleep(1);
	try_connect();
	}
	}

	static void
	try_connect(void)
	{
	int tries = 10;
	static int num_tries = 0;
	int rc = 0;

	lrmd_conn->cmds->set_callback(lrmd_conn, connection_events);
	for (; num_tries < tries; num_tries++) {
	rc = lrmd_conn->cmds->connect_async(lrmd_conn, "lrmd", 10000);

	if (!rc) {
	num_tries++;
	return; /* we'll hear back in async callback */
	}
	sleep(1);
	}

	crm_err("Failed to connect to pacemaker remote.\n");
	client_exit(PCMK_OCF_UNKNOWN_ERROR);
	}

	static gboolean
	client_start(gpointer user_data)
	{
	int rc = 0;

	if (!lrmd_conn->cmds->is_connected(lrmd_conn)) {
	try_connect();
	/* async connect, this funciton will get called back into. */
	return 0;
	}

	lrmd_conn->cmds->set_callback(lrmd_conn, read_events);


	if (safe_str_eq(options.api_call, "ipc_debug")) {
	/* Do nothing, leave connection up just for debugging ipc proxy */
	return 0;
	}
	if (options.timeout) {
	g_timeout_add(options.timeout, timeout_err, NULL);
	}

	if (safe_str_eq(options.api_call, "metadata")) {
	char *output = NULL;

	rc = lrmd_conn->cmds->get_metadata(lrmd_conn,
	options.class,
	options.provider, options.type, &output, 0);
	if (rc == pcmk_ok) {
	printf("%s", output);
	free(output);
	client_exit(PCMK_OCF_OK);
	}
	client_exit(PCMK_OCF_UNKNOWN_ERROR);

	} else if (safe_str_eq(options.api_call, "poke")) {
	rc = lrmd_conn->cmds->poke_connection(lrmd_conn);
	if (rc != pcmk_ok) {
	client_exit(PCMK_OCF_UNKNOWN_ERROR);
	}
	wait_poke = 1;
	} else {
	lrmd_rsc_info_t *rsc_info = NULL;

	rsc_info = lrmd_conn->cmds->get_rsc_info(lrmd_conn, options.rsc_id, 0);
	if (rsc_info == NULL) {
	rc = lrmd_conn->cmds->register_rsc(lrmd_conn, options.rsc_id,
	options.class, options.provider, options.type, 0);

	if (rc != 0){
	crm_err("failed to register resource %s with pacemaker_remote. rc: %d\n", options.rsc_id, rc);
	client_exit(1);
	}
	}
	lrmd_free_rsc_info(rsc_info);

	rc = lrmd_conn->cmds->exec(lrmd_conn,
	options.rsc_id,
	options.action,
	NULL,
	options.interval,
	options.timeout,
	0, 0, options.params);

	if (rc > 0) {
	exec_call_id = rc;
	} else {
	crm_err("execution of rsc %s failed. rc = %d\n", options.rsc_id, rc);
	client_exit(PCMK_OCF_UNKNOWN_ERROR);
	}
	}

	return 0;
	}

	static int
	remote_proxy_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	/* Async responses from cib and friends back to clients via pacemaker_remoted */
	xmlNode *xml = NULL;
	remote_proxy_t *proxy = userdata;
	uint32_t flags;

	xml = string2xml(buffer);
	if (xml == NULL) {
	crm_warn("Received a NULL msg from IPC service.");
	return 1;
	}

	flags = crm_ipc_buffer_flags(proxy->ipc);
	if (flags & crm_ipc_proxied_relay_response) {
	crm_trace("Passing response back to %.8s on %s: %.200s - request id: %d", proxy->session_id, proxy->node_name, buffer, proxy->last_request_id);
	remote_proxy_relay_response(lrmd_conn, proxy->session_id, xml, proxy->last_request_id);
	proxy->last_request_id = 0;

	} else {
	crm_trace("Passing event back to %.8s on %s: %.200s", proxy->session_id, proxy->node_name, buffer);
	remote_proxy_relay_event(lrmd_conn, proxy->session_id, xml);
	}
	free_xml(xml);
	return 1;
	}

	static void
	remote_proxy_disconnected(void *userdata)
	{
	remote_proxy_t *proxy = userdata;

	crm_trace("destroying %p", userdata);

	proxy->source = NULL;
	proxy->ipc = NULL;

	remote_proxy_notify_destroy(lrmd_conn, proxy->session_id);
	g_hash_table_remove(proxy_table, proxy->session_id);
	}

	static remote_proxy_t *
	remote_proxy_new(const char node_name, const char session_id, const char *channel)
	{
	static struct ipc_client_callbacks proxy_callbacks = {
	.dispatch = remote_proxy_dispatch_internal,
	.destroy = remote_proxy_disconnected
	};
	remote_proxy_t *proxy = calloc(1, sizeof(remote_proxy_t));

	proxy->node_name = strdup(node_name);
	proxy->session_id = strdup(session_id);

	if (safe_str_eq(channel, CRM_SYSTEM_CRMD)) {
	proxy->is_local = TRUE;
	} else {
	proxy->source = mainloop_add_ipc_client(channel, G_PRIORITY_LOW, 0, proxy, &proxy_callbacks);
	proxy->ipc = mainloop_get_ipc_client(proxy->source);

	if (proxy->source == NULL) {
	remote_proxy_free(proxy);
	return NULL;
	}
	}

	crm_trace("created proxy session ID %s", proxy->session_id);
	g_hash_table_insert(proxy_table, proxy->session_id, proxy);

	return proxy;
	}

	static void
	remote_proxy_cb(lrmd_t lrmd, void userdata, xmlNode *msg)
	{
	const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
	const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
	int msg_id = 0;

	/* sessions are raw ipc connections to IPC,
	* all we do is proxy requests/responses exactly
	* like they are given to us at the ipc level. */

	CRM_CHECK(op != NULL, return);
	CRM_CHECK(session != NULL, return);

	crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);

	/* This is msg from remote ipc client going to real ipc server */
	- if (safe_str_eq(op, "new")) {
	+ if (safe_str_eq(op, LRMD_IPC_OP_NEW)) {
	const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);

	CRM_CHECK(channel != NULL, return);

	if (remote_proxy_new(options.node_name, session, channel) == NULL) {
	remote_proxy_notify_destroy(lrmd, session);
	}
	crm_info("new remote proxy client established to %s, session id %s", channel, session);
	- } else if (safe_str_eq(op, "destroy")) {
	+ } else if (safe_str_eq(op, LRMD_IPC_OP_DESTROY)) {
	remote_proxy_end_session(session);

	- } else if (safe_str_eq(op, "request")) {
	+ } else if (safe_str_eq(op, LRMD_IPC_OP_REQUEST)) {
	int flags = 0;
	xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
	const char *name = crm_element_value(msg, F_LRMD_IPC_CLIENT);
	remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);

	CRM_CHECK(request != NULL, return);

	if (proxy == NULL) {
	/* proxy connection no longer exists */
	remote_proxy_notify_destroy(lrmd, session);
	return;
	} else if ((proxy->is_local == FALSE) && (crm_ipc_connected(proxy->ipc) == FALSE)) {
	remote_proxy_end_session(session);
	return;
	}
	proxy->last_request_id = 0;
	crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
	crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");

	#if ENABLE_ACL
	CRM_ASSERT(options.node_name);
	crm_acl_get_set_user(request, F_LRMD_IPC_USER, options.node_name);
	#endif

	if (is_set(flags, crm_ipc_proxied)) {
	int rc = crm_ipc_send(proxy->ipc, request, flags, 5000, NULL);

	if(rc < 0) {
	xmlNode *op_reply = create_xml_node(NULL, "nack");

	crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);

	/* Send a n'ack so the caller doesn't block */
	crm_xml_add(op_reply, "function", __FUNCTION__);
	crm_xml_add_int(op_reply, "line", __LINE__);
	crm_xml_add_int(op_reply, "rc", rc);
	remote_proxy_relay_response(lrmd, session, op_reply, msg_id);
	free_xml(op_reply);

	} else {
	crm_trace("Relayed %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
	proxy->last_request_id = msg_id;
	}
	}
	} else {
	crm_err("Unknown proxy operation: %s", op);
	}
	}

	int
	main(int argc, char **argv)
	{
	int option_index = 0;
	int argerr = 0;
	int flag;
	char *key = NULL;
	char *val = NULL;
	gboolean use_tls = FALSE;
	crm_trigger_t *trig;

	crm_set_options(NULL, "mode [options]", long_options,
	"Inject commands into the lrmd and watch for events\n");

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch (flag) {
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'V':
	options.verbose = 1;
	break;
	case 'Q':
	options.quiet = 1;
	options.verbose = 0;
	break;
	case 'n':
	options.node_name = optarg;
	break;
	case 'c':
	options.api_call = optarg;
	break;
	case 'a':
	options.action = optarg;
	break;
	case 'r':
	options.rsc_id = optarg;
	break;
	case 'P':
	options.provider = optarg;
	break;
	case 'C':
	options.class = optarg;
	break;
	case 'T':
	options.type = optarg;
	break;
	case 't':
	if(optarg) {
	options.timeout = atoi(optarg);
	}
	break;
	case 'k':
	key = optarg;
	if (key && val) {
	options.params = lrmd_key_value_add(options.params, key, val);
	key = val = NULL;
	}
	break;
	case 'v':
	val = optarg;
	if (key && val) {
	options.params = lrmd_key_value_add(options.params, key, val);
	key = val = NULL;
	}
	break;
	case 'S':
	options.tls_host = optarg;
	use_tls = TRUE;
	break;
	case 'p':
	if(optarg) {
	options.port = atoi(optarg);
	}
	use_tls = TRUE;
	break;
	default:
	++argerr;
	break;
	}
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}
	if (optind > argc) {
	++argerr;
	}
	crm_log_init("remote_client", LOG_INFO, FALSE, options.verbose ? TRUE : FALSE, argc, argv, FALSE);

	/* if we can't perform an api_call or listen for events,
	* there is nothing to do */
	if (!options.api_call ) {
	crm_err("Nothing to be done. Please specify 'api-call'\n");
	return PCMK_OCF_UNKNOWN_ERROR;
	}

	if (!options.timeout ) {
	options.timeout = 20000;
	}

	if (use_tls) {
	if (options.node_name == NULL) {
	crm_err("\"node\" option required when tls is in use.\n");
	return PCMK_OCF_UNKNOWN_ERROR;
	}
	proxy_table =
	g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, remote_proxy_free);
	lrmd_conn = lrmd_remote_api_new(NULL, options.tls_host ? options.tls_host : "localhost", options.port);
	lrmd_internal_set_proxy_callback(lrmd_conn, NULL, remote_proxy_cb);
	} else {
	lrmd_conn = lrmd_api_new();
	}
	trig = mainloop_add_trigger(G_PRIORITY_HIGH, client_start, NULL);
	mainloop_set_trigger(trig);
	mainloop_add_signal(SIGTERM, client_shutdown);

	mainloop = g_main_new(FALSE);
	g_main_run(mainloop);

	client_exit(0);
	return 0;
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:32 PM (12 h, 8 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002680
Default Alt Text: (74 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions