No OneTemporary
Actions

Size

52 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
	index fcd73176b3..1e81742c06 100644
	--- a/crmd/remote_lrmd_ra.c
	+++ b/crmd/remote_lrmd_ra.c
	@@ -1,689 +1,775 @@
	/*
	* Copyright (C) 2013 David Vossel <dvossel@redhat.com>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>

	#define REMOTE_LRMD_RA "remote"

	/* The max start timeout before cmd retry */
	#define MAX_START_TIMEOUT_MS 10000

	typedef struct remote_ra_cmd_s {
	/! the local node the cmd is issued from /
	char *owner;
	/! the remote node the cmd is executed on /
	char *rsc_id;
	/! the action to execute /
	char *action;
	/! some string the client wants us to give it back /
	char *userdata;
	/! start delay in ms /
	int start_delay;
	/! timer id used for start delay. /
	int delay_id;
	/! timeout in ms for cmd /
	int timeout;
	int remaining_timeout;
	/! recurring interval in ms /
	int interval;
	/! interval timer id /
	int interval_id;
	int reported_success;
	int monitor_timeout_id;
	+ int takeover_timeout_id;
	/! action parameters /
	lrmd_key_value_t *params;
	/! executed rc /
	int rc;
	int op_status;
	int call_id;
	time_t start_time;
	gboolean cancel;
	} remote_ra_cmd_t;

	+enum remote_migration_status {
	+ expect_takeover = 1,
	+ takeover_complete,
	+};
	+
	typedef struct remote_ra_data_s {
	crm_trigger_t *work;
	remote_ra_cmd_t *cur_cmd;
	GList *cmds;
	GList *recurring_cmds;

	+ enum remote_migration_status migrate_status;
	} remote_ra_data_t;

	static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
	+static void handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd);

	static void
	free_cmd(gpointer user_data)
	{
	remote_ra_cmd_t *cmd = user_data;

	if (!cmd) {
	return;
	}
	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}
	if (cmd->interval_id) {
	g_source_remove(cmd->interval_id);
	}
	if (cmd->monitor_timeout_id) {
	g_source_remove(cmd->monitor_timeout_id);
	}
	+ if (cmd->takeover_timeout_id) {
	+ g_source_remove(cmd->takeover_timeout_id);
	+ }
	free(cmd->owner);
	free(cmd->rsc_id);
	free(cmd->action);
	free(cmd->userdata);
	lrmd_key_value_freeall(cmd->params);
	free(cmd);
	}

	static int
	generate_callid(void)
	{
	static int remote_ra_callid = 0;

	remote_ra_callid++;
	if (remote_ra_callid <= 0) {
	remote_ra_callid = 1;
	}

	return remote_ra_callid;
	}

	static gboolean
	recurring_helper(gpointer data)
	{
	remote_ra_cmd_t *cmd = data;
	lrm_state_t *connection_rsc = NULL;

	cmd->interval_id = 0;
	connection_rsc = lrm_state_find(cmd->rsc_id);
	if (connection_rsc && connection_rsc->remote_ra_data) {
	remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;

	ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);

	cmd->call_id = generate_callid();

	ra_data->cmds = g_list_append(ra_data->cmds, cmd);
	mainloop_set_trigger(ra_data->work);
	}
	return FALSE;
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	remote_ra_cmd_t *cmd = data;
	lrm_state_t *connection_rsc = NULL;

	cmd->delay_id = 0;
	connection_rsc = lrm_state_find(cmd->rsc_id);
	if (connection_rsc && connection_rsc->remote_ra_data) {
	remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;

	mainloop_set_trigger(ra_data->work);
	}
	return FALSE;
	}

	static void
	report_remote_ra_result(remote_ra_cmd_t * cmd)
	{
	lrmd_event_data_t op = { 0, };

	op.type = lrmd_event_exec_complete;
	op.rsc_id = cmd->rsc_id;
	op.op_type = cmd->action;
	op.user_data = cmd->userdata;
	op.timeout = cmd->timeout;
	op.interval = cmd->interval;
	op.rc = cmd->rc;
	op.op_status = cmd->op_status;
	if (cmd->params) {
	lrmd_key_value_t *tmp;

	op.params = g_hash_table_new_full(crm_str_hash,
	g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
	for (tmp = cmd->params; tmp; tmp = tmp->next) {
	g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
	}

	}
	op.call_id = cmd->call_id;
	op.remote_nodename = cmd->owner;

	lrm_op_callback(&op);

	if (op.params) {
	g_hash_table_destroy(op.params);
	}
	}

	static void
	update_remaining_timeout(remote_ra_cmd_t * cmd)
	{
	cmd->remaining_timeout = ((cmd->timeout / 1000) - (time(NULL) - cmd->start_time)) * 1000;
	}

	static gboolean
	retry_start_cmd_cb(gpointer data)
	{
	lrm_state_t *lrm_state = data;
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	remote_ra_cmd_t *cmd = NULL;
	int rc = -1;

	if (!ra_data \|\| !ra_data->cur_cmd) {
	return FALSE;
	}
	cmd = ra_data->cur_cmd;
	if (safe_str_neq(cmd->action, "start")) {
	return FALSE;
	}
	update_remaining_timeout(cmd);

	if (cmd->remaining_timeout > 0) {
	rc = handle_remote_ra_start(lrm_state, cmd, cmd->remaining_timeout);
	}

	if (rc != 0) {
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	report_remote_ra_result(cmd);

	if (ra_data->cmds) {
	mainloop_set_trigger(ra_data->work);
	}
	ra_data->cur_cmd = NULL;
	free_cmd(cmd);
	} else {
	/* wait for connection event */
	}

	return FALSE;
	}

	+
	+static gboolean
	+connection_takeover_timeout_cb(gpointer data)
	+{
	+ lrm_state_t *lrm_state = NULL;
	+ remote_ra_cmd_t *cmd = data;
	+
	+ crm_debug("takeover event timed out for node %s", cmd->rsc_id);
	+ cmd->takeover_timeout_id = 0;
	+
	+ handle_remote_ra_stop(lrm_state, cmd);
	+ free_cmd(cmd);
	+
	+ return FALSE;
	+}
	+
	static gboolean
	monitor_timeout_cb(gpointer data)
	{
	lrm_state_t *lrm_state = NULL;
	remote_ra_cmd_t *cmd = data;

	crm_debug("Poke async response timed out for node %s", cmd->rsc_id);
	cmd->monitor_timeout_id = 0;
	cmd->op_status = PCMK_LRM_OP_TIMEOUT;
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;

	lrm_state = lrm_state_find(cmd->rsc_id);
	if (lrm_state && lrm_state->remote_ra_data) {
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;

	if (ra_data->cur_cmd == cmd) {
	ra_data->cur_cmd = NULL;
	}
	if (ra_data->cmds) {
	mainloop_set_trigger(ra_data->work);
	}
	}

	report_remote_ra_result(cmd);
	free_cmd(cmd);
	return FALSE;
	}

	xmlNode *
	simple_remote_node_status(const char node_name, xmlNode parent, const char *source)
	{
	xmlNode *state = create_xml_node(parent, XML_CIB_TAG_STATE);

	crm_xml_add(state, XML_NODE_IS_REMOTE, "true");
	crm_xml_add(state, XML_ATTR_UUID, node_name);
	crm_xml_add(state, XML_ATTR_UNAME, node_name);
	crm_xml_add(state, XML_ATTR_ORIGIN, source);

	return state;
	}

	static void
	remote_init_cib_status(const char *node_name)
	{
	int call_id = 0;
	int call_opt = cib_quorum_override;
	xmlNode *update = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	simple_remote_node_status(node_name, update,__FUNCTION__);

	if (fsa_state == S_ELECTION \|\| fsa_state == S_PENDING) {
	call_opt \|= cib_scope_local;
	}

	fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
	if (call_id != pcmk_ok) {
	crm_debug("Failed to init status section for remote-node %s", node_name);
	}
	free_xml(update);
	}

	void
	remote_lrm_op_callback(lrmd_event_data_t * op)
	{
	lrm_state_t *lrm_state = NULL;
	remote_ra_data_t *ra_data = NULL;
	remote_ra_cmd_t *cmd = NULL;

	crm_debug("remote connection event - event_type:%s node:%s action:%s rc:%s op_status:%s",
	lrmd_event_type2str(op->type),
	op->remote_nodename,
	op->op_type ? op->op_type : "none",
	services_ocf_exitcode_str(op->rc), services_lrm_status_str(op->op_status));

	- /* filter all EXEC events up */
	- if (op->type == lrmd_event_exec_complete) {
	- lrm_op_callback(op);
	- return;
	- }
	-
	lrm_state = lrm_state_find(op->remote_nodename);
	if (!lrm_state \|\| !lrm_state->remote_ra_data) {
	crm_debug("lrm_state info not found for remote lrmd connection event");
	return;
	}
	-
	ra_data = lrm_state->remote_ra_data;
	+
	+ /* Another client has connected to the remote daemon,
	+ * determine if this is expected. */
	+ if (op->type == lrmd_event_new_client) {
	+ /* great, we new this was coming */
	+ if (ra_data->migrate_status == expect_takeover) {
	+ ra_data->migrate_status = takeover_complete;
	+ } else {
	+ crm_err("Unexpected pacemaker_remote client takeover. Disconnecting");
	+ lrm_state_disconnect(lrm_state);
	+ }
	+ return;
	+ }
	+
	+ /* filter all EXEC events up */
	+ if (op->type == lrmd_event_exec_complete) {
	+ if (ra_data->migrate_status == takeover_complete) {
	+ crm_debug("ignoring event, this connection is taken over by another node");
	+ } else {
	+ lrm_op_callback(op);
	+ }
	+ return;
	+ }
	+
	if (!ra_data->cur_cmd) {
	crm_debug("no event to match");
	return;
	}
	cmd = ra_data->cur_cmd;

	/* Start actions and migrate from actions complete after connection
	* comes back to us. */
	if (op->type == lrmd_event_connect && (safe_str_eq(cmd->action, "start") \|\|
	safe_str_eq(cmd->action, "migrate_from"))) {

	if (op->connection_rc < 0) {
	update_remaining_timeout(cmd);
	/* There isn't much of a reason to reschedule if the timeout is too small */
	if (cmd->remaining_timeout > 3000) {
	crm_trace("rescheduling start, remaining timeout %d", cmd->remaining_timeout);
	g_timeout_add(1000, retry_start_cmd_cb, lrm_state);
	return;
	} else {
	crm_trace("can't reschedule start, remaining timeout too small %d",
	cmd->remaining_timeout);
	}
	cmd->op_status = PCMK_LRM_OP_TIMEOUT;
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;

	} else {
	- /* make sure we have a clean status section to start with */
	lrm_state_reset_tables(lrm_state);
	- remote_init_cib_status(lrm_state->node_name);
	- erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);
	- erase_status_tag(lrm_state->node_name, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
	-
	+ /* make sure we have a clean status section to start with */
	+ if (safe_str_eq(cmd->action, "start")) {
	+ remote_init_cib_status(lrm_state->node_name);
	+ erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);
	+ erase_status_tag(lrm_state->node_name, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
	+ }
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;
	}

	crm_debug("remote lrmd connect event matched %s action. ", cmd->action);
	report_remote_ra_result(cmd);
	if (ra_data->cmds) {
	mainloop_set_trigger(ra_data->work);
	}
	ra_data->cur_cmd = NULL;
	free_cmd(cmd);
	return;
	} else if (op->type == lrmd_event_poke && safe_str_eq(cmd->action, "monitor")) {

	if (cmd->monitor_timeout_id) {
	g_source_remove(cmd->monitor_timeout_id);
	cmd->monitor_timeout_id = 0;
	}

	/* Only report success the first time, after that only worry about failures.
	* For this function, if we get the poke pack, it is always a success. Pokes
	* only fail if the send fails, or the response times out. */
	if (!cmd->reported_success) {
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;
	report_remote_ra_result(cmd);
	cmd->reported_success = 1;
	}

	crm_debug("remote lrmd poke event matched %s action. ", cmd->action);

	/* success, keep rescheduling if interval is present. */
	if (cmd->interval && (cmd->cancel == FALSE)) {
	ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
	cmd->interval_id = g_timeout_add(cmd->interval, recurring_helper, cmd);
	cmd = NULL; /* prevent free */
	}

	if (ra_data->cmds) {
	mainloop_set_trigger(ra_data->work);
	}
	ra_data->cur_cmd = NULL;
	free_cmd(cmd);
	return;
	+ } else if (op->type == lrmd_event_new_client && safe_str_eq(cmd->action, "stop")) {
	+
	+ if (cmd->takeover_timeout_id) {
	+ g_source_remove(cmd->takeover_timeout_id);
	+ cmd->takeover_timeout_id = 0;
	+ }
	+
	+ handle_remote_ra_stop(lrm_state, cmd);
	+
	+ if (ra_data->cmds) {
	+ mainloop_set_trigger(ra_data->work);
	+ }
	+ ra_data->cur_cmd = NULL;
	+ free_cmd(cmd);
	}

	crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
	}

	+static void
	+handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd)
	+{
	+ remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	+
	+ if (ra_data->migrate_status != takeover_complete) {
	+ /* only clear the status if this stop is not apart of a successful migration */
	+ update_attrd_remote_node_removed(lrm_state->node_name, NULL);
	+ }
	+
	+ lrm_state_disconnect(lrm_state);
	+ cmd->rc = PCMK_OCF_OK;
	+ cmd->op_status = PCMK_LRM_OP_DONE;
	+
	+ if (ra_data->cmds) {
	+ g_list_free_full(ra_data->cmds, free_cmd);
	+ }
	+ if (ra_data->recurring_cmds) {
	+ g_list_free_full(ra_data->recurring_cmds, free_cmd);
	+ }
	+ ra_data->cmds = NULL;
	+ ra_data->recurring_cmds = NULL;
	+ ra_data->cur_cmd = NULL;
	+
	+ report_remote_ra_result(cmd);
	+}
	+
	static int
	handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms)
	{
	const char *server = NULL;
	lrmd_key_value_t *tmp = NULL;
	int port = 0;
	int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;

	for (tmp = cmd->params; tmp; tmp = tmp->next) {
	const char *key = tmp->key;
	/* skip over 'remote-' prefix if it exists */
	if (strstr(key, "remote-")) {
	key += 7;
	}

	if (safe_str_eq(tmp->key, "addr") \|\| safe_str_eq(tmp->key, "server")) {
	server = tmp->value;
	}
	if (safe_str_eq(tmp->key, "port")) {
	port = atoi(tmp->value);
	}
	}

	return lrm_state_remote_connect_async(lrm_state, server, port, timeout_used);
	}

	static gboolean
	handle_remote_ra_exec(gpointer user_data)
	{
	int rc = 0;
	lrm_state_t *lrm_state = user_data;
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
	remote_ra_cmd_t *cmd;
	GList *first = NULL;

	if (ra_data->cur_cmd) {
	/* still waiting on previous cmd */
	return TRUE;
	}

	while (ra_data->cmds) {
	first = ra_data->cmds;
	cmd = first->data;
	if (cmd->delay_id) {
	/* still waiting for start delay timer to trip */
	return TRUE;
	}

	ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
	g_list_free_1(first);

	if (!strcmp(cmd->action, "start") \|\| !strcmp(cmd->action, "migrate_from")) {
	-
	+ ra_data->migrate_status = 0;
	rc = handle_remote_ra_start(lrm_state, cmd, cmd->timeout);
	if (rc == 0) {
	/* take care of this later when we get async connection result */
	crm_debug("began remote lrmd connect, waiting for connect event.");
	ra_data->cur_cmd = cmd;
	return TRUE;
	} else {
	crm_debug("connect failed, not expecting to match any connection event later");
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	}
	report_remote_ra_result(cmd);

	} else if (!strcmp(cmd->action, "monitor")) {

	if (lrm_state_is_connected(lrm_state) == TRUE) {
	rc = lrm_state_poke_connection(lrm_state);
	if (rc < 0) {
	cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
	cmd->op_status = PCMK_LRM_OP_ERROR;
	}
	} else {
	rc = -1;
	cmd->op_status = PCMK_LRM_OP_DONE;
	cmd->rc = PCMK_OCF_NOT_RUNNING;
	}

	if (rc == 0) {
	crm_debug("poked remote lrmd at node %s, waiting for async response.", cmd->rsc_id);
	ra_data->cur_cmd = cmd;
	cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
	return TRUE;
	}
	report_remote_ra_result(cmd);

	} else if (!strcmp(cmd->action, "stop")) {
	- lrm_state_disconnect(lrm_state);
	- update_attrd_remote_node_removed(lrm_state->node_name, NULL);
	- cmd->rc = PCMK_OCF_OK;
	- cmd->op_status = PCMK_LRM_OP_DONE;

	- if (ra_data->cmds) {
	- g_list_free_full(ra_data->cmds, free_cmd);
	- }
	- if (ra_data->recurring_cmds) {
	- g_list_free_full(ra_data->recurring_cmds, free_cmd);
	+ if (ra_data->migrate_status == expect_takeover) {
	+ /* briefly wait on stop for the takeover event to occur. If the
	+ * takeover event does not occur during the wait period, that's fine.
	+ * It just means that the remote-node's lrm_status section is going to get
	+ * cleared which will require all the resources running in the remote-node
	+ * to be explicitly re-detected via probe actions. If the takeover does occur
	+ * successfully, then we can leave the status section intact. */
	+ cmd->monitor_timeout_id = g_timeout_add((cmd->timeout/2), connection_takeover_timeout_cb, cmd);
	+ ra_data->cur_cmd = cmd;
	+ return TRUE;
	}
	- ra_data->cmds = NULL;
	- ra_data->recurring_cmds = NULL;
	- report_remote_ra_result(cmd);
	+
	+ handle_remote_ra_stop(lrm_state, cmd);

	} else if (!strcmp(cmd->action, "migrate_to")) {
	- /* no-op. */
	+ ra_data->migrate_status = expect_takeover;
	cmd->rc = PCMK_OCF_OK;
	cmd->op_status = PCMK_LRM_OP_DONE;
	report_remote_ra_result(cmd);
	}

	free_cmd(cmd);
	}

	return TRUE;
	}

	static void
	remote_ra_data_init(lrm_state_t * lrm_state)
	{
	remote_ra_data_t *ra_data = NULL;

	if (lrm_state->remote_ra_data) {
	return;
	}

	ra_data = calloc(1, sizeof(remote_ra_data_t));
	ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
	lrm_state->remote_ra_data = ra_data;
	}

	void
	remote_ra_cleanup(lrm_state_t * lrm_state)
	{
	remote_ra_data_t *ra_data = lrm_state->remote_ra_data;

	if (!ra_data) {
	return;
	}

	if (ra_data->cmds) {
	g_list_free_full(ra_data->cmds, free_cmd);
	}

	if (ra_data->recurring_cmds) {
	g_list_free_full(ra_data->recurring_cmds, free_cmd);
	}
	mainloop_destroy_trigger(ra_data->work);
	free(ra_data);
	lrm_state->remote_ra_data = NULL;
	}

	gboolean
	is_remote_lrmd_ra(const char agent, const char provider, const char *id)
	{
	if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
	return TRUE;
	}
	if (id && lrm_state_find(id)) {
	return TRUE;
	}

	return FALSE;
	}

	lrmd_rsc_info_t *
	remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id)
	{
	lrmd_rsc_info_t *info = NULL;

	if ((lrm_state_find(rsc_id))) {
	info = calloc(1, sizeof(lrmd_rsc_info_t));

	info->id = strdup(rsc_id);
	info->type = strdup(REMOTE_LRMD_RA);
	info->class = strdup("ocf");
	info->provider = strdup("pacemaker");
	}

	return info;
	}

	static gboolean
	is_remote_ra_supported_action(const char *action)
	{
	if (!action) {
	return FALSE;
	} else if (strcmp(action, "start") &&
	strcmp(action, "stop") &&
	strcmp(action, "migrate_to") &&
	strcmp(action, "migrate_from") && strcmp(action, "monitor")) {
	return FALSE;
	}

	return TRUE;
	}

	static GList *
	remove_cmd(GList * list, const char *action, int interval)
	{
	remote_ra_cmd_t *cmd = NULL;
	GListPtr gIter = NULL;

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	cmd = gIter->data;
	if (cmd->interval == interval && safe_str_eq(cmd->action, action)) {
	break;
	}
	cmd = NULL;
	}
	if (cmd) {
	list = g_list_remove(list, cmd);
	free_cmd(cmd);
	}
	return list;
	}

	int
	remote_ra_cancel(lrm_state_t * lrm_state, const char rsc_id, const char action, int interval)
	{
	lrm_state_t *connection_rsc = NULL;
	remote_ra_data_t *ra_data = NULL;

	connection_rsc = lrm_state_find(rsc_id);
	if (!connection_rsc \|\| !connection_rsc->remote_ra_data) {
	return -EINVAL;
	}

	ra_data = connection_rsc->remote_ra_data;
	ra_data->cmds = remove_cmd(ra_data->cmds, action, interval);
	ra_data->recurring_cmds = remove_cmd(ra_data->recurring_cmds, action, interval);
	if (ra_data->cur_cmd &&
	(ra_data->cur_cmd->interval == interval) &&
	(safe_str_eq(ra_data->cur_cmd->action, action))) {

	ra_data->cur_cmd->cancel = TRUE;
	}

	return 0;
	}

	int
	remote_ra_exec(lrm_state_t * lrm_state, const char rsc_id, const char action, const char userdata, int interval, / ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	lrmd_key_value_t * params)
	{
	int rc = 0;
	lrm_state_t *connection_rsc = NULL;
	remote_ra_cmd_t *cmd = NULL;
	remote_ra_data_t *ra_data = NULL;

	if (is_remote_ra_supported_action(action) == FALSE) {
	rc = -EINVAL;
	goto exec_done;
	}

	connection_rsc = lrm_state_find(rsc_id);
	if (!connection_rsc) {
	rc = -EINVAL;
	goto exec_done;
	}

	remote_ra_data_init(connection_rsc);

	cmd = calloc(1, sizeof(remote_ra_cmd_t));
	cmd->owner = strdup(lrm_state->node_name);
	cmd->rsc_id = strdup(rsc_id);
	cmd->action = strdup(action);
	cmd->userdata = strdup(userdata);
	cmd->interval = interval;
	cmd->timeout = timeout;
	cmd->start_delay = start_delay;
	cmd->params = params;
	cmd->start_time = time(NULL);

	cmd->call_id = generate_callid();

	if (cmd->start_delay) {
	cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
	}
	ra_data = connection_rsc->remote_ra_data;

	ra_data->cmds = g_list_append(ra_data->cmds, cmd);
	mainloop_set_trigger(ra_data->work);

	return cmd->call_id;
	exec_done:

	lrmd_key_value_freeall(params);
	return rc;
	}
	diff --git a/include/crm/lrmd.h b/include/crm/lrmd.h
	index ae76c18787..e2e4b04908 100644
	--- a/include/crm/lrmd.h
	+++ b/include/crm/lrmd.h
	@@ -1,461 +1,463 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	/**
	* \file
	* \brief Local Resource Manager
	* \ingroup lrm
	*/
	#include <crm/services.h>

	#ifndef LRMD__H
	# define LRMD__H

	typedef struct lrmd_s lrmd_t;
	typedef struct lrmd_key_value_s {
	char *key;
	char *value;
	struct lrmd_key_value_s *next;
	} lrmd_key_value_t;

	#define LRMD_PROTOCOL_VERSION "1.0"

	/* INDENT-OFF */
	#define DEFAULT_REMOTE_KEY_LOCATION "/etc/pacemaker/authkey"
	#define ALT_REMOTE_KEY_LOCATION "/etc/corosync/authkey"
	#define DEFAULT_REMOTE_PORT 3121
	#define DEFAULT_REMOTE_USERNAME "lrmd"

	#define F_LRMD_OPERATION "lrmd_op"
	#define F_LRMD_CLIENTNAME "lrmd_clientname"
	#define F_LRMD_IS_IPC_PROVIDER "lrmd_is_ipc_provider"
	#define F_LRMD_CLIENTID "lrmd_clientid"
	#define F_LRMD_PROTOCOL_VERSION "lrmd_protocol_version"
	#define F_LRMD_REMOTE_MSG_TYPE "lrmd_remote_msg_type"
	#define F_LRMD_REMOTE_MSG_ID "lrmd_remote_msg_id"
	#define F_LRMD_CALLBACK_TOKEN "lrmd_async_id"
	#define F_LRMD_CALLID "lrmd_callid"
	#define F_LRMD_CANCEL_CALLID "lrmd_cancel_callid"
	#define F_LRMD_CALLOPTS "lrmd_callopt"
	#define F_LRMD_CALLDATA "lrmd_calldata"
	#define F_LRMD_RC "lrmd_rc"
	#define F_LRMD_EXEC_RC "lrmd_exec_rc"
	#define F_LRMD_OP_STATUS "lrmd_exec_op_status"
	#define F_LRMD_TIMEOUT "lrmd_timeout"
	#define F_LRMD_CLASS "lrmd_class"
	#define F_LRMD_PROVIDER "lrmd_provider"
	#define F_LRMD_TYPE "lrmd_type"
	#define F_LRMD_ORIGIN "lrmd_origin"

	#define F_LRMD_RSC_RUN_TIME "lrmd_run_time"
	#define F_LRMD_RSC_RCCHANGE_TIME "lrmd_rcchange_time"
	#define F_LRMD_RSC_EXEC_TIME "lrmd_exec_time"
	#define F_LRMD_RSC_QUEUE_TIME "lrmd_queue_time"

	#define F_LRMD_RSC_ID "lrmd_rsc_id"
	#define F_LRMD_RSC_ACTION "lrmd_rsc_action"
	#define F_LRMD_RSC_USERDATA_STR "lrmd_rsc_userdata_str"
	#define F_LRMD_RSC_OUTPUT "lrmd_rsc_output"
	#define F_LRMD_RSC_START_DELAY "lrmd_rsc_start_delay"
	#define F_LRMD_RSC_INTERVAL "lrmd_rsc_interval"
	#define F_LRMD_RSC_METADATA "lrmd_rsc_metadata_res"
	#define F_LRMD_RSC_DELETED "lrmd_rsc_deleted"
	#define F_LRMD_RSC "lrmd_rsc"

	#define LRMD_OP_RSC_CHK_REG "lrmd_rsc_check_register"
	#define LRMD_OP_RSC_REG "lrmd_rsc_register"
	#define LRMD_OP_RSC_EXEC "lrmd_rsc_exec"
	#define LRMD_OP_RSC_CANCEL "lrmd_rsc_cancel"
	#define LRMD_OP_RSC_UNREG "lrmd_rsc_unregister"
	#define LRMD_OP_RSC_INFO "lrmd_rsc_info"
	#define LRMD_OP_RSC_METADATA "lrmd_rsc_metadata"
	#define LRMD_OP_POKE "lrmd_rsc_poke"
	#define LRMD_OP_NEW_CLIENT "lrmd_rsc_new_client"

	#define F_LRMD_IPC_OP "lrmd_ipc_op"
	#define F_LRMD_IPC_IPC_SERVER "lrmd_ipc_server"
	#define F_LRMD_IPC_SESSION "lrmd_ipc_session"
	#define F_LRMD_IPC_CLIENT "lrmd_ipc_client"
	#define F_LRMD_IPC_PROXY_NODE "lrmd_ipc_proxy_node"
	#define F_LRMD_IPC_USER "lrmd_ipc_user"
	#define F_LRMD_IPC_MSG "lrmd_ipc_msg"
	#define F_LRMD_IPC_MSG_ID "lrmd_ipc_msg_id"
	#define F_LRMD_IPC_MSG_FLAGS "lrmd_ipc_msg_flags"

	#define T_LRMD "lrmd"
	#define T_LRMD_REPLY "lrmd_reply"
	#define T_LRMD_NOTIFY "lrmd_notify"
	#define T_LRMD_IPC_PROXY "lrmd_ipc_proxy"
	/* INDENT-ON */

	/*!
	* \brief Create a new local lrmd connection
	*/
	lrmd_t *lrmd_api_new(void);

	/*!
	* \brief Create a new remote lrmd connection using tls backend
	*
	* \note nodename and server may be the same value.
	*
	* \param nodename, the remote node name identified with this connection.
	* \param server, the server to connect to.
	* \param port, the port to connect to.
	*/
	lrmd_t lrmd_remote_api_new(const char nodename, const char *server, int port);

	/*!
	* \brief Use after lrmd_poll returns 1.
	*
	* \param fd to poll on
	* \param timeout in ms
	*
	* \retval true - connection is still up
	* \retval false - disconnected
	*/
	bool lrmd_dispatch(lrmd_t * lrmd);

	/*!
	* \brief Poll for a specified timeout period to determine if a message
	* is ready for dispatch.
	* \retval 1 msg is ready
	* \retval 0 timeout occured
	* \retval negative error code
	*/
	int lrmd_poll(lrmd_t * lrmd, int timeout);

	/*!
	* \brief Destroy lrmd object
	*/
	void lrmd_api_delete(lrmd_t * lrmd);
	lrmd_key_value_t lrmd_key_value_add(lrmd_key_value_t kvp, const char key, const char value);

	/* INDENT-OFF */
	/* Reserved for future use */
	enum lrmd_call_options {
	lrmd_opt_none = 0x00000000,
	/* lrmd_opt_sync_call = 0x00000001, //Not implemented, patches welcome. */
	/! Only notify the client originating a exec() the results /
	lrmd_opt_notify_orig_only = 0x00000002,
	/*! Drop recurring operations initiated by a client when client disconnects.
	- * This call_option is only valid when registering a resource. */
	+ * This call_option is only valid when registering a resource. When used
	+ * remotely with the pacemaker_remote daemon, this option means that recurring
	+ * operations will be dropped once all the remote connections disconnect. */
	lrmd_opt_drop_recurring = 0x00000003,
	/! Only send out notifications for recurring operations whenthe result changes /
	lrmd_opt_notify_changes_only = 0x00000004,
	};

	enum lrmd_callback_event {
	lrmd_event_register,
	lrmd_event_unregister,
	lrmd_event_exec_complete,
	lrmd_event_disconnect,
	lrmd_event_connect,
	lrmd_event_poke,
	lrmd_event_new_client,
	};

	/* INDENT-ON */

	typedef struct lrmd_event_data_s {
	/! Type of event, register, unregister, call_completed... /
	enum lrmd_callback_event type;

	/! The resource this event occurred on. /
	const char *rsc_id;
	/! The action performed, start, stop, monitor... /
	const char *op_type;
	/! The userdata string given do exec() api function /
	const char *user_data;

	/! The client api call id associated with this event /
	int call_id;
	/! The operation's timeout period in ms. /
	int timeout;
	/! The operation's recurring interval in ms. /
	int interval;
	/! The operation's start delay value in ms. /
	int start_delay;
	/! This operation that just completed is on a deleted rsc. /
	int rsc_deleted;

	/! The executed ra return code mapped to OCF /
	enum ocf_exitcode rc;
	/! The lrmd status returned for exec_complete events /
	int op_status;
	/! stdout from resource agent operation /
	const char *output;
	/! Timestamp of when op ran /
	unsigned int t_run;
	/! Timestamp of last rc change /
	unsigned int t_rcchange;
	/! Time in length op took to execute /
	unsigned int exec_time;
	/! Time in length spent in queue /
	unsigned int queue_time;

	/! int connection result. Used for connection and poke events /
	int connection_rc;

	/* This is a GHashTable containing the
	* parameters given to the operation */
	void *params;

	/* client node name associated with this conneciton.
	* This is useful if multiple clients are being utilized by
	* a single process. This name allows the actions to be matched
	* to the proper client. */
	const char *remote_nodename;

	} lrmd_event_data_t;

	lrmd_event_data_t lrmd_copy_event(lrmd_event_data_t event);
	void lrmd_free_event(lrmd_event_data_t * event);

	typedef struct lrmd_rsc_info_s {
	char *id;
	char *type;
	char *class;
	char *provider;
	} lrmd_rsc_info_t;

	lrmd_rsc_info_t lrmd_copy_rsc_info(lrmd_rsc_info_t rsc_info);
	void lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info);

	typedef void (lrmd_event_callback) (lrmd_event_data_t event);

	typedef struct lrmd_list_s {
	const char *val;
	struct lrmd_list_s *next;
	} lrmd_list_t;

	void lrmd_list_freeall(lrmd_list_t * head);
	void lrmd_key_value_freeall(lrmd_key_value_t * head);

	typedef struct lrmd_api_operations_s {
	/*!
	* \brief Connect from the lrmd.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (connect) (lrmd_t lrmd, const char client_name, int fd);

	/*!
	* \brief Establish an connection to lrmd, don't block while connecting.
	* \note this function requires the use of mainloop.
	*
	* \note The is returned using the event callback.
	* \note When this function returns 0, the callback will be invoked
	* to report the final result of the connect.
	* \retval 0, connect in progress, wait for event callback
	* \retval -1, failure.
	*/
	int (connect_async) (lrmd_t lrmd, const char client_name, int timeout /ms */ );

	/*!
	* \brief Is connected to lrmd daemon?
	*
	* \retval 0, false
	* \retval 1, true
	*/
	int (is_connected) (lrmd_t lrmd);

	/*!
	* \brief Poke lrmd connection to verify it is still capable of serving requests
	* \note The response comes in the form of a poke event to the callback.
	*
	* \retval 0, wait for response in callback
	* \retval -1, connection failure, callback may not be invoked
	*/
	int (poke_connection) (lrmd_t lrmd);

	/*!
	* \brief Disconnect from the lrmd.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (disconnect) (lrmd_t lrmd);

	/*!
	* \brief Register a resource with the lrmd.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \retval 0, success
	* \retval negative error code on failure
	*/
	int (register_rsc) (lrmd_t lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char agent, enum lrmd_call_options options);

	/*!
	* \brief Retrieve registration info for a rsc
	*
	* \retval info on success
	* \retval NULL on failure
	*/
	lrmd_rsc_info_t (get_rsc_info) (lrmd_t * lrmd,
	const char *rsc_id, enum lrmd_call_options options);

	/*!
	* \brief Unregister a resource from the lrmd.
	*
	* \note All pending and recurring operations will be cancelled
	* automatically.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \retval 0, success
	* \retval -1, success, but operations are currently executing on the rsc which will
	* return once they are completed.
	* \retval negative error code on failure
	*
	*/
	int (unregister_rsc) (lrmd_t lrmd, const char *rsc_id, enum lrmd_call_options options);

	/*!
	* \brief Sets the callback to receive lrmd events on.
	*/
	void (set_callback) (lrmd_t lrmd, lrmd_event_callback callback);

	/*!
	* \brief Issue a command on a resource
	*
	* \note Asynchronous, command is queued in daemon on function return, but
	* execution of command is not synced.
	*
	* \note Operations on individual resources are guaranteed to occur
	* in the order the client api calls them in.
	*
	* \note Operations between different resources are not guaranteed
	* to occur in any specific order in relation to one another
	* regardless of what order the client api is called in.
	* \retval call_id to track async event result on success
	* \retval negative error code on failure
	*/
	int (exec) (lrmd_t lrmd, const char rsc_id, const char action, const char userdata, / userdata string given back in event notification */
	int interval, /* ms */
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params); /* ownership of params is given up to api here */

	/*!
	* \brief Cancel a recurring command.
	*
	* \note Synchronous, guaranteed to occur in daemon before function returns.
	*
	* \note The cancel is completed async from this call.
	* We can be guaranteed the cancel has completed once
	* the callback receives an exec_complete event with
	* the lrmd_op_status signifying that the operation is
	* cancelled.
	* \note For each resource, cancel operations and exec operations
	* are processed in the order they are received.
	* It is safe to assume that for a single resource, a cancel
	* will occur in the lrmd before an exec if the client's cancel
	* api call occurs before the exec api call.
	*
	* It is not however safe to assume any operation on one resource will
	* occur before an operation on another resource regardless of
	* the order the client api is called in.
	*
	* \retval 0, cancel command sent.
	* \retval negative error code on failure
	*/
	int (cancel) (lrmd_t lrmd, const char rsc_id, const char action, int interval);

	/*!
	* \brief Get the metadata documentation for a resource.
	*
	* \note Value is returned in output. Output must be freed when set
	*
	* \retval lrmd_ok success
	* \retval negative error code on failure
	*/
	int (get_metadata) (lrmd_t lrmd,
	const char *class,
	const char *provider,
	const char agent, char *output, enum lrmd_call_options options);

	/*!
	* \brief Retrieve a list of installed resource agents.
	*
	* \note if class is not provided, all known agents will be returned
	* \note list must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_agents) (lrmd_t lrmd, lrmd_list_t ** agents, const char *class,
	const char *provider);

	/*!
	* \brief Retrieve a list of resource agent providers
	*
	* \note When the agent is provided, only the agent's provider will be returned
	* \note When no agent is supplied, all providers will be returned.
	* \note List must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_ocf_providers) (lrmd_t lrmd, const char agent, lrmd_list_t * providers);

	/*!
	* \brief Retrieve a list of standards supported by this machine/installation
	*
	* \note List must be freed using lrmd_list_freeall()
	*
	* \retval num items in list on success
	* \retval negative error code on failure
	*/
	int (list_standards) (lrmd_t lrmd, lrmd_list_t ** standards);

	} lrmd_api_operations_t;

	struct lrmd_s {
	lrmd_api_operations_t *cmds;
	void *private;
	};

	static inline const char *
	lrmd_event_type2str(enum lrmd_callback_event type)
	{
	switch (type) {
	case lrmd_event_register:
	return "register";
	case lrmd_event_unregister:
	return "unregister";
	case lrmd_event_exec_complete:
	return "exec_complete";
	case lrmd_event_disconnect:
	return "disconnect";
	case lrmd_event_connect:
	return "connect";
	case lrmd_event_poke:
	return "poke";
	case lrmd_event_new_client:
	return "new_client";
	}
	return "unknown";
	}

	#endif
	diff --git a/lrmd/tls_backend.c b/lrmd/tls_backend.c
	index 5b57efbab7..e46ed497a6 100644
	--- a/lrmd/tls_backend.c
	+++ b/lrmd/tls_backend.c
	@@ -1,404 +1,409 @@
	/*
	* Copyright (c) 2012 David Vossel <dvossel@redhat.com>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/mainloop.h>

	#include <lrmd_private.h>

	#include <netdb.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# define LRMD_REMOTE_AUTH_TIMEOUT 10000
	gnutls_psk_server_credentials_t psk_cred_s;
	gnutls_dh_params_t dh_params;
	static int ssock = -1;
	extern int lrmd_call_id;

	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}

	static int
	lrmd_remote_client_msg(gpointer data)
	{
	int id = 0;
	int rc = 0;
	int disconnected = 0;
	xmlNode *request = NULL;
	crm_client_t *client = data;

	if (client->remote->tls_handshake_complete == FALSE) {
	int rc = 0;

	/* Muliple calls to handshake will be required, this callback
	* will be invoked once the client sends more handshake data. */
	do {
	rc = gnutls_handshake(*client->remote->tls_session);

	if (rc < 0 && rc != GNUTLS_E_AGAIN) {
	crm_err("Remote lrmd tls handshake failed");
	return -1;
	}
	} while (rc == GNUTLS_E_INTERRUPTED);

	if (rc == 0) {
	crm_debug("Remote lrmd tls handshake completed");
	client->remote->tls_handshake_complete = TRUE;
	if (client->remote->auth_timeout) {
	g_source_remove(client->remote->auth_timeout);
	}
	client->remote->auth_timeout = 0;
	}
	return 0;
	}

	rc = crm_remote_ready(client->remote, 0);
	if (rc == 0) {
	/* no msg to read */
	return 0;
	} else if (rc < 0) {
	crm_info("Client disconnected during remote client read");
	return -1;
	}

	crm_remote_recv(client->remote, -1, &disconnected);

	request = crm_remote_parse_buffer(client->remote);
	while (request) {
	crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
	crm_trace("processing request from remote client with remote msg id %d", id);
	if (!client->name) {
	const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);

	if (value) {
	client->name = strdup(value);
	}
	}

	lrmd_call_id++;
	if (lrmd_call_id < 1) {
	lrmd_call_id = 1;
	}

	crm_xml_add(request, F_LRMD_CLIENTID, client->id);
	crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
	crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);

	process_lrmd_message(client, id, request);
	free_xml(request);

	/* process all the messages in the current buffer */
	request = crm_remote_parse_buffer(client->remote);
	}

	if (disconnected) {
	crm_info("Client disconnect detected in tls msg dispatcher.");
	return -1;
	}

	return 0;
	}

	static void
	lrmd_remote_client_destroy(gpointer user_data)
	{
	crm_client_t *client = user_data;

	if (client == NULL) {
	return;
	}

	ipc_proxy_remove_provider(client);
	- client_disconnect_cleanup(client->id);
	+
	+ /* if this is the last remote connection, stop recurring
	+ * operations */
	+ if (crm_hash_table_size(client_connections) == 1) {
	+ client_disconnect_cleanup(NULL);
	+ }

	crm_notice("LRMD client disconnecting remote client - name: %s id: %s",
	client->name ? client->name : "<unknown>", client->id);

	if (client->remote->tls_session) {
	void *sock_ptr;
	int csock;

	sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
	csock = GPOINTER_TO_INT(sock_ptr);

	gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*client->remote->tls_session);
	gnutls_free(client->remote->tls_session);
	close(csock);
	}

	crm_client_destroy(client);

	return;
	}

	static gboolean
	lrmd_auth_timeout_cb(gpointer data)
	{
	crm_client_t *client = data;

	client->remote->auth_timeout = 0;

	if (client->remote->tls_handshake_complete == TRUE) {
	return FALSE;
	}

	mainloop_del_fd(client->remote->source);
	client->remote->source = NULL;
	crm_err("Remote client authentication timed out");

	return FALSE;
	}

	static int
	lrmd_remote_listen(gpointer data)
	{
	int csock = 0;
	int flag = 0;
	unsigned laddr;
	struct sockaddr_in addr;
	gnutls_session_t *session = NULL;
	crm_client_t *new_client = NULL;

	static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
	.dispatch = lrmd_remote_client_msg,
	.destroy = lrmd_remote_client_destroy,
	};

	/* accept the connection */
	laddr = sizeof(addr);
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	crm_debug("New remote connection from %s", inet_ntoa(addr.sin_addr));

	if (csock == -1) {
	crm_err("accept socket failed");
	return TRUE;
	}

	if ((flag = fcntl(csock, F_GETFL)) >= 0) {
	if (fcntl(csock, F_SETFL, flag \| O_NONBLOCK) < 0) {
	crm_err("fcntl() write failed");
	close(csock);
	return TRUE;
	}
	} else {
	crm_err("fcntl() read failed");
	close(csock);
	return TRUE;
	}

	session = create_psk_tls_session(csock, GNUTLS_SERVER, psk_cred_s);
	if (session == NULL) {
	crm_err("TLS session creation failed");
	close(csock);
	return TRUE;
	}

	new_client = calloc(1, sizeof(crm_client_t));
	new_client->remote = calloc(1, sizeof(crm_remote_t));
	new_client->kind = CRM_CLIENT_TLS;
	new_client->remote->tls_session = session;
	new_client->id = crm_generate_uuid();
	new_client->remote->auth_timeout =
	g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client);
	crm_notice("LRMD client connection established. %p id: %s", new_client, new_client->id);

	new_client->remote->source =
	mainloop_add_fd("lrmd-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
	&lrmd_remote_fd_cb);
	g_hash_table_insert(client_connections, new_client->id, new_client);

	/* Alert other clients of the new connection */
	notify_of_new_client(new_client);
	return TRUE;
	}

	static void
	lrmd_remote_connection_destroy(gpointer user_data)
	{
	crm_notice("Remote tls server disconnected");
	return;
	}

	static int
	lrmd_tls_server_key_cb(gnutls_session_t session, const char username, gnutls_datum_t key)
	{
	return lrmd_tls_set_key(key);
	}

	static int
	bind_and_listen(struct addrinfo *addr)
	{
	int optval;
	int fd;
	int rc;
	char buffer[256] = { 0, };

	if (addr->ai_family == AF_INET6) {
	struct sockaddr_in6 addr_in = (struct sockaddr_in6 )addr->ai_addr;
	inet_ntop(addr->ai_family, &addr_in->sin6_addr, buffer, DIMOF(buffer));

	} else {
	struct sockaddr_in addr_in = (struct sockaddr_in )addr->ai_addr;
	inet_ntop(addr->ai_family, &addr_in->sin_addr, buffer, DIMOF(buffer));
	}

	crm_trace("Attempting to bind on address %s", buffer);

	fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
	if (fd < 0) {
	return -1;
	}

	/* reuse address */
	optval = 1;
	rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener, bind address %s", buffer);
	close(fd);
	return -1;
	}

	if (addr->ai_family == AF_INET6) {
	optval = 0;
	rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't disable IPV6 only on address %s", buffer);
	close(fd);
	return -1;
	}
	}

	if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
	close(fd);
	return -1;
	}

	if (listen(fd, 10) == -1) {
	crm_err("Can not start listen on address %s", buffer);
	close(fd);
	return -1;
	}

	crm_notice("Listening on address %s", buffer);

	return fd;
	}

	int
	lrmd_init_remote_tls_server(int port)
	{
	int rc;
	int filter;
	struct addrinfo hints, res = NULL, iter;
	char port_str[16];

	static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
	.dispatch = lrmd_remote_listen,
	.destroy = lrmd_remote_connection_destroy,
	};

	crm_notice("Starting a tls listener on port %d.", port);
	gnutls_global_init();
	gnutls_global_set_log_function(debug_log);

	gnutls_dh_params_init(&dh_params);
	gnutls_dh_params_generate2(dh_params, 1024);
	gnutls_psk_allocate_server_credentials(&psk_cred_s);
	gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
	gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);

	memset(&hints, 0, sizeof(struct addrinfo));
	hints.ai_flags = AI_PASSIVE; /* Only return socket addresses with wildcard INADDR_ANY or IN6ADDR_ANY_INIT */
	hints.ai_family = AF_UNSPEC; /* Return IPv6 or IPv4 */
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_protocol = IPPROTO_TCP;

	snprintf(port_str, sizeof(port_str), "%d", port);
	rc = getaddrinfo(NULL, port_str, &hints, &res);
	if (rc) {
	crm_err("getaddrinfo: %s", gai_strerror(rc));
	return -1;
	}

	iter = res;
	filter = AF_INET6;
	/* Try IPv6 addresses first, then IPv4 */
	while (iter) {
	if (iter->ai_family == filter) {
	ssock = bind_and_listen(iter);
	}
	if (ssock != -1) {
	break;
	}

	iter = iter->ai_next;
	if (iter == NULL && filter == AF_INET6) {
	iter = res;
	filter = AF_INET;
	}
	}

	if (ssock < 0) {
	crm_err("unable to bind to address");
	goto init_remote_cleanup;
	}

	mainloop_add_fd("lrmd-remote", G_PRIORITY_DEFAULT, ssock, NULL, &remote_listen_fd_callbacks);

	rc = ssock;
	init_remote_cleanup:
	if (rc < 0) {
	close(ssock);
	ssock = 0;
	}
	freeaddrinfo(res);
	return rc;

	}

	void
	lrmd_tls_server_destroy(void)
	{
	if (psk_cred_s) {
	gnutls_psk_free_server_credentials(psk_cred_s);
	psk_cred_s = 0;
	}

	if (ssock > 0) {
	close(ssock);
	ssock = 0;
	}
	}
	#endif

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:25 PM (11 h, 51 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002642
Default Alt Text: (52 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions