diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
index ed60d8e61a..d38d7f0437 100644
--- a/crmd/remote_lrmd_ra.c
+++ b/crmd/remote_lrmd_ra.c
@@ -1,670 +1,680 @@
 /* 
  * Copyright (C) 2013 David Vossel <dvossel@redhat.com>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 
 #include <crmd.h>
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 #include <crm/lrmd.h>
 #include <crm/services.h>
 
 #define REMOTE_LRMD_RA "remote"
 
 /* The max start timeout before cmd retry */
 #define MAX_START_TIMEOUT_MS 10000
 
-int remote_ra_callid = 0;
-
 typedef struct remote_ra_cmd_s {
     /*! the local node the cmd is issued from */
     char *owner;
     /*! the remote node the cmd is executed on */
     char *rsc_id;
     /*! the action to execute */
     char *action;
     /*! some string the client wants us to give it back */
     char *userdata;
     /*! start delay in ms */
     int start_delay;
     /*! timer id used for start delay. */
     int delay_id;
     /*! timeout in ms for cmd */
     int timeout;
     int remaining_timeout;
     /*! recurring interval in ms */
     int interval;
     /*! interval timer id */
     int interval_id;
     int reported_success;
     int monitor_timeout_id;
     /*! action parameters */
     lrmd_key_value_t *params;
     /*! executed rc */
     int rc;
     int op_status;
     int call_id;
     time_t start_time;
     gboolean cancel;
 } remote_ra_cmd_t;
 
 typedef struct remote_ra_data_s {
     crm_trigger_t *work;
     remote_ra_cmd_t *cur_cmd;
     GList *cmds;
     GList *recurring_cmds;
 
 } remote_ra_data_t;
 
 static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
 
 static void
 free_cmd(gpointer user_data)
 {
     remote_ra_cmd_t *cmd = user_data;
 
     if (!cmd) {
         return;
     }
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
     if (cmd->interval_id) {
         g_source_remove(cmd->interval_id);
     }
     if (cmd->monitor_timeout_id) {
         g_source_remove(cmd->monitor_timeout_id);
     }
     free(cmd->owner);
     free(cmd->rsc_id);
     free(cmd->action);
     free(cmd->userdata);
     lrmd_key_value_freeall(cmd->params);
     free(cmd);
 }
 
+static int
+generate_callid(void)
+{
+    static int remote_ra_callid = 0;
+
+    remote_ra_callid++;
+    if (remote_ra_callid <= 0) {
+        remote_ra_callid = 1;
+    }
+
+    return remote_ra_callid;
+}
+
 static gboolean
 recurring_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->interval_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);
+
+        cmd->call_id = generate_callid();
+
         ra_data->cmds = g_list_append(ra_data->cmds, cmd);
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->delay_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 static void
 report_remote_ra_result(remote_ra_cmd_t * cmd)
 {
     lrmd_event_data_t op = { 0, };
 
     op.type = lrmd_event_exec_complete;
     op.rsc_id = cmd->rsc_id;
     op.op_type = cmd->action;
     op.user_data = cmd->userdata;
     op.timeout = cmd->timeout;
     op.interval = cmd->interval;
     op.rc = cmd->rc;
     op.op_status = cmd->op_status;
     if (cmd->params) {
         lrmd_key_value_t *tmp;
 
         op.params = g_hash_table_new_full(crm_str_hash,
                                           g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
         for (tmp = cmd->params; tmp; tmp = tmp->next) {
             g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
         }
 
     }
     op.call_id = cmd->call_id;
     op.remote_nodename = cmd->owner;
 
     lrm_op_callback(&op);
 
     if (op.params) {
         g_hash_table_destroy(op.params);
     }
 }
 
 static void
 update_remaining_timeout(remote_ra_cmd_t * cmd)
 {
     cmd->remaining_timeout = ((cmd->timeout / 1000) - (time(NULL) - cmd->start_time)) * 1000;
 }
 
 static gboolean
 retry_start_cmd_cb(gpointer data)
 {
     lrm_state_t *lrm_state = data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd = NULL;
     int rc = -1;
 
     if (!ra_data || !ra_data->cur_cmd) {
         return FALSE;
     }
     cmd = ra_data->cur_cmd;
     if (safe_str_neq(cmd->action, "start")) {
         return FALSE;
     }
     update_remaining_timeout(cmd);
 
     if (cmd->remaining_timeout > 0) {
         rc = handle_remote_ra_start(lrm_state, cmd, cmd->remaining_timeout);
     }
 
     if (rc != 0) {
         cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
         cmd->op_status = PCMK_LRM_OP_ERROR;
         report_remote_ra_result(cmd);
 
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         ra_data->cur_cmd = NULL;
         free_cmd(cmd);
     } else {
         /* wait for connection event */
     }
 
     return FALSE;
 }
 
 static gboolean
 monitor_timeout_cb(gpointer data)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_cmd_t *cmd = data;
 
     crm_debug("Poke async response timed out for node %s", cmd->rsc_id);
     cmd->monitor_timeout_id = 0;
     cmd->op_status = PCMK_LRM_OP_TIMEOUT;
     cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
 
     lrm_state = lrm_state_find(cmd->rsc_id);
     if (lrm_state && lrm_state->remote_ra_data) {
         remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
         if (ra_data->cur_cmd == cmd) {
             ra_data->cur_cmd = NULL;
         }
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
     }
 
     report_remote_ra_result(cmd);
     free_cmd(cmd);
     return FALSE;
 }
 
 static void
 remote_init_cib_status(lrm_state_t *lrm_state)
 {
     xmlNode *update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     xmlNode *state = NULL;
     int call_id = 0;
     int call_opt = cib_quorum_override;
 
     if (fsa_state == S_ELECTION || fsa_state == S_PENDING) {
         call_opt |= cib_scope_local;
     }
     state = create_xml_node(update, XML_CIB_TAG_STATE);
 
     crm_xml_add(state, XML_NODE_IS_REMOTE, "true");
     crm_xml_add(state, XML_ATTR_UUID,  lrm_state->node_name);
     crm_xml_add(state, XML_ATTR_UNAME, lrm_state->node_name);
     crm_xml_add(state, XML_ATTR_ORIGIN, __FUNCTION__);
 
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id != pcmk_ok) {
         crm_debug("Failed to init status section for remote-node %s", lrm_state->node_name);
     }
     free_xml(update);
 }
 
 void
 remote_lrm_op_callback(lrmd_event_data_t * op)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_data_t *ra_data = NULL;
     remote_ra_cmd_t *cmd = NULL;
 
     crm_debug("remote connection event - event_type:%s node:%s action:%s rc:%s op_status:%s",
               lrmd_event_type2str(op->type),
               op->remote_nodename,
               op->op_type ? op->op_type : "none",
               lrmd_event_rc2str(op->rc), services_lrm_status_str(op->op_status));
 
     /* filter all EXEC events up */
     if (op->type == lrmd_event_exec_complete) {
         lrm_op_callback(op);
         return;
     }
 
     lrm_state = lrm_state_find(op->remote_nodename);
     if (!lrm_state || !lrm_state->remote_ra_data) {
         crm_debug("lrm_state info not found for remote lrmd connection event");
         return;
     }
 
     ra_data = lrm_state->remote_ra_data;
     if (!ra_data->cur_cmd) {
         crm_debug("no event to match");
         return;
     }
     cmd = ra_data->cur_cmd;
 
     /* Start actions and migrate from actions complete after connection
      * comes back to us. */
     if (op->type == lrmd_event_connect && (safe_str_eq(cmd->action, "start") ||
                                            safe_str_eq(cmd->action, "migrate_from"))) {
 
         if (op->connection_rc < 0) {
             update_remaining_timeout(cmd);
             /* There isn't much of a reason to reschedule if the timeout is too small */
             if (cmd->remaining_timeout > 3000) {
                 crm_trace("rescheduling start, remaining timeout %d", cmd->remaining_timeout);
                 g_timeout_add(1000, retry_start_cmd_cb, lrm_state);
                 return;
             } else {
                 crm_trace("can't reschedule start, remaining timeout too small %d",
                           cmd->remaining_timeout);
             }
             cmd->op_status = PCMK_LRM_OP_TIMEOUT;
             cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
 
         } else {
             /* make sure we have a clean status section to start with */
             remote_init_cib_status(lrm_state);
 
             cmd->rc = PCMK_EXECRA_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
         }
 
         crm_debug("remote lrmd connect event matched %s action. ", cmd->action);
         report_remote_ra_result(cmd);
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         ra_data->cur_cmd = NULL;
         free_cmd(cmd);
         return;
     } else if (op->type == lrmd_event_poke && safe_str_eq(cmd->action, "monitor")) {
 
         if (cmd->monitor_timeout_id) {
             g_source_remove(cmd->monitor_timeout_id);
             cmd->monitor_timeout_id = 0;
         }
 
         /* Only report success the first time, after that only worry about failures.
          * For this function, if we get the poke pack, it is always a success. Pokes
          * only fail if the send fails, or the response times out. */
         if (!cmd->reported_success) {
             cmd->rc = PCMK_EXECRA_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
             report_remote_ra_result(cmd);
             cmd->reported_success = 1;
         }
 
         crm_debug("remote lrmd poke event matched %s action. ", cmd->action);
 
         /* success, keep rescheduling if interval is present. */
         if (cmd->interval && (cmd->cancel == FALSE)) {
             ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
             cmd->interval_id = g_timeout_add(cmd->interval, recurring_helper, cmd);
             cmd = NULL;         /* prevent free */
         }
 
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         ra_data->cur_cmd = NULL;
         free_cmd(cmd);
         return;
     }
 
     crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
 }
 
 static int
 handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms)
 {
     const char *server = NULL;
     lrmd_key_value_t *tmp = NULL;
     int port = 0;
     int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;
 
     for (tmp = cmd->params; tmp; tmp = tmp->next) {
         if (safe_str_eq(tmp->key, "addr") || safe_str_eq(tmp->key, "server")) {
             server = tmp->value;
         }
         if (safe_str_eq(tmp->key, "port")) {
             port = atoi(tmp->value);
         }
     }
 
     return lrm_state_remote_connect_async(lrm_state, server, port, timeout_used);
 }
 
 static gboolean
 handle_remote_ra_exec(gpointer user_data)
 {
     int rc = 0;
     lrm_state_t *lrm_state = user_data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd;
     GList *first = NULL;
 
     if (ra_data->cur_cmd) {
         /* still waiting on previous cmd */
         return TRUE;
     }
 
     while (ra_data->cmds) {
         first = ra_data->cmds;
         cmd = first->data;
         if (cmd->delay_id) {
             /* still waiting for start delay timer to trip */
             return TRUE;
         }
 
         ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
         g_list_free_1(first);
 
         if (!strcmp(cmd->action, "start") || !strcmp(cmd->action, "migrate_from")) {
             xmlNode *status = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
             /* clear node status in cib */
             crm_xml_add(status, XML_ATTR_ID, lrm_state->node_name);
             lrm_state_reset_tables(lrm_state);
             fsa_cib_delete(XML_CIB_TAG_STATUS, status, cib_quorum_override, rc, NULL);
             crm_info("Forced a remote LRM refresh before connection start: call=%d", rc);
             crm_log_xml_trace(status, "CLEAR LRM");
             free_xml(status);
 
             rc = handle_remote_ra_start(lrm_state, cmd, cmd->timeout);
             if (rc == 0) {
                 /* take care of this later when we get async connection result */
                 crm_debug("began remote lrmd connect, waiting for connect event.");
                 ra_data->cur_cmd = cmd;
                 return TRUE;
             } else {
                 crm_debug("connect failed, not expecting to match any connection event later");
                 cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
                 cmd->op_status = PCMK_LRM_OP_ERROR;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "monitor")) {
 
             if (lrm_state_is_connected(lrm_state) == TRUE) {
                 rc = lrm_state_poke_connection(lrm_state);
                 if (rc < 0) {
                     cmd->rc = PCMK_EXECRA_UNKNOWN_ERROR;
                     cmd->op_status = PCMK_LRM_OP_ERROR;
                 }
             } else {
                 rc = -1;
                 cmd->op_status = PCMK_LRM_OP_DONE;
                 cmd->rc = PCMK_EXECRA_NOT_RUNNING;
             }
 
             if (rc == 0) {
                 crm_debug("poked remote lrmd at node %s, waiting for async response.", cmd->rsc_id);
                 ra_data->cur_cmd = cmd;
                 cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
                 return TRUE;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "stop")) {
             lrm_state_disconnect(lrm_state);
             cmd->rc = PCMK_EXECRA_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
 
             if (ra_data->cmds) {
                 g_list_free_full(ra_data->cmds, free_cmd);
             }
             if (ra_data->recurring_cmds) {
                 g_list_free_full(ra_data->recurring_cmds, free_cmd);
             }
             ra_data->cmds = NULL;
             ra_data->recurring_cmds = NULL;
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "migrate_to")) {
             /* no-op. */
             cmd->rc = PCMK_EXECRA_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
             report_remote_ra_result(cmd);
         }
 
         free_cmd(cmd);
     }
 
     return TRUE;
 }
 
 static void
 remote_ra_data_init(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = NULL;
 
     if (lrm_state->remote_ra_data) {
         return;
     }
 
     ra_data = calloc(1, sizeof(remote_ra_data_t));
     ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
     lrm_state->remote_ra_data = ra_data;
 }
 
 void
 remote_ra_cleanup(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     if (!ra_data) {
         return;
     }
 
     if (ra_data->cmds) {
         g_list_free_full(ra_data->cmds, free_cmd);
     }
 
     if (ra_data->recurring_cmds) {
         g_list_free_full(ra_data->recurring_cmds, free_cmd);
     }
     mainloop_destroy_trigger(ra_data->work);
     free(ra_data);
     lrm_state->remote_ra_data = NULL;
 }
 
 gboolean
 is_remote_lrmd_ra(const char *agent, const char *provider, const char *id)
 {
     if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
         return TRUE;
     }
     if (id && lrm_state_find(id)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 lrmd_rsc_info_t *
 remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id)
 {
     lrmd_rsc_info_t *info = NULL;
 
     if ((lrm_state_find(rsc_id))) {
         info = calloc(1, sizeof(lrmd_rsc_info_t));
 
         info->id = strdup(rsc_id);
         info->type = strdup(REMOTE_LRMD_RA);
         info->class = strdup("ocf");
         info->provider = strdup("pacemaker");
     }
 
     return info;
 }
 
 static gboolean
 is_remote_ra_supported_action(const char *action)
 {
     if (!action) {
         return FALSE;
     } else if (strcmp(action, "start") &&
                strcmp(action, "stop") &&
                strcmp(action, "migrate_to") &&
                strcmp(action, "migrate_from") && strcmp(action, "monitor")) {
         return FALSE;
     }
 
     return TRUE;
 }
 
 static GList *
 remove_cmd(GList * list, const char *action, int interval)
 {
     remote_ra_cmd_t *cmd = NULL;
     GListPtr gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if (cmd->interval == interval && safe_str_eq(cmd->action, action)) {
             break;
         }
         cmd = NULL;
     }
     if (cmd) {
         list = g_list_remove(list, cmd);
         free_cmd(cmd);
     }
     return list;
 }
 
 int
 remote_ra_cancel(lrm_state_t * lrm_state, const char *rsc_id, const char *action, int interval)
 {
     lrm_state_t *connection_rsc = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     connection_rsc = lrm_state_find(rsc_id);
     if (!connection_rsc || !connection_rsc->remote_ra_data) {
         return -EINVAL;
     }
 
     ra_data = connection_rsc->remote_ra_data;
     ra_data->cmds = remove_cmd(ra_data->cmds, action, interval);
     ra_data->recurring_cmds = remove_cmd(ra_data->recurring_cmds, action, interval);
     if (ra_data->cur_cmd &&
         (ra_data->cur_cmd->interval == interval) &&
         (safe_str_eq(ra_data->cur_cmd->action, action))) {
 
         ra_data->cur_cmd->cancel = TRUE;
     }
 
     return 0;
 }
 
 int
 remote_ra_exec(lrm_state_t * lrm_state, const char *rsc_id, const char *action, const char *userdata, int interval,     /* ms */
                int timeout,     /* ms */
                int start_delay, /* ms */
                lrmd_key_value_t * params)
 {
     int rc = 0;
     lrm_state_t *connection_rsc = NULL;
     remote_ra_cmd_t *cmd = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     if (is_remote_ra_supported_action(action) == FALSE) {
         rc = -EINVAL;
         goto exec_done;
     }
 
     connection_rsc = lrm_state_find(rsc_id);
     if (!connection_rsc) {
         rc = -EINVAL;
         goto exec_done;
     }
 
     remote_ra_data_init(connection_rsc);
 
     cmd = calloc(1, sizeof(remote_ra_cmd_t));
     cmd->owner = strdup(lrm_state->node_name);
     cmd->rsc_id = strdup(rsc_id);
     cmd->action = strdup(action);
     cmd->userdata = strdup(userdata);
     cmd->interval = interval;
     cmd->timeout = timeout;
     cmd->start_delay = start_delay;
     cmd->params = params;
     cmd->start_time = time(NULL);
 
-    remote_ra_callid++;
-    if (remote_ra_callid <= 0) {
-        remote_ra_callid = 1;
-    }
-    cmd->call_id = remote_ra_callid;
+    cmd->call_id = generate_callid();
 
     if (cmd->start_delay) {
         cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
     }
     ra_data = connection_rsc->remote_ra_data;
 
     ra_data->cmds = g_list_append(ra_data->cmds, cmd);
     mainloop_set_trigger(ra_data->work);
 
     return cmd->call_id;
   exec_done:
 
     lrmd_key_value_freeall(params);
     return rc;
 }
diff --git a/fencing/commands.c b/fencing/commands.c
index 2e0ec388ba..9fe00c8a18 100644
--- a/fencing/commands.c
+++ b/fencing/commands.c
@@ -1,2047 +1,2050 @@
 /*
  * Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/utsname.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <ctype.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipcs.h>
 #include <crm/cluster/internal.h>
 #include <crm/common/mainloop.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <crm/common/xml.h>
 
 #if SUPPORT_CIBSECRETS
 #  include <crm/common/cib_secrets.h>
 #endif
 
 #include <internal.h>
 
 GHashTable *device_list = NULL;
 GHashTable *topology = NULL;
 GList *cmd_list = NULL;
 
 static int active_children = 0;
 
 struct device_search_s {
     char *host;
     char *action;
     int per_device_timeout;
     int replies_needed;
     int replies_received;
 
     void *user_data;
     void (*callback) (GList * devices, void *user_data);
     GListPtr capable;
 };
 
 static gboolean stonith_device_dispatch(gpointer user_data);
 static void st_child_done(GPid pid, int rc, const char *output, gpointer user_data);
 static void stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
                                const char *client_id);
 
 static void search_devices_record_result(struct device_search_s *search, const char *device,
                                          gboolean can_fence);
 
 typedef struct async_command_s {
 
     int id;
     int pid;
     int fd_stdout;
     int options;
     int default_timeout;
     int timeout;
 
     char *op;
     char *origin;
     char *client;
     char *client_name;
     char *remote_op_id;
 
     char *victim;
     uint32_t victim_nodeid;
     char *action;
     char *device;
     char *mode;
 
     GListPtr device_list;
     GListPtr device_next;
 
     void *internal_user_data;
     void (*done_cb) (GPid pid, int rc, const char *output, gpointer user_data);
     guint timer_sigterm;
     guint timer_sigkill;
     /*! If the operation timed out, this is the last signal
      *  we sent to the process to get it to terminate */
     int last_timeout_signo;
 } async_command_t;
 
 static xmlNode *stonith_construct_async_reply(async_command_t * cmd, const char *output,
                                               xmlNode * data, int rc);
 
 static int
 get_action_timeout(stonith_device_t * device, const char *action, int default_timeout)
 {
     char buffer[512] = { 0, };
     char *value = NULL;
 
     CRM_CHECK(action != NULL, return default_timeout);
 
     if (!device->params) {
         return default_timeout;
     }
 
     snprintf(buffer, sizeof(buffer) - 1, "pcmk_%s_timeout", action);
     value = g_hash_table_lookup(device->params, buffer);
 
     if (!value) {
         return default_timeout;
     }
 
     return atoi(value);
 }
 
 static void
 free_async_command(async_command_t * cmd)
 {
     if (!cmd) {
         return;
     }
     cmd_list = g_list_remove(cmd_list, cmd);
 
     g_list_free_full(cmd->device_list, free);
     free(cmd->device);
     free(cmd->action);
     free(cmd->victim);
     free(cmd->remote_op_id);
     free(cmd->client);
     free(cmd->client_name);
     free(cmd->origin);
     free(cmd->mode);
     free(cmd->op);
     free(cmd);
 }
 
 static async_command_t *
 create_async_command(xmlNode * msg)
 {
     async_command_t *cmd = NULL;
     xmlNode *op = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_ERR);
     const char *action = crm_element_value(op, F_STONITH_ACTION);
 
     CRM_CHECK(action != NULL, crm_log_xml_warn(msg, "NoAction"); return NULL);
 
     crm_log_xml_trace(msg, "Command");
     cmd = calloc(1, sizeof(async_command_t));
     crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
     crm_element_value_int(msg, F_STONITH_CALLOPTS, &(cmd->options));
     crm_element_value_int(msg, F_STONITH_TIMEOUT, &(cmd->default_timeout));
     cmd->timeout = cmd->default_timeout;
 
     cmd->origin = crm_element_value_copy(msg, F_ORIG);
     cmd->remote_op_id = crm_element_value_copy(msg, F_STONITH_REMOTE_OP_ID);
     cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
     cmd->client_name = crm_element_value_copy(msg, F_STONITH_CLIENTNAME);
     cmd->op = crm_element_value_copy(msg, F_STONITH_OPERATION);
     cmd->action = strdup(action);
     cmd->victim = crm_element_value_copy(op, F_STONITH_TARGET);
     cmd->mode = crm_element_value_copy(op, F_STONITH_MODE);
     cmd->device = crm_element_value_copy(op, F_STONITH_DEVICE);
 
     CRM_CHECK(cmd->op != NULL, crm_log_xml_warn(msg, "NoOp"); free_async_command(cmd); return NULL);
     CRM_CHECK(cmd->client != NULL, crm_log_xml_warn(msg, "NoClient"));
 
     cmd->done_cb = st_child_done;
     cmd_list = g_list_append(cmd_list, cmd);
     return cmd;
 }
 
 static gboolean
 stonith_device_execute(stonith_device_t * device)
 {
     int exec_rc = 0;
     async_command_t *cmd = NULL;
     stonith_action_t *action = NULL;
 
     CRM_CHECK(device != NULL, return FALSE);
 
     if (device->active_pid) {
         crm_trace("%s is still active with pid %u", device->id, device->active_pid);
         return TRUE;
     }
 
     if (device->pending_ops) {
         GList *first = device->pending_ops;
 
         device->pending_ops = g_list_remove_link(device->pending_ops, first);
         cmd = first->data;
         g_list_free_1(first);
     }
 
     if (cmd == NULL) {
         crm_trace("Nothing further to do for %s", device->id);
         return TRUE;
     }
 
 #if SUPPORT_CIBSECRETS
     if (replace_secret_params(device->id, device->params) < 0) {
         /* replacing secrets failed! */
         if (safe_str_eq(cmd->action,"stop")) {
             /* don't fail on stop! */
             crm_info("proceeding with the stop operation for %s", device->id);
 
         } else {
             crm_err("failed to get secrets for %s, "
                     "considering resource not configured", device->id);
             exec_rc = PCMK_EXECRA_NOT_CONFIGURED;
             cmd->done_cb(0, exec_rc, NULL, cmd);
             return TRUE;
         }
     }
 #endif
 
     action = stonith_action_create(device->agent,
                                    cmd->action,
                                    cmd->victim,
                                    cmd->victim_nodeid,
                                    cmd->timeout, device->params, device->aliases);
 
     /* for async exec, exec_rc is pid if positive and error code if negative/zero */
     exec_rc = stonith_action_execute_async(action, (void *)cmd, cmd->done_cb);
 
     if (exec_rc > 0) {
         crm_debug("Operation %s%s%s on %s now running with pid=%d, timeout=%dms",
                   cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "",
                   device->id, exec_rc, cmd->timeout);
         device->active_pid = exec_rc;
 
     } else {
         crm_warn("Operation %s%s%s on %s failed: %s (%d)",
                  cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "",
                  device->id, pcmk_strerror(exec_rc), exec_rc);
         cmd->done_cb(0, exec_rc, NULL, cmd);
     }
     return TRUE;
 }
 
 static gboolean
 stonith_device_dispatch(gpointer user_data)
 {
     return stonith_device_execute(user_data);
 }
 
 static void
 schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
 {
     CRM_CHECK(cmd != NULL, return);
     CRM_CHECK(device != NULL, return);
 
     if (cmd->device) {
         free(cmd->device);
     }
 
     if (device->include_nodeid && cmd->victim) {
         crm_node_t *node = crm_get_peer(0, cmd->victim);
 
         cmd->victim_nodeid = node->id;
     }
 
     cmd->device = strdup(device->id);
     cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);
 
     if (cmd->remote_op_id) {
         crm_debug("Scheduling %s on %s for remote peer %s with op id (%s) (timeout=%ds)",
                   cmd->action, device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
     } else {
         crm_debug("Scheduling %s on %s for %s (timeout=%ds)",
                   cmd->action, device->id, cmd->client, cmd->timeout);
     }
 
     device->pending_ops = g_list_append(device->pending_ops, cmd);
     mainloop_set_trigger(device->work);
 }
 
 void
 free_device(gpointer data)
 {
     GListPtr gIter = NULL;
     stonith_device_t *device = data;
 
     g_hash_table_destroy(device->params);
     g_hash_table_destroy(device->aliases);
 
     for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
         async_command_t *cmd = gIter->data;
 
         crm_warn("Removal of device '%s' purged operation %s", device->id, cmd->action);
         cmd->done_cb(0, -ENODEV, NULL, cmd);
         free_async_command(cmd);
     }
     g_list_free(device->pending_ops);
 
     g_list_free_full(device->targets, free);
+
+    mainloop_destroy_trigger(device->work);
+
     free_xml(device->agent_metadata);
     free(device->namespace);
     free(device->on_target_actions);
     free(device->agent);
     free(device->id);
     free(device);
 }
 
 static GHashTable *
 build_port_aliases(const char *hostmap, GListPtr * targets)
 {
     char *name = NULL;
     int last = 0, lpc = 0, max = 0, added = 0;
     GHashTable *aliases =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
     if (hostmap == NULL) {
         return aliases;
     }
 
     max = strlen(hostmap);
     for (; lpc <= max; lpc++) {
         switch (hostmap[lpc]) {
                 /* Assignment chars */
             case '=':
             case ':':
                 if (lpc > last) {
                     free(name);
                     name = calloc(1, 1 + lpc - last);
                     memcpy(name, hostmap + last, lpc - last);
                 }
                 last = lpc + 1;
                 break;
 
                 /* Delimeter chars */
                 /* case ',': Potentially used to specify multiple ports */
             case 0:
             case ';':
             case ' ':
             case '\t':
                 if (name) {
                     char *value = NULL;
 
                     value = calloc(1, 1 + lpc - last);
                     memcpy(value, hostmap + last, lpc - last);
 
                     crm_debug("Adding alias '%s'='%s'", name, value);
                     g_hash_table_replace(aliases, name, value);
                     if (targets) {
                         *targets = g_list_append(*targets, strdup(value));
                     }
                     value = NULL;
                     name = NULL;
                     added++;
 
                 } else if (lpc > last) {
                     crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
                 }
 
                 last = lpc + 1;
                 break;
         }
 
         if (hostmap[lpc] == 0) {
             break;
         }
     }
 
     if (added == 0) {
         crm_info("No host mappings detected in '%s'", hostmap);
     }
 
     free(name);
     return aliases;
 }
 
 static void
 parse_host_line(const char *line, int max, GListPtr * output)
 {
     int lpc = 0;
     int last = 0;
 
     if (max <= 0) {
         return;
     }
 
     /* Check for any complaints about additional parameters that the device doesn't understand */
     if (strstr(line, "invalid") || strstr(line, "variable")) {
         crm_debug("Skipping: %s", line);
         return;
     }
 
     crm_trace("Processing %d bytes: [%s]", max, line);
     /* Skip initial whitespace */
     for (lpc = 0; lpc <= max && isspace(line[lpc]); lpc++) {
         last = lpc + 1;
     }
 
     /* Now the actual content */
     for (lpc = 0; lpc <= max; lpc++) {
         gboolean a_space = isspace(line[lpc]);
 
         if (a_space && lpc < max && isspace(line[lpc + 1])) {
             /* fast-forward to the end of the spaces */
 
         } else if (a_space || line[lpc] == ',' || line[lpc] == 0) {
             int rc = 1;
             char *entry = NULL;
 
             if (lpc != last) {
                 entry = calloc(1, 1 + lpc - last);
                 rc = sscanf(line + last, "%[a-zA-Z0-9_-.]", entry);
             }
 
             if (entry == NULL) {
                 /* Skip */
             } else if (rc != 1) {
                 crm_warn("Could not parse (%d %d): %s", last, lpc, line + last);
             } else if (safe_str_neq(entry, "on") && safe_str_neq(entry, "off")) {
                 crm_trace("Adding '%s'", entry);
                 *output = g_list_append(*output, entry);
                 entry = NULL;
             }
 
             free(entry);
             last = lpc + 1;
         }
     }
 }
 
 static GListPtr
 parse_host_list(const char *hosts)
 {
     int lpc = 0;
     int max = 0;
     int last = 0;
     GListPtr output = NULL;
 
     if (hosts == NULL) {
         return output;
     }
 
     max = strlen(hosts);
     for (lpc = 0; lpc <= max; lpc++) {
         if (hosts[lpc] == '\n' || hosts[lpc] == 0) {
             char *line = NULL;
             int len = lpc - last;
 
             if(len > 1) {
                 line = malloc(1 + len);
             }
 
             if(line) {
                 snprintf(line, 1 + len, "%s", hosts + last);
                 line[len] = 0; /* Because it might be '\n' */
                 parse_host_line(line, len, &output);
                 free(line);
             }
 
             last = lpc + 1;
         }
     }
 
     crm_trace("Parsed %d entries from '%s'", g_list_length(output), hosts);
     return output;
 }
 
 static xmlNode *
 get_agent_metadata(const char *agent)
 {
     stonith_t *st = stonith_api_new();
     xmlNode *xml = NULL;
     char *buffer = NULL;
     int rc = 0;
 
     rc = st->cmds->metadata(st, st_opt_sync_call, agent, NULL, &buffer, 10);
     if (rc || !buffer) {
         crm_err("Could not retrieve metadata for fencing agent %s", agent);
         return NULL;
     }
     xml = string2xml(buffer);
     free(buffer);
     stonith_api_delete(st);
 
     return xml;
 }
 
 static gboolean
 is_nodeid_required(xmlNode * xml)
 {
     xmlXPathObjectPtr xpath = NULL;
 
     if (stand_alone) {
         return FALSE;
     }
 
     if (!xml) {
         return FALSE;
     }
 
     xpath = xpath_search(xml, "//parameter[@name='nodeid']");
     if (numXpathResults(xpath)  <= 0) {
         freeXpathObject(xpath);
         return FALSE;
     }
 
     freeXpathObject(xpath);
     return TRUE;
 }
 
 static char *
 get_on_target_actions(xmlNode * xml)
 {
     char *actions = NULL;
     xmlXPathObjectPtr xpath = NULL;
     int max = 0;
     int lpc = 0;
 
     if (!xml) {
         return NULL;
     }
 
     xpath = xpath_search(xml, "//action");
     max = numXpathResults(xpath);
 
     if (max <= 0) {
         freeXpathObject(xpath);
         return NULL;
     }
 
     actions = calloc(1, 512);
 
     for (lpc = 0; lpc < max; lpc++) {
         const char *on_target = NULL;
         const char *action = NULL;
         xmlNode *match = getXpathResult(xpath, lpc);
 
         CRM_CHECK(match != NULL, continue);
 
         on_target = crm_element_value(match, "on_target");
         action = crm_element_value(match, "name");
 
         if (action && crm_is_true(on_target)) {
             if (strlen(actions)) {
                 g_strlcat(actions, " ", 512);
             }
             g_strlcat(actions, action, 512);
         }
     }
 
     freeXpathObject(xpath);
 
     if (!strlen(actions)) {
         free(actions);
         actions = NULL;
     }
 
     return actions;
 }
 
 static stonith_device_t *
 build_device_from_xml(xmlNode * msg)
 {
     const char *value = NULL;
     xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
     stonith_device_t *device = NULL;
 
     device = calloc(1, sizeof(stonith_device_t));
     device->id = crm_element_value_copy(dev, XML_ATTR_ID);
     device->agent = crm_element_value_copy(dev, "agent");
     device->namespace = crm_element_value_copy(dev, "namespace");
     device->params = xml2list(dev);
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTLIST);
     if (value) {
         device->targets = parse_host_list(value);
     }
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTMAP);
     device->aliases = build_port_aliases(value, &(device->targets));
 
     device->agent_metadata = get_agent_metadata(device->agent);
     device->on_target_actions = get_on_target_actions(device->agent_metadata);
 
     value = g_hash_table_lookup(device->params, "nodeid");
     if (!value) {
         device->include_nodeid = is_nodeid_required(device->agent_metadata);
     }
 
     if (device->on_target_actions) {
         crm_info("The fencing device '%s' requires actions (%s) to be executed on the target node",
                  device->id, device->on_target_actions);
     }
 
     device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
     /* TODO: Hook up priority */
 
     return device;
 }
 
 static const char *
 target_list_type(stonith_device_t * dev)
 {
     const char *check_type = NULL;
 
     check_type = g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTCHECK);
 
     if (check_type == NULL) {
 
         if (g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTLIST)) {
             check_type = "static-list";
         } else if (g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTMAP)) {
             check_type = "static-list";
         } else {
             check_type = "dynamic-list";
         }
     }
 
     return check_type;
 }
 
 static void
 schedule_internal_command(stonith_device_t * device,
                           const char *action,
                           const char *victim,
                           int timeout,
                           void *internal_user_data,
                           void (*done_cb) (GPid pid, int rc, const char *output,
                                            gpointer user_data))
 {
     async_command_t *cmd = NULL;
 
     cmd = calloc(1, sizeof(async_command_t));
 
     cmd->id = -1;
     cmd->default_timeout = timeout ? timeout : 60;
     cmd->timeout = cmd->default_timeout;
     cmd->action = strdup(action);
     cmd->victim = victim ? strdup(victim) : NULL;
     cmd->device = strdup(device->id);
     cmd->origin = strdup("st_internal_cmd");
     cmd->client = strdup("st_internal_client");
     cmd->client_name = strdup("st_internal_client_name");
 
     cmd->internal_user_data = internal_user_data;
     cmd->done_cb = done_cb;
 
     schedule_stonith_command(cmd, device);
 }
 
 static gboolean
 string_in_list(GListPtr list, const char *item)
 {
     int lpc = 0;
     int max = g_list_length(list);
 
     for (lpc = 0; lpc < max; lpc++) {
         const char *value = g_list_nth_data(list, lpc);
 
         if (safe_str_eq(item, value)) {
             return TRUE;
         } else {
             crm_trace("%d: '%s' != '%s'", lpc, item, value);
         }
     }
     return FALSE;
 }
 
 static void
 status_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
 {
     async_command_t *cmd = user_data;
     struct device_search_s *search = cmd->internal_user_data;
     stonith_device_t *dev = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
     gboolean can = FALSE;
 
     free_async_command(cmd);
 
     if (!dev) {
         search_devices_record_result(search, NULL, FALSE);
         return;
     }
 
     dev->active_pid = 0;
     mainloop_set_trigger(dev->work);
 
     if (rc == 1 /* unkown */ ) {
         crm_trace("Host %s is not known by %s", search->host, dev->id);
 
     } else if (rc == 0 /* active */  || rc == 2 /* inactive */ ) {
         can = TRUE;
 
     } else {
         crm_notice("Unkown result when testing if %s can fence %s: rc=%d", dev->id, search->host,
                    rc);
     }
     search_devices_record_result(search, dev->id, can);
 }
 
 static void
 dynamic_list_search_cb(GPid pid, int rc, const char *output, gpointer user_data)
 {
     async_command_t *cmd = user_data;
     struct device_search_s *search = cmd->internal_user_data;
     stonith_device_t *dev = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
     gboolean can_fence = FALSE;
 
     free_async_command(cmd);
 
     /* Host/alias must be in the list output to be eligable to be fenced
      *
      * Will cause problems if down'd nodes aren't listed or (for virtual nodes)
      *  if the guest is still listed despite being moved to another machine
      */
     if (!dev) {
         search_devices_record_result(search, NULL, FALSE);
         return;
     }
 
     dev->active_pid = 0;
     mainloop_set_trigger(dev->work);
 
     /* If we successfully got the targets earlier, don't disable. */
     if (rc != 0 && !dev->targets) {
         crm_notice("Disabling port list queries for %s (%d): %s", dev->id, rc, output);
         /* Fall back to status */
         g_hash_table_replace(dev->params, strdup(STONITH_ATTR_HOSTCHECK), strdup("status"));
 
         g_list_free_full(dev->targets, free);
         dev->targets = NULL;
     } else if (!rc) {
         crm_info("Refreshing port list for %s", dev->id);
         g_list_free_full(dev->targets, free);
         dev->targets = parse_host_list(output);
         dev->targets_age = time(NULL);
     }
 
     if (dev->targets) {
         const char *alias = g_hash_table_lookup(dev->aliases, search->host);
 
         if (!alias) {
             alias = search->host;
         }
         if (string_in_list(dev->targets, alias)) {
             can_fence = TRUE;
         }
     }
     search_devices_record_result(search, dev->id, can_fence);
 }
 
 /*!
  * \internal
  * \brief Checks to see if an identical device already exists in the device_list
  */
 static stonith_device_t *
 device_has_duplicate(stonith_device_t * device)
 {
     char *key = NULL;
     char *value = NULL;
     GHashTableIter gIter;
     stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);
 
     if (!dup) {
         crm_trace("No match for %s", device->id);
         return NULL;
 
     } else if (safe_str_neq(dup->agent, device->agent)) {
         crm_trace("Different agent: %s != %s", dup->agent, device->agent);
         return NULL;
     }
 
     /* Use calculate_operation_digest() here? */
     g_hash_table_iter_init(&gIter, device->params);
     while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&value)) {
 
         if(strstr(key, "CRM_meta") == key) {
             continue;
         } else if(strcmp(key, "crm_feature_set") == 0) {
             continue;
         } else {
             char *other_value = g_hash_table_lookup(dup->params, key);
 
             if (!other_value || safe_str_neq(other_value, value)) {
                 crm_trace("Different value for %s: %s != %s", key, other_value, value);
                 return NULL;
             }
         }
     }
 
     crm_trace("Match");
     return dup;
 }
 
 int
 stonith_device_register(xmlNode * msg, const char **desc, gboolean from_cib)
 {
     stonith_device_t *dup = NULL;
     stonith_device_t *device = build_device_from_xml(msg);
 
     dup = device_has_duplicate(device);
     if (dup) {
         crm_notice("Device '%s' already existed in device list (%d active devices)", device->id,
                    g_hash_table_size(device_list));
         free_device(device);
         device = dup;
 
     } else {
         stonith_device_t *old = g_hash_table_lookup(device_list, device->id);
 
         if (from_cib && old && old->api_registered) {
             /* If the cib is writing over an entry that is shared with a stonith client,
              * copy any pending ops that currently exist on the old entry to the new one.
              * Otherwise the pending ops will be reported as failures
              */
             crm_trace("Overwriting an existing entry for %s from the cib", device->id);
             device->pending_ops = old->pending_ops;
             device->api_registered = TRUE;
             old->pending_ops = NULL;
             if (device->pending_ops) {
                 mainloop_set_trigger(device->work);
             }
         }
         g_hash_table_replace(device_list, device->id, device);
 
         crm_notice("Added '%s' to the device list (%d active devices)", device->id,
                    g_hash_table_size(device_list));
     }
     if (desc) {
         *desc = device->id;
     }
 
     if (from_cib) {
         device->cib_registered = TRUE;
     } else {
         device->api_registered = TRUE;
     }
 
     return pcmk_ok;
 }
 
 int
 stonith_device_remove(const char *id, gboolean from_cib)
 {
     stonith_device_t *device = g_hash_table_lookup(device_list, id);
 
     if (!device) {
         crm_info("Device '%s' not found (%d active devices)", id, g_hash_table_size(device_list));
         return pcmk_ok;
     }
 
     if (from_cib) {
         device->cib_registered = FALSE;
     } else {
         device->verified = FALSE;
         device->api_registered = FALSE;
     }
 
     if (!device->cib_registered && !device->api_registered) {
         g_hash_table_remove(device_list, id);
         crm_info("Removed '%s' from the device list (%d active devices)",
                  id, g_hash_table_size(device_list));
     }
     return pcmk_ok;
 }
 
 static int
 count_active_levels(stonith_topology_t * tp)
 {
     int lpc = 0;
     int count = 0;
 
     for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
         if (tp->levels[lpc] != NULL) {
             count++;
         }
     }
     return count;
 }
 
 void
 free_topology_entry(gpointer data)
 {
     stonith_topology_t *tp = data;
 
     int lpc = 0;
 
     for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
         if (tp->levels[lpc] != NULL) {
             g_list_free_full(tp->levels[lpc], free);
         }
     }
     free(tp->node);
     free(tp);
 }
 
 int
 stonith_level_register(xmlNode * msg, char **desc)
 {
     int id = 0;
     int rc = pcmk_ok;
     xmlNode *child = NULL;
 
     xmlNode *level = get_xpath_object("//" F_STONITH_LEVEL, msg, LOG_ERR);
     const char *node = crm_element_value(level, F_STONITH_TARGET);
     stonith_topology_t *tp = g_hash_table_lookup(topology, node);
 
     crm_element_value_int(level, XML_ATTR_ID, &id);
     if (desc) {
         *desc = g_strdup_printf("%s[%d]", node, id);
     }
     if (id <= 0 || id >= ST_LEVEL_MAX) {
         return -EINVAL;
     }
 
     if (tp == NULL) {
         tp = calloc(1, sizeof(stonith_topology_t));
         tp->node = strdup(node);
         g_hash_table_replace(topology, tp->node, tp);
         crm_trace("Added %s to the topology (%d active entries)", node,
                   g_hash_table_size(topology));
     }
 
     if (tp->levels[id] != NULL) {
         crm_info("Adding to the existing %s[%d] topology entry (%d active entries)", node, id,
                  count_active_levels(tp));
     }
 
     for (child = __xml_first_child(level); child != NULL; child = __xml_next(child)) {
         const char *device = ID(child);
 
         crm_trace("Adding device '%s' for %s (%d)", device, node, id);
         tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
     }
 
     crm_info("Node %s has %d active fencing levels", node, count_active_levels(tp));
     return rc;
 }
 
 int
 stonith_level_remove(xmlNode * msg, char **desc)
 {
     int id = 0;
     xmlNode *level = get_xpath_object("//" F_STONITH_LEVEL, msg, LOG_ERR);
     const char *node = crm_element_value(level, F_STONITH_TARGET);
     stonith_topology_t *tp = g_hash_table_lookup(topology, node);
 
     if (desc) {
         *desc = g_strdup_printf("%s[%d]", node, id);
     }
     crm_element_value_int(level, XML_ATTR_ID, &id);
 
     if (tp == NULL) {
         crm_info("Node %s not found (%d active entries)", node, g_hash_table_size(topology));
         return pcmk_ok;
 
     } else if (id < 0 || id >= ST_LEVEL_MAX) {
         return -EINVAL;
     }
 
     if (id == 0 && g_hash_table_remove(topology, node)) {
         crm_info("Removed all %s related entries from the topology (%d active entries)",
                  node, g_hash_table_size(topology));
 
     } else if (id > 0 && tp->levels[id] != NULL) {
         g_list_free_full(tp->levels[id], free);
         tp->levels[id] = NULL;
 
         crm_info("Removed entry '%d' from %s's topology (%d active entries remaining)",
                  id, node, count_active_levels(tp));
     }
     return pcmk_ok;
 }
 
 static int
 stonith_device_action(xmlNode * msg, char **output)
 {
     int rc = pcmk_ok;
     xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, msg, LOG_ERR);
     const char *id = crm_element_value(dev, F_STONITH_DEVICE);
 
     async_command_t *cmd = NULL;
     stonith_device_t *device = NULL;
 
     if (id) {
         crm_trace("Looking for '%s'", id);
         device = g_hash_table_lookup(device_list, id);
     }
 
     if (device) {
         cmd = create_async_command(msg);
         if (cmd == NULL) {
             free_device(device);
             return -EPROTO;
         }
 
         schedule_stonith_command(cmd, device);
         rc = -EINPROGRESS;
 
     } else {
         crm_info("Device %s not found", id ? id : "<none>");
         rc = -ENODEV;
     }
     return rc;
 }
 
 static void
 search_devices_record_result(struct device_search_s *search, const char *device, gboolean can_fence)
 {
     search->replies_received++;
 
     if (can_fence && device) {
         search->capable = g_list_append(search->capable, strdup(device));
     }
 
     if (search->replies_needed == search->replies_received) {
 
         crm_debug("Finished Search. %d devices can perform action (%s) on node %s",
                   g_list_length(search->capable),
                   search->action ? search->action : "<unknown>",
                   search->host ? search->host : "<anyone>");
 
         search->callback(search->capable, search->user_data);
         free(search->host);
         free(search->action);
         free(search);
     }
 }
 
 static void
 can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *search)
 {
     gboolean can = FALSE;
     const char *check_type = NULL;
     const char *host = search->host;
     const char *alias = NULL;
 
     CRM_LOG_ASSERT(dev != NULL);
 
     if (dev == NULL) {
         goto search_report_results;
     } else if (host == NULL) {
         can = TRUE;
         goto search_report_results;
     }
 
     if (dev->on_target_actions &&
         search->action &&
         strstr(dev->on_target_actions, search->action) && safe_str_neq(host, stonith_our_uname)) {
         /* this device can only execute this action on the target node */
         goto search_report_results;
     }
 
     alias = g_hash_table_lookup(dev->aliases, host);
     if (alias == NULL) {
         alias = host;
     }
 
     check_type = target_list_type(dev);
 
     if (safe_str_eq(check_type, "none")) {
         can = TRUE;
 
     } else if (safe_str_eq(check_type, "static-list")) {
 
         /* Presence in the hostmap is sufficient
          * Only use if all hosts on which the device can be active can always fence all listed hosts
          */
 
         if (string_in_list(dev->targets, host)) {
             can = TRUE;
         } else if (g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTMAP)
                    && g_hash_table_lookup(dev->aliases, host)) {
             can = TRUE;
         }
 
     } else if (safe_str_eq(check_type, "dynamic-list")) {
         time_t now = time(NULL);
 
         if (dev->targets == NULL || dev->targets_age + 60 < now) {
             schedule_internal_command(dev,
                                       "list",
                                       NULL,
                                       search->per_device_timeout, search, dynamic_list_search_cb);
 
             /* we'll respond to this search request async in the cb */
             return;
         }
 
         if (string_in_list(dev->targets, alias)) {
             can = TRUE;
         }
 
     } else if (safe_str_eq(check_type, "status")) {
         schedule_internal_command(dev,
                                   "status",
                                   search->host,
                                   search->per_device_timeout, search, status_search_cb);
         /* we'll respond to this search request async in the cb */
         return;
     } else {
         crm_err("Unknown check type: %s", check_type);
     }
 
     if (safe_str_eq(host, alias)) {
         crm_notice("%s can%s fence %s: %s", dev->id, can ? "" : " not", host, check_type);
     } else {
         crm_notice("%s can%s fence %s (aka. '%s'): %s", dev->id, can ? "" : " not", host, alias,
                    check_type);
     }
 
   search_report_results:
     search_devices_record_result(search, dev ? dev->id : NULL, can);
 }
 
 static void
 search_devices(gpointer key, gpointer value, gpointer user_data)
 {
     stonith_device_t *dev = value;
     struct device_search_s *search = user_data;
 
     can_fence_host_with_device(dev, search);
 }
 
 #define DEFAULT_QUERY_TIMEOUT 20
 static void
 get_capable_devices(const char *host, const char *action, int timeout, void *user_data,
                     void (*callback) (GList * devices, void *user_data))
 {
     struct device_search_s *search;
     int per_device_timeout = DEFAULT_QUERY_TIMEOUT;
     int devices_needing_async_query = 0;
     char *key = NULL;
     const char *check_type = NULL;
     GHashTableIter gIter;
     stonith_device_t *device = NULL;
 
     if (!g_hash_table_size(device_list)) {
         callback(NULL, user_data);
         return;
     }
 
     search = calloc(1, sizeof(struct device_search_s));
     if (!search) {
         callback(NULL, user_data);
         return;
     }
 
     g_hash_table_iter_init(&gIter, device_list);
     while (g_hash_table_iter_next(&gIter, (void **)&key, (void **)&device)) {
         check_type = target_list_type(device);
         if (safe_str_eq(check_type, "status") || safe_str_eq(check_type, "dynamic-list")) {
             devices_needing_async_query++;
         }
     }
 
     /* If we have devices that require an async event in order to know what
      * nodes they can fence, we have to give the events a timeout. The total
      * query timeout is divided among those events. */
     if (devices_needing_async_query) {
         per_device_timeout = timeout / devices_needing_async_query;
         if (!per_device_timeout) {
             crm_err("stonith-timeout duration %d is too low, raise the duration to %d seconds",
                     timeout, DEFAULT_QUERY_TIMEOUT * devices_needing_async_query);
             per_device_timeout = DEFAULT_QUERY_TIMEOUT;
         } else if (per_device_timeout < DEFAULT_QUERY_TIMEOUT) {
             crm_notice
                 ("stonith-timeout duration %d is low for the current configuration. Consider raising it to %d seconds",
                  timeout, DEFAULT_QUERY_TIMEOUT * devices_needing_async_query);
         }
     }
 
     search->host = host ? strdup(host) : NULL;
     search->action = action ? strdup(action) : NULL;
     search->per_device_timeout = per_device_timeout;
     /* We are guaranteed this many replies. Even if a device gets
      * unregistered some how during the async search, we will get
      * the correct number of replies. */
     search->replies_needed = g_hash_table_size(device_list);
     search->callback = callback;
     search->user_data = user_data;
     /* kick off the search */
 
     crm_debug("Searching through %d devices to see what is capable of action (%s) for target %s",
               search->replies_needed,
               search->action ? search->action : "<unknown>",
               search->host ? search->host : "<anyone>");
     g_hash_table_foreach(device_list, search_devices, search);
 }
 
 struct st_query_data {
     xmlNode *reply;
     char *remote_peer;
     char *client_id;
     char *target;
     char *action;
     int call_options;
 };
 
 static void
 stonith_query_capable_device_cb(GList * devices, void *user_data)
 {
     struct st_query_data *query = user_data;
     int available_devices = 0;
     xmlNode *dev = NULL;
     xmlNode *list = NULL;
     GListPtr lpc = NULL;
 
     /* Pack the results into data */
     list = create_xml_node(NULL, __FUNCTION__);
     crm_xml_add(list, F_STONITH_TARGET, query->target);
     for (lpc = devices; lpc != NULL; lpc = lpc->next) {
         stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
         int action_specific_timeout;
 
         if (!device) {
             /* It is possible the device got unregistered while
              * determining who can fence the target */
             continue;
         }
 
         available_devices++;
 
         action_specific_timeout = get_action_timeout(device, query->action, 0);
         dev = create_xml_node(list, F_STONITH_DEVICE);
         crm_xml_add(dev, XML_ATTR_ID, device->id);
         crm_xml_add(dev, "namespace", device->namespace);
         crm_xml_add(dev, "agent", device->agent);
         crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified);
         if (action_specific_timeout) {
             crm_xml_add_int(dev, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
         }
         if (query->target == NULL) {
             xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
 
             g_hash_table_foreach(device->params, hash2field, attrs);
         }
     }
 
     crm_xml_add_int(list, "st-available-devices", available_devices);
     if (query->target) {
         crm_debug("Found %d matching devices for '%s'", available_devices, query->target);
     } else {
         crm_debug("%d devices installed", available_devices);
     }
 
     if (list != NULL) {
         crm_trace("Attaching query list output");
         add_message_xml(query->reply, F_STONITH_CALLDATA, list);
     }
     stonith_send_reply(query->reply, query->call_options, query->remote_peer, query->client_id);
 
     free_xml(query->reply);
     free(query->remote_peer);
     free(query->client_id);
     free(query->target);
     free(query->action);
     free(query);
     free_xml(list);
     g_list_free_full(devices, free);
 }
 
 static void
 stonith_query(xmlNode * msg, const char *remote_peer, const char *client_id, int call_options)
 {
     struct st_query_data *query = NULL;
     const char *action = NULL;
     const char *target = NULL;
     int timeout = 0;
     xmlNode *dev = get_xpath_object("//@" F_STONITH_ACTION, msg, LOG_DEBUG_3);
 
     crm_element_value_int(msg, F_STONITH_TIMEOUT, &timeout);
     if (dev) {
         const char *device = crm_element_value(dev, F_STONITH_DEVICE);
 
         target = crm_element_value(dev, F_STONITH_TARGET);
         action = crm_element_value(dev, F_STONITH_ACTION);
         if (device && safe_str_eq(device, "manual_ack")) {
             /* No query or reply necessary */
             return;
         }
     }
 
     crm_log_xml_debug(msg, "Query");
     query = calloc(1, sizeof(struct st_query_data));
 
     query->reply = stonith_construct_reply(msg, NULL, NULL, pcmk_ok);
     query->remote_peer = remote_peer ? strdup(remote_peer) : NULL;
     query->client_id = client_id ? strdup(client_id) : NULL;
     query->target = target ? strdup(target) : NULL;
     query->action = action ? strdup(action) : NULL;
     query->call_options = call_options;
 
     get_capable_devices(target, action, timeout, query, stonith_query_capable_device_cb);
 }
 
 #define ST_LOG_OUTPUT_MAX 512
 static void
 log_operation(async_command_t * cmd, int rc, int pid, const char *next, const char *output)
 {
     if (rc == 0) {
         next = NULL;
     }
 
     if (cmd->victim != NULL) {
         do_crm_log(rc == 0 ? LOG_NOTICE : LOG_ERR,
                    "Operation '%s' [%d] (call %d from %s) for host '%s' with device '%s' returned: %d (%s)%s%s",
                    cmd->action, pid, cmd->id, cmd->client_name, cmd->victim, cmd->device, rc,
                    pcmk_strerror(rc), next ? ". Trying: " : "", next ? next : "");
     } else {
         do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
                             "Operation '%s' [%d] for device '%s' returned: %d (%s)%s%s",
                             cmd->action, pid, cmd->device, rc, pcmk_strerror(rc),
                             next ? ". Trying: " : "", next ? next : "");
     }
 
     if (output) {
         /* Logging the whole string confuses syslog when the string is xml */
         char *prefix = g_strdup_printf("%s:%d", cmd->device, pid);
 
         crm_log_output(rc == 0 ? LOG_INFO : LOG_WARNING, prefix, output);
         g_free(prefix);
     }
 }
 
 static void
 stonith_send_async_reply(async_command_t * cmd, const char *output, int rc, GPid pid)
 {
     xmlNode *reply = NULL;
     gboolean bcast = FALSE;
 
     reply = stonith_construct_async_reply(cmd, output, NULL, rc);
 
     if (safe_str_eq(cmd->action, "metadata")) {
         /* Too verbose to log */
         crm_trace("Metadata query for %s", cmd->device);
         output = NULL;
 
     } else if (crm_str_eq(cmd->action, "monitor", TRUE) ||
                crm_str_eq(cmd->action, "list", TRUE) || crm_str_eq(cmd->action, "status", TRUE)) {
         crm_trace("Never broadcast %s replies", cmd->action);
 
     } else if (!stand_alone && safe_str_eq(cmd->origin, cmd->victim)) {
         crm_trace("Broadcast %s reply for %s", cmd->action, cmd->victim);
         crm_xml_add(reply, F_SUBTYPE, "broadcast");
         bcast = TRUE;
     }
 
     log_operation(cmd, rc, pid, NULL, output);
     crm_log_xml_trace(reply, "Reply");
 
     if (bcast) {
         crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
         send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
 
     } else if (cmd->origin) {
         crm_trace("Directed reply to %s", cmd->origin);
         send_cluster_message(crm_get_peer(0, cmd->origin), crm_msg_stonith_ng, reply, FALSE);
 
     } else {
         crm_trace("Directed local %ssync reply to %s",
                   (cmd->options & st_opt_sync_call) ? "" : "a-", cmd->client_name);
         do_local_reply(reply, cmd->client, cmd->options & st_opt_sync_call, FALSE);
     }
 
     if (stand_alone) {
         /* Do notification with a clean data object */
         xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
 
         crm_xml_add_int(notify_data, F_STONITH_RC, rc);
         crm_xml_add(notify_data, F_STONITH_TARGET, cmd->victim);
         crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op);
         crm_xml_add(notify_data, F_STONITH_DELEGATE, cmd->device);
         crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
         crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client);
 
         do_stonith_notify(0, T_STONITH_NOTIFY_FENCE, rc, notify_data);
     }
 
     free_xml(reply);
 }
 
 static void
 cancel_stonith_command(async_command_t * cmd)
 {
     stonith_device_t *device;
 
     CRM_CHECK(cmd != NULL, return);
 
     if (!cmd->device) {
         return;
     }
 
     device = g_hash_table_lookup(device_list, cmd->device);
 
     if (device) {
         crm_trace("Cancel scheduled %s on %s", cmd->action, device->id);
         device->pending_ops = g_list_remove(device->pending_ops, cmd);
     }
 }
 
 #define READ_MAX 500
 static void
 st_child_done(GPid pid, int rc, const char *output, gpointer user_data)
 {
     stonith_device_t *device = NULL;
     async_command_t *cmd = user_data;
 
     GListPtr gIter = NULL;
     GListPtr gIterNext = NULL;
 
     CRM_CHECK(cmd != NULL, return);
 
     active_children--;
 
     /* The device is ready to do something else now */
     device = g_hash_table_lookup(device_list, cmd->device);
     if (device) {
         device->active_pid = 0;
         if (rc == pcmk_ok &&
             (safe_str_eq(cmd->action, "list") ||
              safe_str_eq(cmd->action, "monitor") || safe_str_eq(cmd->action, "status"))) {
 
             device->verified = TRUE;
         }
 
         mainloop_set_trigger(device->work);
     }
 
     crm_trace("Operation %s on %s completed with rc=%d (%d remaining)",
               cmd->action, cmd->device, rc, g_list_length(cmd->device_next));
 
     if (rc != 0 && cmd->device_next) {
         stonith_device_t *dev = g_hash_table_lookup(device_list, cmd->device_next->data);
 
         if (dev) {
             log_operation(cmd, rc, pid, dev->id, output);
 
             cmd->device_next = cmd->device_next->next;
             schedule_stonith_command(cmd, dev);
             /* Prevent cmd from being freed */
             cmd = NULL;
             goto done;
         }
     }
 
     if (rc > 0) {
         /* Try to provide _something_ useful */
         if(output == NULL) {
             rc = -ENODATA;
 
         } else if(strstr(output, "imed out")) {
             rc = -ETIMEDOUT;
 
         } else if(strstr(output, "Unrecognised action")) {
             rc = -EOPNOTSUPP;
 
         } else {
             rc = -pcmk_err_generic;
         }
     }
 
     stonith_send_async_reply(cmd, output, rc, pid);
 
     if (rc != 0) {
         goto done;
     }
 
     /* Check to see if any operations are scheduled to do the exact
      * same thing that just completed.  If so, rather than
      * performing the same fencing operation twice, return the result
      * of this operation for all pending commands it matches. */
     for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
         async_command_t *cmd_other = gIter->data;
 
         gIterNext = gIter->next;
 
         if (cmd == cmd_other) {
             continue;
         }
 
         /* A pending scheduled command matches the command that just finished if.
          * 1. The client connections are different.
          * 2. The node victim is the same.
          * 3. The fencing action is the same.
          * 4. The device scheduled to execute the action is the same.
          */
         if (safe_str_eq(cmd->client, cmd_other->client) ||
             safe_str_neq(cmd->victim, cmd_other->victim) ||
             safe_str_neq(cmd->action, cmd_other->action) ||
             safe_str_neq(cmd->device, cmd_other->device)) {
 
             continue;
         }
 
         crm_notice
             ("Merging stonith action %s for node %s originating from client %s with identical stonith request from client %s",
              cmd_other->action, cmd_other->victim, cmd_other->client_name, cmd->client_name);
 
         cmd_list = g_list_remove_link(cmd_list, gIter);
 
         stonith_send_async_reply(cmd_other, output, rc, pid);
         cancel_stonith_command(cmd_other);
 
         free_async_command(cmd_other);
         g_list_free_1(gIter);
     }
 
   done:
     free_async_command(cmd);
 }
 
 static gint
 sort_device_priority(gconstpointer a, gconstpointer b)
 {
     const stonith_device_t *dev_a = a;
     const stonith_device_t *dev_b = b;
 
     if (dev_a->priority > dev_b->priority) {
         return -1;
     } else if (dev_a->priority < dev_b->priority) {
         return 1;
     }
     return 0;
 }
 
 static void
 stonith_fence_get_devices_cb(GList * devices, void *user_data)
 {
     async_command_t *cmd = user_data;
     stonith_device_t *device = NULL;
 
     crm_info("Found %d matching devices for '%s'", g_list_length(devices), cmd->victim);
 
     if (g_list_length(devices) > 0) {
         /* Order based on priority */
         devices = g_list_sort(devices, sort_device_priority);
         device = g_hash_table_lookup(device_list, devices->data);
 
         if (device) {
             cmd->device_list = devices;
             cmd->device_next = devices->next;
             devices = NULL;     /* list owned by cmd now */
         }
     }
 
     /* we have a device, schedule it for fencing. */
     if (device) {
         schedule_stonith_command(cmd, device);
         /* in progress */
         return;
     }
 
     /* no device found! */
     stonith_send_async_reply(cmd, NULL, -ENODEV, 0);
 
     free_async_command(cmd);
     g_list_free_full(devices, free);
 }
 
 static int
 stonith_fence(xmlNode * msg)
 {
     const char *device_id = NULL;
     stonith_device_t *device = NULL;
     async_command_t *cmd = create_async_command(msg);
     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
 
     if (cmd == NULL) {
         return -EPROTO;
     }
 
     device_id = crm_element_value(dev, F_STONITH_DEVICE);
     if (device_id) {
         device = g_hash_table_lookup(device_list, device_id);
         if (device == NULL) {
             crm_err("Requested device '%s' is not available", device_id);
             return -ENODEV;
         }
         schedule_stonith_command(cmd, device);
 
     } else {
         const char *host = crm_element_value(dev, F_STONITH_TARGET);
 
         if (cmd->options & st_opt_cs_nodeid) {
             int nodeid = crm_atoi(host, NULL);
             crm_node_t *node = crm_get_peer(nodeid, NULL);
 
             if (node) {
                 host = node->uname;
             }
         }
         get_capable_devices(host, cmd->action, cmd->default_timeout, cmd,
                             stonith_fence_get_devices_cb);
     }
 
     return -EINPROGRESS;
 }
 
 xmlNode *
 stonith_construct_reply(xmlNode * request, const char *output, xmlNode * data, int rc)
 {
     int lpc = 0;
     xmlNode *reply = NULL;
 
     const char *name = NULL;
     const char *value = NULL;
 
     const char *names[] = {
         F_STONITH_OPERATION,
         F_STONITH_CALLID,
         F_STONITH_CLIENTID,
         F_STONITH_CLIENTNAME,
         F_STONITH_REMOTE_OP_ID,
         F_STONITH_CALLOPTS
     };
 
     crm_trace("Creating a basic reply");
     reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __FUNCTION__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
     crm_xml_add(reply, "st_output", output);
     crm_xml_add_int(reply, F_STONITH_RC, rc);
 
     CRM_CHECK(request != NULL, crm_warn("Can't create a sane reply"); return reply);
     for (lpc = 0; lpc < DIMOF(names); lpc++) {
         name = names[lpc];
         value = crm_element_value(request, name);
         crm_xml_add(reply, name, value);
     }
 
     if (data != NULL) {
         crm_trace("Attaching reply output");
         add_message_xml(reply, F_STONITH_CALLDATA, data);
     }
     return reply;
 }
 
 static xmlNode *
 stonith_construct_async_reply(async_command_t * cmd, const char *output, xmlNode * data, int rc)
 {
     xmlNode *reply = NULL;
 
     crm_trace("Creating a basic reply");
     reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __FUNCTION__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
 
     crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
     crm_xml_add(reply, F_STONITH_DEVICE, cmd->device);
     crm_xml_add(reply, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id);
     crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
     crm_xml_add(reply, F_STONITH_CLIENTNAME, cmd->client_name);
     crm_xml_add(reply, F_STONITH_TARGET, cmd->victim);
     crm_xml_add(reply, F_STONITH_ACTION, cmd->op);
     crm_xml_add(reply, F_STONITH_ORIGIN, cmd->origin);
     crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
     crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);
 
     crm_xml_add_int(reply, F_STONITH_RC, rc);
 
     crm_xml_add(reply, "st_output", output);
 
     if (data != NULL) {
         crm_info("Attaching reply output");
         add_message_xml(reply, F_STONITH_CALLDATA, data);
     }
     return reply;
 }
 
 bool fencing_peer_active(crm_node_t *peer)
 {
     if (peer == NULL) {
         return FALSE;
     } else if (peer->uname == NULL) {
         return FALSE;
     } else if(peer->processes & (crm_proc_plugin | crm_proc_heartbeat | crm_proc_cpg)) {
         return TRUE;
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Determine if we need to use an alternate node to
  * fence the target. If so return that node's uname
  *
  * \retval NULL, no alternate host
  * \retval uname, uname of alternate host to use
  */
 static const char *
 check_alternate_host(const char *target)
 {
     const char *alternate_host = NULL;
 
     if (g_hash_table_lookup(topology, target) && safe_str_eq(target, stonith_our_uname)) {
         GHashTableIter gIter;
         crm_node_t *entry = NULL;
 
         g_hash_table_iter_init(&gIter, crm_peer_cache);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
             crm_trace("Checking for %s.%d != %s", entry->uname, entry->id, target);
             if (fencing_peer_active(entry)
                 && safe_str_neq(entry->uname, target)) {
                 alternate_host = entry->uname;
                 break;
             }
         }
         if (alternate_host == NULL) {
             crm_err("No alternate host available to handle complex self fencing request");
             g_hash_table_iter_init(&gIter, crm_peer_cache);
             while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
                 crm_notice("Peer[%d] %s", entry->id, entry->uname);
             }
         }
     }
 
     return alternate_host;
 }
 
 static void
 stonith_send_reply(xmlNode * reply, int call_options, const char *remote_peer,
                    const char *client_id)
 {
     if (remote_peer) {
         send_cluster_message(crm_get_peer(0, remote_peer), crm_msg_stonith_ng, reply, FALSE);
     } else {
         do_local_reply(reply, client_id, is_set(call_options, st_opt_sync_call), remote_peer != NULL);
     }
 }
 
 static int
 handle_request(crm_client_t * client, uint32_t id, uint32_t flags, xmlNode * request,
                const char *remote_peer)
 {
     int call_options = 0;
     int rc = -EOPNOTSUPP;
 
     xmlNode *data = NULL;
     xmlNode *reply = NULL;
 
     char *output = NULL;
     const char *op = crm_element_value(request, F_STONITH_OPERATION);
     const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);
 
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
 
     if (is_set(call_options, st_opt_sync_call)) {
         CRM_ASSERT(client == NULL || client->request_id == id);
     }
 
     if (crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
         xmlNode *reply = create_xml_node(NULL, "reply");
 
         CRM_ASSERT(client);
         crm_xml_add(reply, F_STONITH_OPERATION, CRM_OP_REGISTER);
         crm_xml_add(reply, F_STONITH_CLIENTID, client->id);
         crm_ipcs_send(client, id, reply, FALSE);
         client->request_id = 0;
         free_xml(reply);
         return 0;
 
     } else if (crm_str_eq(op, STONITH_OP_EXEC, TRUE)) {
         rc = stonith_device_action(request, &output);
 
     } else if (crm_str_eq(op, STONITH_OP_TIMEOUT_UPDATE, TRUE)) {
         const char *call_id = crm_element_value(request, F_STONITH_CALLID);
         const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);
         int op_timeout = 0;
 
         crm_element_value_int(request, F_STONITH_TIMEOUT, &op_timeout);
         do_stonith_async_timeout_update(client_id, call_id, op_timeout);
         return 0;
 
     } else if (crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
         if (remote_peer) {
             create_remote_stonith_op(client_id, request, TRUE); /* Record it for the future notification */
         }
         stonith_query(request, remote_peer, client_id, call_options);
         return 0;
 
     } else if (crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
         const char *flag_name = NULL;
 
         CRM_ASSERT(client);
         flag_name = crm_element_value(request, F_STONITH_NOTIFY_ACTIVATE);
         if (flag_name) {
             crm_debug("Setting %s callbacks for %s (%s): ON", flag_name, client->name, client->id);
             client->options |= get_stonith_flag(flag_name);
         }
 
         flag_name = crm_element_value(request, F_STONITH_NOTIFY_DEACTIVATE);
         if (flag_name) {
             crm_debug("Setting %s callbacks for %s (%s): off", flag_name, client->name, client->id);
             client->options |= get_stonith_flag(flag_name);
         }
 
         if (flags & crm_ipc_client_response) {
             crm_ipcs_send_ack(client, id, "ack", __FUNCTION__, __LINE__);
             client->request_id = 0;
         }
         return 0;
 
     } else if (crm_str_eq(op, STONITH_OP_RELAY, TRUE)) {
         xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
 
         crm_notice("Peer %s has received a forwarded fencing request from %s to fence (%s) peer %s",
                    stonith_our_uname,
                    client ? client->name : remote_peer,
                    crm_element_value(dev, F_STONITH_ACTION),
                    crm_element_value(dev, F_STONITH_TARGET));
 
         if (initiate_remote_stonith_op(NULL, request, FALSE) != NULL) {
             rc = -EINPROGRESS;
         }
 
     } else if (crm_str_eq(op, STONITH_OP_FENCE, TRUE)) {
 
         if (remote_peer || stand_alone) {
             rc = stonith_fence(request);
 
         } else if (call_options & st_opt_manual_ack) {
             remote_fencing_op_t *rop = NULL;
             xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
             const char *target = crm_element_value(dev, F_STONITH_TARGET);
 
             crm_notice("Recieved manual confirmation that %s is fenced", target);
             rop = initiate_remote_stonith_op(client, request, TRUE);
             rc = stonith_manual_ack(request, rop);
 
         } else {
             const char *alternate_host = NULL;
             xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
             const char *target = crm_element_value(dev, F_STONITH_TARGET);
             const char *action = crm_element_value(dev, F_STONITH_ACTION);
             const char *device = crm_element_value(dev, F_STONITH_DEVICE);
 
             if (client) {
                 int tolerance = 0;
 
                 crm_notice("Client %s.%.8s wants to fence (%s) '%s' with device '%s'",
                            client->name, client->id, action, target, device ? device : "(any)");
 
                 crm_element_value_int(dev, F_STONITH_TOLERANCE, &tolerance);
 
                 if (stonith_check_fence_tolerance(tolerance, target, action)) {
                     rc = 0;
                     goto done;
                 }
 
             } else {
                 crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
                            remote_peer, action, target, device ? device : "(any)");
             }
 
             alternate_host = check_alternate_host(target);
 
             if (alternate_host && client) {
                 const char *client_id = NULL;
 
                 crm_notice("Forwarding complex self fencing request to peer %s", alternate_host);
 
                 if (client) {
                     client_id = client->id;
                 } else {
                     client_id = crm_element_value(request, F_STONITH_CLIENTID);
                 }
                 /* Create a record of it, otherwise call_id will be 0 if we need to notify of failures */
                 create_remote_stonith_op(client_id, request, FALSE);
 
                 crm_xml_add(request, F_STONITH_OPERATION, STONITH_OP_RELAY);
                 crm_xml_add(request, F_STONITH_CLIENTID, client->id);
                 send_cluster_message(crm_get_peer(0, alternate_host), crm_msg_stonith_ng, request,
                                      FALSE);
                 rc = -EINPROGRESS;
 
             } else if (initiate_remote_stonith_op(client, request, FALSE) != NULL) {
                 rc = -EINPROGRESS;
             }
         }
 
     } else if (crm_str_eq(op, STONITH_OP_FENCE_HISTORY, TRUE)) {
         rc = stonith_fence_history(request, &data);
 
     } else if (crm_str_eq(op, STONITH_OP_DEVICE_ADD, TRUE)) {
         const char *id = NULL;
         xmlNode *notify_data = create_xml_node(NULL, op);
 
         rc = stonith_device_register(request, &id, FALSE);
 
         crm_xml_add(notify_data, F_STONITH_DEVICE, id);
         crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(device_list));
 
         do_stonith_notify(call_options, op, rc, notify_data);
         free_xml(notify_data);
 
     } else if (crm_str_eq(op, STONITH_OP_DEVICE_DEL, TRUE)) {
         xmlNode *dev = get_xpath_object("//" F_STONITH_DEVICE, request, LOG_ERR);
         const char *id = crm_element_value(dev, XML_ATTR_ID);
         xmlNode *notify_data = create_xml_node(NULL, op);
 
         rc = stonith_device_remove(id, FALSE);
 
         crm_xml_add(notify_data, F_STONITH_DEVICE, id);
         crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(device_list));
 
         do_stonith_notify(call_options, op, rc, notify_data);
         free_xml(notify_data);
 
     } else if (crm_str_eq(op, STONITH_OP_LEVEL_ADD, TRUE)) {
         char *id = NULL;
         xmlNode *notify_data = create_xml_node(NULL, op);
 
         rc = stonith_level_register(request, &id);
 
         crm_xml_add(notify_data, F_STONITH_DEVICE, id);
         crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(topology));
 
         do_stonith_notify(call_options, op, rc, notify_data);
         free_xml(notify_data);
 
     } else if (crm_str_eq(op, STONITH_OP_LEVEL_DEL, TRUE)) {
         char *id = NULL;
         xmlNode *notify_data = create_xml_node(NULL, op);
 
         rc = stonith_level_remove(request, &id);
 
         crm_xml_add(notify_data, F_STONITH_DEVICE, id);
         crm_xml_add_int(notify_data, F_STONITH_ACTIVE, g_hash_table_size(topology));
 
         do_stonith_notify(call_options, op, rc, notify_data);
         free_xml(notify_data);
 
     } else if (crm_str_eq(op, STONITH_OP_CONFIRM, TRUE)) {
         async_command_t *cmd = create_async_command(request);
         xmlNode *reply = stonith_construct_async_reply(cmd, NULL, NULL, 0);
 
         crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
         crm_notice("Broadcasting manual fencing confirmation for node %s", cmd->victim);
         send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
 
         free_async_command(cmd);
         free_xml(reply);
 
     } else {
         crm_err("Unknown %s from %s", op, client ? client->name : remote_peer);
         crm_log_xml_warn(request, "UnknownOp");
     }
 
   done:
 
     /* Always reply unles the request is in process still.
      * If in progress, a reply will happen async after the request
      * processing is finished */
     if (rc != -EINPROGRESS) {
         crm_trace("Reply handling: %p %u %u %d %d %s", client, client?client->request_id:0,
                   id, is_set(call_options, st_opt_sync_call), call_options,
                   crm_element_value(request, F_STONITH_CALLOPTS));
 
         if (is_set(call_options, st_opt_sync_call)) {
             CRM_ASSERT(client == NULL || client->request_id == id);
         }
         reply = stonith_construct_reply(request, output, data, rc);
         stonith_send_reply(reply, call_options, remote_peer, client_id);
     }
 
     free(output);
     free_xml(data);
     free_xml(reply);
 
     return rc;
 }
 
 static void
 handle_reply(crm_client_t * client, xmlNode * request, const char *remote_peer)
 {
     const char *op = crm_element_value(request, F_STONITH_OPERATION);
 
     if (crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
         process_remote_stonith_query(request);
     } else if (crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
         process_remote_stonith_exec(request);
     } else if (crm_str_eq(op, STONITH_OP_FENCE, TRUE)) {
         /* Reply to a complex fencing op */
         process_remote_stonith_exec(request);
     } else {
         crm_err("Unknown %s reply from %s", op, client ? client->name : remote_peer);
         crm_log_xml_warn(request, "UnknownOp");
     }
 }
 
 void
 stonith_command(crm_client_t * client, uint32_t id, uint32_t flags, xmlNode * request,
                 const char *remote_peer)
 {
     int call_options = 0;
     int rc = 0;
     gboolean is_reply = FALSE;
     char *op = crm_element_value_copy(request, F_STONITH_OPERATION);
 
     /* F_STONITH_OPERATION can be overwritten in remote_op_done() with crm_xml_add()
      *
      * by 0x4C2E934: crm_xml_add (xml.c:377)
      * by 0x40C5E9: remote_op_done (remote.c:178)
      * by 0x40F1D3: process_remote_stonith_exec (remote.c:1084)
      * by 0x40AD4F: stonith_command (commands.c:1891)
      *
      */
 
     if (get_xpath_object("//" T_STONITH_REPLY, request, LOG_DEBUG_3)) {
         is_reply = TRUE;
     }
 
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
     crm_debug("Processing %s%s %u from %s (%16x)", op, is_reply ? " reply" : "",
               id, client ? client->name : remote_peer, call_options);
 
     if (is_set(call_options, st_opt_sync_call)) {
         CRM_ASSERT(client == NULL || client->request_id == id);
     }
 
     if (is_reply) {
         handle_reply(client, request, remote_peer);
     } else {
         rc = handle_request(client, id, flags, request, remote_peer);
     }
 
     do_crm_log_unlikely(rc > 0 ? LOG_DEBUG : LOG_INFO, "Processed %s%s from %s: %s (%d)", op,
                         is_reply ? " reply" : "", client ? client->name : remote_peer,
                         rc > 0 ? "" : pcmk_strerror(rc), rc);
 
     free(op);
 }
diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c
index e3a6057a34..ae16682d75 100644
--- a/lib/cluster/cpg.c
+++ b/lib/cluster/cpg.c
@@ -1,675 +1,675 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 #include <bzlib.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <netdb.h>
 
 #include <crm/common/ipc.h>
 #include <crm/cluster/internal.h>
 #include <crm/common/mainloop.h>
 #include <sys/utsname.h>
 
 #include <qb/qbipcc.h>
 #include <qb/qbutil.h>
 
 #include <corosync/corodefs.h>
 #include <corosync/corotypes.h>
 #include <corosync/hdb.h>
 #include <corosync/cpg.h>
 
 #include <crm/msg_xml.h>
 
 cpg_handle_t pcmk_cpg_handle = 0; /* TODO: Remove, use cluster.cpg_handle */
 
 static bool cpg_evicted = FALSE;
 gboolean(*pcmk_cpg_dispatch_fn) (int kind, const char *from, const char *data) = NULL;
 
 #define cs_repeat(counter, max, code) do {		\
 	code;						\
 	if(rc == CS_ERR_TRY_AGAIN || rc == CS_ERR_QUEUE_FULL) {  \
 	    counter++;					\
 	    crm_debug("Retrying operation after %ds", counter);	\
 	    sleep(counter);				\
 	} else {                                        \
             break;                                      \
         }                                               \
     } while(counter < max)
 
 void
 cluster_disconnect_cpg(crm_cluster_t *cluster)
 {
     pcmk_cpg_handle = 0;
     if (cluster->cpg_handle) {
         crm_trace("Disconnecting CPG");
         cpg_leave(cluster->cpg_handle, &cluster->group);
         cpg_finalize(cluster->cpg_handle);
         cluster->cpg_handle = 0;
 
     } else {
         crm_info("No CPG connection");
     }
 }
 
 uint32_t get_local_nodeid(cpg_handle_t handle)
 {
     int rc = CS_OK;
     int retries = 0;
     static uint32_t local_nodeid = 0;
     cpg_handle_t local_handle = handle;
     cpg_callbacks_t cb = { };
 
     if(local_nodeid != 0) {
         return local_nodeid;
     }
 
 #if 0
     /* Should not be necessary */
     if(get_cluster_type() == pcmk_cluster_classic_ais) {
         get_ais_details(&local_nodeid, NULL);
         goto done;
     }
 #endif
 
     if(handle == 0) {
         crm_trace("Creating connection");
         cs_repeat(retries, 5, rc = cpg_initialize(&local_handle, &cb));
     }
 
     if (rc == CS_OK) {
         retries = 0;
         crm_trace("Performing lookup");
         cs_repeat(retries, 5, rc = cpg_local_get(local_handle, &local_nodeid));
     }
 
     if (rc != CS_OK) {
         crm_err("Could not get local node id from the CPG API: %s (%d)", ais_error2text(rc), rc);
     }
     if(handle == 0) {
         crm_trace("Closing connection");
         cpg_finalize(local_handle);
     }
     crm_debug("Local nodeid is %u", local_nodeid);
     return local_nodeid;
 }
 
 
 GListPtr cs_message_queue = NULL;
 int cs_message_timer = 0;
 
 static ssize_t crm_cs_flush(gpointer data);
 
 static gboolean
 crm_cs_flush_cb(gpointer data)
 {
     cs_message_timer = 0;
     crm_cs_flush(data);
     return FALSE;
 }
 
 #define CS_SEND_MAX 200
 static ssize_t
 crm_cs_flush(gpointer data)
 {
     int sent = 0;
     ssize_t rc = 0;
     int queue_len = 0;
     static unsigned int last_sent = 0;
     cpg_handle_t *handle = (cpg_handle_t *)data;
 
     if (*handle == 0) {
         crm_trace("Connection is dead");
         return pcmk_ok;
     }
 
     queue_len = g_list_length(cs_message_queue);
     if ((queue_len % 1000) == 0 && queue_len > 1) {
         crm_err("CPG queue has grown to %d", queue_len);
 
     } else if (queue_len == CS_SEND_MAX) {
         crm_warn("CPG queue has grown to %d", queue_len);
     }
 
     if (cs_message_timer) {
         /* There is already a timer, wait until it goes off */
         crm_trace("Timer active %d", cs_message_timer);
         return pcmk_ok;
     }
 
     while (cs_message_queue && sent < CS_SEND_MAX) {
         struct iovec *iov = cs_message_queue->data;
 
         errno = 0;
         rc = cpg_mcast_joined(*handle, CPG_TYPE_AGREED, iov, 1);
 
         if (rc != CS_OK) {
             break;
         }
 
         sent++;
         last_sent++;
         crm_trace("CPG message sent, size=%d", iov->iov_len);
 
         cs_message_queue = g_list_remove(cs_message_queue, iov);
         free(iov[0].iov_base);
         free(iov);
     }
 
     queue_len -= sent;
     if (sent > 1 || cs_message_queue) {
         crm_info("Sent %d CPG messages  (%d remaining, last=%u): %s (%d)",
                  sent, queue_len, last_sent, ais_error2text(rc), rc);
     } else {
         crm_trace("Sent %d CPG messages  (%d remaining, last=%u): %s (%d)",
                   sent, queue_len, last_sent, ais_error2text(rc), rc);
     }
 
     if (cs_message_queue) {
         uint32_t delay_ms = 100;
         if(rc != CS_OK) {
             /* Proportionally more if sending failed but cap at 1s */
             delay_ms = QB_MIN(1000, CS_SEND_MAX + (10 * queue_len));
         }
         cs_message_timer = g_timeout_add(delay_ms, crm_cs_flush_cb, data);
     }
 
     return rc;
 }
 
 gboolean
 send_cpg_iov(struct iovec * iov)
 {
     static unsigned int queued = 0;
 
     queued++;
     crm_trace("Queueing CPG message %u (%d bytes)", queued, iov->iov_len);
     cs_message_queue = g_list_append(cs_message_queue, iov);
     crm_cs_flush(&pcmk_cpg_handle);
     return TRUE;
 }
 
 static int
 pcmk_cpg_dispatch(gpointer user_data)
 {
     int rc = 0;
     crm_cluster_t *cluster = (crm_cluster_t*) user_data;
 
     rc = cpg_dispatch(cluster->cpg_handle, CS_DISPATCH_ALL);
     if (rc != CS_OK) {
         crm_err("Connection to the CPG API failed: %s (%d)", ais_error2text(rc), rc);
         cluster->cpg_handle = 0;
         return -1;
 
     } else if(cpg_evicted) {
         crm_err("Evicted from CPG membership");
         return -1;
     }
     return 0;
 }
 
 char *
 pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void *content,
                         uint32_t *kind, const char **from)
 {
     char *data = NULL;
     AIS_Message *msg = (AIS_Message *) content;
 
     if(handle) {
         /* 'msg' came from CPG not the plugin
          * Do filtering and field massaging
          */
         uint32_t local_nodeid = get_local_nodeid(handle);
         const char *local_name = get_local_node_name();
 
         if (msg->sender.id > 0 && msg->sender.id != nodeid) {
             crm_err("Nodeid mismatch from %d.%d: claimed nodeid=%u", nodeid, pid, msg->sender.id);
             return NULL;
 
         } else if (msg->host.id != 0 && (local_nodeid != msg->host.id)) {
             /* Not for us */
             crm_trace("Not for us: %u != %u", msg->host.id, local_nodeid);
             return NULL;
         } else if (msg->host.size != 0 && safe_str_neq(msg->host.uname, local_name)) {
             /* Not for us */
             crm_trace("Not for us: %s != %s", msg->host.uname, local_name);
             return NULL;
         }
 
         msg->sender.id = nodeid;
         if (msg->sender.size == 0) {
             crm_node_t *peer = crm_get_peer(nodeid, NULL);
 
             if (peer == NULL) {
                 crm_err("Peer with nodeid=%u is unknown", nodeid);
 
             } else if (peer->uname == NULL) {
                 crm_err("No uname for peer with nodeid=%u", nodeid);
 
             } else {
                 crm_notice("Fixing uname for peer with nodeid=%u", nodeid);
                 msg->sender.size = strlen(peer->uname);
                 memset(msg->sender.uname, 0, MAX_NAME);
                 memcpy(msg->sender.uname, peer->uname, msg->sender.size);
             }
         }
     }
 
     crm_trace("Got new%s message (size=%d, %d, %d)",
               msg->is_compressed ? " compressed" : "",
               ais_data_len(msg), msg->size, msg->compressed_size);
 
     if (kind != NULL) {
         *kind = msg->header.id;
     }
     if (from != NULL) {
         *from = msg->sender.uname;
     }
 
     if (msg->is_compressed && msg->size > 0) {
         int rc = BZ_OK;
         char *uncompressed = NULL;
         unsigned int new_size = msg->size + 1;
 
         if (check_message_sanity(msg, NULL) == FALSE) {
             goto badmsg;
         }
 
         crm_trace("Decompressing message data");
         uncompressed = calloc(1, new_size);
         rc = BZ2_bzBuffToBuffDecompress(uncompressed, &new_size, msg->data, msg->compressed_size, 1, 0);
 
         if (rc != BZ_OK) {
             crm_err("Decompression failed: %d", rc);
             goto badmsg;
         }
 
         CRM_ASSERT(rc == BZ_OK);
         CRM_ASSERT(new_size == msg->size);
 
         data = uncompressed;
 
     } else if (check_message_sanity(msg, data) == FALSE) {
         goto badmsg;
 
     } else if (safe_str_eq("identify", data)) {
         int pid = getpid();
         char *pid_s = crm_itoa(pid);
 
         send_cluster_text(crm_class_cluster, pid_s, TRUE, NULL, crm_msg_ais);
         free(pid_s);
         return NULL;
 
     } else {
         data = strdup(msg->data);
     }
 
     if (msg->header.id != crm_class_members) {
         /* Is this even needed anymore? */
         crm_get_peer(msg->sender.id, msg->sender.uname);
     }
 
     if (msg->header.id == crm_class_rmpeer) {
         uint32_t id = crm_int_helper(data, NULL);
 
         crm_info("Removing peer %s/%u", data, id);
         reap_crm_member(id, NULL);
         free(data);
         return NULL;
 
 #if SUPPORT_PLUGIN
     } else if (is_classic_ais_cluster()) {
         plugin_handle_membership(msg);
 #endif
     }
 
     crm_trace("Payload: %.200s", data);
     return data;
 
   badmsg:
     crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
             " min=%d, total=%d, size=%d, bz2_size=%d",
             msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
             ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
             msg->sender.pid, (int)sizeof(AIS_Message),
             msg->header.size, msg->size, msg->compressed_size);
 
     free(data);
     return NULL;
 }
 
 void
 pcmk_cpg_membership(cpg_handle_t handle,
                     const struct cpg_name *groupName,
                     const struct cpg_address *member_list, size_t member_list_entries,
                     const struct cpg_address *left_list, size_t left_list_entries,
                     const struct cpg_address *joined_list, size_t joined_list_entries)
 {
     int i;
     gboolean found = FALSE;
     static int counter = 0;
     uint32_t local_nodeid = get_local_nodeid(handle);
 
     for (i = 0; i < left_list_entries; i++) {
         crm_node_t *peer = crm_get_peer(left_list[i].nodeid, NULL);
 
         crm_info("Left[%d.%d] %s.%u ", counter, i, groupName->value, left_list[i].nodeid);
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, OFFLINESTATUS);
     }
 
     for (i = 0; i < joined_list_entries; i++) {
         crm_info("Joined[%d.%d] %s.%u ", counter, i, groupName->value, joined_list[i].nodeid);
     }
 
     for (i = 0; i < member_list_entries; i++) {
         crm_node_t *peer = crm_get_peer(member_list[i].nodeid, NULL);
 
         crm_info("Member[%d.%d] %s.%u ", counter, i, groupName->value, member_list[i].nodeid);
 
         /* Anyone that is sending us CPG messages must also be a _CPG_ member.
          * But its _not_ safe to assume its in the quorum membership.
          * We may have just found out its dead and are processing the last couple of messages it sent
          */
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);
         if(peer && peer->state && crm_is_peer_active(peer) == FALSE) {
             time_t now = time(NULL);
 
             /* Co-opt the otherwise unused votes field */
             if(peer->votes == 0) {
                 peer->votes = now;
 
             } else if(now > (60 + peer->votes)) {
                 /* On the otherhand, if we're still getting messages, at a certain point
                  * we need to acknowledge our internal cache is probably wrong
                  *
                  * Set the threshold to 1 minute
                  */
                 crm_err("Node %s[%u] appears to be online even though we think it is dead", peer->uname, peer->id);
                 crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_MEMBER, 0);
                 peer->votes = 0;
             }
         }
 
         if (local_nodeid == member_list[i].nodeid) {
             found = TRUE;
         }
     }
 
     if (!found) {
         crm_err("We're not part of CPG group '%s' anymore!", groupName->value);
         cpg_evicted = TRUE;
     }
 
     counter++;
 }
 
 gboolean
 cluster_connect_cpg(crm_cluster_t *cluster)
 {
     int rc = -1;
     int fd = 0;
     int retries = 0;
     uint32_t id = 0;
     crm_node_t *peer = NULL;
     cpg_handle_t handle = 0;
 
     struct mainloop_fd_callbacks cpg_fd_callbacks = {
         .dispatch = pcmk_cpg_dispatch,
         .destroy = cluster->destroy,
     };
 
     cpg_callbacks_t cpg_callbacks = {
         .cpg_deliver_fn = cluster->cpg.cpg_deliver_fn,
         .cpg_confchg_fn = cluster->cpg.cpg_confchg_fn,
         /* .cpg_deliver_fn = pcmk_cpg_deliver, */
         /* .cpg_confchg_fn = pcmk_cpg_membership, */
     };
 
     cpg_evicted = FALSE;
     cluster->group.length = 0;
     cluster->group.value[0] = 0;
 
     strncpy(cluster->group.value, crm_system_name, 128);
     cluster->group.length = strlen(crm_system_name) + 1;
 
     cs_repeat(retries, 30, rc = cpg_initialize(&handle, &cpg_callbacks));
     if (rc != CS_OK) {
         crm_err("Could not connect to the Cluster Process Group API: %d\n", rc);
         goto bail;
     }
 
     id = get_local_nodeid(handle);
     if (id == 0) {
         crm_err("Could not get local node id from the CPG API");
         goto bail;
 
     }
     cluster->nodeid = id;
 
     retries = 0;
     cs_repeat(retries, 30, rc = cpg_join(handle, &cluster->group));
     if (rc != CS_OK) {
         crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc);
         goto bail;
     }
 
     rc = cpg_fd_get(handle, &fd);
     if (rc != CS_OK) {
         crm_err("Could not obtain the CPG API connection: %d\n", rc);
         goto bail;
     }
 
     pcmk_cpg_handle = handle;
     cluster->cpg_handle = handle;
     mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks);
 
   bail:
     if (rc != CS_OK) {
         cpg_finalize(handle);
         return FALSE;
     }
 
     peer = crm_get_peer(id, NULL);
     crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cpg, ONLINESTATUS);
     return TRUE;
 }
 
 gboolean
 send_cluster_message_cs(xmlNode * msg, gboolean local, crm_node_t * node, enum crm_ais_msg_types dest)
 {
     gboolean rc = TRUE;
     char *data = NULL;
 
     data = dump_xml_unformatted(msg);
     rc = send_cluster_text(crm_class_cluster, data, local, node, dest);
     free(data);
     return rc;
 }
 
 gboolean
 send_cluster_text(int class, const char *data,
               gboolean local, crm_node_t * node, enum crm_ais_msg_types dest)
 {
     static int msg_id = 0;
     static int local_pid = 0;
     static int local_name_len = 0;
     static const char *local_name = NULL;
 
     char *target = NULL;
     struct iovec *iov;
     AIS_Message *msg = NULL;
     enum crm_ais_msg_types sender = text2msg_type(crm_system_name);
 
     /* There are only 6 handlers registered to crm_lib_service in plugin.c */
     CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class);
               return FALSE);
 
 #if !SUPPORT_PLUGIN
     CRM_CHECK(dest != crm_msg_ais, return FALSE);
 #endif
 
     if(local_name == NULL) {
         local_name = get_local_node_name();
     }
     if(local_name_len == 0 && local_name) {
         local_name_len = strlen(local_name);
     }
 
     if (data == NULL) {
         data = "";
     }
 
     if (local_pid == 0) {
         local_pid = getpid();
     }
 
     if (sender == crm_msg_none) {
         sender = local_pid;
     }
 
     msg = calloc(1, sizeof(AIS_Message));
 
     msg_id++;
     msg->id = msg_id;
     msg->header.id = class;
     msg->header.error = CS_OK;
 
     msg->host.type = dest;
     msg->host.local = local;
 
     if (node) {
         if (node->uname) {
             target = strdup(node->uname);
             msg->host.size = strlen(node->uname);
             memset(msg->host.uname, 0, MAX_NAME);
             memcpy(msg->host.uname, node->uname, msg->host.size);
         } else {
             target = g_strdup_printf("%u", node->id);
         }
         msg->host.id = node->id;
     } else {
         target = strdup("all");
     }
 
     msg->sender.id = 0;
     msg->sender.type = sender;
     msg->sender.pid = local_pid;
     msg->sender.size = local_name_len;
     memset(msg->sender.uname, 0, MAX_NAME);
     memcpy(msg->sender.uname, local_name, msg->sender.size);
 
     msg->size = 1 + strlen(data);
     msg->header.size = sizeof(AIS_Message) + msg->size;
 
     if (msg->size < CRM_BZ2_THRESHOLD) {
         msg = realloc(msg, msg->header.size);
         memcpy(msg->data, data, msg->size);
 
     } else {
         char *compressed = NULL;
         unsigned int new_size = 0;
         char *uncompressed = strdup(data);
 
         if (crm_compress_string(uncompressed, msg->size, 0, &compressed, &new_size)) {
 
             msg->header.size = sizeof(AIS_Message) + new_size + 1;
             msg = realloc(msg, msg->header.size);
             memcpy(msg->data, compressed, new_size);
             msg->data[new_size] = 0;
 
             msg->is_compressed = TRUE;
             msg->compressed_size = new_size;
 
         } else {
             msg = realloc(msg, msg->header.size);
             memcpy(msg->data, data, msg->size);
         }
 
         free(uncompressed);
         free(compressed);
     }
 
     iov = calloc(1, sizeof(struct iovec));
     iov->iov_base = msg;
     iov->iov_len = msg->header.size;
 
     if (msg->compressed_size) {
         crm_trace("Queueing CPG message %u to %s (%d bytes, %d bytes compressed payload): %.200s",
                   msg->id, target, iov->iov_len, msg->compressed_size, data);
     } else {
         crm_trace("Queueing CPG message %u to %s (%d bytes, %d bytes payload): %.200s",
                   msg->id, target, iov->iov_len, msg->size, data);
     }
     free(target);
 
 #if SUPPORT_PLUGIN
     /* The plugin is the only time we dont use CPG messaging */
     if(get_cluster_type() == pcmk_cluster_classic_ais) {
         return send_plugin_text(class, iov);
     }
 #endif
 
     send_cpg_iov(iov);
 
     return TRUE;
 }
 
 enum crm_ais_msg_types
 text2msg_type(const char *text)
 {
     int type = crm_msg_none;
 
     CRM_CHECK(text != NULL, return type);
     if (safe_str_eq(text, "ais")) {
         type = crm_msg_ais;
     } else if (safe_str_eq(text, "crm_plugin")) {
         type = crm_msg_ais;
     } else if (safe_str_eq(text, CRM_SYSTEM_CIB)) {
         type = crm_msg_cib;
     } else if (safe_str_eq(text, CRM_SYSTEM_CRMD)) {
         type = crm_msg_crmd;
     } else if (safe_str_eq(text, CRM_SYSTEM_DC)) {
         type = crm_msg_crmd;
     } else if (safe_str_eq(text, CRM_SYSTEM_TENGINE)) {
         type = crm_msg_te;
     } else if (safe_str_eq(text, CRM_SYSTEM_PENGINE)) {
         type = crm_msg_pe;
     } else if (safe_str_eq(text, CRM_SYSTEM_LRMD)) {
         type = crm_msg_lrmd;
     } else if (safe_str_eq(text, CRM_SYSTEM_STONITHD)) {
         type = crm_msg_stonithd;
     } else if (safe_str_eq(text, "stonith-ng")) {
         type = crm_msg_stonith_ng;
     } else if (safe_str_eq(text, "attrd")) {
         type = crm_msg_attrd;
 
     } else {
         /* This will normally be a transient client rather than
          * a cluster daemon.  Set the type to the pid of the client
          */
         int scan_rc = sscanf(text, "%d", &type);
 
-        if (scan_rc != 1) {
+        if (scan_rc != 1 || type <= crm_msg_stonith_ng) {
             /* Ensure its sane */
             type = crm_msg_none;
         }
     }
     return type;
 }
diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
index 6e99d88187..bdaf18ccd5 100644
--- a/lib/lrmd/lrmd_client.c
+++ b/lib/lrmd/lrmd_client.c
@@ -1,2016 +1,2009 @@
 /*
  * Copyright (c) 2012 David Vossel <dvossel@redhat.com>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  *
  */
 
 #include <crm_internal.h>
 
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 #include <ctype.h>
 
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <glib.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>
 #include <crm/services.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/ipcs.h>
 #include <crm/msg_xml.h>
 
 #include <crm/stonith-ng.h>
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  undef KEYFILE
 #  include <gnutls/gnutls.h>
 #endif
 
 #include <sys/socket.h>
 #include <netinet/ip.h>
 #include <arpa/inet.h>
 #include <netdb.h>
 
 CRM_TRACE_INIT_DATA(lrmd);
 
-static stonith_t *stonith_api = NULL;
-
 static int lrmd_api_disconnect(lrmd_t * lrmd);
 static int lrmd_api_is_connected(lrmd_t * lrmd);
 
 /* IPC proxy functions */
 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
 gnutls_psk_client_credentials_t psk_cred_s;
 int lrmd_tls_set_key(gnutls_datum_t * key);
 static void lrmd_tls_disconnect(lrmd_t * lrmd);
 static int global_remote_msg_id = 0;
 int lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type);
 static void lrmd_tls_connection_destroy(gpointer userdata);
 #endif
 
 typedef struct lrmd_private_s {
     enum client_type type;
     char *token;
     mainloop_io_t *source;
 
     /* IPC parameters */
     crm_ipc_t *ipc;
 
     crm_remote_t *remote;
 
     /* Extra TLS parameters */
     char *remote_nodename;
 #ifdef HAVE_GNUTLS_GNUTLS_H
     char *server;
     int port;
     gnutls_psk_client_credentials_t psk_cred_c;
 
     int sock;
     GList *pending_notify;
     crm_trigger_t *process_notify;
 #endif
 
     lrmd_event_callback callback;
 
     /* Internal IPC proxy msg passing for remote guests */
     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
     void *proxy_callback_userdata;
 } lrmd_private_t;
 
 static lrmd_list_t *
 lrmd_list_add(lrmd_list_t * head, const char *value)
 {
     lrmd_list_t *p, *end;
 
     p = calloc(1, sizeof(lrmd_list_t));
     p->val = strdup(value);
 
     end = head;
     while (end && end->next) {
         end = end->next;
     }
 
     if (end) {
         end->next = p;
     } else {
         head = p;
     }
 
     return head;
 }
 
 void
 lrmd_list_freeall(lrmd_list_t * head)
 {
     lrmd_list_t *p;
 
     while (head) {
         char *val = (char *)head->val;
 
         p = head->next;
         free(val);
         free(head);
         head = p;
     }
 }
 
 lrmd_key_value_t *
 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
 {
     lrmd_key_value_t *p, *end;
 
     p = calloc(1, sizeof(lrmd_key_value_t));
     p->key = strdup(key);
     p->value = strdup(value);
 
     end = head;
     while (end && end->next) {
         end = end->next;
     }
 
     if (end) {
         end->next = p;
     } else {
         head = p;
     }
 
     return head;
 }
 
 void
 lrmd_key_value_freeall(lrmd_key_value_t * head)
 {
     lrmd_key_value_t *p;
 
     while (head) {
         p = head->next;
         free(head->key);
         free(head->value);
         free(head);
         head = p;
     }
 }
 
 static void
 dup_attr(gpointer key, gpointer value, gpointer user_data)
 {
     g_hash_table_replace(user_data, strdup(key), strdup(value));
 }
 
 lrmd_event_data_t *
 lrmd_copy_event(lrmd_event_data_t * event)
 {
     lrmd_event_data_t *copy = NULL;
 
     copy = calloc(1, sizeof(lrmd_event_data_t));
 
     /* This will get all the int values.
      * we just have to be careful not to leave any
      * dangling pointers to strings. */
     memcpy(copy, event, sizeof(lrmd_event_data_t));
 
     copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
     copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
     copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
     copy->output = event->output ? strdup(event->output) : NULL;
     copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
 
     if (event->params) {
         copy->params = g_hash_table_new_full(crm_str_hash,
                                              g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
         if (copy->params != NULL) {
             g_hash_table_foreach(event->params, dup_attr, copy->params);
         }
     }
 
     return copy;
 }
 
 void
 lrmd_free_event(lrmd_event_data_t * event)
 {
     if (!event) {
         return;
     }
 
     /* free gives me grief if i try to cast */
     free((char *)event->rsc_id);
     free((char *)event->op_type);
     free((char *)event->user_data);
     free((char *)event->output);
     free((char *)event->remote_nodename);
     if (event->params) {
         g_hash_table_destroy(event->params);
     }
     free(event);
 }
 
 static int
 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
 {
     const char *type;
     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
     lrmd_private_t *native = lrmd->private;
     lrmd_event_data_t event = { 0, };
 
     if (proxy_session != NULL) {
         /* this is proxy business */
         lrmd_internal_proxy_dispatch(lrmd, msg);
         return 1;
     }
 
     if (!native->callback) {
         /* no callback set */
         crm_trace("notify event received but client has not set callback");
         return 1;
     }
 
     event.remote_nodename = native->remote_nodename;
     type = crm_element_value(msg, F_LRMD_OPERATION);
     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 
     if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) {
         event.type = lrmd_event_register;
     } else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) {
         event.type = lrmd_event_unregister;
     } else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) {
         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
         crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval);
         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 
         crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run);
         crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange);
         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 
         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
         event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
         event.type = lrmd_event_exec_complete;
 
         event.params = xml2list(msg);
     } else if (crm_str_eq(type, LRMD_OP_POKE, TRUE)) {
         event.type = lrmd_event_poke;
     } else {
         return 1;
     }
 
     crm_trace("op %s notify event received", type);
     native->callback(&event);
 
     if (event.params) {
         g_hash_table_destroy(event.params);
     }
     return 1;
 }
 
 static int
 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->private;
     xmlNode *msg;
     int rc;
 
     if (!native->callback) {
         /* no callback set */
         return 1;
     }
 
     msg = string2xml(buffer);
     rc = lrmd_dispatch_internal(lrmd, msg);
     free_xml(msg);
     return rc;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_free_xml(gpointer userdata)
 {
     free_xml((xmlNode *) userdata);
 }
 
 static int
 lrmd_tls_connected(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->private;
 
     if (native->remote->tls_session) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 static int
 lrmd_tls_dispatch(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->private;
     xmlNode *xml = NULL;
     int rc = 0;
     int disconnected = 0;
 
     if (lrmd_tls_connected(lrmd) == FALSE) {
         crm_trace("tls dispatch triggered after disconnect");
         return 0;
     }
 
     crm_trace("tls_dispatch triggered");
 
     /* First check if there are any pending notifies to process that came
      * while we were waiting for replies earlier. */
     if (native->pending_notify) {
         GList *iter = NULL;
 
         crm_trace("Processing pending notifies");
         for (iter = native->pending_notify; iter; iter = iter->next) {
             lrmd_dispatch_internal(lrmd, iter->data);
         }
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 
     /* Next read the current buffer and see if there are any messages to handle. */
     rc = crm_remote_ready(native->remote, 0);
     if (rc == 0) {
         /* nothing to read, see if any full messages are already in buffer. */
         xml = crm_remote_parse_buffer(native->remote);
     } else if (rc < 0) {
         disconnected = 1;
     } else {
         crm_remote_recv(native->remote, -1, &disconnected);
         xml = crm_remote_parse_buffer(native->remote);
     }
     while (xml) {
         lrmd_dispatch_internal(lrmd, xml);
         free_xml(xml);
         xml = crm_remote_parse_buffer(native->remote);
     }
 
     if (disconnected) {
         crm_info("Server disconnected while reading remote server msg.");
         lrmd_tls_disconnect(lrmd);
         return 0;
     }
     return 1;
 }
 #endif
 
 /* Not used with mainloop */
 int
 lrmd_poll(lrmd_t * lrmd, int timeout)
 {
     lrmd_private_t *native = lrmd->private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             return crm_ipc_ready(native->ipc);
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             if (native->pending_notify) {
                 return 1;
             } else if (native->remote->buffer
                        && strstr(native->remote->buffer, REMOTE_MSG_TERMINATOR)) {
                 return 1;
             }
 
             return crm_remote_ready(native->remote, 0);
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return 0;
 }
 
 /* Not used with mainloop */
 bool
 lrmd_dispatch(lrmd_t * lrmd)
 {
     lrmd_private_t *private = NULL;
 
     CRM_ASSERT(lrmd != NULL);
 
     private = lrmd->private;
     switch (private->type) {
         case CRM_CLIENT_IPC:
             while (crm_ipc_ready(private->ipc)) {
                 if (crm_ipc_read(private->ipc) > 0) {
                     const char *msg = crm_ipc_buffer(private->ipc);
 
                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
                 }
             }
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             lrmd_tls_dispatch(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", private->type);
     }
 
     if (lrmd_api_is_connected(lrmd) == FALSE) {
         crm_err("Connection closed");
         return FALSE;
     }
 
     return TRUE;
 }
 
 static xmlNode *
 lrmd_create_op(const char *token, const char *op, xmlNode * data, enum lrmd_call_options options)
 {
     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 
     CRM_CHECK(op_msg != NULL, return NULL);
     CRM_CHECK(token != NULL, return NULL);
 
     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 
     crm_xml_add(op_msg, F_TYPE, T_LRMD);
     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
     crm_trace("Sending call options: %.8lx, %d", (long)options, options);
     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 
     if (data != NULL) {
         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
     }
 
     return op_msg;
 }
 
 static void
 lrmd_ipc_connection_destroy(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->private;
 
     crm_info("IPC connection destroyed");
 
     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
     native->ipc = NULL;
     native->source = NULL;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.type = lrmd_event_disconnect;
         event.remote_nodename = native->remote_nodename;
         native->callback(&event);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tls_connection_destroy(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->private;
 
     crm_info("TLS connection destroyed");
 
     if (native->remote->tls_session) {
         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
     }
     if (native->psk_cred_c) {
         gnutls_psk_free_client_credentials(native->psk_cred_c);
     }
     if (native->sock) {
         close(native->sock);
     }
     if (native->process_notify) {
         mainloop_destroy_trigger(native->process_notify);
         native->process_notify = NULL;
     }
     if (native->pending_notify) {
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 
     free(native->remote->buffer);
     native->remote->buffer = NULL;
     native->source = 0;
     native->sock = 0;
     native->psk_cred_c = NULL;
     native->remote->tls_session = NULL;
     native->sock = 0;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.remote_nodename = native->remote_nodename;
         event.type = lrmd_event_disconnect;
         native->callback(&event);
     }
     return;
 }
 
 int
 lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type)
 {
     int rc = -1;
 
     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 
     rc = crm_remote_send(session, msg);
 
     if (rc < 0) {
         crm_err("Failed to send remote lrmd tls msg, rc = %d", rc);
         return rc;
     }
 
     return rc;
 }
 
 static xmlNode *
 lrmd_tls_recv_reply(lrmd_t * lrmd, int total_timeout, int expected_reply_id, int *disconnected)
 {
     lrmd_private_t *native = lrmd->private;
     xmlNode *xml = NULL;
     time_t start = time(NULL);
     const char *msg_type = NULL;
     int reply_id = 0;
     int remaining_timeout = 0;
 
     /* A timeout of 0 here makes no sense.  We have to wait a period of time
      * for the response to come back.  If -1 or 0, default to 10 seconds. */
     if (total_timeout <= 0) {
         total_timeout = 10000;
     }
 
     while (!xml) {
 
         xml = crm_remote_parse_buffer(native->remote);
         if (!xml) {
             /* read some more off the tls buffer if we still have time left. */
             if (remaining_timeout) {
                 remaining_timeout = remaining_timeout - ((time(NULL) - start) * 1000);
             } else {
                 remaining_timeout = total_timeout;
             }
             if (remaining_timeout <= 0) {
                 return NULL;
             }
 
             crm_remote_recv(native->remote, remaining_timeout, disconnected);
             xml = crm_remote_parse_buffer(native->remote);
             if (!xml || *disconnected) {
                 return NULL;
             }
         }
 
         CRM_ASSERT(xml != NULL);
 
         crm_element_value_int(xml, F_LRMD_REMOTE_MSG_ID, &reply_id);
         msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 
         if (!msg_type) {
             crm_err("Empty msg type received while waiting for reply");
             free_xml(xml);
             xml = NULL;
         } else if (safe_str_eq(msg_type, "notify")) {
             /* got a notify while waiting for reply, trigger the notify to be processed later */
             crm_info("queueing notify");
             native->pending_notify = g_list_append(native->pending_notify, xml);
             if (native->process_notify) {
                 crm_info("notify trigger set.");
                 mainloop_set_trigger(native->process_notify);
             }
             xml = NULL;
         } else if (safe_str_neq(msg_type, "reply")) {
             /* msg isn't a reply, make some noise */
             crm_err("Expected a reply, got %s", msg_type);
             free_xml(xml);
             xml = NULL;
         } else if (reply_id != expected_reply_id) {
             crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
             free_xml(xml);
             xml = NULL;
         }
     }
 
     if (native->remote->buffer && native->process_notify) {
         mainloop_set_trigger(native->process_notify);
     }
 
     return xml;
 }
 
 static int
 lrmd_tls_send(lrmd_t * lrmd, xmlNode * msg)
 {
     int rc = 0;
     lrmd_private_t *native = lrmd->private;
 
     global_remote_msg_id++;
     if (global_remote_msg_id <= 0) {
         global_remote_msg_id = 1;
     }
 
     rc = lrmd_tls_send_msg(native->remote, msg, global_remote_msg_id, "request");
     if (rc <= 0) {
         crm_err("Remote lrmd send failed, disconnecting");
         lrmd_tls_disconnect(lrmd);
         return -ENOTCONN;
     }
     return pcmk_ok;
 }
 
 static int
 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
 {
     int rc = 0;
     int disconnected = 0;
     xmlNode *xml = NULL;
 
     if (lrmd_tls_connected(lrmd) == FALSE) {
         return -1;
     }
 
     rc = lrmd_tls_send(lrmd, msg);
     if (rc < 0) {
         return rc;
     }
 
     xml = lrmd_tls_recv_reply(lrmd, timeout, global_remote_msg_id, &disconnected);
 
     if (disconnected) {
         crm_err("Remote lrmd server disconnected while waiting for reply with id %d. ",
                 global_remote_msg_id);
         lrmd_tls_disconnect(lrmd);
         rc = -ENOTCONN;
     } else if (!xml) {
         crm_err("Remote lrmd never received reply for request id %d. timeout: %dms ",
                 global_remote_msg_id, timeout);
         rc = -ECOMM;
     }
 
     if (reply) {
         *reply = xml;
     } else {
         free_xml(xml);
     }
 
     return rc;
 }
 #endif
 
 static int
 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
 {
     int rc = -1;
     lrmd_private_t *native = lrmd->private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return rc;
 }
 
 static int
 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
 {
     int rc = -1;
     lrmd_private_t *native = lrmd->private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_none, 0, NULL);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_send(lrmd, msg);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return rc;
 }
 
 static int
 lrmd_api_is_connected(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             return crm_ipc_connected(native->ipc);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             return lrmd_tls_connected(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return 0;
 }
 
 static int
 lrmd_send_command(lrmd_t * lrmd, const char *op, xmlNode * data, xmlNode ** output_data, int timeout,   /* ms. defaults to 1000 if set to 0 */
                   enum lrmd_call_options options, gboolean expect_reply)
 {                               /* TODO we need to reduce usage of this boolean */
     int rc = pcmk_ok;
     int reply_id = -1;
     lrmd_private_t *native = lrmd->private;
     xmlNode *op_msg = NULL;
     xmlNode *op_reply = NULL;
 
     if (!lrmd_api_is_connected(lrmd)) {
         return -ENOTCONN;
     }
 
     if (op == NULL) {
         crm_err("No operation specified");
         return -EINVAL;
     }
 
     CRM_CHECK(native->token != NULL,;
         );
     crm_trace("sending %s op to lrmd", op);
 
     op_msg = lrmd_create_op(native->token, op, data, options);
 
     if (op_msg == NULL) {
         return -EINVAL;
     }
 
     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 
     if (expect_reply) {
         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
     } else {
         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
         goto done;
     }
 
     if (rc < 0) {
         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
         rc = -ECOMM;
         goto done;
     }
 
     rc = pcmk_ok;
     crm_element_value_int(op_reply, F_LRMD_CALLID, &reply_id);
     crm_trace("%s op reply received", op);
     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
         rc = -ENOMSG;
         goto done;
     }
 
     crm_log_xml_trace(op_reply, "Reply");
 
     if (output_data) {
         *output_data = op_reply;
         op_reply = NULL;        /* Prevent subsequent free */
     }
 
   done:
     if (lrmd_api_is_connected(lrmd) == FALSE) {
         crm_err("LRMD disconnected");
     }
 
     free_xml(op_msg);
     free_xml(op_reply);
     return rc;
 }
 
 static int
 lrmd_api_poke_connection(lrmd_t * lrmd)
 {
     int rc;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0, FALSE);
     free_xml(data);
 
     return rc;
 }
 
 static int
 lrmd_handshake(lrmd_t * lrmd, const char *name)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->private;
     xmlNode *reply = NULL;
     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 
     crm_xml_add(hello, F_TYPE, T_LRMD);
     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 
     /* advertise that we are a proxy provider */
     if (native->proxy_callback) {
         crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
     }
 
     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 
     if (rc < 0) {
         crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc);
         rc = -ECOMM;
     } else if (reply == NULL) {
         crm_err("Did not receive registration reply");
         rc = -EPROTO;
     } else {
         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 
         crm_element_value_int(reply, F_LRMD_RC, &rc);
 
         if (rc == -EPROTO) {
             crm_err("LRMD protocol mismatch client version %s, server version %s",
                 LRMD_PROTOCOL_VERSION, crm_element_value(reply, F_LRMD_PROTOCOL_VERSION));
             crm_log_xml_err(reply, "Protocol Error");
 
         } else if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
             crm_err("Invalid registration message: %s", msg_type);
             crm_log_xml_err(reply, "Bad reply");
             rc = -EPROTO;
         } else if (tmp_ticket == NULL) {
             crm_err("No registration token provided");
             crm_log_xml_err(reply, "Bad reply");
             rc = -EPROTO;
         } else {
             crm_trace("Obtained registration token: %s", tmp_ticket);
             native->token = strdup(tmp_ticket);
             rc = pcmk_ok;
         }
     }
 
     free_xml(reply);
     free_xml(hello);
 
     if (rc != pcmk_ok) {
         lrmd_api_disconnect(lrmd);
     }
     return rc;
 }
 
 static int
 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->private;
 
     static struct ipc_client_callbacks lrmd_callbacks = {
         .dispatch = lrmd_ipc_dispatch,
         .destroy = lrmd_ipc_connection_destroy
     };
 
     crm_info("Connecting to lrmd");
 
     if (fd) {
         /* No mainloop */
         native->ipc = crm_ipc_new("lrmd", 0);
         if (native->ipc && crm_ipc_connect(native->ipc)) {
             *fd = crm_ipc_get_fd(native->ipc);
         } else if (native->ipc) {
             rc = -ENOTCONN;
         }
     } else {
         native->source = mainloop_add_ipc_client("lrmd", G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
         native->ipc = mainloop_get_ipc_client(native->source);
     }
 
     if (native->ipc == NULL) {
         crm_debug("Could not connect to the LRMD API");
         rc = -ENOTCONN;
     }
 
     return rc;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static int
 set_key(gnutls_datum_t * key, const char *location)
 {
     FILE *stream;
     int read_len = 256;
     int cur_len = 0;
     int buf_len = read_len;
     static char *key_cache = NULL;
     static size_t key_cache_len = 0;
     static time_t key_cache_updated;
 
     if (location == NULL) {
         return -1;
     }
 
     if (key_cache) {
         time_t now = time(NULL);
 
         if ((now - key_cache_updated) < 60) {
             key->data = gnutls_malloc(key_cache_len + 1);
             key->size = key_cache_len;
             memcpy(key->data, key_cache, key_cache_len);
 
             crm_debug("using cached LRMD key");
             return 0;
         } else {
             key_cache_len = 0;
             key_cache_updated = 0;
             free(key_cache);
             key_cache = NULL;
             crm_debug("clearing lrmd key cache");
         }
     }
 
     stream = fopen(location, "r");
     if (!stream) {
         return -1;
     }
 
     key->data = gnutls_malloc(read_len);
     while (!feof(stream)) {
         int next;
 
         if (cur_len == buf_len) {
             buf_len = cur_len + read_len;
             key->data = gnutls_realloc(key->data, buf_len);
         }
         next = fgetc(stream);
         if (next == EOF && feof(stream)) {
             break;
         }
 
         key->data[cur_len] = next;
         cur_len++;
     }
     fclose(stream);
 
     key->size = cur_len;
     if (!cur_len) {
         gnutls_free(key->data);
         key->data = 0;
         return -1;
     }
 
     if (!key_cache) {
         key_cache = calloc(1, key->size + 1);
         memcpy(key_cache, key->data, key->size);
 
         key_cache_len = key->size;
         key_cache_updated = time(NULL);
     }
 
     return 0;
 }
 
 int
 lrmd_tls_set_key(gnutls_datum_t * key)
 {
     int rc = 0;
     const char *specific_location = getenv("PCMK_authkey_location");
 
     if (set_key(key, specific_location) == 0) {
         crm_debug("Using custom authkey location %s", specific_location);
         return 0;
     }
 
     if (set_key(key, DEFAULT_REMOTE_KEY_LOCATION)) {
         rc = set_key(key, ALT_REMOTE_KEY_LOCATION);
     }
     if (rc) {
         crm_err("No lrmd remote key found");
         return -1;
     }
 
     return rc;
 }
 
 static void
 lrmd_gnutls_global_init(void)
 {
     static int gnutls_init = 0;
 
     if (!gnutls_init) {
         gnutls_global_init();
     }
     gnutls_init = 1;
 }
 #endif
 
 static void
 report_async_connection_result(lrmd_t * lrmd, int rc)
 {
     lrmd_private_t *native = lrmd->private;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.type = lrmd_event_connect;
         event.remote_nodename = native->remote_nodename;
         event.connection_rc = rc;
         native->callback(&event);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tcp_connect_cb(void *userdata, int sock)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->private;
     char name[256] = { 0, };
     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
         .dispatch = lrmd_tls_dispatch,
         .destroy = lrmd_tls_connection_destroy,
     };
     int rc = sock;
     gnutls_datum_t psk_key = { NULL, 0 };
 
     if (rc < 0) {
         lrmd_tls_connection_destroy(lrmd);
         crm_info("remote lrmd connect to %s at port %d failed", native->server, native->port);
         report_async_connection_result(lrmd, rc);
         return;
     }
 
     /* TODO continue with tls stuff now that tcp connect passed. make this async as well soon
      * to avoid all blocking code in the client. */
     native->sock = sock;
 
     if (lrmd_tls_set_key(&psk_key) != 0) {
         lrmd_tls_connection_destroy(lrmd);
         return;
     }
 
     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
     gnutls_free(psk_key.data);
 
     native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);
 
     if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
         crm_warn("Client tls handshake failed for server %s:%d. Disconnecting", native->server,
                  native->port);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = NULL;
         lrmd_tls_connection_destroy(lrmd);
         report_async_connection_result(lrmd, -1);
         return;
     }
 
     crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server,
              native->port);
 
     snprintf(name, 128, "remote-lrmd-%s:%d", native->server, native->port);
 
     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
     native->source =
         mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
 
     rc = lrmd_handshake(lrmd, name);
     report_async_connection_result(lrmd, rc);
 
     return;
 }
 
 static int
 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
 {
     int rc = 0;
     lrmd_private_t *native = lrmd->private;
 
     lrmd_gnutls_global_init();
 
     rc = crm_remote_tcp_connect_async(native->server, native->port, timeout, lrmd,
                                       lrmd_tcp_connect_cb);
 
     return rc;
 }
 
 static int
 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
 {
     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
         .dispatch = lrmd_tls_dispatch,
         .destroy = lrmd_tls_connection_destroy,
     };
 
     lrmd_private_t *native = lrmd->private;
     int sock;
     gnutls_datum_t psk_key = { NULL, 0 };
 
     lrmd_gnutls_global_init();
 
     sock = crm_remote_tcp_connect(native->server, native->port);
     if (sock < 0) {
         crm_warn("Could not establish remote lrmd connection to %s", native->server);
         lrmd_tls_connection_destroy(lrmd);
         return -ENOTCONN;
     }
 
     native->sock = sock;
 
     if (lrmd_tls_set_key(&psk_key) != 0) {
         lrmd_tls_connection_destroy(lrmd);
         return -1;
     }
 
     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
     gnutls_free(psk_key.data);
 
     native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);
 
     if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
         crm_err("Session creation for %s:%d failed", native->server, native->port);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = NULL;
         lrmd_tls_connection_destroy(lrmd);
         return -1;
     }
 
     crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server,
              native->port);
 
     if (fd) {
         *fd = sock;
     } else {
         char name[256] = { 0, };
         snprintf(name, 128, "remote-lrmd-%s:%d", native->server, native->port);
 
         native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
         native->source =
             mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
     }
     return pcmk_ok;
 }
 #endif
 
 static int
 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
 {
     int rc = -ENOTCONN;
     lrmd_private_t *native = lrmd->private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             rc = lrmd_ipc_connect(lrmd, fd);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_connect(lrmd, fd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     if (rc == pcmk_ok) {
         rc = lrmd_handshake(lrmd, name);
     }
 
     return rc;
 }
 
 static int
 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
 {
     int rc = 0;
     lrmd_private_t *native = lrmd->private;
 
     if (!native->callback) {
         crm_err("Async connect not possible, no lrmd client callback set.");
         return -1;
     }
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             /* fake async connection with ipc.  it should be fast
              * enough that we gain very little from async */
             rc = lrmd_api_connect(lrmd, name, NULL);
             if (!rc) {
                 report_async_connection_result(lrmd, rc);
             }
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_connect_async(lrmd, timeout);
             if (rc) {
                 /* connection failed, report rc now */
                 report_async_connection_result(lrmd, rc);
             }
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return rc;
 }
 
 static void
 lrmd_ipc_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->private;
 
     if (native->source != NULL) {
         /* Attached to mainloop */
         mainloop_del_ipc_client(native->source);
         native->source = NULL;
         native->ipc = NULL;
 
     } else if (native->ipc) {
         /* Not attached to mainloop */
         crm_ipc_t *ipc = native->ipc;
 
         native->ipc = NULL;
         crm_ipc_close(ipc);
         crm_ipc_destroy(ipc);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tls_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->private;
 
     if (native->remote->tls_session) {
         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = 0;
     }
 
     if (native->source != NULL) {
         /* Attached to mainloop */
         mainloop_del_ipc_client(native->source);
         native->source = NULL;
 
     } else if (native->sock) {
         close(native->sock);
     }
 
     if (native->pending_notify) {
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 }
 #endif
 
 static int
 lrmd_api_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->private;
 
     crm_info("Disconnecting from lrmd service");
     switch (native->type) {
         case CRM_CLIENT_IPC:
             lrmd_ipc_disconnect(lrmd);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             lrmd_tls_disconnect(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     free(native->token);
     native->token = NULL;
     return 0;
 }
 
 static int
 lrmd_api_register_rsc(lrmd_t * lrmd,
                       const char *rsc_id,
                       const char *class,
                       const char *provider, const char *type, enum lrmd_call_options options)
 {
     int rc = pcmk_ok;
     xmlNode *data = NULL;
 
     if (!class || !type || !rsc_id) {
         return -EINVAL;
     }
     if (safe_str_eq(class, "ocf") && !provider) {
         return -EINVAL;
     }
 
     data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add(data, F_LRMD_CLASS, class);
     crm_xml_add(data, F_LRMD_PROVIDER, provider);
     crm_xml_add(data, F_LRMD_TYPE, type);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
     free_xml(data);
 
     return rc;
 }
 
 static int
 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
     free_xml(data);
 
     return rc;
 }
 
 lrmd_rsc_info_t *
 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
 {
     lrmd_rsc_info_t *copy = NULL;
 
     copy = calloc(1, sizeof(lrmd_rsc_info_t));
 
     copy->id = strdup(rsc_info->id);
     copy->type = strdup(rsc_info->type);
     copy->class = strdup(rsc_info->class);
     if (rsc_info->provider) {
         copy->provider = strdup(rsc_info->provider);
     }
 
     return copy;
 }
 
 void
 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
 {
     if (!rsc_info) {
         return;
     }
     free(rsc_info->id);
     free(rsc_info->type);
     free(rsc_info->class);
     free(rsc_info->provider);
     free(rsc_info);
 }
 
 static lrmd_rsc_info_t *
 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
 {
     lrmd_rsc_info_t *rsc_info = NULL;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
     xmlNode *output = NULL;
     const char *class = NULL;
     const char *provider = NULL;
     const char *type = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
     free_xml(data);
 
     if (!output) {
         return NULL;
     }
 
     class = crm_element_value(output, F_LRMD_CLASS);
     provider = crm_element_value(output, F_LRMD_PROVIDER);
     type = crm_element_value(output, F_LRMD_TYPE);
 
     if (!class || !type) {
         free_xml(output);
         return NULL;
     } else if (safe_str_eq(class, "ocf") && !provider) {
         free_xml(output);
         return NULL;
     }
 
     rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
     rsc_info->id = strdup(rsc_id);
     rsc_info->class = strdup(class);
     if (provider) {
         rsc_info->provider = strdup(provider);
     }
     rsc_info->type = strdup(type);
 
     free_xml(output);
     return rsc_info;
 }
 
 static void
 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
 {
     lrmd_private_t *native = lrmd->private;
 
     native->callback = callback;
 }
 
 void
 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
 {
     lrmd_private_t *native = lrmd->private;
 
     native->proxy_callback = callback;
     native->proxy_callback_userdata = userdata;
 }
 
 void
 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
 {
     lrmd_private_t *native = lrmd->private;
 
     if (native->proxy_callback) {
         crm_log_xml_trace(msg, "PROXY_INBOUND");
         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
     }
 }
 
 int
 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
 {
     if (lrmd == NULL) {
         return -ENOTCONN;
     }
     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
 
     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
     return lrmd_send_xml_no_reply(lrmd, msg);
 }
 
 static int
 stonith_get_metadata(const char *provider, const char *type, char **output)
 {
     int rc = pcmk_ok;
+    stonith_t *stonith_api = stonith_api_new();
 
     if(stonith_api) {
         stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0);
+        stonith_api->cmds->free(stonith_api);
     }
     if (*output == NULL) {
         rc = -EIO;
     }
     return rc;
 }
 
 #define lsb_metadata_template  \
     "<?xml version='1.0'?>\n"                                           \
     "<!DOCTYPE resource-agent SYSTEM 'ra-api-1.dtd'>\n"                 \
     "<resource-agent name='%s' version='0.1'>\n"                        \
     "  <version>1.0</version>\n"                                        \
     "  <longdesc lang='en'>\n"                                          \
     "    %s\n"                                                          \
     "  </longdesc>\n"                                                   \
     "  <shortdesc lang='en'>%s</shortdesc>\n"                           \
     "  <parameters>\n"                                                  \
     "  </parameters>\n"                                                 \
     "  <actions>\n"                                                     \
     "    <action name='meta-data'    timeout='5' />\n"                  \
     "    <action name='start'        timeout='15' />\n"                 \
     "    <action name='stop'         timeout='15' />\n"                 \
     "    <action name='status'       timeout='15' />\n"                 \
     "    <action name='restart'      timeout='15' />\n"                 \
     "    <action name='force-reload' timeout='15' />\n"                 \
     "    <action name='monitor'      timeout='15' interval='15' />\n"   \
     "  </actions>\n"                                                    \
     "  <special tag='LSB'>\n"                                           \
     "    <Provides>%s</Provides>\n"                                     \
     "    <Required-Start>%s</Required-Start>\n"                         \
     "    <Required-Stop>%s</Required-Stop>\n"                           \
     "    <Should-Start>%s</Should-Start>\n"                             \
     "    <Should-Stop>%s</Should-Stop>\n"                               \
     "    <Default-Start>%s</Default-Start>\n"                           \
     "    <Default-Stop>%s</Default-Stop>\n"                             \
     "  </special>\n"                                                    \
     "</resource-agent>\n"
 
 #define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
 #define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
 #define PROVIDES    "# Provides:"
 #define REQ_START   "# Required-Start:"
 #define REQ_STOP    "# Required-Stop:"
 #define SHLD_START  "# Should-Start:"
 #define SHLD_STOP   "# Should-Stop:"
 #define DFLT_START  "# Default-Start:"
 #define DFLT_STOP   "# Default-Stop:"
 #define SHORT_DSCR  "# Short-Description:"
 #define DESCRIPTION "# Description:"
 
 #define lsb_meta_helper_free_value(m)   \
     if ((m) != NULL) {                  \
         xmlFree(m);                     \
         (m) = NULL;                     \
     }
 
 #define lsb_meta_helper_get_value(buffer, ptr, keyword)                 \
     if (!ptr && !strncasecmp(buffer, keyword, strlen(keyword))) {       \
         (ptr) = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST buffer+strlen(keyword)); \
         continue;                                                       \
     }
 
 static int
 lsb_get_metadata(const char *type, char **output)
 {
     char ra_pathname[PATH_MAX] = { 0, };
     FILE *fp;
     GString *meta_data = NULL;
     char buffer[1024];
     char *provides = NULL;
     char *req_start = NULL;
     char *req_stop = NULL;
     char *shld_start = NULL;
     char *shld_stop = NULL;
     char *dflt_start = NULL;
     char *dflt_stop = NULL;
     char *s_dscrpt = NULL;
     char *xml_l_dscrpt = NULL;
     GString *l_dscrpt = NULL;
 
     if(type[0] == '/') {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s", type);
     } else {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s/%s", LSB_ROOT_DIR, type);
     }
 
     crm_trace("Looking into %s", ra_pathname);
     if (!(fp = fopen(ra_pathname, "r"))) {
         return -errno;
     }
 
     /* Enter into the lsb-compliant comment block */
     while (fgets(buffer, sizeof(buffer), fp)) {
         /* Now suppose each of the following eight arguments contain only one line */
         lsb_meta_helper_get_value(buffer, provides, PROVIDES)
             lsb_meta_helper_get_value(buffer, req_start, REQ_START)
             lsb_meta_helper_get_value(buffer, req_stop, REQ_STOP)
             lsb_meta_helper_get_value(buffer, shld_start, SHLD_START)
             lsb_meta_helper_get_value(buffer, shld_stop, SHLD_STOP)
             lsb_meta_helper_get_value(buffer, dflt_start, DFLT_START)
             lsb_meta_helper_get_value(buffer, dflt_stop, DFLT_STOP)
             lsb_meta_helper_get_value(buffer, s_dscrpt, SHORT_DSCR)
 
             /* Long description may cross multiple lines */
             if ((l_dscrpt == NULL) && (0 == strncasecmp(buffer, DESCRIPTION, strlen(DESCRIPTION)))) {
             l_dscrpt = g_string_new(buffer + strlen(DESCRIPTION));
             /* Between # and keyword, more than one space, or a tab character,
              * indicates the continuation line.     Extracted from LSB init script standard */
             while (fgets(buffer, sizeof(buffer), fp)) {
                 if (!strncmp(buffer, "#  ", 3) || !strncmp(buffer, "#\t", 2)) {
                     buffer[0] = ' ';
                     l_dscrpt = g_string_append(l_dscrpt, buffer);
                 } else {
                     fputs(buffer, fp);
                     break;      /* Long description ends */
                 }
             }
             continue;
         }
         if (l_dscrpt) {
             xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(l_dscrpt->str));
         }
         if (!strncasecmp(buffer, LSB_INITSCRIPT_INFOEND_TAG, strlen(LSB_INITSCRIPT_INFOEND_TAG))) {
             /* Get to the out border of LSB comment block */
             break;
         }
         if (buffer[0] != '#') {
             break;              /* Out of comment block in the beginning */
         }
     }
     fclose(fp);
 
     meta_data = g_string_new("");
     g_string_sprintf(meta_data, lsb_metadata_template, type,
                      (xml_l_dscrpt == NULL) ? type : xml_l_dscrpt,
                      (s_dscrpt == NULL) ? type : s_dscrpt, (provides == NULL) ? "" : provides,
                      (req_start == NULL) ? "" : req_start, (req_stop == NULL) ? "" : req_stop,
                      (shld_start == NULL) ? "" : shld_start, (shld_stop == NULL) ? "" : shld_stop,
                      (dflt_start == NULL) ? "" : dflt_start, (dflt_stop == NULL) ? "" : dflt_stop);
 
     lsb_meta_helper_free_value(xml_l_dscrpt);
     lsb_meta_helper_free_value(s_dscrpt);
     lsb_meta_helper_free_value(provides);
     lsb_meta_helper_free_value(req_start);
     lsb_meta_helper_free_value(req_stop);
     lsb_meta_helper_free_value(shld_start);
     lsb_meta_helper_free_value(shld_stop);
     lsb_meta_helper_free_value(dflt_start);
     lsb_meta_helper_free_value(dflt_stop);
 
     if (l_dscrpt) {
         g_string_free(l_dscrpt, TRUE);
     }
 
     *output = strdup(meta_data->str);
     g_string_free(meta_data, TRUE);
 
     crm_trace("Created fake metadata: %d", strlen(*output));
     return pcmk_ok;
 }
 
 #if SUPPORT_NAGIOS
 static int
 nagios_get_metadata(const char *type, char **output)
 {
     int rc = pcmk_ok;
     FILE *file_strm = NULL;
     int start = 0, length = 0, read_len = 0;
     char *metadata_file = NULL;
     int len = 36;
 
     len += strlen(NAGIOS_METADATA_DIR);
     len += strlen(type);
     metadata_file = calloc(1, len);
     CRM_CHECK(metadata_file != NULL, return -ENOMEM);
 
     sprintf(metadata_file, "%s/%s.xml", NAGIOS_METADATA_DIR, type);
     file_strm = fopen(metadata_file, "r");
     if (file_strm == NULL) {
         crm_err("Metadata file %s does not exist", metadata_file);
         free(metadata_file);
         return -EIO;
     }
 
     /* see how big the file is */
     start = ftell(file_strm);
     fseek(file_strm, 0L, SEEK_END);
     length = ftell(file_strm);
     fseek(file_strm, 0L, start);
 
     CRM_ASSERT(length >= 0);
     CRM_ASSERT(start == ftell(file_strm));
 
     if (length <= 0) {
         crm_info("%s was not valid", metadata_file);
         free(*output);
         *output = NULL;
         rc = -EIO;
 
     } else {
         crm_trace("Reading %d bytes from file", length);
         *output = calloc(1, (length + 1));
         read_len = fread(*output, 1, length, file_strm);
         if (read_len != length) {
             crm_err("Calculated and read bytes differ: %d vs. %d", length, read_len);
             free(*output);
             *output = NULL;
             rc = -EIO;
         }
     }
 
     fclose(file_strm);
     free(metadata_file);
     return rc;
 }
 #endif
 
 static int
 generic_get_metadata(const char *standard, const char *provider, const char *type, char **output)
 {
     svc_action_t *action = resources_action_create(type,
                                                    standard,
                                                    provider,
                                                    type,
                                                    "meta-data",
                                                    0,
                                                    5000,
                                                    NULL);
 
     if (!(services_action_sync(action))) {
         crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
         services_action_free(action);
         return -EIO;
     }
 
     if (!action->stdout_data) {
         crm_err("Failed to retrieve meta-data for %s:%s:%s", standard, provider, type);
         services_action_free(action);
         return -EIO;
     }
 
     *output = strdup(action->stdout_data);
     services_action_free(action);
 
     return pcmk_ok;
 }
 
 static int
 lrmd_api_get_metadata(lrmd_t * lrmd,
                       const char *class,
                       const char *provider,
                       const char *type, char **output, enum lrmd_call_options options)
 {
     if (!class || !type) {
         return -EINVAL;
     }
 
     if (safe_str_eq(class, "stonith")) {
         return stonith_get_metadata(provider, type, output);
     } else if (safe_str_eq(class, "lsb")) {
         return lsb_get_metadata(type, output);
 #if SUPPORT_NAGIOS
     } else if (safe_str_eq(class, "nagios")) {
         return nagios_get_metadata(type, output);
 #endif
     }
     return generic_get_metadata(class, provider, type, output);
 }
 
 static int
 lrmd_api_exec(lrmd_t * lrmd, const char *rsc_id, const char *action, const char *userdata, int interval,        /* ms */
               int timeout,      /* ms */
               int start_delay,  /* ms */
               enum lrmd_call_options options, lrmd_key_value_t * params)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
     lrmd_key_value_t *tmp = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
     crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
 
     for (tmp = params; tmp; tmp = tmp->next) {
         hash2field((gpointer) tmp->key, (gpointer) tmp->value, args);
     }
 
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
     free_xml(data);
 
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 static int
 lrmd_api_cancel(lrmd_t * lrmd, const char *rsc_id, const char *action, int interval)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
     free_xml(data);
     return rc;
 }
 
 static int
 list_stonith_agents(lrmd_list_t ** resources)
 {
     int rc = 0;
+    stonith_t *stonith_api = stonith_api_new();
     stonith_key_value_t *stonith_resources = NULL;
     stonith_key_value_t *dIter = NULL;
 
     if(stonith_api) {
         stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);
+        stonith_api->cmds->free(stonith_api);
     }
 
     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
         rc++;
         if (resources) {
             *resources = lrmd_list_add(*resources, dIter->value);
         }
     }
 
     stonith_key_value_freeall(stonith_resources, 1, 0);
     return rc;
 }
 
 static int
 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
                      const char *provider)
 {
     int rc = 0;
 
     if (safe_str_eq(class, "stonith")) {
         rc += list_stonith_agents(resources);
 
     } else {
         GListPtr gIter = NULL;
         GList *agents = resources_list_agents(class, provider);
 
         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
             rc++;
         }
         g_list_free_full(agents, free);
 
         if (!class) {
             rc += list_stonith_agents(resources);
         }
     }
 
     if (rc == 0) {
         crm_notice("No agents found for class %s", class);
         rc = -EPROTONOSUPPORT;
     }
     return rc;
 }
 
 static int
 does_provider_have_agent(const char *agent, const char *provider, const char *class)
 {
     int found = 0;
     GList *agents = NULL;
     GListPtr gIter2 = NULL;
 
     agents = resources_list_agents(class, provider);
     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
         if (safe_str_eq(agent, gIter2->data)) {
             found = 1;
         }
     }
     g_list_free_full(agents, free);
 
     return found;
 }
 
 static int
 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
 {
     int rc = pcmk_ok;
     char *provider = NULL;
     GList *ocf_providers = NULL;
     GListPtr gIter = NULL;
 
     ocf_providers = resources_list_providers("ocf");
 
     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
         provider = gIter->data;
         if (!agent || does_provider_have_agent(agent, provider, "ocf")) {
             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
             rc++;
         }
     }
 
     g_list_free_full(ocf_providers, free);
     return rc;
 }
 
 static int
 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
 {
     int rc = 0;
     GList *standards = NULL;
     GListPtr gIter = NULL;
 
     standards = resources_list_standards();
 
     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
         rc++;
     }
 
     if (list_stonith_agents(NULL) > 0) {
         *supported = lrmd_list_add(*supported, "stonith");
         rc++;
     }
 
     g_list_free_full(standards, free);
     return rc;
 }
 
 lrmd_t *
 lrmd_api_new(void)
 {
     lrmd_t *new_lrmd = NULL;
     lrmd_private_t *pvt = NULL;
 
     new_lrmd = calloc(1, sizeof(lrmd_t));
     pvt = calloc(1, sizeof(lrmd_private_t));
     pvt->remote = calloc(1, sizeof(crm_remote_t));
     new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));
 
     pvt->type = CRM_CLIENT_IPC;
     new_lrmd->private = pvt;
 
     new_lrmd->cmds->connect = lrmd_api_connect;
     new_lrmd->cmds->connect_async = lrmd_api_connect_async;
     new_lrmd->cmds->is_connected = lrmd_api_is_connected;
     new_lrmd->cmds->poke_connection = lrmd_api_poke_connection;
     new_lrmd->cmds->disconnect = lrmd_api_disconnect;
     new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
     new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
     new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
     new_lrmd->cmds->set_callback = lrmd_api_set_callback;
     new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
     new_lrmd->cmds->exec = lrmd_api_exec;
     new_lrmd->cmds->cancel = lrmd_api_cancel;
     new_lrmd->cmds->list_agents = lrmd_api_list_agents;
     new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
     new_lrmd->cmds->list_standards = lrmd_api_list_standards;
 
-    if (!stonith_api) {
-        stonith_api = stonith_api_new();
-    }
-
     return new_lrmd;
 }
 
 lrmd_t *
 lrmd_remote_api_new(const char *nodename, const char *server, int port)
 {
 #ifdef HAVE_GNUTLS_GNUTLS_H
     lrmd_t *new_lrmd = lrmd_api_new();
     lrmd_private_t *native = new_lrmd->private;
 
     if (!nodename && !server) {
         lrmd_api_delete(new_lrmd);
         return NULL;
     }
 
     native->type = CRM_CLIENT_TLS;
     native->remote_nodename = nodename ? strdup(nodename) : strdup(server);
     native->server = server ? strdup(server) : strdup(nodename);
     native->port = port;
     if (native->port == 0) {
         const char *remote_port_str = getenv("PCMK_remote_port");
         native->port = remote_port_str ? atoi(remote_port_str) : DEFAULT_REMOTE_PORT;
     }
 
     return new_lrmd;
 #else
     crm_err("GNUTLS is not enabled for this build, remote LRMD client can not be created");
     return NULL;
 #endif
 
 }
 
 void
 lrmd_api_delete(lrmd_t * lrmd)
 {
     if (!lrmd) {
         return;
     }
     lrmd->cmds->disconnect(lrmd);       /* no-op if already disconnected */
     free(lrmd->cmds);
     if (lrmd->private) {
         lrmd_private_t *native = lrmd->private;
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
         free(native->server);
 #endif
         free(native->remote_nodename);
         free(native->remote);
     }
 
-    if (stonith_api) {
-        stonith_api->cmds->free(stonith_api);
-        stonith_api = NULL;
-    }
-
     free(lrmd->private);
     free(lrmd);
 }