diff --git a/daemons/based/based_remote.c b/daemons/based/based_remote.c
index 71c62aff97..ee2d4d99f5 100644
--- a/daemons/based/based_remote.c
+++ b/daemons/based/based_remote.c
@@ -1,685 +1,685 @@
 /*
  * Copyright 2004-2018 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/socket.h>
 #include <arpa/inet.h>
 
 #include <netinet/ip.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <glib.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipcs.h>
 #include <crm/common/xml.h>
 #include <crm/common/remote_internal.h>
 #include <crm/cib/internal.h>
 
 #include "pacemaker-based.h"
 
 /* #undef HAVE_PAM_PAM_APPL_H */
 /* #undef HAVE_GNUTLS_GNUTLS_H */
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  undef KEYFILE
 #  include <gnutls/gnutls.h>
 #endif
 
 #include <pwd.h>
 #include <grp.h>
 #if HAVE_SECURITY_PAM_APPL_H
 #  include <security/pam_appl.h>
 #  define HAVE_PAM 1
 #else
 #  if HAVE_PAM_PAM_APPL_H
 #    include <pam/pam_appl.h>
 #    define HAVE_PAM 1
 #  endif
 #endif
 
 extern int remote_tls_fd;
 extern gboolean cib_shutdown_flag;
 
 int init_remote_listener(int port, gboolean encrypted);
 void cib_remote_connection_destroy(gpointer user_data);
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 gnutls_dh_params_t dh_params;
 gnutls_anon_server_credentials_t anon_cred_s;
 static void
 debug_log(int level, const char *str)
 {
     fputs(str, stderr);
 }
 #endif
 
 #define REMOTE_AUTH_TIMEOUT 10000
 
 int num_clients;
 int authenticate_user(const char *user, const char *passwd);
-int cib_remote_listen(gpointer data);
-int cib_remote_msg(gpointer data);
+static int cib_remote_listen(gpointer data);
+static int cib_remote_msg(gpointer data);
 
 static void
 remote_connection_destroy(gpointer user_data)
 {
     return;
 }
 
 #define ERROR_SUFFIX "  Shutting down remote listener"
 int
 init_remote_listener(int port, gboolean encrypted)
 {
     int rc;
     int *ssock = NULL;
     struct sockaddr_in saddr;
     int optval;
 
     static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
         .dispatch = cib_remote_listen,
         .destroy = remote_connection_destroy,
     };
 
     if (port <= 0) {
         /* don't start it */
         return 0;
     }
 
     if (encrypted) {
 #ifndef HAVE_GNUTLS_GNUTLS_H
         crm_warn("TLS support is not available");
         return 0;
 #else
         crm_notice("Starting a tls listener on port %d.", port);
         crm_gnutls_global_init();
         /* gnutls_global_set_log_level (10); */
         gnutls_global_set_log_function(debug_log);
         if (pcmk__init_tls_dh(&dh_params) != GNUTLS_E_SUCCESS) {
             return -1;
         }
         gnutls_anon_allocate_server_credentials(&anon_cred_s);
         gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
 #endif
     } else {
         crm_warn("Starting a plain_text listener on port %d.", port);
     }
 #ifndef HAVE_PAM
     crm_warn("PAM is _not_ enabled!");
 #endif
 
     /* create server socket */
     ssock = malloc(sizeof(int));
     if(ssock == NULL) {
         crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
         return -1;
     }
 
     *ssock = socket(AF_INET, SOCK_STREAM, 0);
     if (*ssock == -1) {
         crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
         free(ssock);
         return -1;
     }
 
     /* reuse address */
     optval = 1;
     rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
     if (rc < 0) {
         crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener");
     }
 
     /* bind server socket */
     memset(&saddr, '\0', sizeof(saddr));
     saddr.sin_family = AF_INET;
     saddr.sin_addr.s_addr = INADDR_ANY;
     saddr.sin_port = htons(port);
     if (bind(*ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) {
         crm_perror(LOG_ERR, "Can not bind server socket." ERROR_SUFFIX);
         close(*ssock);
         free(ssock);
         return -2;
     }
     if (listen(*ssock, 10) == -1) {
         crm_perror(LOG_ERR, "Can not start listen." ERROR_SUFFIX);
         close(*ssock);
         free(ssock);
         return -3;
     }
 
     mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks);
 
     return *ssock;
 }
 
 static int
 check_group_membership(const char *usr, const char *grp)
 {
     int index = 0;
     struct passwd *pwd = NULL;
     struct group *group = NULL;
 
     CRM_CHECK(usr != NULL, return FALSE);
     CRM_CHECK(grp != NULL, return FALSE);
 
     pwd = getpwnam(usr);
     if (pwd == NULL) {
         crm_err("No user named '%s' exists!", usr);
         return FALSE;
     }
 
     group = getgrgid(pwd->pw_gid);
     if (group != NULL && crm_str_eq(grp, group->gr_name, TRUE)) {
         return TRUE;
     }
 
     group = getgrnam(grp);
     if (group == NULL) {
         crm_err("No group named '%s' exists!", grp);
         return FALSE;
     }
 
     while (TRUE) {
         char *member = group->gr_mem[index++];
 
         if (member == NULL) {
             break;
 
         } else if (crm_str_eq(usr, member, TRUE)) {
             return TRUE;
         }
     };
 
     return FALSE;
 }
 
 static gboolean
 cib_remote_auth(xmlNode * login)
 {
     const char *user = NULL;
     const char *pass = NULL;
     const char *tmp = NULL;
 
     crm_log_xml_info(login, "Login: ");
     if (login == NULL) {
         return FALSE;
     }
 
     tmp = crm_element_name(login);
     if (safe_str_neq(tmp, "cib_command")) {
         crm_err("Wrong tag: %s", tmp);
         return FALSE;
     }
 
     tmp = crm_element_value(login, "op");
     if (safe_str_neq(tmp, "authenticate")) {
         crm_err("Wrong operation: %s", tmp);
         return FALSE;
     }
 
     user = crm_element_value(login, "user");
     pass = crm_element_value(login, "password");
 
     if (!user || !pass) {
         crm_err("missing auth credentials");
         return FALSE;
     }
 
     /* Non-root daemons can only validate the password of the
      * user they're running as
      */
     if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
         crm_err("User is not a member of the required group");
         return FALSE;
 
     } else if (authenticate_user(user, pass) == FALSE) {
         crm_err("PAM auth failed");
         return FALSE;
     }
 
     return TRUE;
 }
 
 static gboolean
 remote_auth_timeout_cb(gpointer data)
 {
     crm_client_t *client = data;
 
     client->remote->auth_timeout = 0;
 
     if (client->remote->authenticated == TRUE) {
         return FALSE;
     }
 
     mainloop_del_fd(client->remote->source);
     crm_err("Remote client authentication timed out");
 
     return FALSE;
 }
 
-int
+static int
 cib_remote_listen(gpointer data)
 {
     int csock = 0;
     unsigned laddr;
     struct sockaddr_storage addr;
     char ipstr[INET6_ADDRSTRLEN];
     int ssock = *(int *)data;
     int rc;
 
     crm_client_t *new_client = NULL;
 
     static struct mainloop_fd_callbacks remote_client_fd_callbacks = {
         .dispatch = cib_remote_msg,
         .destroy = cib_remote_connection_destroy,
     };
 
     /* accept the connection */
     laddr = sizeof(addr);
     memset(&addr, 0, sizeof(addr));
     csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
     if (csock == -1) {
         crm_perror(LOG_ERR, "Could not accept socket connection");
         return TRUE;
     }
 
     crm_sockaddr2str(&addr, ipstr);
     crm_debug("New %s connection from %s",
               ((ssock == remote_tls_fd)? "secure" : "clear-text"), ipstr);
 
     rc = crm_set_nonblocking(csock);
     if (rc < 0) {
         crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
                 pcmk_strerror(rc), rc);
         close(csock);
         return TRUE;
     }
 
     num_clients++;
 
     crm_client_init();
     new_client = crm_client_alloc(NULL);
     new_client->remote = calloc(1, sizeof(crm_remote_t));
 
     if (ssock == remote_tls_fd) {
 #ifdef HAVE_GNUTLS_GNUTLS_H
         new_client->kind = CRM_CLIENT_TLS;
 
         /* create gnutls session for the server socket */
         new_client->remote->tls_session = pcmk__new_tls_session(csock,
                                                                 GNUTLS_SERVER,
                                                                 GNUTLS_CRD_ANON,
                                                                 anon_cred_s);
         if (new_client->remote->tls_session == NULL) {
             close(csock);
             return TRUE;
         }
 #endif
     } else {
         new_client->kind = CRM_CLIENT_TCP;
         new_client->remote->tcp_socket = csock;
     }
 
     /* clients have a few seconds to perform handshake. */
     new_client->remote->auth_timeout =
         g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client);
 
     new_client->remote->source =
         mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
                         &remote_client_fd_callbacks);
 
     return TRUE;
 }
 
 void
 cib_remote_connection_destroy(gpointer user_data)
 {
     crm_client_t *client = user_data;
     int csock = 0;
 
     if (client == NULL) {
         return;
     }
 
     crm_trace("Cleaning up after client disconnect: %s/%s", crm_str(client->name), client->id);
 
     num_clients--;
     crm_trace("Num unfree'd clients: %d", num_clients);
 
     switch (client->kind) {
         case CRM_CLIENT_TCP:
             csock = client->remote->tcp_socket;
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             if (client->remote->tls_session) {
                 void *sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
 
                 csock = GPOINTER_TO_INT(sock_ptr);
                 if (client->remote->tls_handshake_complete) {
                     gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_WR);
                 }
                 gnutls_deinit(*client->remote->tls_session);
                 gnutls_free(client->remote->tls_session);
                 client->remote->tls_session = NULL;
             }
             break;
 #endif
         default:
             crm_warn("Unexpected client type %d", client->kind);
     }
 
     if (csock > 0) {
         close(csock);
     }
 
     crm_client_destroy(client);
 
     crm_trace("Freed the cib client");
 
     if (cib_shutdown_flag) {
         cib_shutdown(0);
     }
     return;
 }
 
 static void
 cib_handle_remote_msg(crm_client_t * client, xmlNode * command)
 {
     const char *value = NULL;
 
     value = crm_element_name(command);
     if (safe_str_neq(value, "cib_command")) {
         crm_log_xml_trace(command, "Bad command: ");
         return;
     }
 
     if (client->name == NULL) {
         value = crm_element_value(command, F_CLIENTNAME);
         if (value == NULL) {
             client->name = strdup(client->id);
         } else {
             client->name = strdup(value);
         }
     }
 
     if (client->userdata == NULL) {
         value = crm_element_value(command, F_CIB_CALLBACK_TOKEN);
         if (value != NULL) {
             client->userdata = strdup(value);
             crm_trace("Callback channel for %s is %s", client->id, (char*)client->userdata);
 
         } else {
             client->userdata = strdup(client->id);
         }
     }
 
     /* unset dangerous options */
     xml_remove_prop(command, F_ORIG);
     xml_remove_prop(command, F_CIB_HOST);
     xml_remove_prop(command, F_CIB_GLOBAL_UPDATE);
 
     crm_xml_add(command, F_TYPE, T_CIB);
     crm_xml_add(command, F_CIB_CLIENTID, client->id);
     crm_xml_add(command, F_CIB_CLIENTNAME, client->name);
 #if ENABLE_ACL
     crm_xml_add(command, F_CIB_USER, client->user);
 #endif
 
     if (crm_element_value(command, F_CIB_CALLID) == NULL) {
         char *call_uuid = crm_generate_uuid();
 
         /* fix the command */
         crm_xml_add(command, F_CIB_CALLID, call_uuid);
         free(call_uuid);
     }
 
     if (crm_element_value(command, F_CIB_CALLOPTS) == NULL) {
         crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
     }
 
     crm_log_xml_trace(command, "Remote command: ");
     cib_common_callback_worker(0, 0, command, client, TRUE);
 }
 
-int
+static int
 cib_remote_msg(gpointer data)
 {
     xmlNode *command = NULL;
     crm_client_t *client = data;
     int disconnected = 0;
     int timeout = client->remote->authenticated ? -1 : 1000;
 
     crm_trace("%s callback", client->kind != CRM_CLIENT_TCP ? "secure" : "clear-text");
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
     if (client->kind == CRM_CLIENT_TLS && (client->remote->tls_handshake_complete == FALSE)) {
-        int rc = 0;
-
-        /* Muliple calls to handshake will be required, this callback
-         * will be invoked once the client sends more handshake data. */
-        do {
-            rc = gnutls_handshake(*client->remote->tls_session);
-
-            if (rc < 0 && rc != GNUTLS_E_AGAIN) {
-                crm_err("TLS handshake with remote CIB manager failed");
-                return -1;
-            }
-        } while (rc == GNUTLS_E_INTERRUPTED);
+        int rc = pcmk__read_handshake_data(client);
 
         if (rc == 0) {
-            crm_debug("TLS handshake with remote CIB manager completed");
-            client->remote->tls_handshake_complete = TRUE;
-            if (client->remote->auth_timeout) {
-                g_source_remove(client->remote->auth_timeout);
-            }
-            /* after handshake, clients must send auth in a few seconds */
-            client->remote->auth_timeout =
-                g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client);
+            /* No more data is available at the moment. Just return for now;
+             * we'll get invoked again once the client sends more.
+             */
+            return 0;
+        } else if (rc < 0) {
+            crm_err("TLS handshake with remote CIB client failed: %s "
+                    CRM_XS " rc=%d", gnutls_strerror(rc), rc);
+            return -1;
         }
+
+        crm_debug("TLS handshake with remote CIB client completed");
+        client->remote->tls_handshake_complete = TRUE;
+        if (client->remote->auth_timeout) {
+            g_source_remove(client->remote->auth_timeout);
+        }
+
+        // Require the client to authenticate within this time
+        client->remote->auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT,
+                                                     remote_auth_timeout_cb,
+                                                     client);
         return 0;
     }
 #endif
 
     crm_remote_recv(client->remote, timeout, &disconnected);
 
     /* must pass auth before we will process anything else */
     if (client->remote->authenticated == FALSE) {
         xmlNode *reg;
 
 #if ENABLE_ACL
         const char *user = NULL;
 #endif
         command = crm_remote_parse_buffer(client->remote);
         if (cib_remote_auth(command) == FALSE) {
             free_xml(command);
             return -1;
         }
 
         crm_debug("remote connection authenticated successfully");
         client->remote->authenticated = TRUE;
         g_source_remove(client->remote->auth_timeout);
         client->remote->auth_timeout = 0;
         client->name = crm_element_value_copy(command, "name");
 
 #if ENABLE_ACL
         user = crm_element_value(command, "user");
         if (user) {
             client->user = strdup(user);
         }
 #endif
 
         /* send ACK */
         reg = create_xml_node(NULL, "cib_result");
         crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
         crm_xml_add(reg, F_CIB_CLIENTID, client->id);
         crm_remote_send(client->remote, reg);
         free_xml(reg);
         free_xml(command);
     }
 
     command = crm_remote_parse_buffer(client->remote);
     while (command) {
         crm_trace("command received");
         cib_handle_remote_msg(client, command);
         free_xml(command);
         command = crm_remote_parse_buffer(client->remote);
     }
 
     if (disconnected) {
-        crm_trace("Disconnected while receiving message from remote CIB manager");
+        crm_trace("Disconnected while receiving message from remote CIB client");
         return -1;
     }
 
     return 0;
 }
 
 #ifdef HAVE_PAM
 /*
  * Useful Examples:
  *    http://www.kernel.org/pub/linux/libs/pam/Linux-PAM-html
  *    http://developer.apple.com/samplecode/CryptNoMore/index.html
  */
 static int
 construct_pam_passwd(int num_msg, const struct pam_message **msg,
                      struct pam_response **response, void *data)
 {
     int count = 0;
     struct pam_response *reply;
     char *string = (char *)data;
 
     CRM_CHECK(data, return PAM_CONV_ERR);
     CRM_CHECK(num_msg == 1, return PAM_CONV_ERR);       /* We only want to handle one message */
 
     reply = calloc(1, sizeof(struct pam_response));
     CRM_ASSERT(reply != NULL);
 
     for (count = 0; count < num_msg; ++count) {
         switch (msg[count]->msg_style) {
             case PAM_TEXT_INFO:
                 crm_info("PAM: %s", msg[count]->msg);
                 break;
             case PAM_PROMPT_ECHO_OFF:
             case PAM_PROMPT_ECHO_ON:
                 reply[count].resp_retcode = 0;
                 reply[count].resp = string;     /* We already made a copy */
             case PAM_ERROR_MSG:
                 /* In theory we'd want to print this, but then
                  * we see the password prompt in the logs
                  */
                 /* crm_err("PAM error: %s", msg[count]->msg); */
                 break;
             default:
                 crm_err("Unhandled conversation type: %d", msg[count]->msg_style);
                 goto bail;
         }
     }
 
     *response = reply;
     reply = NULL;
 
     return PAM_SUCCESS;
 
   bail:
     for (count = 0; count < num_msg; ++count) {
         if (reply[count].resp != NULL) {
             switch (msg[count]->msg_style) {
                 case PAM_PROMPT_ECHO_ON:
                 case PAM_PROMPT_ECHO_OFF:
                     /* Erase the data - it contained a password */
                     while (*(reply[count].resp)) {
                         *(reply[count].resp)++ = '\0';
                     }
                     free(reply[count].resp);
                     break;
             }
             reply[count].resp = NULL;
         }
     }
     free(reply);
     reply = NULL;
 
     return PAM_CONV_ERR;
 }
 #endif
 
 int
 authenticate_user(const char *user, const char *passwd)
 {
 #ifndef HAVE_PAM
     gboolean pass = TRUE;
 #else
     int rc = 0;
     gboolean pass = FALSE;
     const void *p_user = NULL;
 
     struct pam_conv p_conv;
     struct pam_handle *pam_h = NULL;
     static const char *pam_name = NULL;
 
     if (pam_name == NULL) {
         pam_name = getenv("CIB_pam_service");
     }
     if (pam_name == NULL) {
         pam_name = "login";
     }
 
     p_conv.conv = construct_pam_passwd;
     p_conv.appdata_ptr = strdup(passwd);
 
     rc = pam_start(pam_name, user, &p_conv, &pam_h);
     if (rc != PAM_SUCCESS) {
         crm_err("Could not initialize PAM: %s (%d)", pam_strerror(pam_h, rc), rc);
         goto bail;
     }
 
     rc = pam_authenticate(pam_h, 0);
     if (rc != PAM_SUCCESS) {
         crm_err("Authentication failed for %s: %s (%d)", user, pam_strerror(pam_h, rc), rc);
         goto bail;
     }
 
     /* Make sure we authenticated the user we wanted to authenticate.
      * Since we also run as non-root, it might be worth pre-checking
      * the user has the same EID as us, since that the only user we
      * can authenticate.
      */
     rc = pam_get_item(pam_h, PAM_USER, &p_user);
     if (rc != PAM_SUCCESS) {
         crm_err("Internal PAM error: %s (%d)", pam_strerror(pam_h, rc), rc);
         goto bail;
 
     } else if (p_user == NULL) {
         crm_err("Unknown user authenticated.");
         goto bail;
 
     } else if (safe_str_neq(p_user, user)) {
         crm_err("User mismatch: %s vs. %s.", (const char *)p_user, (const char *)user);
         goto bail;
     }
 
     rc = pam_acct_mgmt(pam_h, 0);
     if (rc != PAM_SUCCESS) {
         crm_err("Access denied: %s (%d)", pam_strerror(pam_h, rc), rc);
         goto bail;
     }
     pass = TRUE;
 
   bail:
     pam_end(pam_h, rc);
 #endif
     return pass;
 }
diff --git a/daemons/execd/remoted_tls.c b/daemons/execd/remoted_tls.c
index 6e838567f5..c69381e801 100644
--- a/daemons/execd/remoted_tls.c
+++ b/daemons/execd/remoted_tls.c
@@ -1,407 +1,403 @@
 /*
  * Copyright 2012-2018 David Vossel <davidvossel@gmail.com>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 #include <unistd.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/remote_internal.h>
 
 #include <netdb.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <arpa/inet.h>
 
 #include "pacemaker-execd.h"
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 
 #  include <gnutls/gnutls.h>
 
 // Hidden in liblrmd
 extern int lrmd_tls_set_key(gnutls_datum_t *key);
 
 #  define LRMD_REMOTE_AUTH_TIMEOUT 10000
 gnutls_psk_server_credentials_t psk_cred_s;
 gnutls_dh_params_t dh_params;
 static int ssock = -1;
 extern int lrmd_call_id;
 
 static void
 debug_log(int level, const char *str)
 {
     fputs(str, stderr);
 }
 
 /*!
  * \internal
  * \brief Read (more) TLS handshake data from client
  */
 static int
 remoted__read_handshake_data(crm_client_t *client)
 {
-    int rc = 0;
-
-    do {
-        rc = gnutls_handshake(*client->remote->tls_session);
-    } while (rc == GNUTLS_E_INTERRUPTED);
+    int rc = pcmk__read_handshake_data(client);
 
-    if (rc == GNUTLS_E_AGAIN) {
+    if (rc == 0) {
         /* No more data is available at the moment. Just return for now;
          * we'll get invoked again once the client sends more.
          */
         return 0;
-    } else if (rc != GNUTLS_E_SUCCESS) {
+    } else if (rc < 0) {
         crm_err("TLS handshake with Pacemaker Remote failed: %s "
                 CRM_XS " rc=%d", gnutls_strerror(rc), rc);
         return -1;
     }
 
     if (client->remote->auth_timeout) {
         g_source_remove(client->remote->auth_timeout);
     }
     client->remote->auth_timeout = 0;
 
     client->remote->tls_handshake_complete = TRUE;
     crm_debug("TLS handshake with Pacemaker Remote completed");
 
     // Alert other clients of the new connection
     notify_of_new_client(client);
     return 0;
 }
 
 static int
 lrmd_remote_client_msg(gpointer data)
 {
     int id = 0;
     int rc = 0;
     int disconnected = 0;
     xmlNode *request = NULL;
     crm_client_t *client = data;
 
     if (client->remote->tls_handshake_complete == FALSE) {
         return remoted__read_handshake_data(client);
     }
 
     rc = crm_remote_ready(client->remote, 0);
     if (rc == 0) {
         /* no msg to read */
         return 0;
     } else if (rc < 0) {
         crm_info("Client disconnected while polling it");
         return -1;
     }
 
     crm_remote_recv(client->remote, -1, &disconnected);
 
     request = crm_remote_parse_buffer(client->remote);
     while (request) {
         crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
         crm_trace("processing request from remote client with remote msg id %d", id);
         if (!client->name) {
             const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
 
             if (value) {
                 client->name = strdup(value);
             }
         }
 
         lrmd_call_id++;
         if (lrmd_call_id < 1) {
             lrmd_call_id = 1;
         }
 
         crm_xml_add(request, F_LRMD_CLIENTID, client->id);
         crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
         crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
 
         process_lrmd_message(client, id, request);
         free_xml(request);
 
         /* process all the messages in the current buffer */
         request = crm_remote_parse_buffer(client->remote);
     }
 
     if (disconnected) {
         crm_info("Client disconnected while reading from it");
         return -1;
     }
 
     return 0;
 }
 
 static void
 lrmd_remote_client_destroy(gpointer user_data)
 {
     crm_client_t *client = user_data;
 
     if (client == NULL) {
         return;
     }
 
     crm_notice("Cleaning up after remote client %s disconnected "
                CRM_XS " id=%s",
                (client->name? client->name : ""), client->id);
 
     ipc_proxy_remove_provider(client);
 
     /* if this is the last remote connection, stop recurring
      * operations */
     if (crm_hash_table_size(client_connections) == 1) {
         client_disconnect_cleanup(NULL);
     }
 
     if (client->remote->tls_session) {
         void *sock_ptr;
         int csock;
 
         sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
         csock = GPOINTER_TO_INT(sock_ptr);
 
         gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
         gnutls_deinit(*client->remote->tls_session);
         gnutls_free(client->remote->tls_session);
         close(csock);
     }
 
     lrmd_client_destroy(client);
     return;
 }
 
 static gboolean
 lrmd_auth_timeout_cb(gpointer data)
 {
     crm_client_t *client = data;
 
     client->remote->auth_timeout = 0;
 
     if (client->remote->tls_handshake_complete == TRUE) {
         return FALSE;
     }
 
     mainloop_del_fd(client->remote->source);
     client->remote->source = NULL;
     crm_err("Remote client authentication timed out");
 
     return FALSE;
 }
 
 static int
 lrmd_remote_listen(gpointer data)
 {
     int csock = 0;
     gnutls_session_t *session = NULL;
     crm_client_t *new_client = NULL;
 
     static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
         .dispatch = lrmd_remote_client_msg,
         .destroy = lrmd_remote_client_destroy,
     };
 
     csock = crm_remote_accept(ssock);
     if (csock < 0) {
         return TRUE;
     }
 
     session = pcmk__new_tls_session(csock, GNUTLS_SERVER, GNUTLS_CRD_PSK,
                                     psk_cred_s);
     if (session == NULL) {
         close(csock);
         return TRUE;
     }
 
     new_client = crm_client_alloc(NULL);
     new_client->remote = calloc(1, sizeof(crm_remote_t));
     new_client->kind = CRM_CLIENT_TLS;
     new_client->remote->tls_session = session;
     new_client->remote->auth_timeout =
         g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client);
     crm_notice("Client connection to Pacemaker Remote established "
                CRM_XS " %p id: %s", new_client, new_client->id);
 
     new_client->remote->source =
         mainloop_add_fd("pacemaker-remote-client", G_PRIORITY_DEFAULT, csock,
                         new_client, &lrmd_remote_fd_cb);
     return TRUE;
 }
 
 static void
 lrmd_remote_connection_destroy(gpointer user_data)
 {
     crm_notice("Remote tls server disconnected");
     return;
 }
 
 static int
 lrmd_tls_server_key_cb(gnutls_session_t session, const char *username, gnutls_datum_t * key)
 {
     return lrmd_tls_set_key(key);
 }
 
 static int
 bind_and_listen(struct addrinfo *addr)
 {
     int optval;
     int fd;
     int rc;
     char buffer[INET6_ADDRSTRLEN] = { 0, };
 
     crm_sockaddr2str(addr->ai_addr, buffer);
     crm_trace("Attempting to bind on address %s", buffer);
 
     fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
     if (fd < 0) {
         return -1;
     }
 
     /* reuse address */
     optval = 1;
     rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
     if (rc < 0) {
         crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener, bind address %s", buffer);
         close(fd);
         return -1;
     }
 
     if (addr->ai_family == AF_INET6) {
         optval = 0;
         rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
         if (rc < 0) {
             crm_perror(LOG_INFO, "Couldn't disable IPV6 only on address %s", buffer);
             close(fd);
             return -1;
         }
     }
 
     if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
         close(fd);
         return -1;
     }
 
     if (listen(fd, 10) == -1) {
         crm_err("Can not start listen on address %s", buffer);
         close(fd);
         return -1;
     }
 
     crm_notice("Listening on address %s", buffer);
 
     return fd;
 }
 
 int
 lrmd_init_remote_tls_server()
 {
     int rc;
     int filter;
     int port = crm_default_remote_port();
     struct addrinfo hints, *res = NULL, *iter;
     char port_str[6]; // at most "65535"
     gnutls_datum_t psk_key = { NULL, 0 };
 
     static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
         .dispatch = lrmd_remote_listen,
         .destroy = lrmd_remote_connection_destroy,
     };
 
     crm_notice("Starting TLS listener on port %d", port);
     crm_gnutls_global_init();
     gnutls_global_set_log_function(debug_log);
 
     if (pcmk__init_tls_dh(&dh_params) != GNUTLS_E_SUCCESS) {
         return -1;
     }
     gnutls_psk_allocate_server_credentials(&psk_cred_s);
     gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
     gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);
 
     /* The key callback won't get called until the first client connection
      * attempt. Do it once here, so we can warn the user at start-up if we can't
      * read the key. We don't error out, though, because it's fine if the key is
      * going to be added later.
      */
     rc = lrmd_tls_set_key(&psk_key);
     if (rc != 0) {
         crm_warn("A cluster connection will not be possible until the key is available");
     }
     gnutls_free(psk_key.data);
 
     memset(&hints, 0, sizeof(struct addrinfo));
     /* Bind to the wildcard address (INADDR_ANY or IN6ADDR_ANY_INIT).
      * @TODO allow user to specify a specific address
      */
     hints.ai_flags = AI_PASSIVE;
     hints.ai_family = AF_UNSPEC; /* Return IPv6 or IPv4 */
     hints.ai_socktype = SOCK_STREAM;
     hints.ai_protocol = IPPROTO_TCP;
 
     snprintf(port_str, sizeof(port_str), "%d", port);
     rc = getaddrinfo(NULL, port_str, &hints, &res);
     if (rc) {
         crm_err("Unable to get IP address info for local node: %s",
                 gai_strerror(rc));
         return -1;
     }
 
     iter = res;
     filter = AF_INET6;
     /* Try IPv6 addresses first, then IPv4 */
     while (iter) {
         if (iter->ai_family == filter) {
             ssock = bind_and_listen(iter);
         }
         if (ssock != -1) {
             break;
         }
 
         iter = iter->ai_next;
         if (iter == NULL && filter == AF_INET6) {
             iter = res;
             filter = AF_INET;
         }
     }
 
     if (ssock < 0) {
         crm_err("unable to bind to address");
         goto init_remote_cleanup;
     }
 
     mainloop_add_fd("pacemaker-remote-server", G_PRIORITY_DEFAULT, ssock, NULL,
                     &remote_listen_fd_callbacks);
 
     rc = ssock;
   init_remote_cleanup:
     if (rc < 0) {
         close(ssock);
         ssock = 0;
     }
     freeaddrinfo(res);
     return rc;
 
 }
 
 void
 lrmd_tls_server_destroy(void)
 {
     if (psk_cred_s) {
         gnutls_psk_free_server_credentials(psk_cred_s);
         psk_cred_s = 0;
     }
 
     if (ssock > 0) {
         close(ssock);
         ssock = 0;
     }
 }
 #endif
diff --git a/include/crm/common/remote_internal.h b/include/crm/common/remote_internal.h
index b1ded10834..2f76ea07cf 100644
--- a/include/crm/common/remote_internal.h
+++ b/include/crm/common/remote_internal.h
@@ -1,47 +1,48 @@
 /*
  * Copyright 2008-2018 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__REMOTE__H
 #  define PCMK__REMOTE__H
 
 // internal functions from remote.c
 
 typedef struct crm_remote_s crm_remote_t;
 
 int crm_remote_send(crm_remote_t *remote, xmlNode *msg);
 int crm_remote_ready(crm_remote_t *remote, int total_timeout /*ms */ );
 gboolean crm_remote_recv(crm_remote_t *remote, int total_timeout /*ms */,
                          int *disconnected);
 xmlNode *crm_remote_parse_buffer(crm_remote_t *remote);
 int crm_remote_tcp_connect(const char *host, int port);
 int crm_remote_tcp_connect_async(const char *host, int port,
                                  int timeout /*ms */,
                                  int *timer_id, void *userdata,
                                  void (*callback) (void *userdata, int sock));
 int crm_remote_accept(int ssock);
 void crm_sockaddr2str(void *sa, char *s);
 
 #  ifdef HAVE_GNUTLS_GNUTLS_H
 #    include <gnutls/gnutls.h>
 
 gnutls_session_t *pcmk__new_tls_session(int csock, unsigned int conn_type,
                                         gnutls_credentials_type_t cred_type,
                                         void *credentials);
 int pcmk__init_tls_dh(gnutls_dh_params_t *dh_params);
+int pcmk__read_handshake_data(crm_client_t *client);
 
 /*!
  * \internal
  * \brief Initiate the client handshake after establishing the tcp socket
  *
  * \return 0 on success, negative number on failure
  * \note This function will block until the entire handshake is complete or
  *        until the timeout period is reached.
  */
 int crm_initiate_client_tls_handshake(crm_remote_t *remote, int timeout_ms);
 
 #  endif    // HAVE_GNUTLS_GNUTLS_H
 #endif      // PCMK__REMOTE__H
diff --git a/lib/common/remote.c b/lib/common/remote.c
index 88eae34af9..4f06567551 100644
--- a/lib/common/remote.c
+++ b/lib/common/remote.c
@@ -1,1108 +1,1142 @@
 /*
  * Copyright 2008-2018 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/socket.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <netinet/tcp.h>
 #include <netdb.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <inttypes.h>  /* X32T ~ PRIx32 */
 
 #include <glib.h>
 #include <bzlib.h>
 
 #include <crm/common/ipcs.h>
 #include <crm/common/xml.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/remote_internal.h>
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  undef KEYFILE
 #  include <gnutls/gnutls.h>
 
 const int psk_tls_kx_order[] = {
     GNUTLS_KX_DHE_PSK,
     GNUTLS_KX_PSK,
 };
 
 const int anon_tls_kx_order[] = {
     GNUTLS_KX_ANON_DH,
     GNUTLS_KX_DHE_RSA,
     GNUTLS_KX_DHE_DSS,
     GNUTLS_KX_RSA,
     0
 };
 #endif
 
 /* Swab macros from linux/swab.h */
 #ifdef HAVE_LINUX_SWAB_H
 #  include <linux/swab.h>
 #else
 /*
  * casts are necessary for constants, because we never know how for sure
  * how U/UL/ULL map to __u16, __u32, __u64. At least not in a portable way.
  */
 #define __swab16(x) ((uint16_t)(                                      \
         (((uint16_t)(x) & (uint16_t)0x00ffU) << 8) |                  \
         (((uint16_t)(x) & (uint16_t)0xff00U) >> 8)))
 
 #define __swab32(x) ((uint32_t)(                                      \
         (((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) |            \
         (((uint32_t)(x) & (uint32_t)0x0000ff00UL) <<  8) |            \
         (((uint32_t)(x) & (uint32_t)0x00ff0000UL) >>  8) |            \
         (((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24)))
 
 #define __swab64(x) ((uint64_t)(                                      \
         (((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) |   \
         (((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) |   \
         (((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) |   \
         (((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) <<  8) |   \
         (((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >>  8) |   \
         (((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) |   \
         (((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) |   \
         (((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56)))
 #endif
 
 #define REMOTE_MSG_VERSION 1
 #define ENDIAN_LOCAL 0xBADADBBD
 
 struct crm_remote_header_v0 
 {
     uint32_t endian;    /* Detect messages from hosts with different endian-ness */
     uint32_t version;
     uint64_t id;
     uint64_t flags;
     uint32_t size_total;
     uint32_t payload_offset;
     uint32_t payload_compressed;
     uint32_t payload_uncompressed;
 
         /* New fields get added here */
 
 } __attribute__ ((packed));
 
 static struct crm_remote_header_v0 *
 crm_remote_header(crm_remote_t * remote)
 {
     struct crm_remote_header_v0 *header = (struct crm_remote_header_v0 *)remote->buffer;
     if(remote->buffer_offset < sizeof(struct crm_remote_header_v0)) {
         return NULL;
 
     } else if(header->endian != ENDIAN_LOCAL) {
         uint32_t endian = __swab32(header->endian);
 
         CRM_LOG_ASSERT(endian == ENDIAN_LOCAL);
         if(endian != ENDIAN_LOCAL) {
             crm_err("Invalid message detected, endian mismatch: %" X32T
                     " is neither %" X32T " nor the swab'd %" X32T,
                     ENDIAN_LOCAL, header->endian, endian);
             return NULL;
         }
 
         header->id = __swab64(header->id);
         header->flags = __swab64(header->flags);
         header->endian = __swab32(header->endian);
 
         header->version = __swab32(header->version);
         header->size_total = __swab32(header->size_total);
         header->payload_offset = __swab32(header->payload_offset);
         header->payload_compressed = __swab32(header->payload_compressed);
         header->payload_uncompressed = __swab32(header->payload_uncompressed);
     }
 
     return header;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 
 int
 crm_initiate_client_tls_handshake(crm_remote_t * remote, int timeout_ms)
 {
     int rc = 0;
     int pollrc = 0;
     time_t start = time(NULL);
 
     do {
         rc = gnutls_handshake(*remote->tls_session);
         if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
             pollrc = crm_remote_ready(remote, 1000);
             if (pollrc < 0) {
                 /* poll returned error, there is no hope */
                 rc = -1;
             }
         }
 
     } while (((time(NULL) - start) < (timeout_ms / 1000)) &&
              (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN));
 
     if (rc < 0) {
         crm_trace("gnutls_handshake() failed with %d", rc);
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Initialize a new TLS session
  *
  * \param[in] csock       Connected socket for TLS session
  * \param[in] conn_type   GNUTLS_SERVER or GNUTLS_CLIENT
  * \param[in] cred_type   GNUTLS_CRD_ANON or GNUTLS_CRD_PSK
  * \param[in] credentials TLS session credentials
  *
  * \return Pointer to newly created session object, or NULL on error
  */
 gnutls_session_t *
 pcmk__new_tls_session(int csock, unsigned int conn_type,
                       gnutls_credentials_type_t cred_type, void *credentials)
 {
     int rc = GNUTLS_E_SUCCESS;
     const char *prio = NULL;
     gnutls_session_t *session = NULL;
 
     if (cred_type == GNUTLS_CRD_ANON) {
         // http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication
         prio = PCMK_GNUTLS_PRIORITIES ":+ANON-DH";
     } else {
         prio = PCMK_GNUTLS_PRIORITIES ":+DHE-PSK:+PSK";
     }
 
     session = gnutls_malloc(sizeof(gnutls_session_t));
     if (session == NULL) {
         rc = GNUTLS_E_MEMORY_ERROR;
         goto error;
     }
 
     rc = gnutls_init(session, conn_type);
     if (rc != GNUTLS_E_SUCCESS) {
         goto error;
     }
 
     /* @TODO On the server side, it would be more efficient to cache the
      * priority with gnutls_priority_init2() and set it with
      * gnutls_priority_set() for all sessions.
      */
     rc = gnutls_priority_set_direct(*session, prio, NULL);
     if (rc != GNUTLS_E_SUCCESS) {
         goto error;
     }
 
     gnutls_transport_set_ptr(*session,
                              (gnutls_transport_ptr_t) GINT_TO_POINTER(csock));
 
     rc = gnutls_credentials_set(*session, cred_type, credentials);
     if (rc != GNUTLS_E_SUCCESS) {
         goto error;
     }
     return session;
 
 error:
     crm_err("Could not initialize %s TLS %s session: %s "
             CRM_XS " rc=%d priority='%s'",
             (cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK",
             (conn_type == GNUTLS_SERVER)? "server" : "client",
             gnutls_strerror(rc), rc, prio);
     if (session != NULL) {
         gnutls_free(session);
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Initialize Diffie-Hellman parameters for a TLS server
  *
  * \param[out] dh_params  Parameter object to initialize
  *
  * \return GNUTLS_E_SUCCESS on success, GnuTLS error code on error
  * \todo The current best practice is to allow the client and server to
  *       negotiate the Diffie-Hellman parameters via a TLS extension (RFC 7919).
  *       However, we have to support both older versions of GnuTLS (<3.6) that
  *       don't support the extension on our side, and older Pacemaker versions
  *       that don't support the extension on the other side. The next best
  *       practice would be to use a known good prime (see RFC 5114 section 2.2),
  *       possibly stored in a file distributed with Pacemaker.
  */
 int
 pcmk__init_tls_dh(gnutls_dh_params_t *dh_params)
 {
     int rc = GNUTLS_E_SUCCESS;
     unsigned int dh_bits = 0;
 
     rc = gnutls_dh_params_init(dh_params);
     if (rc != GNUTLS_E_SUCCESS) {
         goto error;
     }
 
 #ifdef HAVE_GNUTLS_SEC_PARAM_TO_PK_BITS
     dh_bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
                                           GNUTLS_SEC_PARAM_NORMAL);
     if (dh_bits == 0) {
         rc = GNUTLS_E_DH_PRIME_UNACCEPTABLE;
         goto error;
     }
 #else
     dh_bits = 1024;
 #endif
 
     crm_info("Generating Diffie-Hellman parameters with %u-bit prime for TLS",
              dh_bits);
     rc = gnutls_dh_params_generate2(*dh_params, dh_bits);
     if (rc != GNUTLS_E_SUCCESS) {
         goto error;
     }
 
     return rc;
 
 error:
     crm_err("Could not initialize Diffie-Hellman parameters for TLS: %s "
             CRM_XS " rc=%d", gnutls_strerror(rc), rc);
     CRM_ASSERT(rc == GNUTLS_E_SUCCESS);
     return rc;
 }
 
+/*!
+ * \internal
+ * \brief Process handshake data from TLS client
+ *
+ * Read as much TLS handshake data as is available.
+ *
+ * \param[in] client  Client connection
+ *
+ * \retval GnuTLS error code on error
+ * \retval 0 if more data is needed
+ * \retval 1 if handshake is successfully completed
+ */
+int
+pcmk__read_handshake_data(crm_client_t *client)
+{
+    int rc = 0;
+
+    CRM_ASSERT(client && client->remote && client->remote->tls_session);
+
+    do {
+        rc = gnutls_handshake(*client->remote->tls_session);
+    } while (rc == GNUTLS_E_INTERRUPTED);
+
+    if (rc == GNUTLS_E_AGAIN) {
+        /* No more data is available at the moment. This function should be
+         * invoked again once the client sends more.
+         */
+        return 0;
+    } else if (rc != GNUTLS_E_SUCCESS) {
+        return rc;
+    }
+    return 1;
+}
+
 static int
 crm_send_tls(gnutls_session_t * session, const char *buf, size_t len)
 {
     const char *unsent = buf;
     int rc = 0;
     int total_send;
 
     if (buf == NULL) {
         return -EINVAL;
     }
 
     total_send = len;
     crm_trace("Message size: %llu", (unsigned long long) len);
 
     while (TRUE) {
         rc = gnutls_record_send(*session, unsent, len);
 
         if (rc == GNUTLS_E_INTERRUPTED || rc == GNUTLS_E_AGAIN) {
             crm_trace("Retrying to send %llu bytes",
                       (unsigned long long) len);
 
         } else if (rc < 0) {
             // Caller can log as error if necessary
             crm_info("TLS connection terminated: %s " CRM_XS " rc=%d",
                      gnutls_strerror(rc), rc);
             rc = -ECONNABORTED;
             break;
 
         } else if (rc < len) {
             crm_debug("Sent %d of %llu bytes", rc, (unsigned long long) len);
             len -= rc;
             unsent += rc;
         } else {
             crm_trace("Sent all %d bytes", rc);
             break;
         }
     }
 
     return rc < 0 ? rc : total_send;
 }
 #endif
 
 static int
 crm_send_plaintext(int sock, const char *buf, size_t len)
 {
 
     int rc = 0;
     const char *unsent = buf;
     int total_send;
 
     if (buf == NULL) {
         return -EINVAL;
     }
     total_send = len;
 
     crm_trace("Message on socket %d: size=%llu",
               sock, (unsigned long long) len);
   retry:
     rc = write(sock, unsent, len);
     if (rc < 0) {
         rc = -errno;
         switch (errno) {
             case EINTR:
             case EAGAIN:
                 crm_trace("Retry");
                 goto retry;
             default:
                 crm_perror(LOG_INFO,
                            "Could only write %d of the remaining %llu bytes",
                            rc, (unsigned long long) len);
                 break;
         }
 
     } else if (rc < len) {
         crm_trace("Only sent %d of %llu remaining bytes",
                   rc, (unsigned long long) len);
         len -= rc;
         unsent += rc;
         goto retry;
 
     } else {
         crm_trace("Sent %d bytes: %.100s", rc, buf);
     }
 
     return rc < 0 ? rc : total_send;
 
 }
 
 static int
 crm_remote_sendv(crm_remote_t * remote, struct iovec * iov, int iovs)
 {
     int rc = 0;
 
     for (int lpc = 0; (lpc < iovs) && (rc >= 0); lpc++) {
 #ifdef HAVE_GNUTLS_GNUTLS_H
         if (remote->tls_session) {
             rc = crm_send_tls(remote->tls_session, iov[lpc].iov_base, iov[lpc].iov_len);
             continue;
         }
 #endif
         if (remote->tcp_socket) {
             rc = crm_send_plaintext(remote->tcp_socket, iov[lpc].iov_base, iov[lpc].iov_len);
         } else {
             rc = -ESOCKTNOSUPPORT;
         }
     }
     return rc;
 }
 
 int
 crm_remote_send(crm_remote_t * remote, xmlNode * msg)
 {
     int rc = pcmk_ok;
     static uint64_t id = 0;
     char *xml_text = dump_xml_unformatted(msg);
 
     struct iovec iov[2];
     struct crm_remote_header_v0 *header;
 
     if (xml_text == NULL) {
         crm_err("Could not send remote message: no message provided");
         return -EINVAL;
     }
 
     header = calloc(1, sizeof(struct crm_remote_header_v0));
     iov[0].iov_base = header;
     iov[0].iov_len = sizeof(struct crm_remote_header_v0);
 
     iov[1].iov_base = xml_text;
     iov[1].iov_len = 1 + strlen(xml_text);
 
     id++;
     header->id = id;
     header->endian = ENDIAN_LOCAL;
     header->version = REMOTE_MSG_VERSION;
     header->payload_offset = iov[0].iov_len;
     header->payload_uncompressed = iov[1].iov_len;
     header->size_total = iov[0].iov_len + iov[1].iov_len;
 
     crm_trace("Sending len[0]=%d, start=%x",
               (int)iov[0].iov_len, *(int*)(void*)xml_text);
     rc = crm_remote_sendv(remote, iov, 2);
     if (rc < 0) {
         crm_err("Could not send remote message: %s " CRM_XS " rc=%d",
                 pcmk_strerror(rc), rc);
     }
 
     free(iov[0].iov_base);
     free(iov[1].iov_base);
     return rc;
 }
 
 
 /*!
  * \internal
  * \brief handles the recv buffer and parsing out msgs.
  * \note new_data is owned by this function once it is passed in.
  */
 xmlNode *
 crm_remote_parse_buffer(crm_remote_t * remote)
 {
     xmlNode *xml = NULL;
     struct crm_remote_header_v0 *header = crm_remote_header(remote);
 
     if (remote->buffer == NULL || header == NULL) {
         return NULL;
     }
 
     /* Support compression on the receiving end now, in case we ever want to add it later */
     if (header->payload_compressed) {
         int rc = 0;
         unsigned int size_u = 1 + header->payload_uncompressed;
         char *uncompressed = calloc(1, header->payload_offset + size_u);
 
         crm_trace("Decompressing message data %d bytes into %d bytes",
                  header->payload_compressed, size_u);
 
         rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset, &size_u,
                                         remote->buffer + header->payload_offset,
                                         header->payload_compressed, 1, 0);
 
         if (rc != BZ_OK && header->version > REMOTE_MSG_VERSION) {
             crm_warn("Couldn't decompress v%d message, we only understand v%d",
                      header->version, REMOTE_MSG_VERSION);
             free(uncompressed);
             return NULL;
 
         } else if (rc != BZ_OK) {
             crm_err("Decompression failed: %s " CRM_XS " bzerror=%d",
                     bz2_strerror(rc), rc);
             free(uncompressed);
             return NULL;
         }
 
         CRM_ASSERT(size_u == header->payload_uncompressed);
 
         memcpy(uncompressed, remote->buffer, header->payload_offset);       /* Preserve the header */
         remote->buffer_size = header->payload_offset + size_u;
 
         free(remote->buffer);
         remote->buffer = uncompressed;
         header = crm_remote_header(remote);
     }
 
     /* take ownership of the buffer */
     remote->buffer_offset = 0;
 
     CRM_LOG_ASSERT(remote->buffer[sizeof(struct crm_remote_header_v0) + header->payload_uncompressed - 1] == 0);
 
     xml = string2xml(remote->buffer + header->payload_offset);
     if (xml == NULL && header->version > REMOTE_MSG_VERSION) {
         crm_warn("Couldn't parse v%d message, we only understand v%d",
                  header->version, REMOTE_MSG_VERSION);
 
     } else if (xml == NULL) {
         crm_err("Couldn't parse: '%.120s'", remote->buffer + header->payload_offset);
     }
 
     return xml;
 }
 
 /*!
  * \internal
  * \brief Wait for a remote session to have data to read
  *
  * \param[in] remote         Connection to check
  * \param[in] total_timeout  Maximum time (in ms) to wait
  *
  * \return Positive value if ready to be read, 0 on timeout, -errno on error
  */
 int
 crm_remote_ready(crm_remote_t *remote, int total_timeout)
 {
     struct pollfd fds = { 0, };
     int sock = 0;
     int rc = 0;
     time_t start;
     int timeout = total_timeout;
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
     if (remote->tls_session) {
         void *sock_ptr = gnutls_transport_get_ptr(*remote->tls_session);
 
         sock = GPOINTER_TO_INT(sock_ptr);
     } else if (remote->tcp_socket) {
 #else
     if (remote->tcp_socket) {
 #endif
         sock = remote->tcp_socket;
     } else {
         crm_err("Unsupported connection type");
     }
 
     if (sock <= 0) {
         crm_trace("No longer connected");
         return -ENOTCONN;
     }
 
     start = time(NULL);
     errno = 0;
     do {
         fds.fd = sock;
         fds.events = POLLIN;
 
         /* If we got an EINTR while polling, and we have a
          * specific timeout we are trying to honor, attempt
          * to adjust the timeout to the closest second. */
         if (errno == EINTR && (timeout > 0)) {
             timeout = total_timeout - ((time(NULL) - start) * 1000);
             if (timeout < 1000) {
                 timeout = 1000;
             }
         }
 
         rc = poll(&fds, 1, timeout);
     } while (rc < 0 && errno == EINTR);
 
     return (rc < 0)? -errno : rc;
 }
 
 
 /*!
  * \internal
  * \brief Read bytes off non blocking remote connection.
  *
  * \note only use with NON-Blocking sockets. Should only be used after polling socket.
  *       This function will return once max_size is met, the socket read buffer
  *       is empty, or an error is encountered.
  *
  * \retval number of bytes received
  */
 static size_t
 crm_remote_recv_once(crm_remote_t * remote)
 {
     int rc = 0;
     size_t read_len = sizeof(struct crm_remote_header_v0);
     struct crm_remote_header_v0 *header = crm_remote_header(remote);
 
     if(header) {
         /* Stop at the end of the current message */
         read_len = header->size_total;
     }
 
     /* automatically grow the buffer when needed */
     if(remote->buffer_size < read_len) {
            remote->buffer_size = 2 * read_len;
         crm_trace("Expanding buffer to %llu bytes",
                   (unsigned long long) remote->buffer_size);
 
         remote->buffer = realloc_safe(remote->buffer, remote->buffer_size + 1);
         CRM_ASSERT(remote->buffer != NULL);
     }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
     if (remote->tls_session) {
         rc = gnutls_record_recv(*(remote->tls_session),
                                 remote->buffer + remote->buffer_offset,
                                 remote->buffer_size - remote->buffer_offset);
         if (rc == GNUTLS_E_INTERRUPTED) {
             rc = -EINTR;
         } else if (rc == GNUTLS_E_AGAIN) {
             rc = -EAGAIN;
         } else if (rc < 0) {
             crm_debug("TLS receive failed: %s (%d)", gnutls_strerror(rc), rc);
             rc = -pcmk_err_generic;
         }
     } else if (remote->tcp_socket) {
 #else
     if (remote->tcp_socket) {
 #endif
         errno = 0;
         rc = read(remote->tcp_socket,
                   remote->buffer + remote->buffer_offset,
                   remote->buffer_size - remote->buffer_offset);
         if(rc < 0) {
             rc = -errno;
         }
 
     } else {
         crm_err("Unsupported connection type");
         return -ESOCKTNOSUPPORT;
     }
 
     /* process any errors. */
     if (rc > 0) {
         remote->buffer_offset += rc;
         /* always null terminate buffer, the +1 to alloc always allows for this. */
         remote->buffer[remote->buffer_offset] = '\0';
         crm_trace("Received %u more bytes, %llu total",
                   rc, (unsigned long long) remote->buffer_offset);
 
     } else if (rc == -EINTR || rc == -EAGAIN) {
         crm_trace("non-blocking, exiting read: %s (%d)", pcmk_strerror(rc), rc);
 
     } else if (rc == 0) {
         crm_debug("EOF encoutered after %llu bytes",
                   (unsigned long long) remote->buffer_offset);
         return -ENOTCONN;
 
     } else {
         crm_debug("Error receiving message after %llu bytes: %s (%d)",
                   (unsigned long long) remote->buffer_offset,
                   pcmk_strerror(rc), rc);
         return -ENOTCONN;
     }
 
     header = crm_remote_header(remote);
     if(header) {
         if(remote->buffer_offset < header->size_total) {
             crm_trace("Read less than the advertised length: %llu < %u bytes",
                       (unsigned long long) remote->buffer_offset,
                       header->size_total);
         } else {
             crm_trace("Read full message of %llu bytes",
                       (unsigned long long) remote->buffer_offset);
             return remote->buffer_offset;
         }
     }
 
     return -EAGAIN;
 }
 
 /*!
  * \internal
  * \brief Read message(s) from a remote connection
  *
  * \param[in]  remote         Remote connection to read
  * \param[in]  total_timeout  Fail if message not read in this time (ms)
  * \param[out] disconnected   Will be set to 1 if disconnect detected
  *
  * \return TRUE if at least one full message read, FALSE otherwise
  */
 gboolean
 crm_remote_recv(crm_remote_t *remote, int total_timeout, int *disconnected)
 {
     int rc;
     time_t start = time(NULL);
     int remaining_timeout = 0;
 
     if (total_timeout == 0) {
         total_timeout = 10000;
     } else if (total_timeout < 0) {
         total_timeout = 60000;
     }
     *disconnected = 0;
 
     remaining_timeout = total_timeout;
     while ((remaining_timeout > 0) && !(*disconnected)) {
 
         crm_trace("Waiting for remote data (%d of %d ms timeout remaining)",
                   remaining_timeout, total_timeout);
         rc = crm_remote_ready(remote, remaining_timeout);
 
         if (rc == 0) {
             crm_err("Timed out (%d ms) while waiting for remote data",
                     remaining_timeout);
             return FALSE;
 
         } else if (rc < 0) {
             crm_debug("Wait for remote data aborted, will try again: %s "
                       CRM_XS " rc=%d", pcmk_strerror(rc), rc);
 
         } else {
             rc = crm_remote_recv_once(remote);
             if (rc > 0) {
                 return TRUE;
             } else if (rc == -EAGAIN) {
                 crm_trace("Still waiting for remote data");
             } else if (rc < 0) {
                 crm_debug("Could not receive remote data: %s " CRM_XS " rc=%d",
                           pcmk_strerror(rc), rc);
             }
         }
 
         if (rc == -ENOTCONN) {
             *disconnected = 1;
             return FALSE;
         }
 
         remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
     }
 
     return FALSE;
 }
 
 struct tcp_async_cb_data {
     gboolean success;
     int sock;
     void *userdata;
     void (*callback) (void *userdata, int sock);
     int timeout;                /*ms */
     time_t start;
 };
 
 static gboolean
 check_connect_finished(gpointer userdata)
 {
     struct tcp_async_cb_data *cb_data = userdata;
     int cb_arg = 0; // socket fd on success, -errno on error
     int sock = cb_data->sock;
     int error = 0;
 
     fd_set rset, wset;
     socklen_t len = sizeof(error);
     struct timeval ts = { 0, };
 
     if (cb_data->success == TRUE) {
         goto dispatch_done;
     }
 
     FD_ZERO(&rset);
     FD_SET(sock, &rset);
     wset = rset;
 
     crm_trace("fd %d: checking to see if connect finished", sock);
     cb_arg = select(sock + 1, &rset, &wset, NULL, &ts);
 
     if (cb_arg < 0) {
         cb_arg = -errno;
         if ((errno == EINPROGRESS) || (errno == EAGAIN)) {
             /* reschedule if there is still time left */
             if ((time(NULL) - cb_data->start) < (cb_data->timeout / 1000)) {
                 goto reschedule;
             } else {
                 cb_arg = -ETIMEDOUT;
             }
         }
         crm_trace("fd %d: select failed %d connect dispatch ", sock, cb_arg);
         goto dispatch_done;
     } else if (cb_arg == 0) {
         if ((time(NULL) - cb_data->start) < (cb_data->timeout / 1000)) {
             goto reschedule;
         }
         crm_debug("fd %d: timeout during select", sock);
         cb_arg = -ETIMEDOUT;
         goto dispatch_done;
     } else {
         crm_trace("fd %d: select returned success", sock);
         cb_arg = 0;
     }
 
     /* can we read or write to the socket now? */
     if (FD_ISSET(sock, &rset) || FD_ISSET(sock, &wset)) {
         if (getsockopt(sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) {
             cb_arg = -errno;
             crm_trace("fd %d: call to getsockopt failed", sock);
             goto dispatch_done;
         }
         if (error) {
             crm_trace("fd %d: error returned from getsockopt: %d", sock, error);
             cb_arg = -error;
             goto dispatch_done;
         }
     } else {
         crm_trace("neither read nor write set after select");
         cb_arg = -EAGAIN;
         goto dispatch_done;
     }
 
   dispatch_done:
     if (!cb_arg) {
         crm_trace("fd %d: connected", sock);
         /* Success, set the return code to the sock to report to the callback */
         cb_arg = cb_data->sock;
         cb_data->sock = 0;
     } else {
         close(sock);
     }
 
     if (cb_data->callback) {
         cb_data->callback(cb_data->userdata, cb_arg);
     }
     free(cb_data);
     return FALSE;
 
   reschedule:
 
     /* will check again next interval */
     return TRUE;
 }
 
 static int
 internal_tcp_connect_async(int sock,
                            const struct sockaddr *addr, socklen_t addrlen, int timeout /* ms */ ,
                            int *timer_id, void *userdata, void (*callback) (void *userdata, int sock))
 {
     int rc = 0;
     int interval = 500;
     int timer;
     struct tcp_async_cb_data *cb_data = NULL;
 
     rc = crm_set_nonblocking(sock);
     if (rc < 0) {
         crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
                  pcmk_strerror(rc), rc);
         close(sock);
         return -1;
     }
 
     rc = connect(sock, addr, addrlen);
     if (rc < 0 && (errno != EINPROGRESS) && (errno != EAGAIN)) {
         crm_perror(LOG_WARNING, "connect");
         return -1;
     }
 
     cb_data = calloc(1, sizeof(struct tcp_async_cb_data));
     cb_data->userdata = userdata;
     cb_data->callback = callback;
     cb_data->sock = sock;
     cb_data->timeout = timeout;
     cb_data->start = time(NULL);
 
     if (rc == 0) {
         /* The connect was successful immediately, we still return to mainloop
          * and let this callback get called later. This avoids the user of this api
          * to have to account for the fact the callback could be invoked within this
          * function before returning. */
         cb_data->success = TRUE;
         interval = 1;
     }
 
     /* Check connect finished is mostly doing a non-block poll on the socket
      * to see if we can read/write to it. Once we can, the connect has completed.
      * This method allows us to connect to the server without blocking mainloop.
      *
      * This is a poor man's way of polling to see when the connection finished.
      * At some point we should figure out a way to use a mainloop fd callback for this.
      * Something about the way mainloop is currently polling prevents this from working at the
      * moment though. */
     crm_trace("Scheduling check in %dms for whether connect to fd %d finished",
               interval, sock);
     timer = g_timeout_add(interval, check_connect_finished, cb_data);
     if (timer_id) {
         *timer_id = timer;
     }
 
     return 0;
 }
 
 static int
 internal_tcp_connect(int sock, const struct sockaddr *addr, socklen_t addrlen)
 {
     int rc = connect(sock, addr, addrlen);
 
     if (rc < 0) {
         rc = -errno;
         crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
                  pcmk_strerror(rc), rc);
         return rc;
     }
 
     rc = crm_set_nonblocking(sock);
     if (rc < 0) {
         crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
                  pcmk_strerror(rc), rc);
         return rc;
     }
 
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Connect to server at specified TCP port
  *
  * \param[in]  host      Name of server to connect to
  * \param[in]  port      Server port to connect to
  * \param[in]  timeout   Report error if not connected in this many milliseconds
  * \param[out] timer_id  If non-NULL, will be set to timer ID, if asynchronous
  * \param[in]  userdata  Data to pass to callback, if asynchronous
  * \param[in]  callback  If non-NULL, connect asynchronously then call this
  *
  * \return File descriptor of connected socket on success, -ENOTCONN otherwise
  */
 int
 crm_remote_tcp_connect_async(const char *host, int port, int timeout,
                              int *timer_id, void *userdata,
                              void (*callback) (void *userdata, int sock))
 {
     char buffer[INET6_ADDRSTRLEN];
     struct addrinfo *res = NULL;
     struct addrinfo *rp = NULL;
     struct addrinfo hints;
     const char *server = host;
     int ret_ga;
     int sock = -ENOTCONN;
 
     // Get host's IP address(es)
     memset(&hints, 0, sizeof(struct addrinfo));
     hints.ai_family = AF_UNSPEC;        /* Allow IPv4 or IPv6 */
     hints.ai_socktype = SOCK_STREAM;
     hints.ai_flags = AI_CANONNAME;
     ret_ga = getaddrinfo(server, NULL, &hints, &res);
     if (ret_ga) {
         crm_err("Unable to get IP address info for %s: %s",
                 server, gai_strerror(ret_ga));
         goto async_cleanup;
     }
     if (!res || !res->ai_addr) {
         crm_err("Unable to get IP address info for %s: no result", server);
         goto async_cleanup;
     }
 
     // getaddrinfo() returns a list of host's addresses, try them in order
     for (rp = res; rp != NULL; rp = rp->ai_next) {
         struct sockaddr *addr = rp->ai_addr;
 
         if (!addr) {
             continue;
         }
 
         if (rp->ai_canonname) {
             server = res->ai_canonname;
         }
         crm_debug("Got canonical name %s for %s", server, host);
 
         sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
         if (sock == -1) {
             crm_perror(LOG_WARNING, "creating socket for connection to %s",
                        server);
             sock = -ENOTCONN;
             continue;
         }
 
         /* Set port appropriately for address family */
         /* (void*) casts avoid false-positive compiler alignment warnings */
         if (addr->sa_family == AF_INET6) {
             ((struct sockaddr_in6 *)(void*)addr)->sin6_port = htons(port);
         } else {
             ((struct sockaddr_in *)(void*)addr)->sin_port = htons(port);
         }
 
         memset(buffer, 0, DIMOF(buffer));
         crm_sockaddr2str(addr, buffer);
         crm_info("Attempting TCP connection to %s:%d", buffer, port);
 
         if (callback) {
             if (internal_tcp_connect_async
                 (sock, rp->ai_addr, rp->ai_addrlen, timeout, timer_id, userdata, callback) == 0) {
                 goto async_cleanup; /* Success for now, we'll hear back later in the callback */
             }
 
         } else if (internal_tcp_connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) {
             break;          /* Success */
         }
 
         close(sock);
         sock = -ENOTCONN;
     }
 
 async_cleanup:
 
     if (res) {
         freeaddrinfo(res);
     }
     return sock;
 }
 
 int
 crm_remote_tcp_connect(const char *host, int port)
 {
     return crm_remote_tcp_connect_async(host, port, -1, NULL, NULL, NULL);
 }
 
 /*!
  * \brief Convert an IP address (IPv4 or IPv6) to a string for logging
  *
  * \param[in]  sa  Socket address for IP
  * \param[out] s   Storage for at least INET6_ADDRSTRLEN bytes
  *
  * \note sa The socket address can be a pointer to struct sockaddr_in (IPv4),
  *          struct sockaddr_in6 (IPv6) or struct sockaddr_storage (either),
  *          as long as its sa_family member is set correctly.
  */
 void
 crm_sockaddr2str(void *sa, char *s)
 {
     switch (((struct sockaddr*)sa)->sa_family) {
         case AF_INET:
             inet_ntop(AF_INET, &(((struct sockaddr_in *)sa)->sin_addr),
                       s, INET6_ADDRSTRLEN);
             break;
 
         case AF_INET6:
             inet_ntop(AF_INET6, &(((struct sockaddr_in6 *)sa)->sin6_addr),
                       s, INET6_ADDRSTRLEN);
             break;
 
         default:
             strcpy(s, "<invalid>");
     }
 }
 
 int
 crm_remote_accept(int ssock)
 {
     int csock = 0;
     int rc = 0;
     unsigned laddr = 0;
     struct sockaddr_storage addr;
     char addr_str[INET6_ADDRSTRLEN];
 #ifdef TCP_USER_TIMEOUT
     int optval;
     long sbd_timeout = crm_get_sbd_timeout();
 #endif
 
     /* accept the connection */
     laddr = sizeof(addr);
     memset(&addr, 0, sizeof(addr));
     csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
     crm_sockaddr2str(&addr, addr_str);
     crm_info("New remote connection from %s", addr_str);
 
     if (csock == -1) {
         crm_err("accept socket failed");
         return -1;
     }
 
     rc = crm_set_nonblocking(csock);
     if (rc < 0) {
         crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
                 pcmk_strerror(rc), rc);
         close(csock);
         return rc;
     }
 
 #ifdef TCP_USER_TIMEOUT
     if (sbd_timeout > 0) {
         optval = sbd_timeout / 2; /* time to fail and retry before watchdog */
         rc = setsockopt(csock, SOL_TCP, TCP_USER_TIMEOUT,
                         &optval, sizeof(optval));
         if (rc < 0) {
             crm_err("setting TCP_USER_TIMEOUT (%d) on client socket failed",
                     optval);
             close(csock);
             return rc;
         }
     }
 #endif
 
     return csock;
 }
 
 /*!
  * \brief Get the default remote connection TCP port on this host
  *
  * \return Remote connection TCP port number
  */
 int
 crm_default_remote_port()
 {
     static int port = 0;
 
     if (port == 0) {
         const char *env = getenv("PCMK_remote_port");
 
         if (env) {
             errno = 0;
             port = strtol(env, NULL, 10);
             if (errno || (port < 1) || (port > 65535)) {
                 crm_warn("Environment variable PCMK_remote_port has invalid value '%s', using %d instead",
                          env, DEFAULT_REMOTE_PORT);
                 port = DEFAULT_REMOTE_PORT;
             }
         } else {
             port = DEFAULT_REMOTE_PORT;
         }
     }
     return port;
 }