No OneTemporary
Actions

Size

67 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/based/based_remote.c b/daemons/based/based_remote.c
	index 71c62aff97..ee2d4d99f5 100644
	--- a/daemons/based/based_remote.c
	+++ b/daemons/based/based_remote.c
	@@ -1,685 +1,685 @@
	/*
	* Copyright 2004-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/socket.h>
	#include <arpa/inet.h>

	#include <netinet/ip.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <glib.h>

	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipcs.h>
	#include <crm/common/xml.h>
	#include <crm/common/remote_internal.h>
	#include <crm/cib/internal.h>

	#include "pacemaker-based.h"

	/* #undef HAVE_PAM_PAM_APPL_H */
	/* #undef HAVE_GNUTLS_GNUTLS_H */

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>
	#endif

	#include <pwd.h>
	#include <grp.h>
	#if HAVE_SECURITY_PAM_APPL_H
	# include <security/pam_appl.h>
	# define HAVE_PAM 1
	#else
	# if HAVE_PAM_PAM_APPL_H
	# include <pam/pam_appl.h>
	# define HAVE_PAM 1
	# endif
	#endif

	extern int remote_tls_fd;
	extern gboolean cib_shutdown_flag;

	int init_remote_listener(int port, gboolean encrypted);
	void cib_remote_connection_destroy(gpointer user_data);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	gnutls_dh_params_t dh_params;
	gnutls_anon_server_credentials_t anon_cred_s;
	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}
	#endif

	#define REMOTE_AUTH_TIMEOUT 10000

	int num_clients;
	int authenticate_user(const char user, const char passwd);
	-int cib_remote_listen(gpointer data);
	-int cib_remote_msg(gpointer data);
	+static int cib_remote_listen(gpointer data);
	+static int cib_remote_msg(gpointer data);

	static void
	remote_connection_destroy(gpointer user_data)
	{
	return;
	}

	#define ERROR_SUFFIX " Shutting down remote listener"
	int
	init_remote_listener(int port, gboolean encrypted)
	{
	int rc;
	int *ssock = NULL;
	struct sockaddr_in saddr;
	int optval;

	static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
	.dispatch = cib_remote_listen,
	.destroy = remote_connection_destroy,
	};

	if (port <= 0) {
	/* don't start it */
	return 0;
	}

	if (encrypted) {
	#ifndef HAVE_GNUTLS_GNUTLS_H
	crm_warn("TLS support is not available");
	return 0;
	#else
	crm_notice("Starting a tls listener on port %d.", port);
	crm_gnutls_global_init();
	/* gnutls_global_set_log_level (10); */
	gnutls_global_set_log_function(debug_log);
	if (pcmk__init_tls_dh(&dh_params) != GNUTLS_E_SUCCESS) {
	return -1;
	}
	gnutls_anon_allocate_server_credentials(&anon_cred_s);
	gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
	#endif
	} else {
	crm_warn("Starting a plain_text listener on port %d.", port);
	}
	#ifndef HAVE_PAM
	crm_warn("PAM is _not_ enabled!");
	#endif

	/* create server socket */
	ssock = malloc(sizeof(int));
	if(ssock == NULL) {
	crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
	return -1;
	}

	*ssock = socket(AF_INET, SOCK_STREAM, 0);
	if (*ssock == -1) {
	crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
	free(ssock);
	return -1;
	}

	/* reuse address */
	optval = 1;
	rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener");
	}

	/* bind server socket */
	memset(&saddr, '\0', sizeof(saddr));
	saddr.sin_family = AF_INET;
	saddr.sin_addr.s_addr = INADDR_ANY;
	saddr.sin_port = htons(port);
	if (bind(ssock, (struct sockaddr )&saddr, sizeof(saddr)) == -1) {
	crm_perror(LOG_ERR, "Can not bind server socket." ERROR_SUFFIX);
	close(*ssock);
	free(ssock);
	return -2;
	}
	if (listen(*ssock, 10) == -1) {
	crm_perror(LOG_ERR, "Can not start listen." ERROR_SUFFIX);
	close(*ssock);
	free(ssock);
	return -3;
	}

	mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks);

	return *ssock;
	}

	static int
	check_group_membership(const char usr, const char grp)
	{
	int index = 0;
	struct passwd *pwd = NULL;
	struct group *group = NULL;

	CRM_CHECK(usr != NULL, return FALSE);
	CRM_CHECK(grp != NULL, return FALSE);

	pwd = getpwnam(usr);
	if (pwd == NULL) {
	crm_err("No user named '%s' exists!", usr);
	return FALSE;
	}

	group = getgrgid(pwd->pw_gid);
	if (group != NULL && crm_str_eq(grp, group->gr_name, TRUE)) {
	return TRUE;
	}

	group = getgrnam(grp);
	if (group == NULL) {
	crm_err("No group named '%s' exists!", grp);
	return FALSE;
	}

	while (TRUE) {
	char *member = group->gr_mem[index++];

	if (member == NULL) {
	break;

	} else if (crm_str_eq(usr, member, TRUE)) {
	return TRUE;
	}
	};

	return FALSE;
	}

	static gboolean
	cib_remote_auth(xmlNode * login)
	{
	const char *user = NULL;
	const char *pass = NULL;
	const char *tmp = NULL;

	crm_log_xml_info(login, "Login: ");
	if (login == NULL) {
	return FALSE;
	}

	tmp = crm_element_name(login);
	if (safe_str_neq(tmp, "cib_command")) {
	crm_err("Wrong tag: %s", tmp);
	return FALSE;
	}

	tmp = crm_element_value(login, "op");
	if (safe_str_neq(tmp, "authenticate")) {
	crm_err("Wrong operation: %s", tmp);
	return FALSE;
	}

	user = crm_element_value(login, "user");
	pass = crm_element_value(login, "password");

	if (!user \|\| !pass) {
	crm_err("missing auth credentials");
	return FALSE;
	}

	/* Non-root daemons can only validate the password of the
	* user they're running as
	*/
	if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
	crm_err("User is not a member of the required group");
	return FALSE;

	} else if (authenticate_user(user, pass) == FALSE) {
	crm_err("PAM auth failed");
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	remote_auth_timeout_cb(gpointer data)
	{
	crm_client_t *client = data;

	client->remote->auth_timeout = 0;

	if (client->remote->authenticated == TRUE) {
	return FALSE;
	}

	mainloop_del_fd(client->remote->source);
	crm_err("Remote client authentication timed out");

	return FALSE;
	}

	-int
	+static int
	cib_remote_listen(gpointer data)
	{
	int csock = 0;
	unsigned laddr;
	struct sockaddr_storage addr;
	char ipstr[INET6_ADDRSTRLEN];
	int ssock = (int )data;
	int rc;

	crm_client_t *new_client = NULL;

	static struct mainloop_fd_callbacks remote_client_fd_callbacks = {
	.dispatch = cib_remote_msg,
	.destroy = cib_remote_connection_destroy,
	};

	/* accept the connection */
	laddr = sizeof(addr);
	memset(&addr, 0, sizeof(addr));
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	if (csock == -1) {
	crm_perror(LOG_ERR, "Could not accept socket connection");
	return TRUE;
	}

	crm_sockaddr2str(&addr, ipstr);
	crm_debug("New %s connection from %s",
	((ssock == remote_tls_fd)? "secure" : "clear-text"), ipstr);

	rc = crm_set_nonblocking(csock);
	if (rc < 0) {
	crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	close(csock);
	return TRUE;
	}

	num_clients++;

	crm_client_init();
	new_client = crm_client_alloc(NULL);
	new_client->remote = calloc(1, sizeof(crm_remote_t));

	if (ssock == remote_tls_fd) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	new_client->kind = CRM_CLIENT_TLS;

	/* create gnutls session for the server socket */
	new_client->remote->tls_session = pcmk__new_tls_session(csock,
	GNUTLS_SERVER,
	GNUTLS_CRD_ANON,
	anon_cred_s);
	if (new_client->remote->tls_session == NULL) {
	close(csock);
	return TRUE;
	}
	#endif
	} else {
	new_client->kind = CRM_CLIENT_TCP;
	new_client->remote->tcp_socket = csock;
	}

	/* clients have a few seconds to perform handshake. */
	new_client->remote->auth_timeout =
	g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client);

	new_client->remote->source =
	mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
	&remote_client_fd_callbacks);

	return TRUE;
	}

	void
	cib_remote_connection_destroy(gpointer user_data)
	{
	crm_client_t *client = user_data;
	int csock = 0;

	if (client == NULL) {
	return;
	}

	crm_trace("Cleaning up after client disconnect: %s/%s", crm_str(client->name), client->id);

	num_clients--;
	crm_trace("Num unfree'd clients: %d", num_clients);

	switch (client->kind) {
	case CRM_CLIENT_TCP:
	csock = client->remote->tcp_socket;
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case CRM_CLIENT_TLS:
	if (client->remote->tls_session) {
	void sock_ptr = gnutls_transport_get_ptr(client->remote->tls_session);

	csock = GPOINTER_TO_INT(sock_ptr);
	if (client->remote->tls_handshake_complete) {
	gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_WR);
	}
	gnutls_deinit(*client->remote->tls_session);
	gnutls_free(client->remote->tls_session);
	client->remote->tls_session = NULL;
	}
	break;
	#endif
	default:
	crm_warn("Unexpected client type %d", client->kind);
	}

	if (csock > 0) {
	close(csock);
	}

	crm_client_destroy(client);

	crm_trace("Freed the cib client");

	if (cib_shutdown_flag) {
	cib_shutdown(0);
	}
	return;
	}

	static void
	cib_handle_remote_msg(crm_client_t * client, xmlNode * command)
	{
	const char *value = NULL;

	value = crm_element_name(command);
	if (safe_str_neq(value, "cib_command")) {
	crm_log_xml_trace(command, "Bad command: ");
	return;
	}

	if (client->name == NULL) {
	value = crm_element_value(command, F_CLIENTNAME);
	if (value == NULL) {
	client->name = strdup(client->id);
	} else {
	client->name = strdup(value);
	}
	}

	if (client->userdata == NULL) {
	value = crm_element_value(command, F_CIB_CALLBACK_TOKEN);
	if (value != NULL) {
	client->userdata = strdup(value);
	crm_trace("Callback channel for %s is %s", client->id, (char*)client->userdata);

	} else {
	client->userdata = strdup(client->id);
	}
	}

	/* unset dangerous options */
	xml_remove_prop(command, F_ORIG);
	xml_remove_prop(command, F_CIB_HOST);
	xml_remove_prop(command, F_CIB_GLOBAL_UPDATE);

	crm_xml_add(command, F_TYPE, T_CIB);
	crm_xml_add(command, F_CIB_CLIENTID, client->id);
	crm_xml_add(command, F_CIB_CLIENTNAME, client->name);
	#if ENABLE_ACL
	crm_xml_add(command, F_CIB_USER, client->user);
	#endif

	if (crm_element_value(command, F_CIB_CALLID) == NULL) {
	char *call_uuid = crm_generate_uuid();

	/* fix the command */
	crm_xml_add(command, F_CIB_CALLID, call_uuid);
	free(call_uuid);
	}

	if (crm_element_value(command, F_CIB_CALLOPTS) == NULL) {
	crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
	}

	crm_log_xml_trace(command, "Remote command: ");
	cib_common_callback_worker(0, 0, command, client, TRUE);
	}

	-int
	+static int
	cib_remote_msg(gpointer data)
	{
	xmlNode *command = NULL;
	crm_client_t *client = data;
	int disconnected = 0;
	int timeout = client->remote->authenticated ? -1 : 1000;

	crm_trace("%s callback", client->kind != CRM_CLIENT_TCP ? "secure" : "clear-text");

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (client->kind == CRM_CLIENT_TLS && (client->remote->tls_handshake_complete == FALSE)) {
	- int rc = 0;
	-
	- /* Muliple calls to handshake will be required, this callback
	- * will be invoked once the client sends more handshake data. */
	- do {
	- rc = gnutls_handshake(*client->remote->tls_session);
	-
	- if (rc < 0 && rc != GNUTLS_E_AGAIN) {
	- crm_err("TLS handshake with remote CIB manager failed");
	- return -1;
	- }
	- } while (rc == GNUTLS_E_INTERRUPTED);
	+ int rc = pcmk__read_handshake_data(client);

	if (rc == 0) {
	- crm_debug("TLS handshake with remote CIB manager completed");
	- client->remote->tls_handshake_complete = TRUE;
	- if (client->remote->auth_timeout) {
	- g_source_remove(client->remote->auth_timeout);
	- }
	- /* after handshake, clients must send auth in a few seconds */
	- client->remote->auth_timeout =
	- g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client);
	+ /* No more data is available at the moment. Just return for now;
	+ * we'll get invoked again once the client sends more.
	+ */
	+ return 0;
	+ } else if (rc < 0) {
	+ crm_err("TLS handshake with remote CIB client failed: %s "
	+ CRM_XS " rc=%d", gnutls_strerror(rc), rc);
	+ return -1;
	}
	+
	+ crm_debug("TLS handshake with remote CIB client completed");
	+ client->remote->tls_handshake_complete = TRUE;
	+ if (client->remote->auth_timeout) {
	+ g_source_remove(client->remote->auth_timeout);
	+ }
	+
	+ // Require the client to authenticate within this time
	+ client->remote->auth_timeout = g_timeout_add(REMOTE_AUTH_TIMEOUT,
	+ remote_auth_timeout_cb,
	+ client);
	return 0;
	}
	#endif

	crm_remote_recv(client->remote, timeout, &disconnected);

	/* must pass auth before we will process anything else */
	if (client->remote->authenticated == FALSE) {
	xmlNode *reg;

	#if ENABLE_ACL
	const char *user = NULL;
	#endif
	command = crm_remote_parse_buffer(client->remote);
	if (cib_remote_auth(command) == FALSE) {
	free_xml(command);
	return -1;
	}

	crm_debug("remote connection authenticated successfully");
	client->remote->authenticated = TRUE;
	g_source_remove(client->remote->auth_timeout);
	client->remote->auth_timeout = 0;
	client->name = crm_element_value_copy(command, "name");

	#if ENABLE_ACL
	user = crm_element_value(command, "user");
	if (user) {
	client->user = strdup(user);
	}
	#endif

	/* send ACK */
	reg = create_xml_node(NULL, "cib_result");
	crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
	crm_xml_add(reg, F_CIB_CLIENTID, client->id);
	crm_remote_send(client->remote, reg);
	free_xml(reg);
	free_xml(command);
	}

	command = crm_remote_parse_buffer(client->remote);
	while (command) {
	crm_trace("command received");
	cib_handle_remote_msg(client, command);
	free_xml(command);
	command = crm_remote_parse_buffer(client->remote);
	}

	if (disconnected) {
	- crm_trace("Disconnected while receiving message from remote CIB manager");
	+ crm_trace("Disconnected while receiving message from remote CIB client");
	return -1;
	}

	return 0;
	}

	#ifdef HAVE_PAM
	/*
	* Useful Examples:
	* http://www.kernel.org/pub/linux/libs/pam/Linux-PAM-html
	* http://developer.apple.com/samplecode/CryptNoMore/index.html
	*/
	static int
	construct_pam_passwd(int num_msg, const struct pam_message **msg,
	struct pam_response *response, void data)
	{
	int count = 0;
	struct pam_response *reply;
	char string = (char )data;

	CRM_CHECK(data, return PAM_CONV_ERR);
	CRM_CHECK(num_msg == 1, return PAM_CONV_ERR); /* We only want to handle one message */

	reply = calloc(1, sizeof(struct pam_response));
	CRM_ASSERT(reply != NULL);

	for (count = 0; count < num_msg; ++count) {
	switch (msg[count]->msg_style) {
	case PAM_TEXT_INFO:
	crm_info("PAM: %s", msg[count]->msg);
	break;
	case PAM_PROMPT_ECHO_OFF:
	case PAM_PROMPT_ECHO_ON:
	reply[count].resp_retcode = 0;
	reply[count].resp = string; /* We already made a copy */
	case PAM_ERROR_MSG:
	/* In theory we'd want to print this, but then
	* we see the password prompt in the logs
	*/
	/* crm_err("PAM error: %s", msg[count]->msg); */
	break;
	default:
	crm_err("Unhandled conversation type: %d", msg[count]->msg_style);
	goto bail;
	}
	}

	*response = reply;
	reply = NULL;

	return PAM_SUCCESS;

	bail:
	for (count = 0; count < num_msg; ++count) {
	if (reply[count].resp != NULL) {
	switch (msg[count]->msg_style) {
	case PAM_PROMPT_ECHO_ON:
	case PAM_PROMPT_ECHO_OFF:
	/* Erase the data - it contained a password */
	while (*(reply[count].resp)) {
	*(reply[count].resp)++ = '\0';
	}
	free(reply[count].resp);
	break;
	}
	reply[count].resp = NULL;
	}
	}
	free(reply);
	reply = NULL;

	return PAM_CONV_ERR;
	}
	#endif

	int
	authenticate_user(const char user, const char passwd)
	{
	#ifndef HAVE_PAM
	gboolean pass = TRUE;
	#else
	int rc = 0;
	gboolean pass = FALSE;
	const void *p_user = NULL;

	struct pam_conv p_conv;
	struct pam_handle *pam_h = NULL;
	static const char *pam_name = NULL;

	if (pam_name == NULL) {
	pam_name = getenv("CIB_pam_service");
	}
	if (pam_name == NULL) {
	pam_name = "login";
	}

	p_conv.conv = construct_pam_passwd;
	p_conv.appdata_ptr = strdup(passwd);

	rc = pam_start(pam_name, user, &p_conv, &pam_h);
	if (rc != PAM_SUCCESS) {
	crm_err("Could not initialize PAM: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	rc = pam_authenticate(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Authentication failed for %s: %s (%d)", user, pam_strerror(pam_h, rc), rc);
	goto bail;
	}

	/* Make sure we authenticated the user we wanted to authenticate.
	* Since we also run as non-root, it might be worth pre-checking
	* the user has the same EID as us, since that the only user we
	* can authenticate.
	*/
	rc = pam_get_item(pam_h, PAM_USER, &p_user);
	if (rc != PAM_SUCCESS) {
	crm_err("Internal PAM error: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;

	} else if (p_user == NULL) {
	crm_err("Unknown user authenticated.");
	goto bail;

	} else if (safe_str_neq(p_user, user)) {
	crm_err("User mismatch: %s vs. %s.", (const char )p_user, (const char )user);
	goto bail;
	}

	rc = pam_acct_mgmt(pam_h, 0);
	if (rc != PAM_SUCCESS) {
	crm_err("Access denied: %s (%d)", pam_strerror(pam_h, rc), rc);
	goto bail;
	}
	pass = TRUE;

	bail:
	pam_end(pam_h, rc);
	#endif
	return pass;
	}
	diff --git a/daemons/execd/remoted_tls.c b/daemons/execd/remoted_tls.c
	index 6e838567f5..c69381e801 100644
	--- a/daemons/execd/remoted_tls.c
	+++ b/daemons/execd/remoted_tls.c
	@@ -1,407 +1,403 @@
	/*
	* Copyright 2012-2018 David Vossel <davidvossel@gmail.com>
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/remote_internal.h>

	#include <netdb.h>
	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>

	#include "pacemaker-execd.h"

	#ifdef HAVE_GNUTLS_GNUTLS_H

	# include <gnutls/gnutls.h>

	// Hidden in liblrmd
	extern int lrmd_tls_set_key(gnutls_datum_t *key);

	# define LRMD_REMOTE_AUTH_TIMEOUT 10000
	gnutls_psk_server_credentials_t psk_cred_s;
	gnutls_dh_params_t dh_params;
	static int ssock = -1;
	extern int lrmd_call_id;

	static void
	debug_log(int level, const char *str)
	{
	fputs(str, stderr);
	}

	/*!
	* \internal
	* \brief Read (more) TLS handshake data from client
	*/
	static int
	remoted__read_handshake_data(crm_client_t *client)
	{
	- int rc = 0;
	-
	- do {
	- rc = gnutls_handshake(*client->remote->tls_session);
	- } while (rc == GNUTLS_E_INTERRUPTED);
	+ int rc = pcmk__read_handshake_data(client);

	- if (rc == GNUTLS_E_AGAIN) {
	+ if (rc == 0) {
	/* No more data is available at the moment. Just return for now;
	* we'll get invoked again once the client sends more.
	*/
	return 0;
	- } else if (rc != GNUTLS_E_SUCCESS) {
	+ } else if (rc < 0) {
	crm_err("TLS handshake with Pacemaker Remote failed: %s "
	CRM_XS " rc=%d", gnutls_strerror(rc), rc);
	return -1;
	}

	if (client->remote->auth_timeout) {
	g_source_remove(client->remote->auth_timeout);
	}
	client->remote->auth_timeout = 0;

	client->remote->tls_handshake_complete = TRUE;
	crm_debug("TLS handshake with Pacemaker Remote completed");

	// Alert other clients of the new connection
	notify_of_new_client(client);
	return 0;
	}

	static int
	lrmd_remote_client_msg(gpointer data)
	{
	int id = 0;
	int rc = 0;
	int disconnected = 0;
	xmlNode *request = NULL;
	crm_client_t *client = data;

	if (client->remote->tls_handshake_complete == FALSE) {
	return remoted__read_handshake_data(client);
	}

	rc = crm_remote_ready(client->remote, 0);
	if (rc == 0) {
	/* no msg to read */
	return 0;
	} else if (rc < 0) {
	crm_info("Client disconnected while polling it");
	return -1;
	}

	crm_remote_recv(client->remote, -1, &disconnected);

	request = crm_remote_parse_buffer(client->remote);
	while (request) {
	crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
	crm_trace("processing request from remote client with remote msg id %d", id);
	if (!client->name) {
	const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);

	if (value) {
	client->name = strdup(value);
	}
	}

	lrmd_call_id++;
	if (lrmd_call_id < 1) {
	lrmd_call_id = 1;
	}

	crm_xml_add(request, F_LRMD_CLIENTID, client->id);
	crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
	crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);

	process_lrmd_message(client, id, request);
	free_xml(request);

	/* process all the messages in the current buffer */
	request = crm_remote_parse_buffer(client->remote);
	}

	if (disconnected) {
	crm_info("Client disconnected while reading from it");
	return -1;
	}

	return 0;
	}

	static void
	lrmd_remote_client_destroy(gpointer user_data)
	{
	crm_client_t *client = user_data;

	if (client == NULL) {
	return;
	}

	crm_notice("Cleaning up after remote client %s disconnected "
	CRM_XS " id=%s",
	(client->name? client->name : ""), client->id);

	ipc_proxy_remove_provider(client);

	/* if this is the last remote connection, stop recurring
	* operations */
	if (crm_hash_table_size(client_connections) == 1) {
	client_disconnect_cleanup(NULL);
	}

	if (client->remote->tls_session) {
	void *sock_ptr;
	int csock;

	sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
	csock = GPOINTER_TO_INT(sock_ptr);

	gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*client->remote->tls_session);
	gnutls_free(client->remote->tls_session);
	close(csock);
	}

	lrmd_client_destroy(client);
	return;
	}

	static gboolean
	lrmd_auth_timeout_cb(gpointer data)
	{
	crm_client_t *client = data;

	client->remote->auth_timeout = 0;

	if (client->remote->tls_handshake_complete == TRUE) {
	return FALSE;
	}

	mainloop_del_fd(client->remote->source);
	client->remote->source = NULL;
	crm_err("Remote client authentication timed out");

	return FALSE;
	}

	static int
	lrmd_remote_listen(gpointer data)
	{
	int csock = 0;
	gnutls_session_t *session = NULL;
	crm_client_t *new_client = NULL;

	static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
	.dispatch = lrmd_remote_client_msg,
	.destroy = lrmd_remote_client_destroy,
	};

	csock = crm_remote_accept(ssock);
	if (csock < 0) {
	return TRUE;
	}

	session = pcmk__new_tls_session(csock, GNUTLS_SERVER, GNUTLS_CRD_PSK,
	psk_cred_s);
	if (session == NULL) {
	close(csock);
	return TRUE;
	}

	new_client = crm_client_alloc(NULL);
	new_client->remote = calloc(1, sizeof(crm_remote_t));
	new_client->kind = CRM_CLIENT_TLS;
	new_client->remote->tls_session = session;
	new_client->remote->auth_timeout =
	g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client);
	crm_notice("Client connection to Pacemaker Remote established "
	CRM_XS " %p id: %s", new_client, new_client->id);

	new_client->remote->source =
	mainloop_add_fd("pacemaker-remote-client", G_PRIORITY_DEFAULT, csock,
	new_client, &lrmd_remote_fd_cb);
	return TRUE;
	}

	static void
	lrmd_remote_connection_destroy(gpointer user_data)
	{
	crm_notice("Remote tls server disconnected");
	return;
	}

	static int
	lrmd_tls_server_key_cb(gnutls_session_t session, const char username, gnutls_datum_t key)
	{
	return lrmd_tls_set_key(key);
	}

	static int
	bind_and_listen(struct addrinfo *addr)
	{
	int optval;
	int fd;
	int rc;
	char buffer[INET6_ADDRSTRLEN] = { 0, };

	crm_sockaddr2str(addr->ai_addr, buffer);
	crm_trace("Attempting to bind on address %s", buffer);

	fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
	if (fd < 0) {
	return -1;
	}

	/* reuse address */
	optval = 1;
	rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener, bind address %s", buffer);
	close(fd);
	return -1;
	}

	if (addr->ai_family == AF_INET6) {
	optval = 0;
	rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
	if (rc < 0) {
	crm_perror(LOG_INFO, "Couldn't disable IPV6 only on address %s", buffer);
	close(fd);
	return -1;
	}
	}

	if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
	close(fd);
	return -1;
	}

	if (listen(fd, 10) == -1) {
	crm_err("Can not start listen on address %s", buffer);
	close(fd);
	return -1;
	}

	crm_notice("Listening on address %s", buffer);

	return fd;
	}

	int
	lrmd_init_remote_tls_server()
	{
	int rc;
	int filter;
	int port = crm_default_remote_port();
	struct addrinfo hints, res = NULL, iter;
	char port_str[6]; // at most "65535"
	gnutls_datum_t psk_key = { NULL, 0 };

	static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
	.dispatch = lrmd_remote_listen,
	.destroy = lrmd_remote_connection_destroy,
	};

	crm_notice("Starting TLS listener on port %d", port);
	crm_gnutls_global_init();
	gnutls_global_set_log_function(debug_log);

	if (pcmk__init_tls_dh(&dh_params) != GNUTLS_E_SUCCESS) {
	return -1;
	}
	gnutls_psk_allocate_server_credentials(&psk_cred_s);
	gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
	gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);

	/* The key callback won't get called until the first client connection
	* attempt. Do it once here, so we can warn the user at start-up if we can't
	* read the key. We don't error out, though, because it's fine if the key is
	* going to be added later.
	*/
	rc = lrmd_tls_set_key(&psk_key);
	if (rc != 0) {
	crm_warn("A cluster connection will not be possible until the key is available");
	}
	gnutls_free(psk_key.data);

	memset(&hints, 0, sizeof(struct addrinfo));
	/* Bind to the wildcard address (INADDR_ANY or IN6ADDR_ANY_INIT).
	* @TODO allow user to specify a specific address
	*/
	hints.ai_flags = AI_PASSIVE;
	hints.ai_family = AF_UNSPEC; /* Return IPv6 or IPv4 */
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_protocol = IPPROTO_TCP;

	snprintf(port_str, sizeof(port_str), "%d", port);
	rc = getaddrinfo(NULL, port_str, &hints, &res);
	if (rc) {
	crm_err("Unable to get IP address info for local node: %s",
	gai_strerror(rc));
	return -1;
	}

	iter = res;
	filter = AF_INET6;
	/* Try IPv6 addresses first, then IPv4 */
	while (iter) {
	if (iter->ai_family == filter) {
	ssock = bind_and_listen(iter);
	}
	if (ssock != -1) {
	break;
	}

	iter = iter->ai_next;
	if (iter == NULL && filter == AF_INET6) {
	iter = res;
	filter = AF_INET;
	}
	}

	if (ssock < 0) {
	crm_err("unable to bind to address");
	goto init_remote_cleanup;
	}

	mainloop_add_fd("pacemaker-remote-server", G_PRIORITY_DEFAULT, ssock, NULL,
	&remote_listen_fd_callbacks);

	rc = ssock;
	init_remote_cleanup:
	if (rc < 0) {
	close(ssock);
	ssock = 0;
	}
	freeaddrinfo(res);
	return rc;

	}

	void
	lrmd_tls_server_destroy(void)
	{
	if (psk_cred_s) {
	gnutls_psk_free_server_credentials(psk_cred_s);
	psk_cred_s = 0;
	}

	if (ssock > 0) {
	close(ssock);
	ssock = 0;
	}
	}
	#endif
	diff --git a/include/crm/common/remote_internal.h b/include/crm/common/remote_internal.h
	index b1ded10834..2f76ea07cf 100644
	--- a/include/crm/common/remote_internal.h
	+++ b/include/crm/common/remote_internal.h
	@@ -1,47 +1,48 @@
	/*
	* Copyright 2008-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__REMOTE__H
	# define PCMK__REMOTE__H

	// internal functions from remote.c

	typedef struct crm_remote_s crm_remote_t;

	int crm_remote_send(crm_remote_t remote, xmlNode msg);
	int crm_remote_ready(crm_remote_t remote, int total_timeout /ms */ );
	gboolean crm_remote_recv(crm_remote_t remote, int total_timeout /ms */,
	int *disconnected);
	xmlNode crm_remote_parse_buffer(crm_remote_t remote);
	int crm_remote_tcp_connect(const char *host, int port);
	int crm_remote_tcp_connect_async(const char *host, int port,
	int timeout /ms /,
	int timer_id, void userdata,
	void (callback) (void userdata, int sock));
	int crm_remote_accept(int ssock);
	void crm_sockaddr2str(void sa, char s);

	# ifdef HAVE_GNUTLS_GNUTLS_H
	# include <gnutls/gnutls.h>

	gnutls_session_t *pcmk__new_tls_session(int csock, unsigned int conn_type,
	gnutls_credentials_type_t cred_type,
	void *credentials);
	int pcmk__init_tls_dh(gnutls_dh_params_t *dh_params);
	+int pcmk__read_handshake_data(crm_client_t *client);

	/*!
	* \internal
	* \brief Initiate the client handshake after establishing the tcp socket
	*
	* \return 0 on success, negative number on failure
	* \note This function will block until the entire handshake is complete or
	* until the timeout period is reached.
	*/
	int crm_initiate_client_tls_handshake(crm_remote_t *remote, int timeout_ms);

	# endif // HAVE_GNUTLS_GNUTLS_H
	#endif // PCMK__REMOTE__H
	diff --git a/lib/common/remote.c b/lib/common/remote.c
	index 88eae34af9..4f06567551 100644
	--- a/lib/common/remote.c
	+++ b/lib/common/remote.c
	@@ -1,1108 +1,1142 @@
	/*
	* Copyright 2008-2018 Andrew Beekhof <andrew@beekhof.net>
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/socket.h>
	#include <arpa/inet.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <netinet/tcp.h>
	#include <netdb.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <inttypes.h> /* X32T ~ PRIx32 */

	#include <glib.h>
	#include <bzlib.h>

	#include <crm/common/ipcs.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/remote_internal.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# undef KEYFILE
	# include <gnutls/gnutls.h>

	const int psk_tls_kx_order[] = {
	GNUTLS_KX_DHE_PSK,
	GNUTLS_KX_PSK,
	};

	const int anon_tls_kx_order[] = {
	GNUTLS_KX_ANON_DH,
	GNUTLS_KX_DHE_RSA,
	GNUTLS_KX_DHE_DSS,
	GNUTLS_KX_RSA,
	0
	};
	#endif

	/* Swab macros from linux/swab.h */
	#ifdef HAVE_LINUX_SWAB_H
	# include <linux/swab.h>
	#else
	/*
	* casts are necessary for constants, because we never know how for sure
	* how U/UL/ULL map to __u16, __u32, __u64. At least not in a portable way.
	*/
	#define __swab16(x) ((uint16_t)( \
	(((uint16_t)(x) & (uint16_t)0x00ffU) << 8) \| \
	(((uint16_t)(x) & (uint16_t)0xff00U) >> 8)))

	#define __swab32(x) ((uint32_t)( \
	(((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) \| \
	(((uint32_t)(x) & (uint32_t)0x0000ff00UL) << 8) \| \
	(((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) \| \
	(((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24)))

	#define __swab64(x) ((uint64_t)( \
	(((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) \| \
	(((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) \| \
	(((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) \| \
	(((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) << 8) \| \
	(((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >> 8) \| \
	(((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) \| \
	(((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) \| \
	(((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56)))
	#endif

	#define REMOTE_MSG_VERSION 1
	#define ENDIAN_LOCAL 0xBADADBBD

	struct crm_remote_header_v0
	{
	uint32_t endian; /* Detect messages from hosts with different endian-ness */
	uint32_t version;
	uint64_t id;
	uint64_t flags;
	uint32_t size_total;
	uint32_t payload_offset;
	uint32_t payload_compressed;
	uint32_t payload_uncompressed;

	/* New fields get added here */

	} __attribute__ ((packed));

	static struct crm_remote_header_v0 *
	crm_remote_header(crm_remote_t * remote)
	{
	struct crm_remote_header_v0 header = (struct crm_remote_header_v0 )remote->buffer;
	if(remote->buffer_offset < sizeof(struct crm_remote_header_v0)) {
	return NULL;

	} else if(header->endian != ENDIAN_LOCAL) {
	uint32_t endian = __swab32(header->endian);

	CRM_LOG_ASSERT(endian == ENDIAN_LOCAL);
	if(endian != ENDIAN_LOCAL) {
	crm_err("Invalid message detected, endian mismatch: %" X32T
	" is neither %" X32T " nor the swab'd %" X32T,
	ENDIAN_LOCAL, header->endian, endian);
	return NULL;
	}

	header->id = __swab64(header->id);
	header->flags = __swab64(header->flags);
	header->endian = __swab32(header->endian);

	header->version = __swab32(header->version);
	header->size_total = __swab32(header->size_total);
	header->payload_offset = __swab32(header->payload_offset);
	header->payload_compressed = __swab32(header->payload_compressed);
	header->payload_uncompressed = __swab32(header->payload_uncompressed);
	}

	return header;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H

	int
	crm_initiate_client_tls_handshake(crm_remote_t * remote, int timeout_ms)
	{
	int rc = 0;
	int pollrc = 0;
	time_t start = time(NULL);

	do {
	rc = gnutls_handshake(*remote->tls_session);
	if (rc == GNUTLS_E_INTERRUPTED \|\| rc == GNUTLS_E_AGAIN) {
	pollrc = crm_remote_ready(remote, 1000);
	if (pollrc < 0) {
	/* poll returned error, there is no hope */
	rc = -1;
	}
	}

	} while (((time(NULL) - start) < (timeout_ms / 1000)) &&
	(rc == GNUTLS_E_INTERRUPTED \|\| rc == GNUTLS_E_AGAIN));

	if (rc < 0) {
	crm_trace("gnutls_handshake() failed with %d", rc);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Initialize a new TLS session
	*
	* \param[in] csock Connected socket for TLS session
	* \param[in] conn_type GNUTLS_SERVER or GNUTLS_CLIENT
	* \param[in] cred_type GNUTLS_CRD_ANON or GNUTLS_CRD_PSK
	* \param[in] credentials TLS session credentials
	*
	* \return Pointer to newly created session object, or NULL on error
	*/
	gnutls_session_t *
	pcmk__new_tls_session(int csock, unsigned int conn_type,
	gnutls_credentials_type_t cred_type, void *credentials)
	{
	int rc = GNUTLS_E_SUCCESS;
	const char *prio = NULL;
	gnutls_session_t *session = NULL;

	if (cred_type == GNUTLS_CRD_ANON) {
	// http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication
	prio = PCMK_GNUTLS_PRIORITIES ":+ANON-DH";
	} else {
	prio = PCMK_GNUTLS_PRIORITIES ":+DHE-PSK:+PSK";
	}

	session = gnutls_malloc(sizeof(gnutls_session_t));
	if (session == NULL) {
	rc = GNUTLS_E_MEMORY_ERROR;
	goto error;
	}

	rc = gnutls_init(session, conn_type);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}

	/* @TODO On the server side, it would be more efficient to cache the
	* priority with gnutls_priority_init2() and set it with
	* gnutls_priority_set() for all sessions.
	*/
	rc = gnutls_priority_set_direct(*session, prio, NULL);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}

	gnutls_transport_set_ptr(*session,
	(gnutls_transport_ptr_t) GINT_TO_POINTER(csock));

	rc = gnutls_credentials_set(*session, cred_type, credentials);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}
	return session;

	error:
	crm_err("Could not initialize %s TLS %s session: %s "
	CRM_XS " rc=%d priority='%s'",
	(cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK",
	(conn_type == GNUTLS_SERVER)? "server" : "client",
	gnutls_strerror(rc), rc, prio);
	if (session != NULL) {
	gnutls_free(session);
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Initialize Diffie-Hellman parameters for a TLS server
	*
	* \param[out] dh_params Parameter object to initialize
	*
	* \return GNUTLS_E_SUCCESS on success, GnuTLS error code on error
	* \todo The current best practice is to allow the client and server to
	* negotiate the Diffie-Hellman parameters via a TLS extension (RFC 7919).
	* However, we have to support both older versions of GnuTLS (<3.6) that
	* don't support the extension on our side, and older Pacemaker versions
	* that don't support the extension on the other side. The next best
	* practice would be to use a known good prime (see RFC 5114 section 2.2),
	* possibly stored in a file distributed with Pacemaker.
	*/
	int
	pcmk__init_tls_dh(gnutls_dh_params_t *dh_params)
	{
	int rc = GNUTLS_E_SUCCESS;
	unsigned int dh_bits = 0;

	rc = gnutls_dh_params_init(dh_params);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}

	#ifdef HAVE_GNUTLS_SEC_PARAM_TO_PK_BITS
	dh_bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
	GNUTLS_SEC_PARAM_NORMAL);
	if (dh_bits == 0) {
	rc = GNUTLS_E_DH_PRIME_UNACCEPTABLE;
	goto error;
	}
	#else
	dh_bits = 1024;
	#endif

	crm_info("Generating Diffie-Hellman parameters with %u-bit prime for TLS",
	dh_bits);
	rc = gnutls_dh_params_generate2(*dh_params, dh_bits);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}

	return rc;

	error:
	crm_err("Could not initialize Diffie-Hellman parameters for TLS: %s "
	CRM_XS " rc=%d", gnutls_strerror(rc), rc);
	CRM_ASSERT(rc == GNUTLS_E_SUCCESS);
	return rc;
	}

	+/*!
	+ * \internal
	+ * \brief Process handshake data from TLS client
	+ *
	+ * Read as much TLS handshake data as is available.
	+ *
	+ * \param[in] client Client connection
	+ *
	+ * \retval GnuTLS error code on error
	+ * \retval 0 if more data is needed
	+ * \retval 1 if handshake is successfully completed
	+ */
	+int
	+pcmk__read_handshake_data(crm_client_t *client)
	+{
	+ int rc = 0;
	+
	+ CRM_ASSERT(client && client->remote && client->remote->tls_session);
	+
	+ do {
	+ rc = gnutls_handshake(*client->remote->tls_session);
	+ } while (rc == GNUTLS_E_INTERRUPTED);
	+
	+ if (rc == GNUTLS_E_AGAIN) {
	+ /* No more data is available at the moment. This function should be
	+ * invoked again once the client sends more.
	+ */
	+ return 0;
	+ } else if (rc != GNUTLS_E_SUCCESS) {
	+ return rc;
	+ }
	+ return 1;
	+}
	+
	static int
	crm_send_tls(gnutls_session_t * session, const char *buf, size_t len)
	{
	const char *unsent = buf;
	int rc = 0;
	int total_send;

	if (buf == NULL) {
	return -EINVAL;
	}

	total_send = len;
	crm_trace("Message size: %llu", (unsigned long long) len);

	while (TRUE) {
	rc = gnutls_record_send(*session, unsent, len);

	if (rc == GNUTLS_E_INTERRUPTED \|\| rc == GNUTLS_E_AGAIN) {
	crm_trace("Retrying to send %llu bytes",
	(unsigned long long) len);

	} else if (rc < 0) {
	// Caller can log as error if necessary
	crm_info("TLS connection terminated: %s " CRM_XS " rc=%d",
	gnutls_strerror(rc), rc);
	rc = -ECONNABORTED;
	break;

	} else if (rc < len) {
	crm_debug("Sent %d of %llu bytes", rc, (unsigned long long) len);
	len -= rc;
	unsent += rc;
	} else {
	crm_trace("Sent all %d bytes", rc);
	break;
	}
	}

	return rc < 0 ? rc : total_send;
	}
	#endif

	static int
	crm_send_plaintext(int sock, const char *buf, size_t len)
	{

	int rc = 0;
	const char *unsent = buf;
	int total_send;

	if (buf == NULL) {
	return -EINVAL;
	}
	total_send = len;

	crm_trace("Message on socket %d: size=%llu",
	sock, (unsigned long long) len);
	retry:
	rc = write(sock, unsent, len);
	if (rc < 0) {
	rc = -errno;
	switch (errno) {
	case EINTR:
	case EAGAIN:
	crm_trace("Retry");
	goto retry;
	default:
	crm_perror(LOG_INFO,
	"Could only write %d of the remaining %llu bytes",
	rc, (unsigned long long) len);
	break;
	}

	} else if (rc < len) {
	crm_trace("Only sent %d of %llu remaining bytes",
	rc, (unsigned long long) len);
	len -= rc;
	unsent += rc;
	goto retry;

	} else {
	crm_trace("Sent %d bytes: %.100s", rc, buf);
	}

	return rc < 0 ? rc : total_send;

	}

	static int
	crm_remote_sendv(crm_remote_t * remote, struct iovec * iov, int iovs)
	{
	int rc = 0;

	for (int lpc = 0; (lpc < iovs) && (rc >= 0); lpc++) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (remote->tls_session) {
	rc = crm_send_tls(remote->tls_session, iov[lpc].iov_base, iov[lpc].iov_len);
	continue;
	}
	#endif
	if (remote->tcp_socket) {
	rc = crm_send_plaintext(remote->tcp_socket, iov[lpc].iov_base, iov[lpc].iov_len);
	} else {
	rc = -ESOCKTNOSUPPORT;
	}
	}
	return rc;
	}

	int
	crm_remote_send(crm_remote_t * remote, xmlNode * msg)
	{
	int rc = pcmk_ok;
	static uint64_t id = 0;
	char *xml_text = dump_xml_unformatted(msg);

	struct iovec iov[2];
	struct crm_remote_header_v0 *header;

	if (xml_text == NULL) {
	crm_err("Could not send remote message: no message provided");
	return -EINVAL;
	}

	header = calloc(1, sizeof(struct crm_remote_header_v0));
	iov[0].iov_base = header;
	iov[0].iov_len = sizeof(struct crm_remote_header_v0);

	iov[1].iov_base = xml_text;
	iov[1].iov_len = 1 + strlen(xml_text);

	id++;
	header->id = id;
	header->endian = ENDIAN_LOCAL;
	header->version = REMOTE_MSG_VERSION;
	header->payload_offset = iov[0].iov_len;
	header->payload_uncompressed = iov[1].iov_len;
	header->size_total = iov[0].iov_len + iov[1].iov_len;

	crm_trace("Sending len[0]=%d, start=%x",
	(int)iov[0].iov_len, (int)(void*)xml_text);
	rc = crm_remote_sendv(remote, iov, 2);
	if (rc < 0) {
	crm_err("Could not send remote message: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}

	free(iov[0].iov_base);
	free(iov[1].iov_base);
	return rc;
	}


	/*!
	* \internal
	* \brief handles the recv buffer and parsing out msgs.
	* \note new_data is owned by this function once it is passed in.
	*/
	xmlNode *
	crm_remote_parse_buffer(crm_remote_t * remote)
	{
	xmlNode *xml = NULL;
	struct crm_remote_header_v0 *header = crm_remote_header(remote);

	if (remote->buffer == NULL \|\| header == NULL) {
	return NULL;
	}

	/* Support compression on the receiving end now, in case we ever want to add it later */
	if (header->payload_compressed) {
	int rc = 0;
	unsigned int size_u = 1 + header->payload_uncompressed;
	char *uncompressed = calloc(1, header->payload_offset + size_u);

	crm_trace("Decompressing message data %d bytes into %d bytes",
	header->payload_compressed, size_u);

	rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset, &size_u,
	remote->buffer + header->payload_offset,
	header->payload_compressed, 1, 0);

	if (rc != BZ_OK && header->version > REMOTE_MSG_VERSION) {
	crm_warn("Couldn't decompress v%d message, we only understand v%d",
	header->version, REMOTE_MSG_VERSION);
	free(uncompressed);
	return NULL;

	} else if (rc != BZ_OK) {
	crm_err("Decompression failed: %s " CRM_XS " bzerror=%d",
	bz2_strerror(rc), rc);
	free(uncompressed);
	return NULL;
	}

	CRM_ASSERT(size_u == header->payload_uncompressed);

	memcpy(uncompressed, remote->buffer, header->payload_offset); /* Preserve the header */
	remote->buffer_size = header->payload_offset + size_u;

	free(remote->buffer);
	remote->buffer = uncompressed;
	header = crm_remote_header(remote);
	}

	/* take ownership of the buffer */
	remote->buffer_offset = 0;

	CRM_LOG_ASSERT(remote->buffer[sizeof(struct crm_remote_header_v0) + header->payload_uncompressed - 1] == 0);

	xml = string2xml(remote->buffer + header->payload_offset);
	if (xml == NULL && header->version > REMOTE_MSG_VERSION) {
	crm_warn("Couldn't parse v%d message, we only understand v%d",
	header->version, REMOTE_MSG_VERSION);

	} else if (xml == NULL) {
	crm_err("Couldn't parse: '%.120s'", remote->buffer + header->payload_offset);
	}

	return xml;
	}

	/*!
	* \internal
	* \brief Wait for a remote session to have data to read
	*
	* \param[in] remote Connection to check
	* \param[in] total_timeout Maximum time (in ms) to wait
	*
	* \return Positive value if ready to be read, 0 on timeout, -errno on error
	*/
	int
	crm_remote_ready(crm_remote_t *remote, int total_timeout)
	{
	struct pollfd fds = { 0, };
	int sock = 0;
	int rc = 0;
	time_t start;
	int timeout = total_timeout;

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (remote->tls_session) {
	void sock_ptr = gnutls_transport_get_ptr(remote->tls_session);

	sock = GPOINTER_TO_INT(sock_ptr);
	} else if (remote->tcp_socket) {
	#else
	if (remote->tcp_socket) {
	#endif
	sock = remote->tcp_socket;
	} else {
	crm_err("Unsupported connection type");
	}

	if (sock <= 0) {
	crm_trace("No longer connected");
	return -ENOTCONN;
	}

	start = time(NULL);
	errno = 0;
	do {
	fds.fd = sock;
	fds.events = POLLIN;

	/* If we got an EINTR while polling, and we have a
	* specific timeout we are trying to honor, attempt
	* to adjust the timeout to the closest second. */
	if (errno == EINTR && (timeout > 0)) {
	timeout = total_timeout - ((time(NULL) - start) * 1000);
	if (timeout < 1000) {
	timeout = 1000;
	}
	}

	rc = poll(&fds, 1, timeout);
	} while (rc < 0 && errno == EINTR);

	return (rc < 0)? -errno : rc;
	}


	/*!
	* \internal
	* \brief Read bytes off non blocking remote connection.
	*
	* \note only use with NON-Blocking sockets. Should only be used after polling socket.
	* This function will return once max_size is met, the socket read buffer
	* is empty, or an error is encountered.
	*
	* \retval number of bytes received
	*/
	static size_t
	crm_remote_recv_once(crm_remote_t * remote)
	{
	int rc = 0;
	size_t read_len = sizeof(struct crm_remote_header_v0);
	struct crm_remote_header_v0 *header = crm_remote_header(remote);

	if(header) {
	/* Stop at the end of the current message */
	read_len = header->size_total;
	}

	/* automatically grow the buffer when needed */
	if(remote->buffer_size < read_len) {
	remote->buffer_size = 2 * read_len;
	crm_trace("Expanding buffer to %llu bytes",
	(unsigned long long) remote->buffer_size);

	remote->buffer = realloc_safe(remote->buffer, remote->buffer_size + 1);
	CRM_ASSERT(remote->buffer != NULL);
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (remote->tls_session) {
	rc = gnutls_record_recv(*(remote->tls_session),
	remote->buffer + remote->buffer_offset,
	remote->buffer_size - remote->buffer_offset);
	if (rc == GNUTLS_E_INTERRUPTED) {
	rc = -EINTR;
	} else if (rc == GNUTLS_E_AGAIN) {
	rc = -EAGAIN;
	} else if (rc < 0) {
	crm_debug("TLS receive failed: %s (%d)", gnutls_strerror(rc), rc);
	rc = -pcmk_err_generic;
	}
	} else if (remote->tcp_socket) {
	#else
	if (remote->tcp_socket) {
	#endif
	errno = 0;
	rc = read(remote->tcp_socket,
	remote->buffer + remote->buffer_offset,
	remote->buffer_size - remote->buffer_offset);
	if(rc < 0) {
	rc = -errno;
	}

	} else {
	crm_err("Unsupported connection type");
	return -ESOCKTNOSUPPORT;
	}

	/* process any errors. */
	if (rc > 0) {
	remote->buffer_offset += rc;
	/* always null terminate buffer, the +1 to alloc always allows for this. */
	remote->buffer[remote->buffer_offset] = '\0';
	crm_trace("Received %u more bytes, %llu total",
	rc, (unsigned long long) remote->buffer_offset);

	} else if (rc == -EINTR \|\| rc == -EAGAIN) {
	crm_trace("non-blocking, exiting read: %s (%d)", pcmk_strerror(rc), rc);

	} else if (rc == 0) {
	crm_debug("EOF encoutered after %llu bytes",
	(unsigned long long) remote->buffer_offset);
	return -ENOTCONN;

	} else {
	crm_debug("Error receiving message after %llu bytes: %s (%d)",
	(unsigned long long) remote->buffer_offset,
	pcmk_strerror(rc), rc);
	return -ENOTCONN;
	}

	header = crm_remote_header(remote);
	if(header) {
	if(remote->buffer_offset < header->size_total) {
	crm_trace("Read less than the advertised length: %llu < %u bytes",
	(unsigned long long) remote->buffer_offset,
	header->size_total);
	} else {
	crm_trace("Read full message of %llu bytes",
	(unsigned long long) remote->buffer_offset);
	return remote->buffer_offset;
	}
	}

	return -EAGAIN;
	}

	/*!
	* \internal
	* \brief Read message(s) from a remote connection
	*
	* \param[in] remote Remote connection to read
	* \param[in] total_timeout Fail if message not read in this time (ms)
	* \param[out] disconnected Will be set to 1 if disconnect detected
	*
	* \return TRUE if at least one full message read, FALSE otherwise
	*/
	gboolean
	crm_remote_recv(crm_remote_t remote, int total_timeout, int disconnected)
	{
	int rc;
	time_t start = time(NULL);
	int remaining_timeout = 0;

	if (total_timeout == 0) {
	total_timeout = 10000;
	} else if (total_timeout < 0) {
	total_timeout = 60000;
	}
	*disconnected = 0;

	remaining_timeout = total_timeout;
	while ((remaining_timeout > 0) && !(*disconnected)) {

	crm_trace("Waiting for remote data (%d of %d ms timeout remaining)",
	remaining_timeout, total_timeout);
	rc = crm_remote_ready(remote, remaining_timeout);

	if (rc == 0) {
	crm_err("Timed out (%d ms) while waiting for remote data",
	remaining_timeout);
	return FALSE;

	} else if (rc < 0) {
	crm_debug("Wait for remote data aborted, will try again: %s "
	CRM_XS " rc=%d", pcmk_strerror(rc), rc);

	} else {
	rc = crm_remote_recv_once(remote);
	if (rc > 0) {
	return TRUE;
	} else if (rc == -EAGAIN) {
	crm_trace("Still waiting for remote data");
	} else if (rc < 0) {
	crm_debug("Could not receive remote data: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}
	}

	if (rc == -ENOTCONN) {
	*disconnected = 1;
	return FALSE;
	}

	remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
	}

	return FALSE;
	}

	struct tcp_async_cb_data {
	gboolean success;
	int sock;
	void *userdata;
	void (callback) (void userdata, int sock);
	int timeout; /ms /
	time_t start;
	};

	static gboolean
	check_connect_finished(gpointer userdata)
	{
	struct tcp_async_cb_data *cb_data = userdata;
	int cb_arg = 0; // socket fd on success, -errno on error
	int sock = cb_data->sock;
	int error = 0;

	fd_set rset, wset;
	socklen_t len = sizeof(error);
	struct timeval ts = { 0, };

	if (cb_data->success == TRUE) {
	goto dispatch_done;
	}

	FD_ZERO(&rset);
	FD_SET(sock, &rset);
	wset = rset;

	crm_trace("fd %d: checking to see if connect finished", sock);
	cb_arg = select(sock + 1, &rset, &wset, NULL, &ts);

	if (cb_arg < 0) {
	cb_arg = -errno;
	if ((errno == EINPROGRESS) \|\| (errno == EAGAIN)) {
	/* reschedule if there is still time left */
	if ((time(NULL) - cb_data->start) < (cb_data->timeout / 1000)) {
	goto reschedule;
	} else {
	cb_arg = -ETIMEDOUT;
	}
	}
	crm_trace("fd %d: select failed %d connect dispatch ", sock, cb_arg);
	goto dispatch_done;
	} else if (cb_arg == 0) {
	if ((time(NULL) - cb_data->start) < (cb_data->timeout / 1000)) {
	goto reschedule;
	}
	crm_debug("fd %d: timeout during select", sock);
	cb_arg = -ETIMEDOUT;
	goto dispatch_done;
	} else {
	crm_trace("fd %d: select returned success", sock);
	cb_arg = 0;
	}

	/* can we read or write to the socket now? */
	if (FD_ISSET(sock, &rset) \|\| FD_ISSET(sock, &wset)) {
	if (getsockopt(sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) {
	cb_arg = -errno;
	crm_trace("fd %d: call to getsockopt failed", sock);
	goto dispatch_done;
	}
	if (error) {
	crm_trace("fd %d: error returned from getsockopt: %d", sock, error);
	cb_arg = -error;
	goto dispatch_done;
	}
	} else {
	crm_trace("neither read nor write set after select");
	cb_arg = -EAGAIN;
	goto dispatch_done;
	}

	dispatch_done:
	if (!cb_arg) {
	crm_trace("fd %d: connected", sock);
	/* Success, set the return code to the sock to report to the callback */
	cb_arg = cb_data->sock;
	cb_data->sock = 0;
	} else {
	close(sock);
	}

	if (cb_data->callback) {
	cb_data->callback(cb_data->userdata, cb_arg);
	}
	free(cb_data);
	return FALSE;

	reschedule:

	/* will check again next interval */
	return TRUE;
	}

	static int
	internal_tcp_connect_async(int sock,
	const struct sockaddr addr, socklen_t addrlen, int timeout / ms */ ,
	int timer_id, void userdata, void (callback) (void userdata, int sock))
	{
	int rc = 0;
	int interval = 500;
	int timer;
	struct tcp_async_cb_data *cb_data = NULL;

	rc = crm_set_nonblocking(sock);
	if (rc < 0) {
	crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	close(sock);
	return -1;
	}

	rc = connect(sock, addr, addrlen);
	if (rc < 0 && (errno != EINPROGRESS) && (errno != EAGAIN)) {
	crm_perror(LOG_WARNING, "connect");
	return -1;
	}

	cb_data = calloc(1, sizeof(struct tcp_async_cb_data));
	cb_data->userdata = userdata;
	cb_data->callback = callback;
	cb_data->sock = sock;
	cb_data->timeout = timeout;
	cb_data->start = time(NULL);

	if (rc == 0) {
	/* The connect was successful immediately, we still return to mainloop
	* and let this callback get called later. This avoids the user of this api
	* to have to account for the fact the callback could be invoked within this
	* function before returning. */
	cb_data->success = TRUE;
	interval = 1;
	}

	/* Check connect finished is mostly doing a non-block poll on the socket
	* to see if we can read/write to it. Once we can, the connect has completed.
	* This method allows us to connect to the server without blocking mainloop.
	*
	* This is a poor man's way of polling to see when the connection finished.
	* At some point we should figure out a way to use a mainloop fd callback for this.
	* Something about the way mainloop is currently polling prevents this from working at the
	* moment though. */
	crm_trace("Scheduling check in %dms for whether connect to fd %d finished",
	interval, sock);
	timer = g_timeout_add(interval, check_connect_finished, cb_data);
	if (timer_id) {
	*timer_id = timer;
	}

	return 0;
	}

	static int
	internal_tcp_connect(int sock, const struct sockaddr *addr, socklen_t addrlen)
	{
	int rc = connect(sock, addr, addrlen);

	if (rc < 0) {
	rc = -errno;
	crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	return rc;
	}

	rc = crm_set_nonblocking(sock);
	if (rc < 0) {
	crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	return rc;
	}

	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Connect to server at specified TCP port
	*
	* \param[in] host Name of server to connect to
	* \param[in] port Server port to connect to
	* \param[in] timeout Report error if not connected in this many milliseconds
	* \param[out] timer_id If non-NULL, will be set to timer ID, if asynchronous
	* \param[in] userdata Data to pass to callback, if asynchronous
	* \param[in] callback If non-NULL, connect asynchronously then call this
	*
	* \return File descriptor of connected socket on success, -ENOTCONN otherwise
	*/
	int
	crm_remote_tcp_connect_async(const char *host, int port, int timeout,
	int timer_id, void userdata,
	void (callback) (void userdata, int sock))
	{
	char buffer[INET6_ADDRSTRLEN];
	struct addrinfo *res = NULL;
	struct addrinfo *rp = NULL;
	struct addrinfo hints;
	const char *server = host;
	int ret_ga;
	int sock = -ENOTCONN;

	// Get host's IP address(es)
	memset(&hints, 0, sizeof(struct addrinfo));
	hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_flags = AI_CANONNAME;
	ret_ga = getaddrinfo(server, NULL, &hints, &res);
	if (ret_ga) {
	crm_err("Unable to get IP address info for %s: %s",
	server, gai_strerror(ret_ga));
	goto async_cleanup;
	}
	if (!res \|\| !res->ai_addr) {
	crm_err("Unable to get IP address info for %s: no result", server);
	goto async_cleanup;
	}

	// getaddrinfo() returns a list of host's addresses, try them in order
	for (rp = res; rp != NULL; rp = rp->ai_next) {
	struct sockaddr *addr = rp->ai_addr;

	if (!addr) {
	continue;
	}

	if (rp->ai_canonname) {
	server = res->ai_canonname;
	}
	crm_debug("Got canonical name %s for %s", server, host);

	sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
	if (sock == -1) {
	crm_perror(LOG_WARNING, "creating socket for connection to %s",
	server);
	sock = -ENOTCONN;
	continue;
	}

	/* Set port appropriately for address family */
	/* (void) casts avoid false-positive compiler alignment warnings /
	if (addr->sa_family == AF_INET6) {
	((struct sockaddr_in6 )(void)addr)->sin6_port = htons(port);
	} else {
	((struct sockaddr_in )(void)addr)->sin_port = htons(port);
	}

	memset(buffer, 0, DIMOF(buffer));
	crm_sockaddr2str(addr, buffer);
	crm_info("Attempting TCP connection to %s:%d", buffer, port);

	if (callback) {
	if (internal_tcp_connect_async
	(sock, rp->ai_addr, rp->ai_addrlen, timeout, timer_id, userdata, callback) == 0) {
	goto async_cleanup; /* Success for now, we'll hear back later in the callback */
	}

	} else if (internal_tcp_connect(sock, rp->ai_addr, rp->ai_addrlen) == 0) {
	break; /* Success */
	}

	close(sock);
	sock = -ENOTCONN;
	}

	async_cleanup:

	if (res) {
	freeaddrinfo(res);
	}
	return sock;
	}

	int
	crm_remote_tcp_connect(const char *host, int port)
	{
	return crm_remote_tcp_connect_async(host, port, -1, NULL, NULL, NULL);
	}

	/*!
	* \brief Convert an IP address (IPv4 or IPv6) to a string for logging
	*
	* \param[in] sa Socket address for IP
	* \param[out] s Storage for at least INET6_ADDRSTRLEN bytes
	*
	* \note sa The socket address can be a pointer to struct sockaddr_in (IPv4),
	* struct sockaddr_in6 (IPv6) or struct sockaddr_storage (either),
	* as long as its sa_family member is set correctly.
	*/
	void
	crm_sockaddr2str(void sa, char s)
	{
	switch (((struct sockaddr*)sa)->sa_family) {
	case AF_INET:
	inet_ntop(AF_INET, &(((struct sockaddr_in *)sa)->sin_addr),
	s, INET6_ADDRSTRLEN);
	break;

	case AF_INET6:
	inet_ntop(AF_INET6, &(((struct sockaddr_in6 *)sa)->sin6_addr),
	s, INET6_ADDRSTRLEN);
	break;

	default:
	strcpy(s, "<invalid>");
	}
	}

	int
	crm_remote_accept(int ssock)
	{
	int csock = 0;
	int rc = 0;
	unsigned laddr = 0;
	struct sockaddr_storage addr;
	char addr_str[INET6_ADDRSTRLEN];
	#ifdef TCP_USER_TIMEOUT
	int optval;
	long sbd_timeout = crm_get_sbd_timeout();
	#endif

	/* accept the connection */
	laddr = sizeof(addr);
	memset(&addr, 0, sizeof(addr));
	csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
	crm_sockaddr2str(&addr, addr_str);
	crm_info("New remote connection from %s", addr_str);

	if (csock == -1) {
	crm_err("accept socket failed");
	return -1;
	}

	rc = crm_set_nonblocking(csock);
	if (rc < 0) {
	crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	close(csock);
	return rc;
	}

	#ifdef TCP_USER_TIMEOUT
	if (sbd_timeout > 0) {
	optval = sbd_timeout / 2; /* time to fail and retry before watchdog */
	rc = setsockopt(csock, SOL_TCP, TCP_USER_TIMEOUT,
	&optval, sizeof(optval));
	if (rc < 0) {
	crm_err("setting TCP_USER_TIMEOUT (%d) on client socket failed",
	optval);
	close(csock);
	return rc;
	}
	}
	#endif

	return csock;
	}

	/*!
	* \brief Get the default remote connection TCP port on this host
	*
	* \return Remote connection TCP port number
	*/
	int
	crm_default_remote_port()
	{
	static int port = 0;

	if (port == 0) {
	const char *env = getenv("PCMK_remote_port");

	if (env) {
	errno = 0;
	port = strtol(env, NULL, 10);
	if (errno \|\| (port < 1) \|\| (port > 65535)) {
	crm_warn("Environment variable PCMK_remote_port has invalid value '%s', using %d instead",
	env, DEFAULT_REMOTE_PORT);
	port = DEFAULT_REMOTE_PORT;
	}
	} else {
	port = DEFAULT_REMOTE_PORT;
	}
	}
	return port;
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jun 26, 5:41 PM (17 h, 11 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1959264
Default Alt Text: (67 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions