diff --git a/cib/remote.c b/cib/remote.c
index 901155224c..0160c7e14d 100644
--- a/cib/remote.c
+++ b/cib/remote.c
@@ -1,703 +1,699 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netinet/ip.h>
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>
#include <glib.h>
#include <crm/msg_xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipcs.h>
#include <crm/common/xml.h>
#include <crm/cib/internal.h>
#include "callbacks.h"
/* #undef HAVE_PAM_PAM_APPL_H */
/* #undef HAVE_GNUTLS_GNUTLS_H */
#ifdef HAVE_GNUTLS_GNUTLS_H
# undef KEYFILE
# include <gnutls/gnutls.h>
#endif
#include <pwd.h>
#include <grp.h>
#if HAVE_SECURITY_PAM_APPL_H
# include <security/pam_appl.h>
# define HAVE_PAM 1
#else
# if HAVE_PAM_PAM_APPL_H
# include <pam/pam_appl.h>
# define HAVE_PAM 1
# endif
#endif
extern int remote_tls_fd;
extern gboolean cib_shutdown_flag;
int init_remote_listener(int port, gboolean encrypted);
void cib_remote_connection_destroy(gpointer user_data);
#ifdef HAVE_GNUTLS_GNUTLS_H
# define DH_BITS 1024
gnutls_dh_params_t dh_params;
gnutls_anon_server_credentials_t anon_cred_s;
static void
debug_log(int level, const char *str)
{
fputs(str, stderr);
}
#endif
#define REMOTE_AUTH_TIMEOUT 10000
int num_clients;
int authenticate_user(const char *user, const char *passwd);
int cib_remote_listen(gpointer data);
int cib_remote_msg(gpointer data);
static void
remote_connection_destroy(gpointer user_data)
{
return;
}
#define ERROR_SUFFIX " Shutting down remote listener"
int
init_remote_listener(int port, gboolean encrypted)
{
int rc;
int *ssock = NULL;
struct sockaddr_in saddr;
int optval;
static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
.dispatch = cib_remote_listen,
.destroy = remote_connection_destroy,
};
if (port <= 0) {
/* don't start it */
return 0;
}
if (encrypted) {
#ifndef HAVE_GNUTLS_GNUTLS_H
crm_warn("TLS support is not available");
return 0;
#else
crm_notice("Starting a tls listener on port %d.", port);
crm_gnutls_global_init();
/* gnutls_global_set_log_level (10); */
gnutls_global_set_log_function(debug_log);
gnutls_dh_params_init(&dh_params);
gnutls_dh_params_generate2(dh_params, DH_BITS);
gnutls_anon_allocate_server_credentials(&anon_cred_s);
gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
#endif
} else {
crm_warn("Starting a plain_text listener on port %d.", port);
}
#ifndef HAVE_PAM
crm_warn("PAM is _not_ enabled!");
#endif
/* create server socket */
ssock = malloc(sizeof(int));
if(ssock == NULL) {
crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
return -1;
}
*ssock = socket(AF_INET, SOCK_STREAM, 0);
if (*ssock == -1) {
crm_perror(LOG_ERR, "Can not create server socket." ERROR_SUFFIX);
free(ssock);
return -1;
}
/* reuse address */
optval = 1;
rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
if (rc < 0) {
crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener");
}
/* bind server socket */
memset(&saddr, '\0', sizeof(saddr));
saddr.sin_family = AF_INET;
saddr.sin_addr.s_addr = INADDR_ANY;
saddr.sin_port = htons(port);
if (bind(*ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) {
crm_perror(LOG_ERR, "Can not bind server socket." ERROR_SUFFIX);
close(*ssock);
free(ssock);
return -2;
}
if (listen(*ssock, 10) == -1) {
crm_perror(LOG_ERR, "Can not start listen." ERROR_SUFFIX);
close(*ssock);
free(ssock);
return -3;
}
mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks);
return *ssock;
}
static int
check_group_membership(const char *usr, const char *grp)
{
int index = 0;
struct passwd *pwd = NULL;
struct group *group = NULL;
CRM_CHECK(usr != NULL, return FALSE);
CRM_CHECK(grp != NULL, return FALSE);
pwd = getpwnam(usr);
if (pwd == NULL) {
crm_err("No user named '%s' exists!", usr);
return FALSE;
}
group = getgrgid(pwd->pw_gid);
if (group != NULL && crm_str_eq(grp, group->gr_name, TRUE)) {
return TRUE;
}
group = getgrnam(grp);
if (group == NULL) {
crm_err("No group named '%s' exists!", grp);
return FALSE;
}
while (TRUE) {
char *member = group->gr_mem[index++];
if (member == NULL) {
break;
} else if (crm_str_eq(usr, member, TRUE)) {
return TRUE;
}
};
return FALSE;
}
static gboolean
cib_remote_auth(xmlNode * login)
{
const char *user = NULL;
const char *pass = NULL;
const char *tmp = NULL;
crm_log_xml_info(login, "Login: ");
if (login == NULL) {
return FALSE;
}
tmp = crm_element_name(login);
if (safe_str_neq(tmp, "cib_command")) {
crm_err("Wrong tag: %s", tmp);
return FALSE;
}
tmp = crm_element_value(login, "op");
if (safe_str_neq(tmp, "authenticate")) {
crm_err("Wrong operation: %s", tmp);
return FALSE;
}
user = crm_element_value(login, "user");
pass = crm_element_value(login, "password");
if (!user || !pass) {
crm_err("missing auth credentials");
return FALSE;
}
/* Non-root daemons can only validate the password of the
* user they're running as
*/
if (check_group_membership(user, CRM_DAEMON_GROUP) == FALSE) {
crm_err("User is not a member of the required group");
return FALSE;
} else if (authenticate_user(user, pass) == FALSE) {
crm_err("PAM auth failed");
return FALSE;
}
return TRUE;
}
static gboolean
remote_auth_timeout_cb(gpointer data)
{
crm_client_t *client = data;
client->remote->auth_timeout = 0;
if (client->remote->authenticated == TRUE) {
return FALSE;
}
mainloop_del_fd(client->remote->source);
crm_err("Remote client authentication timed out");
return FALSE;
}
int
cib_remote_listen(gpointer data)
{
int csock = 0;
unsigned laddr;
struct sockaddr_storage addr;
char ipstr[INET6_ADDRSTRLEN];
int ssock = *(int *)data;
int flag;
crm_client_t *new_client = NULL;
static struct mainloop_fd_callbacks remote_client_fd_callbacks = {
.dispatch = cib_remote_msg,
.destroy = cib_remote_connection_destroy,
};
/* accept the connection */
laddr = sizeof(addr);
memset(&addr, 0, sizeof(addr));
csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
if (csock == -1) {
crm_perror(LOG_ERR, "Could not accept socket connection");
return TRUE;
}
crm_sockaddr2str(&addr, ipstr);
crm_debug("New %s connection from %s",
((ssock == remote_tls_fd)? "secure" : "clear-text"), ipstr);
if ((flag = fcntl(csock, F_GETFL)) >= 0) {
if (fcntl(csock, F_SETFL, flag | O_NONBLOCK) < 0) {
crm_err("fcntl() write failed");
close(csock);
return TRUE;
}
} else {
crm_err("fcntl() read failed");
close(csock);
return TRUE;
}
num_clients++;
crm_client_init();
- new_client = calloc(1, sizeof(crm_client_t));
+ new_client = crm_client_alloc(NULL);
new_client->remote = calloc(1, sizeof(crm_remote_t));
- new_client->id = crm_generate_uuid();
-
- g_hash_table_insert(client_connections, new_client->id /* Should work */ , new_client);
-
if (ssock == remote_tls_fd) {
#ifdef HAVE_GNUTLS_GNUTLS_H
new_client->kind = CRM_CLIENT_TLS;
/* create gnutls session for the server socket */
new_client->remote->tls_session =
crm_create_anon_tls_session(csock, GNUTLS_SERVER, anon_cred_s);
if (new_client->remote->tls_session == NULL) {
crm_err("TLS session creation failed");
close(csock);
return TRUE;
}
#endif
} else {
new_client->kind = CRM_CLIENT_TCP;
new_client->remote->tcp_socket = csock;
}
/* clients have a few seconds to perform handshake. */
new_client->remote->auth_timeout =
g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, new_client);
new_client->remote->source =
mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
&remote_client_fd_callbacks);
return TRUE;
}
void
cib_remote_connection_destroy(gpointer user_data)
{
crm_client_t *client = user_data;
int csock = 0;
if (client == NULL) {
return;
}
crm_trace("Cleaning up after client disconnect: %s/%s", crm_str(client->name), client->id);
num_clients--;
crm_trace("Num unfree'd clients: %d", num_clients);
switch (client->kind) {
case CRM_CLIENT_TCP:
csock = client->remote->tcp_socket;
break;
#ifdef HAVE_GNUTLS_GNUTLS_H
case CRM_CLIENT_TLS:
if (client->remote->tls_session) {
void *sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
csock = GPOINTER_TO_INT(sock_ptr);
if (client->remote->tls_handshake_complete) {
gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_WR);
}
gnutls_deinit(*client->remote->tls_session);
gnutls_free(client->remote->tls_session);
client->remote->tls_session = NULL;
}
break;
#endif
default:
crm_warn("Unexpected client type %d", client->kind);
}
if (csock > 0) {
close(csock);
}
crm_client_destroy(client);
crm_trace("Freed the cib client");
if (cib_shutdown_flag) {
cib_shutdown(0);
}
return;
}
static void
cib_handle_remote_msg(crm_client_t * client, xmlNode * command)
{
const char *value = NULL;
value = crm_element_name(command);
if (safe_str_neq(value, "cib_command")) {
crm_log_xml_trace(command, "Bad command: ");
return;
}
if (client->name == NULL) {
value = crm_element_value(command, F_CLIENTNAME);
if (value == NULL) {
client->name = strdup(client->id);
} else {
client->name = strdup(value);
}
}
if (client->userdata == NULL) {
value = crm_element_value(command, F_CIB_CALLBACK_TOKEN);
if (value != NULL) {
client->userdata = strdup(value);
crm_trace("Callback channel for %s is %s", client->id, (char*)client->userdata);
} else {
client->userdata = strdup(client->id);
}
}
/* unset dangerous options */
xml_remove_prop(command, F_ORIG);
xml_remove_prop(command, F_CIB_HOST);
xml_remove_prop(command, F_CIB_GLOBAL_UPDATE);
crm_xml_add(command, F_TYPE, T_CIB);
crm_xml_add(command, F_CIB_CLIENTID, client->id);
crm_xml_add(command, F_CIB_CLIENTNAME, client->name);
#if ENABLE_ACL
crm_xml_add(command, F_CIB_USER, client->user);
#endif
if (crm_element_value(command, F_CIB_CALLID) == NULL) {
char *call_uuid = crm_generate_uuid();
/* fix the command */
crm_xml_add(command, F_CIB_CALLID, call_uuid);
free(call_uuid);
}
if (crm_element_value(command, F_CIB_CALLOPTS) == NULL) {
crm_xml_add_int(command, F_CIB_CALLOPTS, 0);
}
crm_log_xml_trace(command, "Remote command: ");
cib_common_callback_worker(0, 0, command, client, TRUE);
}
int
cib_remote_msg(gpointer data)
{
xmlNode *command = NULL;
crm_client_t *client = data;
int disconnected = 0;
int timeout = client->remote->authenticated ? -1 : 1000;
crm_trace("%s callback", client->kind != CRM_CLIENT_TCP ? "secure" : "clear-text");
#ifdef HAVE_GNUTLS_GNUTLS_H
if (client->kind == CRM_CLIENT_TLS && (client->remote->tls_handshake_complete == FALSE)) {
int rc = 0;
/* Muliple calls to handshake will be required, this callback
* will be invoked once the client sends more handshake data. */
do {
rc = gnutls_handshake(*client->remote->tls_session);
if (rc < 0 && rc != GNUTLS_E_AGAIN) {
crm_err("Remote cib tls handshake failed");
return -1;
}
} while (rc == GNUTLS_E_INTERRUPTED);
if (rc == 0) {
crm_debug("Remote cib tls handshake completed");
client->remote->tls_handshake_complete = TRUE;
if (client->remote->auth_timeout) {
g_source_remove(client->remote->auth_timeout);
}
/* after handshake, clients must send auth in a few seconds */
client->remote->auth_timeout =
g_timeout_add(REMOTE_AUTH_TIMEOUT, remote_auth_timeout_cb, client);
}
return 0;
}
#endif
crm_remote_recv(client->remote, timeout, &disconnected);
/* must pass auth before we will process anything else */
if (client->remote->authenticated == FALSE) {
xmlNode *reg;
#if ENABLE_ACL
const char *user = NULL;
#endif
command = crm_remote_parse_buffer(client->remote);
if (cib_remote_auth(command) == FALSE) {
free_xml(command);
return -1;
}
crm_debug("remote connection authenticated successfully");
client->remote->authenticated = TRUE;
g_source_remove(client->remote->auth_timeout);
client->remote->auth_timeout = 0;
client->name = crm_element_value_copy(command, "name");
#if ENABLE_ACL
user = crm_element_value(command, "user");
if (user) {
client->user = strdup(user);
}
#endif
/* send ACK */
reg = create_xml_node(NULL, "cib_result");
crm_xml_add(reg, F_CIB_OPERATION, CRM_OP_REGISTER);
crm_xml_add(reg, F_CIB_CLIENTID, client->id);
crm_remote_send(client->remote, reg);
free_xml(reg);
free_xml(command);
}
command = crm_remote_parse_buffer(client->remote);
while (command) {
crm_trace("command received");
cib_handle_remote_msg(client, command);
free_xml(command);
command = crm_remote_parse_buffer(client->remote);
}
if (disconnected) {
crm_trace("disconnected while receiving remote cib msg.");
return -1;
}
return 0;
}
#ifdef HAVE_PAM
/*
* Useful Examples:
* http://www.kernel.org/pub/linux/libs/pam/Linux-PAM-html
* http://developer.apple.com/samplecode/CryptNoMore/index.html
*/
static int
construct_pam_passwd(int num_msg, const struct pam_message **msg,
struct pam_response **response, void *data)
{
int count = 0;
struct pam_response *reply;
char *string = (char *)data;
CRM_CHECK(data, return PAM_CONV_ERR);
CRM_CHECK(num_msg == 1, return PAM_CONV_ERR); /* We only want to handle one message */
reply = calloc(1, sizeof(struct pam_response));
CRM_ASSERT(reply != NULL);
for (count = 0; count < num_msg; ++count) {
switch (msg[count]->msg_style) {
case PAM_TEXT_INFO:
crm_info("PAM: %s", msg[count]->msg);
break;
case PAM_PROMPT_ECHO_OFF:
case PAM_PROMPT_ECHO_ON:
reply[count].resp_retcode = 0;
reply[count].resp = string; /* We already made a copy */
case PAM_ERROR_MSG:
/* In theory we'd want to print this, but then
* we see the password prompt in the logs
*/
/* crm_err("PAM error: %s", msg[count]->msg); */
break;
default:
crm_err("Unhandled conversation type: %d", msg[count]->msg_style);
goto bail;
}
}
*response = reply;
reply = NULL;
return PAM_SUCCESS;
bail:
for (count = 0; count < num_msg; ++count) {
if (reply[count].resp != NULL) {
switch (msg[count]->msg_style) {
case PAM_PROMPT_ECHO_ON:
case PAM_PROMPT_ECHO_OFF:
/* Erase the data - it contained a password */
while (*(reply[count].resp)) {
*(reply[count].resp)++ = '\0';
}
free(reply[count].resp);
break;
}
reply[count].resp = NULL;
}
}
free(reply);
reply = NULL;
return PAM_CONV_ERR;
}
#endif
int
authenticate_user(const char *user, const char *passwd)
{
#ifndef HAVE_PAM
gboolean pass = TRUE;
#else
int rc = 0;
gboolean pass = FALSE;
const void *p_user = NULL;
struct pam_conv p_conv;
struct pam_handle *pam_h = NULL;
static const char *pam_name = NULL;
if (pam_name == NULL) {
pam_name = getenv("CIB_pam_service");
}
if (pam_name == NULL) {
pam_name = "login";
}
p_conv.conv = construct_pam_passwd;
p_conv.appdata_ptr = strdup(passwd);
rc = pam_start(pam_name, user, &p_conv, &pam_h);
if (rc != PAM_SUCCESS) {
crm_err("Could not initialize PAM: %s (%d)", pam_strerror(pam_h, rc), rc);
goto bail;
}
rc = pam_authenticate(pam_h, 0);
if (rc != PAM_SUCCESS) {
crm_err("Authentication failed for %s: %s (%d)", user, pam_strerror(pam_h, rc), rc);
goto bail;
}
/* Make sure we authenticated the user we wanted to authenticate.
* Since we also run as non-root, it might be worth pre-checking
* the user has the same EID as us, since that the only user we
* can authenticate.
*/
rc = pam_get_item(pam_h, PAM_USER, &p_user);
if (rc != PAM_SUCCESS) {
crm_err("Internal PAM error: %s (%d)", pam_strerror(pam_h, rc), rc);
goto bail;
} else if (p_user == NULL) {
crm_err("Unknown user authenticated.");
goto bail;
} else if (safe_str_neq(p_user, user)) {
crm_err("User mismatch: %s vs. %s.", (const char *)p_user, (const char *)user);
goto bail;
}
rc = pam_acct_mgmt(pam_h, 0);
if (rc != PAM_SUCCESS) {
crm_err("Access denied: %s (%d)", pam_strerror(pam_h, rc), rc);
goto bail;
}
pass = TRUE;
bail:
pam_end(pam_h, rc);
#endif
return pass;
}
diff --git a/include/crm/common/ipcs.h b/include/crm/common/ipcs.h
index ba1ccefc7f..06cade9dbf 100644
--- a/include/crm/common/ipcs.h
+++ b/include/crm/common/ipcs.h
@@ -1,123 +1,124 @@
/*
* Copyright (C) 2013 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#ifndef CRM_COMMON_IPCS__H
# define CRM_COMMON_IPCS__H
# include <qb/qbipcs.h>
# ifdef HAVE_GNUTLS_GNUTLS_H
# undef KEYFILE
# include <gnutls/gnutls.h>
# endif
# include <crm/common/ipc.h>
# include <crm/common/mainloop.h>
typedef struct crm_client_s crm_client_t;
enum client_type {
CRM_CLIENT_IPC = 1,
CRM_CLIENT_TCP = 2,
# ifdef HAVE_GNUTLS_GNUTLS_H
CRM_CLIENT_TLS = 3,
# endif
};
struct crm_remote_s {
/* Shared */
char *buffer;
size_t buffer_size;
size_t buffer_offset;
int auth_timeout;
int tcp_socket;
mainloop_io_t *source;
/* CIB-only */
bool authenticated;
char *token;
/* TLS only */
# ifdef HAVE_GNUTLS_GNUTLS_H
gnutls_session_t *tls_session;
bool tls_handshake_complete;
# endif
};
enum crm_client_flags
{
crm_client_flag_ipc_proxied = 0x00001, /* ipc_proxy code only */
};
struct crm_client_s {
uint pid;
uid_t uid;
gid_t gid;
char *id;
char *name;
char *user;
/* Provided for server use (not used by library) */
/* @TODO merge options, flags, and kind (reserving lower bits for server) */
long long options;
int request_id;
uint32_t flags;
void *userdata;
int event_timer;
GList *event_queue; /* @TODO use GQueue instead */
/* Depending on the value of kind, only some of the following
* will be populated/valid
*/
enum client_type kind;
qb_ipcs_connection_t *ipcs; /* IPC */
struct crm_remote_s *remote; /* TCP/TLS */
unsigned int backlog_len; /* IPC queue length after last flush */
};
extern GHashTable *client_connections;
void crm_client_init(void);
void crm_client_cleanup(void);
crm_client_t *crm_client_get(qb_ipcs_connection_t * c);
crm_client_t *crm_client_get_by_id(const char *id);
const char *crm_client_name(crm_client_t * c);
+crm_client_t *crm_client_alloc(void *key);
crm_client_t *crm_client_new(qb_ipcs_connection_t * c, uid_t uid, gid_t gid);
void crm_client_destroy(crm_client_t * c);
void crm_client_disconnect_all(qb_ipcs_service_t *s);
void crm_ipcs_send_ack(crm_client_t * c, uint32_t request, uint32_t flags,
const char *tag, const char *function, int line);
/* when max_send_size is 0, default ipc buffer size is used */
ssize_t crm_ipc_prepare(uint32_t request, xmlNode * message, struct iovec ** result, uint32_t max_send_size);
ssize_t crm_ipcs_send(crm_client_t * c, uint32_t request, xmlNode * message, enum crm_ipc_flags flags);
ssize_t crm_ipcs_sendv(crm_client_t * c, struct iovec *iov, enum crm_ipc_flags flags);
xmlNode *crm_ipcs_recv(crm_client_t * c, void *data, size_t size, uint32_t * id, uint32_t * flags);
int crm_ipcs_client_pid(qb_ipcs_connection_t * c);
#endif
diff --git a/lib/common/ipc.c b/lib/common/ipc.c
index d32e373903..efe64802b3 100644
--- a/lib/common/ipc.c
+++ b/lib/common/ipc.c
@@ -1,1324 +1,1337 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <grp.h>
#include <errno.h>
#include <fcntl.h>
#include <bzlib.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipcs.h>
#define PCMK_IPC_VERSION 1
/* Evict clients whose event queue grows this large */
#define PCMK_IPC_MAX_QUEUE 500
struct crm_ipc_response_header {
struct qb_ipc_response_header qb;
uint32_t size_uncompressed;
uint32_t size_compressed;
uint32_t flags;
uint8_t version; /* Protect against version changes for anyone that might bother to statically link us */
};
static int hdr_offset = 0;
static unsigned int ipc_buffer_max = 0;
static unsigned int pick_ipc_buffer(unsigned int max);
static inline void
crm_ipc_init(void)
{
if (hdr_offset == 0) {
hdr_offset = sizeof(struct crm_ipc_response_header);
}
if (ipc_buffer_max == 0) {
ipc_buffer_max = pick_ipc_buffer(0);
}
}
unsigned int
crm_ipc_default_buffer_size(void)
{
return pick_ipc_buffer(0);
}
static char *
generateReference(const char *custom1, const char *custom2)
{
static uint ref_counter = 0;
const char *local_cust1 = custom1;
const char *local_cust2 = custom2;
int reference_len = 4;
char *since_epoch = NULL;
reference_len += 20; /* too big */
reference_len += 40; /* too big */
if (local_cust1 == NULL) {
local_cust1 = "_empty_";
}
reference_len += strlen(local_cust1);
if (local_cust2 == NULL) {
local_cust2 = "_empty_";
}
reference_len += strlen(local_cust2);
since_epoch = calloc(1, reference_len);
if (since_epoch != NULL) {
sprintf(since_epoch, "%s-%s-%lu-%u",
local_cust1, local_cust2, (unsigned long)time(NULL), ref_counter++);
}
return since_epoch;
}
xmlNode *
create_request_adv(const char *task, xmlNode * msg_data,
const char *host_to, const char *sys_to,
const char *sys_from, const char *uuid_from, const char *origin)
{
char *true_from = NULL;
xmlNode *request = NULL;
char *reference = generateReference(task, sys_from);
if (uuid_from != NULL) {
true_from = generate_hash_key(sys_from, uuid_from);
} else if (sys_from != NULL) {
true_from = strdup(sys_from);
} else {
crm_err("No sys from specified");
}
/* host_from will get set for us if necessary by CRMd when routed */
request = create_xml_node(NULL, __FUNCTION__);
crm_xml_add(request, F_CRM_ORIGIN, origin);
crm_xml_add(request, F_TYPE, T_CRM);
crm_xml_add(request, F_CRM_VERSION, CRM_FEATURE_SET);
crm_xml_add(request, F_CRM_MSG_TYPE, XML_ATTR_REQUEST);
crm_xml_add(request, F_CRM_REFERENCE, reference);
crm_xml_add(request, F_CRM_TASK, task);
crm_xml_add(request, F_CRM_SYS_TO, sys_to);
crm_xml_add(request, F_CRM_SYS_FROM, true_from);
/* HOSTTO will be ignored if it is to the DC anyway. */
if (host_to != NULL && strlen(host_to) > 0) {
crm_xml_add(request, F_CRM_HOST_TO, host_to);
}
if (msg_data != NULL) {
add_message_xml(request, F_CRM_DATA, msg_data);
}
free(reference);
free(true_from);
return request;
}
/*
* This method adds a copy of xml_response_data
*/
xmlNode *
create_reply_adv(xmlNode * original_request, xmlNode * xml_response_data, const char *origin)
{
xmlNode *reply = NULL;
const char *host_from = crm_element_value(original_request, F_CRM_HOST_FROM);
const char *sys_from = crm_element_value(original_request, F_CRM_SYS_FROM);
const char *sys_to = crm_element_value(original_request, F_CRM_SYS_TO);
const char *type = crm_element_value(original_request, F_CRM_MSG_TYPE);
const char *operation = crm_element_value(original_request, F_CRM_TASK);
const char *crm_msg_reference = crm_element_value(original_request, F_CRM_REFERENCE);
if (type == NULL) {
crm_err("Cannot create new_message, no message type in original message");
CRM_ASSERT(type != NULL);
return NULL;
#if 0
} else if (strcasecmp(XML_ATTR_REQUEST, type) != 0) {
crm_err("Cannot create new_message, original message was not a request");
return NULL;
#endif
}
reply = create_xml_node(NULL, __FUNCTION__);
if (reply == NULL) {
crm_err("Cannot create new_message, malloc failed");
return NULL;
}
crm_xml_add(reply, F_CRM_ORIGIN, origin);
crm_xml_add(reply, F_TYPE, T_CRM);
crm_xml_add(reply, F_CRM_VERSION, CRM_FEATURE_SET);
crm_xml_add(reply, F_CRM_MSG_TYPE, XML_ATTR_RESPONSE);
crm_xml_add(reply, F_CRM_REFERENCE, crm_msg_reference);
crm_xml_add(reply, F_CRM_TASK, operation);
/* since this is a reply, we reverse the from and to */
crm_xml_add(reply, F_CRM_SYS_TO, sys_from);
crm_xml_add(reply, F_CRM_SYS_FROM, sys_to);
/* HOSTTO will be ignored if it is to the DC anyway. */
if (host_from != NULL && strlen(host_from) > 0) {
crm_xml_add(reply, F_CRM_HOST_TO, host_from);
}
if (xml_response_data != NULL) {
add_message_xml(reply, F_CRM_DATA, xml_response_data);
}
return reply;
}
/* Libqb based IPC */
/* Server... */
GHashTable *client_connections = NULL;
crm_client_t *
crm_client_get(qb_ipcs_connection_t * c)
{
if (client_connections) {
return g_hash_table_lookup(client_connections, c);
}
crm_trace("No client found for %p", c);
return NULL;
}
crm_client_t *
crm_client_get_by_id(const char *id)
{
gpointer key;
crm_client_t *client;
GHashTableIter iter;
if (client_connections && id) {
g_hash_table_iter_init(&iter, client_connections);
while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) {
if (strcmp(client->id, id) == 0) {
return client;
}
}
}
crm_trace("No client found with id=%s", id);
return NULL;
}
const char *
crm_client_name(crm_client_t * c)
{
if (c == NULL) {
return "null";
} else if (c->name == NULL && c->id == NULL) {
return "unknown";
} else if (c->name == NULL) {
return c->id;
} else {
return c->name;
}
}
void
crm_client_init(void)
{
if (client_connections == NULL) {
crm_trace("Creating client hash table");
client_connections = g_hash_table_new(g_direct_hash, g_direct_equal);
}
}
void
crm_client_cleanup(void)
{
if (client_connections != NULL) {
int active = g_hash_table_size(client_connections);
if (active) {
crm_err("Exiting with %d active connections", active);
}
g_hash_table_destroy(client_connections); client_connections = NULL;
}
}
void
crm_client_disconnect_all(qb_ipcs_service_t *service)
{
qb_ipcs_connection_t *c = NULL;
if (service == NULL) {
return;
}
c = qb_ipcs_connection_first_get(service);
while (c != NULL) {
qb_ipcs_connection_t *last = c;
c = qb_ipcs_connection_next_get(service, last);
/* There really shouldn't be anyone connected at this point */
crm_notice("Disconnecting client %p, pid=%d...", last, crm_ipcs_client_pid(last));
qb_ipcs_disconnect(last);
qb_ipcs_connection_unref(last);
}
}
+/*!
+ * \brief Allocate a new crm_client_t object and generate its ID
+ *
+ * \param[in] key What to use as connections hash table key (NULL to use ID)
+ *
+ * \return Pointer to new crm_client_t (asserts on failure)
+ */
+crm_client_t *
+crm_client_alloc(void *key)
+{
+ crm_client_t *client = calloc(1, sizeof(crm_client_t));
+
+ CRM_ASSERT(client != NULL);
+ client->id = crm_generate_uuid();
+ g_hash_table_insert(client_connections, (key? key : client->id), client);
+ return client;
+}
+
crm_client_t *
crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client)
{
static gid_t gid_cluster = 0;
crm_client_t *client = NULL;
CRM_LOG_ASSERT(c);
if (c == NULL) {
return NULL;
}
if (gid_cluster == 0) {
if(crm_user_lookup(CRM_DAEMON_USER, NULL, &gid_cluster) < 0) {
static bool have_error = FALSE;
if(have_error == FALSE) {
crm_warn("Could not find group for user %s", CRM_DAEMON_USER);
have_error = TRUE;
}
}
}
if (uid_client != 0) {
crm_trace("Giving access to group %u", gid_cluster);
/* Passing -1 to chown(2) means don't change */
qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
}
crm_client_init();
/* TODO: Do our own auth checking, return NULL if unauthorized */
- client = calloc(1, sizeof(crm_client_t));
-
+ client = crm_client_alloc(c);
client->ipcs = c;
client->kind = CRM_CLIENT_IPC;
client->pid = crm_ipcs_client_pid(c);
- client->id = crm_generate_uuid();
-
crm_debug("Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->pid, client->id);
#if ENABLE_ACL
client->user = uid2username(uid_client);
#endif
-
- g_hash_table_insert(client_connections, c, client);
return client;
}
void
crm_client_destroy(crm_client_t * c)
{
if (c == NULL) {
return;
}
if (client_connections) {
if (c->ipcs) {
crm_trace("Destroying %p/%p (%d remaining)",
c, c->ipcs, crm_hash_table_size(client_connections) - 1);
g_hash_table_remove(client_connections, c->ipcs);
} else {
crm_trace("Destroying remote connection %p (%d remaining)",
c, crm_hash_table_size(client_connections) - 1);
g_hash_table_remove(client_connections, c->id);
}
}
if (c->event_timer) {
g_source_remove(c->event_timer);
}
crm_debug("Destroying %d events", g_list_length(c->event_queue));
while (c->event_queue) {
struct iovec *event = c->event_queue->data;
c->event_queue = g_list_remove(c->event_queue, event);
free(event[0].iov_base);
free(event[1].iov_base);
free(event);
}
free(c->id);
free(c->name);
free(c->user);
if (c->remote) {
if (c->remote->auth_timeout) {
g_source_remove(c->remote->auth_timeout);
}
free(c->remote->buffer);
free(c->remote);
}
free(c);
}
int
crm_ipcs_client_pid(qb_ipcs_connection_t * c)
{
struct qb_ipcs_connection_stats stats;
stats.client_pid = 0;
qb_ipcs_connection_stats_get(c, &stats, 0);
return stats.client_pid;
}
xmlNode *
crm_ipcs_recv(crm_client_t * c, void *data, size_t size, uint32_t * id, uint32_t * flags)
{
xmlNode *xml = NULL;
char *uncompressed = NULL;
char *text = ((char *)data) + sizeof(struct crm_ipc_response_header);
struct crm_ipc_response_header *header = data;
if (id) {
*id = ((struct qb_ipc_response_header *)data)->id;
}
if (flags) {
*flags = header->flags;
}
if (is_set(header->flags, crm_ipc_proxied)) {
/* mark this client as being the endpoint of a proxy connection.
* Proxy connections responses are sent on the event channel to avoid
* blocking the proxy daemon (crmd) */
c->flags |= crm_client_flag_ipc_proxied;
}
if(header->version > PCMK_IPC_VERSION) {
crm_err("Filtering incompatible v%d IPC message, we only support versions <= %d",
header->version, PCMK_IPC_VERSION);
return NULL;
}
if (header->size_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->size_uncompressed;
uncompressed = calloc(1, size_u);
crm_trace("Decompressing message data %u bytes into %u bytes",
header->size_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0);
text = uncompressed;
if (rc != BZ_OK) {
crm_err("Decompression failed: %s (%d)", bz2_strerror(rc), rc);
free(uncompressed);
return NULL;
}
}
CRM_ASSERT(text[header->size_uncompressed - 1] == 0);
crm_trace("Received %.200s", text);
xml = string2xml(text);
free(uncompressed);
return xml;
}
ssize_t crm_ipcs_flush_events(crm_client_t * c);
static gboolean
crm_ipcs_flush_events_cb(gpointer data)
{
crm_client_t *c = data;
c->event_timer = 0;
crm_ipcs_flush_events(c);
return FALSE;
}
/*!
* \internal
* \brief Add progressive delay before next event queue flush
*
* \param[in,out] c Client connection to add delay to
* \param[in] queue_len Current event queue length
*/
static inline void
delay_next_flush(crm_client_t *c, unsigned int queue_len)
{
/* Delay a maximum of 5 seconds */
guint delay = (queue_len < 40)? (1000 + 100 * queue_len) : 5000;
c->event_timer = g_timeout_add(delay, crm_ipcs_flush_events_cb, c);
}
ssize_t
crm_ipcs_flush_events(crm_client_t * c)
{
ssize_t rc = 0;
unsigned int sent = 0;
unsigned int queue_len = 0;
if (c == NULL) {
return pcmk_ok;
} else if (c->event_timer) {
/* There is already a timer, wait until it goes off */
crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer);
return pcmk_ok;
}
queue_len = g_list_length(c->event_queue);
while (c->event_queue && sent < 100) {
struct crm_ipc_response_header *header = NULL;
struct iovec *event = c->event_queue->data;
rc = qb_ipcs_event_sendv(c->ipcs, event, 2);
if (rc < 0) {
break;
}
sent++;
header = event[0].iov_base;
if (header->size_compressed) {
crm_trace("Event %d to %p[%d] (%lld compressed bytes) sent",
header->qb.id, c->ipcs, c->pid, (long long) rc);
} else {
crm_trace("Event %d to %p[%d] (%lld bytes) sent: %.120s",
header->qb.id, c->ipcs, c->pid, (long long) rc,
(char *) (event[1].iov_base));
}
c->event_queue = g_list_remove(c->event_queue, event);
free(event[0].iov_base);
free(event[1].iov_base);
free(event);
}
queue_len -= sent;
if (sent > 0 || queue_len) {
crm_trace("Sent %d events (%d remaining) for %p[%d]: %s (%lld)",
sent, queue_len, c->ipcs, c->pid,
pcmk_strerror(rc < 0 ? rc : 0), (long long) rc);
}
if (queue_len) {
/* We want to allow clients to briefly fall behind on processing
* incoming messages, but drop completely unresponsive clients so the
* connection doesn't consume resources indefinitely.
*
* @TODO It is possible that the queue could reasonably grow large in a
* short time. An example is a reprobe of hundreds of resources on many
* nodes resulting in a surge of CIB replies to the crmd. We could
* possibly give cluster daemons a higher threshold here, and/or prevent
* such a surge by throttling LRM history writes in the crmd.
*/
if (queue_len > PCMK_IPC_MAX_QUEUE) {
if ((c->backlog_len <= 1) || (queue_len < c->backlog_len)) {
/* Don't evict for a new or shrinking backlog */
crm_warn("Client with process ID %u has a backlog of %u messages "
CRM_XS " %p", c->pid, queue_len, c->ipcs);
} else {
crm_err("Evicting client with process ID %u due to backlog of %u messages "
CRM_XS " %p", c->pid, queue_len, c->ipcs);
c->backlog_len = 0;
qb_ipcs_disconnect(c->ipcs);
return rc;
}
}
c->backlog_len = queue_len;
delay_next_flush(c, queue_len);
} else {
/* Event queue is empty, there is no backlog */
c->backlog_len = 0;
}
return rc;
}
ssize_t
crm_ipc_prepare(uint32_t request, xmlNode * message, struct iovec ** result, uint32_t max_send_size)
{
static unsigned int biggest = 0;
struct iovec *iov;
unsigned int total = 0;
char *compressed = NULL;
char *buffer = dump_xml_unformatted(message);
struct crm_ipc_response_header *header = calloc(1, sizeof(struct crm_ipc_response_header));
CRM_ASSERT(result != NULL);
crm_ipc_init();
if (max_send_size == 0) {
max_send_size = ipc_buffer_max;
}
CRM_LOG_ASSERT(max_send_size != 0);
*result = NULL;
iov = calloc(2, sizeof(struct iovec));
iov[0].iov_len = hdr_offset;
iov[0].iov_base = header;
header->version = PCMK_IPC_VERSION;
header->size_uncompressed = 1 + strlen(buffer);
total = iov[0].iov_len + header->size_uncompressed;
if (total < max_send_size) {
iov[1].iov_base = buffer;
iov[1].iov_len = header->size_uncompressed;
} else {
unsigned int new_size = 0;
if (crm_compress_string
(buffer, header->size_uncompressed, max_send_size, &compressed, &new_size)) {
header->flags |= crm_ipc_compressed;
header->size_compressed = new_size;
iov[1].iov_len = header->size_compressed;
iov[1].iov_base = compressed;
free(buffer);
biggest = QB_MAX(header->size_compressed, biggest);
} else {
ssize_t rc = -EMSGSIZE;
crm_log_xml_trace(message, "EMSGSIZE");
biggest = QB_MAX(header->size_uncompressed, biggest);
crm_err
("Could not compress the message (%u bytes) into less than the configured ipc limit (%u bytes). "
"Set PCMK_ipc_buffer to a higher value (%u bytes suggested)",
header->size_uncompressed, max_send_size, 4 * biggest);
free(compressed);
free(buffer);
free(header);
free(iov);
return rc;
}
}
header->qb.size = iov[0].iov_len + iov[1].iov_len;
header->qb.id = (int32_t)request; /* Replying to a specific request */
*result = iov;
CRM_ASSERT(header->qb.size > 0);
return header->qb.size;
}
ssize_t
crm_ipcs_sendv(crm_client_t * c, struct iovec * iov, enum crm_ipc_flags flags)
{
ssize_t rc;
static uint32_t id = 1;
struct crm_ipc_response_header *header = iov[0].iov_base;
if (c->flags & crm_client_flag_ipc_proxied) {
/* _ALL_ replies to proxied connections need to be sent as events */
if (is_not_set(flags, crm_ipc_server_event)) {
flags |= crm_ipc_server_event;
/* this flag lets us know this was originally meant to be a response.
* even though we're sending it over the event channel. */
flags |= crm_ipc_proxied_relay_response;
}
}
header->flags |= flags;
if (flags & crm_ipc_server_event) {
header->qb.id = id++; /* We don't really use it, but doesn't hurt to set one */
if (flags & crm_ipc_server_free) {
crm_trace("Sending the original to %p[%d]", c->ipcs, c->pid);
c->event_queue = g_list_append(c->event_queue, iov);
} else {
struct iovec *iov_copy = calloc(2, sizeof(struct iovec));
crm_trace("Sending a copy to %p[%d]", c->ipcs, c->pid);
iov_copy[0].iov_len = iov[0].iov_len;
iov_copy[0].iov_base = malloc(iov[0].iov_len);
memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len);
iov_copy[1].iov_len = iov[1].iov_len;
iov_copy[1].iov_base = malloc(iov[1].iov_len);
memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len);
c->event_queue = g_list_append(c->event_queue, iov_copy);
}
} else {
CRM_LOG_ASSERT(header->qb.id != 0); /* Replying to a specific request */
rc = qb_ipcs_response_sendv(c->ipcs, iov, 2);
if (rc < header->qb.size) {
crm_notice("Response %d to %p[%d] (%u bytes) failed: %s (%d)",
header->qb.id, c->ipcs, c->pid, header->qb.size, pcmk_strerror(rc), rc);
} else {
crm_trace("Response %d sent, %lld bytes to %p[%d]",
header->qb.id, (long long) rc, c->ipcs, c->pid);
}
if (flags & crm_ipc_server_free) {
free(iov[0].iov_base);
free(iov[1].iov_base);
free(iov);
}
}
if (flags & crm_ipc_server_event) {
rc = crm_ipcs_flush_events(c);
} else {
crm_ipcs_flush_events(c);
}
if (rc == -EPIPE || rc == -ENOTCONN) {
crm_trace("Client %p disconnected", c->ipcs);
}
return rc;
}
ssize_t
crm_ipcs_send(crm_client_t * c, uint32_t request, xmlNode * message,
enum crm_ipc_flags flags)
{
struct iovec *iov = NULL;
ssize_t rc = 0;
if(c == NULL) {
return -EDESTADDRREQ;
}
crm_ipc_init();
rc = crm_ipc_prepare(request, message, &iov, ipc_buffer_max);
if (rc > 0) {
rc = crm_ipcs_sendv(c, iov, flags | crm_ipc_server_free);
} else {
free(iov);
crm_notice("Message to %p[%d] failed: %s (%d)",
c->ipcs, c->pid, pcmk_strerror(rc), rc);
}
return rc;
}
void
crm_ipcs_send_ack(crm_client_t * c, uint32_t request, uint32_t flags, const char *tag, const char *function,
int line)
{
if (flags & crm_ipc_client_response) {
xmlNode *ack = create_xml_node(NULL, tag);
crm_trace("Ack'ing msg from %s (%p)", crm_client_name(c), c);
c->request_id = 0;
crm_xml_add(ack, "function", function);
crm_xml_add_int(ack, "line", line);
crm_ipcs_send(c, request, ack, flags);
free_xml(ack);
}
}
/* Client... */
#define MIN_MSG_SIZE 12336 /* sizeof(struct qb_ipc_connection_response) */
#define MAX_MSG_SIZE 128*1024 /* 128k default */
struct crm_ipc_s {
struct pollfd pfd;
/* the max size we can send/receive over ipc */
unsigned int max_buf_size;
/* Size of the allocated 'buffer' */
unsigned int buf_size;
int msg_size;
int need_reply;
char *buffer;
char *name;
uint32_t buffer_flags;
qb_ipcc_connection_t *ipc;
};
static unsigned int
pick_ipc_buffer(unsigned int max)
{
static unsigned int global_max = 0;
if (global_max == 0) {
const char *env = getenv("PCMK_ipc_buffer");
if (env) {
int env_max = crm_parse_int(env, "0");
global_max = (env_max > 0)? QB_MAX(MIN_MSG_SIZE, env_max) : MAX_MSG_SIZE;
} else {
global_max = MAX_MSG_SIZE;
}
}
return QB_MAX(max, global_max);
}
crm_ipc_t *
crm_ipc_new(const char *name, size_t max_size)
{
crm_ipc_t *client = NULL;
client = calloc(1, sizeof(crm_ipc_t));
client->name = strdup(name);
client->buf_size = pick_ipc_buffer(max_size);
client->buffer = malloc(client->buf_size);
/* Clients initiating connection pick the max buf size */
client->max_buf_size = client->buf_size;
client->pfd.fd = -1;
client->pfd.events = POLLIN;
client->pfd.revents = 0;
return client;
}
/*!
* \brief Establish an IPC connection to a Pacemaker component
*
* \param[in] client Connection instance obtained from crm_ipc_new()
*
* \return TRUE on success, FALSE otherwise (in which case errno will be set)
*/
bool
crm_ipc_connect(crm_ipc_t * client)
{
client->need_reply = FALSE;
client->ipc = qb_ipcc_connect(client->name, client->buf_size);
if (client->ipc == NULL) {
crm_debug("Could not establish %s connection: %s (%d)", client->name, pcmk_strerror(errno), errno);
return FALSE;
}
client->pfd.fd = crm_ipc_get_fd(client);
if (client->pfd.fd < 0) {
crm_debug("Could not obtain file descriptor for %s connection: %s (%d)", client->name, pcmk_strerror(errno), errno);
return FALSE;
}
qb_ipcc_context_set(client->ipc, client);
#ifdef HAVE_IPCS_GET_BUFFER_SIZE
client->max_buf_size = qb_ipcc_get_buffer_size(client->ipc);
if (client->max_buf_size > client->buf_size) {
free(client->buffer);
client->buffer = calloc(1, client->max_buf_size);
client->buf_size = client->max_buf_size;
}
#endif
return TRUE;
}
void
crm_ipc_close(crm_ipc_t * client)
{
if (client) {
crm_trace("Disconnecting %s IPC connection %p (%p)", client->name, client, client->ipc);
if (client->ipc) {
qb_ipcc_connection_t *ipc = client->ipc;
client->ipc = NULL;
qb_ipcc_disconnect(ipc);
}
}
}
void
crm_ipc_destroy(crm_ipc_t * client)
{
if (client) {
if (client->ipc && qb_ipcc_is_connected(client->ipc)) {
crm_notice("Destroying an active IPC connection to %s", client->name);
/* The next line is basically unsafe
*
* If this connection was attached to mainloop and mainloop is active,
* the 'disconnected' callback will end up back here and we'll end
* up free'ing the memory twice - something that can still happen
* even without this if we destroy a connection and it closes before
* we call exit
*/
/* crm_ipc_close(client); */
}
crm_trace("Destroying IPC connection to %s: %p", client->name, client);
free(client->buffer);
free(client->name);
free(client);
}
}
int
crm_ipc_get_fd(crm_ipc_t * client)
{
int fd = 0;
if (client && client->ipc && (qb_ipcc_fd_get(client->ipc, &fd) == 0)) {
return fd;
}
errno = EINVAL;
crm_perror(LOG_ERR, "Could not obtain file IPC descriptor for %s",
(client? client->name : "unspecified client"));
return -errno;
}
bool
crm_ipc_connected(crm_ipc_t * client)
{
bool rc = FALSE;
if (client == NULL) {
crm_trace("No client");
return FALSE;
} else if (client->ipc == NULL) {
crm_trace("No connection");
return FALSE;
} else if (client->pfd.fd < 0) {
crm_trace("Bad descriptor");
return FALSE;
}
rc = qb_ipcc_is_connected(client->ipc);
if (rc == FALSE) {
client->pfd.fd = -EINVAL;
}
return rc;
}
/*!
* \brief Check whether an IPC connection is ready to be read
*
* \param[in] client Connection to check
*
* \return Positive value if ready to be read, 0 if not ready, -errno on error
*/
int
crm_ipc_ready(crm_ipc_t *client)
{
int rc;
CRM_ASSERT(client != NULL);
if (crm_ipc_connected(client) == FALSE) {
return -ENOTCONN;
}
client->pfd.revents = 0;
rc = poll(&(client->pfd), 1, 0);
return (rc < 0)? -errno : rc;
}
static int
crm_ipc_decompress(crm_ipc_t * client)
{
struct crm_ipc_response_header *header = (struct crm_ipc_response_header *)(void*)client->buffer;
if (header->size_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->size_uncompressed;
/* never let buf size fall below our max size required for ipc reads. */
unsigned int new_buf_size = QB_MAX((hdr_offset + size_u), client->max_buf_size);
char *uncompressed = calloc(1, new_buf_size);
crm_trace("Decompressing message data %u bytes into %u bytes",
header->size_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed + hdr_offset, &size_u,
client->buffer + hdr_offset, header->size_compressed, 1, 0);
if (rc != BZ_OK) {
crm_err("Decompression failed: %s (%d)", bz2_strerror(rc), rc);
free(uncompressed);
return -EILSEQ;
}
/*
* This assert no longer holds true. For an identical msg, some clients may
* require compression, and others may not. If that same msg (event) is sent
* to multiple clients, it could result in some clients receiving a compressed
* msg even though compression was not explicitly required for them.
*
* CRM_ASSERT((header->size_uncompressed + hdr_offset) >= ipc_buffer_max);
*/
CRM_ASSERT(size_u == header->size_uncompressed);
memcpy(uncompressed, client->buffer, hdr_offset); /* Preserve the header */
header = (struct crm_ipc_response_header *)(void*)uncompressed;
free(client->buffer);
client->buf_size = new_buf_size;
client->buffer = uncompressed;
}
CRM_ASSERT(client->buffer[hdr_offset + header->size_uncompressed - 1] == 0);
return pcmk_ok;
}
long
crm_ipc_read(crm_ipc_t * client)
{
struct crm_ipc_response_header *header = NULL;
CRM_ASSERT(client != NULL);
CRM_ASSERT(client->ipc != NULL);
CRM_ASSERT(client->buffer != NULL);
crm_ipc_init();
client->buffer[0] = 0;
client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer, client->buf_size - 1, 0);
if (client->msg_size >= 0) {
int rc = crm_ipc_decompress(client);
if (rc != pcmk_ok) {
return rc;
}
header = (struct crm_ipc_response_header *)(void*)client->buffer;
if(header->version > PCMK_IPC_VERSION) {
crm_err("Filtering incompatible v%d IPC message, we only support versions <= %d",
header->version, PCMK_IPC_VERSION);
return -EBADMSG;
}
crm_trace("Received %s event %d, size=%u, rc=%d, text: %.100s",
client->name, header->qb.id, header->qb.size, client->msg_size,
client->buffer + hdr_offset);
} else {
crm_trace("No message from %s received: %s", client->name, pcmk_strerror(client->msg_size));
}
if (crm_ipc_connected(client) == FALSE || client->msg_size == -ENOTCONN) {
crm_err("Connection to %s failed", client->name);
}
if (header) {
/* Data excluding the header */
return header->size_uncompressed;
}
return -ENOMSG;
}
const char *
crm_ipc_buffer(crm_ipc_t * client)
{
CRM_ASSERT(client != NULL);
return client->buffer + sizeof(struct crm_ipc_response_header);
}
uint32_t
crm_ipc_buffer_flags(crm_ipc_t * client)
{
struct crm_ipc_response_header *header = NULL;
CRM_ASSERT(client != NULL);
if (client->buffer == NULL) {
return 0;
}
header = (struct crm_ipc_response_header *)(void*)client->buffer;
return header->flags;
}
const char *
crm_ipc_name(crm_ipc_t * client)
{
CRM_ASSERT(client != NULL);
return client->name;
}
static int
internal_ipc_send_recv(crm_ipc_t * client, const void *iov)
{
int rc = 0;
do {
rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer, client->buf_size, -1);
} while (rc == -EAGAIN && crm_ipc_connected(client));
return rc;
}
static int
internal_ipc_send_request(crm_ipc_t * client, const void *iov, int ms_timeout)
{
int rc = 0;
time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
do {
rc = qb_ipcc_sendv(client->ipc, iov, 2);
} while (rc == -EAGAIN && time(NULL) < timeout && crm_ipc_connected(client));
return rc;
}
static int
internal_ipc_get_reply(crm_ipc_t * client, int request_id, int ms_timeout)
{
time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
int rc = 0;
crm_ipc_init();
/* get the reply */
crm_trace("client %s waiting on reply to msg id %d", client->name, request_id);
do {
rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000);
if (rc > 0) {
struct crm_ipc_response_header *hdr = NULL;
int rc = crm_ipc_decompress(client);
if (rc != pcmk_ok) {
return rc;
}
hdr = (struct crm_ipc_response_header *)(void*)client->buffer;
if (hdr->qb.id == request_id) {
/* Got it */
break;
} else if (hdr->qb.id < request_id) {
xmlNode *bad = string2xml(crm_ipc_buffer(client));
crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id);
crm_log_xml_notice(bad, "OldIpcReply");
} else {
xmlNode *bad = string2xml(crm_ipc_buffer(client));
crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id);
crm_log_xml_notice(bad, "ImpossibleReply");
CRM_ASSERT(hdr->qb.id <= request_id);
}
} else if (crm_ipc_connected(client) == FALSE) {
crm_err("Server disconnected client %s while waiting for msg id %d", client->name,
request_id);
break;
}
} while (time(NULL) < timeout);
return rc;
}
int
crm_ipc_send(crm_ipc_t * client, xmlNode * message, enum crm_ipc_flags flags, int32_t ms_timeout,
xmlNode ** reply)
{
long rc = 0;
struct iovec *iov;
static uint32_t id = 0;
static int factor = 8;
struct crm_ipc_response_header *header;
crm_ipc_init();
if (client == NULL) {
crm_notice("Invalid connection");
return -ENOTCONN;
} else if (crm_ipc_connected(client) == FALSE) {
/* Don't even bother */
crm_notice("Connection to %s closed", client->name);
return -ENOTCONN;
}
if (ms_timeout == 0) {
ms_timeout = 5000;
}
if (client->need_reply) {
crm_trace("Trying again to obtain pending reply from %s", client->name);
rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout);
if (rc < 0) {
crm_warn("Sending to %s (%p) is disabled until pending reply is received", client->name,
client->ipc);
return -EALREADY;
} else {
crm_notice("Lost reply from %s (%p) finally arrived, sending re-enabled", client->name,
client->ipc);
client->need_reply = FALSE;
}
}
id++;
CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */
rc = crm_ipc_prepare(id, message, &iov, client->max_buf_size);
if(rc < 0) {
return rc;
}
header = iov[0].iov_base;
header->flags |= flags;
if(is_set(flags, crm_ipc_proxied)) {
/* Don't look for a synchronous response */
clear_bit(flags, crm_ipc_client_response);
}
if(header->size_compressed) {
if(factor < 10 && (client->max_buf_size / 10) < (rc / factor)) {
crm_notice("Compressed message exceeds %d0%% of the configured ipc limit (%u bytes), "
"consider setting PCMK_ipc_buffer to %u or higher",
factor, client->max_buf_size, 2 * client->max_buf_size);
factor++;
}
}
crm_trace("Sending from client: %s request id: %d bytes: %u timeout:%d msg...",
client->name, header->qb.id, header->qb.size, ms_timeout);
if (ms_timeout > 0 || is_not_set(flags, crm_ipc_client_response)) {
rc = internal_ipc_send_request(client, iov, ms_timeout);
if (rc <= 0) {
crm_trace("Failed to send from client %s request %d with %u bytes...",
client->name, header->qb.id, header->qb.size);
goto send_cleanup;
} else if (is_not_set(flags, crm_ipc_client_response)) {
crm_trace("Message sent, not waiting for reply to %d from %s to %u bytes...",
header->qb.id, client->name, header->qb.size);
goto send_cleanup;
}
rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout);
if (rc < 0) {
/* No reply, for now, disable sending
*
* The alternative is to close the connection since we don't know
* how to detect and discard out-of-sequence replies
*
* TODO - implement the above
*/
client->need_reply = TRUE;
}
} else {
rc = internal_ipc_send_recv(client, iov);
}
if (rc > 0) {
struct crm_ipc_response_header *hdr = (struct crm_ipc_response_header *)(void*)client->buffer;
crm_trace("Received response %d, size=%u, rc=%ld, text: %.200s", hdr->qb.id, hdr->qb.size,
rc, crm_ipc_buffer(client));
if (reply) {
*reply = string2xml(crm_ipc_buffer(client));
}
} else {
crm_trace("Response not received: rc=%ld, errno=%d", rc, errno);
}
send_cleanup:
if (crm_ipc_connected(client) == FALSE) {
crm_notice("Connection to %s closed: %s (%ld)", client->name, pcmk_strerror(rc), rc);
} else if (rc == -ETIMEDOUT) {
crm_warn("Request %d to %s (%p) failed: %s (%ld) after %dms",
header->qb.id, client->name, client->ipc, pcmk_strerror(rc), rc, ms_timeout);
crm_write_blackbox(0, NULL);
} else if (rc <= 0) {
crm_warn("Request %d to %s (%p) failed: %s (%ld)",
header->qb.id, client->name, client->ipc, pcmk_strerror(rc), rc);
}
free(header);
free(iov[1].iov_base);
free(iov);
return rc;
}
/* Utils */
xmlNode *
create_hello_message(const char *uuid,
const char *client_name, const char *major_version, const char *minor_version)
{
xmlNode *hello_node = NULL;
xmlNode *hello = NULL;
if (uuid == NULL || strlen(uuid) == 0
|| client_name == NULL || strlen(client_name) == 0
|| major_version == NULL || strlen(major_version) == 0
|| minor_version == NULL || strlen(minor_version) == 0) {
crm_err("Missing fields, Hello message will not be valid.");
return NULL;
}
hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
crm_xml_add(hello_node, "major_version", major_version);
crm_xml_add(hello_node, "minor_version", minor_version);
crm_xml_add(hello_node, "client_name", client_name);
crm_xml_add(hello_node, "client_uuid", uuid);
crm_trace("creating hello message");
hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
free_xml(hello_node);
return hello;
}
diff --git a/lrmd/tls_backend.c b/lrmd/tls_backend.c
index 8c36434024..7d790cf808 100644
--- a/lrmd/tls_backend.c
+++ b/lrmd/tls_backend.c
@@ -1,381 +1,378 @@
/*
* Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*
*/
#include <crm_internal.h>
#include <glib.h>
#include <unistd.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/mainloop.h>
#include <lrmd_private.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#ifdef HAVE_GNUTLS_GNUTLS_H
# define LRMD_REMOTE_AUTH_TIMEOUT 10000
gnutls_psk_server_credentials_t psk_cred_s;
gnutls_dh_params_t dh_params;
static int ssock = -1;
extern int lrmd_call_id;
static void
debug_log(int level, const char *str)
{
fputs(str, stderr);
}
static int
lrmd_remote_client_msg(gpointer data)
{
int id = 0;
int rc = 0;
int disconnected = 0;
xmlNode *request = NULL;
crm_client_t *client = data;
if (client->remote->tls_handshake_complete == FALSE) {
int rc = 0;
/* Muliple calls to handshake will be required, this callback
* will be invoked once the client sends more handshake data. */
do {
rc = gnutls_handshake(*client->remote->tls_session);
if (rc < 0 && rc != GNUTLS_E_AGAIN) {
crm_err("Remote lrmd tls handshake failed");
return -1;
}
} while (rc == GNUTLS_E_INTERRUPTED);
if (rc == 0) {
crm_debug("Remote lrmd tls handshake completed");
client->remote->tls_handshake_complete = TRUE;
if (client->remote->auth_timeout) {
g_source_remove(client->remote->auth_timeout);
}
client->remote->auth_timeout = 0;
/* Alert other clients of the new connection */
notify_of_new_client(client);
}
return 0;
}
rc = crm_remote_ready(client->remote, 0);
if (rc == 0) {
/* no msg to read */
return 0;
} else if (rc < 0) {
crm_info("Client disconnected during remote client read");
return -1;
}
crm_remote_recv(client->remote, -1, &disconnected);
request = crm_remote_parse_buffer(client->remote);
while (request) {
crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id);
crm_trace("processing request from remote client with remote msg id %d", id);
if (!client->name) {
const char *value = crm_element_value(request, F_LRMD_CLIENTNAME);
if (value) {
client->name = strdup(value);
}
}
lrmd_call_id++;
if (lrmd_call_id < 1) {
lrmd_call_id = 1;
}
crm_xml_add(request, F_LRMD_CLIENTID, client->id);
crm_xml_add(request, F_LRMD_CLIENTNAME, client->name);
crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id);
process_lrmd_message(client, id, request);
free_xml(request);
/* process all the messages in the current buffer */
request = crm_remote_parse_buffer(client->remote);
}
if (disconnected) {
crm_info("Client disconnect detected in tls msg dispatcher.");
return -1;
}
return 0;
}
static void
lrmd_remote_client_destroy(gpointer user_data)
{
crm_client_t *client = user_data;
if (client == NULL) {
return;
}
ipc_proxy_remove_provider(client);
/* if this is the last remote connection, stop recurring
* operations */
if (crm_hash_table_size(client_connections) == 1) {
client_disconnect_cleanup(NULL);
}
crm_notice("LRMD client disconnecting remote client - name: %s id: %s",
client->name ? client->name : "<unknown>", client->id);
if (client->remote->tls_session) {
void *sock_ptr;
int csock;
sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
csock = GPOINTER_TO_INT(sock_ptr);
gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
gnutls_deinit(*client->remote->tls_session);
gnutls_free(client->remote->tls_session);
close(csock);
}
lrmd_client_destroy(client);
return;
}
static gboolean
lrmd_auth_timeout_cb(gpointer data)
{
crm_client_t *client = data;
client->remote->auth_timeout = 0;
if (client->remote->tls_handshake_complete == TRUE) {
return FALSE;
}
mainloop_del_fd(client->remote->source);
client->remote->source = NULL;
crm_err("Remote client authentication timed out");
return FALSE;
}
static int
lrmd_remote_listen(gpointer data)
{
int csock = 0;
gnutls_session_t *session = NULL;
crm_client_t *new_client = NULL;
static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
.dispatch = lrmd_remote_client_msg,
.destroy = lrmd_remote_client_destroy,
};
csock = crm_remote_accept(ssock);
if (csock < 0) {
return TRUE;
}
session = create_psk_tls_session(csock, GNUTLS_SERVER, psk_cred_s);
if (session == NULL) {
crm_err("TLS session creation failed");
close(csock);
return TRUE;
}
- new_client = calloc(1, sizeof(crm_client_t));
+ new_client = crm_client_alloc(NULL);
new_client->remote = calloc(1, sizeof(crm_remote_t));
new_client->kind = CRM_CLIENT_TLS;
new_client->remote->tls_session = session;
- new_client->id = crm_generate_uuid();
new_client->remote->auth_timeout =
g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client);
crm_notice("LRMD client connection established. %p id: %s", new_client, new_client->id);
new_client->remote->source =
mainloop_add_fd("lrmd-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
&lrmd_remote_fd_cb);
- g_hash_table_insert(client_connections, new_client->id, new_client);
-
return TRUE;
}
static void
lrmd_remote_connection_destroy(gpointer user_data)
{
crm_notice("Remote tls server disconnected");
return;
}
static int
lrmd_tls_server_key_cb(gnutls_session_t session, const char *username, gnutls_datum_t * key)
{
return lrmd_tls_set_key(key);
}
static int
bind_and_listen(struct addrinfo *addr)
{
int optval;
int fd;
int rc;
char buffer[INET6_ADDRSTRLEN] = { 0, };
crm_sockaddr2str(addr->ai_addr, buffer);
crm_trace("Attempting to bind on address %s", buffer);
fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
if (fd < 0) {
return -1;
}
/* reuse address */
optval = 1;
rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
if (rc < 0) {
crm_perror(LOG_INFO, "Couldn't allow the reuse of local addresses by our remote listener, bind address %s", buffer);
close(fd);
return -1;
}
if (addr->ai_family == AF_INET6) {
optval = 0;
rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
if (rc < 0) {
crm_perror(LOG_INFO, "Couldn't disable IPV6 only on address %s", buffer);
close(fd);
return -1;
}
}
if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
close(fd);
return -1;
}
if (listen(fd, 10) == -1) {
crm_err("Can not start listen on address %s", buffer);
close(fd);
return -1;
}
crm_notice("Listening on address %s", buffer);
return fd;
}
int
lrmd_init_remote_tls_server(int port)
{
int rc;
int filter;
struct addrinfo hints, *res = NULL, *iter;
char port_str[16];
static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
.dispatch = lrmd_remote_listen,
.destroy = lrmd_remote_connection_destroy,
};
crm_notice("Starting a tls listener on port %d.", port);
crm_gnutls_global_init();
gnutls_global_set_log_function(debug_log);
gnutls_dh_params_init(&dh_params);
gnutls_dh_params_generate2(dh_params, 1024);
gnutls_psk_allocate_server_credentials(&psk_cred_s);
gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_flags = AI_PASSIVE; /* Only return socket addresses with wildcard INADDR_ANY or IN6ADDR_ANY_INIT */
hints.ai_family = AF_UNSPEC; /* Return IPv6 or IPv4 */
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
snprintf(port_str, sizeof(port_str), "%d", port);
rc = getaddrinfo(NULL, port_str, &hints, &res);
if (rc) {
crm_err("getaddrinfo: %s", gai_strerror(rc));
return -1;
}
iter = res;
filter = AF_INET6;
/* Try IPv6 addresses first, then IPv4 */
while (iter) {
if (iter->ai_family == filter) {
ssock = bind_and_listen(iter);
}
if (ssock != -1) {
break;
}
iter = iter->ai_next;
if (iter == NULL && filter == AF_INET6) {
iter = res;
filter = AF_INET;
}
}
if (ssock < 0) {
crm_err("unable to bind to address");
goto init_remote_cleanup;
}
mainloop_add_fd("lrmd-remote", G_PRIORITY_DEFAULT, ssock, NULL, &remote_listen_fd_callbacks);
rc = ssock;
init_remote_cleanup:
if (rc < 0) {
close(ssock);
ssock = 0;
}
freeaddrinfo(res);
return rc;
}
void
lrmd_tls_server_destroy(void)
{
if (psk_cred_s) {
gnutls_psk_free_server_credentials(psk_cred_s);
psk_cred_s = 0;
}
if (ssock > 0) {
close(ssock);
ssock = 0;
}
}
#endif