diff --git a/daemons/based/based_remote.c b/daemons/based/based_remote.c
index d3995e8a9e..42835be279 100644
--- a/daemons/based/based_remote.c
+++ b/daemons/based/based_remote.c
@@ -1,670 +1,667 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <inttypes.h> // PRIx64
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netinet/ip.h>
#include <stdlib.h>
#include <errno.h>
#include <glib.h>
#include <libxml/tree.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/xml.h>
#include <crm/common/remote_internal.h>
+#include <crm/common/tls_internal.h>
#include <crm/cib/internal.h>
#include "pacemaker-based.h"
#include <gnutls/gnutls.h>
#include <pwd.h>
#include <grp.h>
#if HAVE_SECURITY_PAM_APPL_H
# include <security/pam_appl.h>
# define HAVE_PAM 1
#elif HAVE_PAM_PAM_APPL_H
# include <pam/pam_appl.h>
# define HAVE_PAM 1
#endif
+static pcmk__tls_t *tls = NULL;
+
extern int remote_tls_fd;
extern gboolean cib_shutdown_flag;
int init_remote_listener(int port, gboolean encrypted);
void cib_remote_connection_destroy(gpointer user_data);
-gnutls_dh_params_t dh_params;
-gnutls_anon_server_credentials_t anon_cred_s;
-static void
-debug_log(int level, const char *str)
-{
- fputs(str, stderr);
-}
-
// @TODO This is rather short for someone to type their password
#define REMOTE_AUTH_TIMEOUT 10000
int num_clients;
static bool authenticate_user(const char *user, const char *passwd);
static int cib_remote_listen(gpointer data);
static int cib_remote_msg(gpointer data);
static void
remote_connection_destroy(gpointer user_data)
{
crm_info("No longer listening for remote connections");
return;
}
int
init_remote_listener(int port, gboolean encrypted)
{
int rc;
int *ssock = NULL;
struct sockaddr_in saddr;
int optval;
static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
.dispatch = cib_remote_listen,
.destroy = remote_connection_destroy,
};
if (port <= 0) {
/* don't start it */
return 0;
}
if (encrypted) {
+ bool use_cert = pcmk__x509_enabled(true);
+
crm_notice("Starting TLS listener on port %d", port);
- crm_gnutls_global_init();
- /* gnutls_global_set_log_level (10); */
- gnutls_global_set_log_function(debug_log);
- if (pcmk__init_tls_dh(&dh_params) != pcmk_rc_ok) {
+
+ rc = pcmk__init_tls(&tls, true, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_ANON);
+ if (rc != pcmk_rc_ok) {
return -1;
}
- gnutls_anon_allocate_server_credentials(&anon_cred_s);
- gnutls_anon_set_server_dh_params(anon_cred_s, dh_params);
} else {
crm_warn("Starting plain-text listener on port %d", port);
}
#ifndef HAVE_PAM
crm_warn("This build does not support remote administrators "
"because PAM support is not available");
#endif
/* create server socket */
ssock = malloc(sizeof(int));
if(ssock == NULL) {
crm_err("Listener socket allocation failed: %s", pcmk_rc_str(errno));
return -1;
}
*ssock = socket(AF_INET, SOCK_STREAM, 0);
if (*ssock == -1) {
crm_err("Listener socket creation failed: %s", pcmk_rc_str(errno));
free(ssock);
return -1;
}
/* reuse address */
optval = 1;
rc = setsockopt(*ssock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
if (rc < 0) {
crm_err("Local address reuse not allowed on listener socket: %s",
pcmk_rc_str(errno));
}
/* bind server socket */
memset(&saddr, '\0', sizeof(saddr));
saddr.sin_family = AF_INET;
saddr.sin_addr.s_addr = INADDR_ANY;
saddr.sin_port = htons(port);
if (bind(*ssock, (struct sockaddr *)&saddr, sizeof(saddr)) == -1) {
crm_err("Cannot bind to listener socket: %s", pcmk_rc_str(errno));
close(*ssock);
free(ssock);
return -2;
}
if (listen(*ssock, 10) == -1) {
crm_err("Cannot listen on socket: %s", pcmk_rc_str(errno));
close(*ssock);
free(ssock);
return -3;
}
mainloop_add_fd("cib-remote", G_PRIORITY_DEFAULT, *ssock, ssock, &remote_listen_fd_callbacks);
crm_debug("Started listener on port %d", port);
return *ssock;
}
static int
check_group_membership(const char *usr, const char *grp)
{
int index = 0;
struct passwd *pwd = NULL;
struct group *group = NULL;
pwd = getpwnam(usr);
if (pwd == NULL) {
crm_notice("Rejecting remote client: '%s' is not a valid user", usr);
return FALSE;
}
group = getgrgid(pwd->pw_gid);
if (group != NULL && pcmk__str_eq(grp, group->gr_name, pcmk__str_none)) {
return TRUE;
}
group = getgrnam(grp);
if (group == NULL) {
crm_err("Rejecting remote client: '%s' is not a valid group", grp);
return FALSE;
}
while (TRUE) {
char *member = group->gr_mem[index++];
if (member == NULL) {
break;
} else if (pcmk__str_eq(usr, member, pcmk__str_none)) {
return TRUE;
}
}
crm_notice("Rejecting remote client: User '%s' is not a member of "
"group '%s'", usr, grp);
return FALSE;
}
static gboolean
cib_remote_auth(xmlNode * login)
{
const char *user = NULL;
const char *pass = NULL;
const char *tmp = NULL;
if (login == NULL) {
return FALSE;
}
if (!pcmk__xe_is(login, PCMK__XE_CIB_COMMAND)) {
crm_warn("Rejecting remote client: Unrecognizable message "
"(element '%s' not '" PCMK__XE_CIB_COMMAND "')", login->name);
crm_log_xml_debug(login, "bad");
return FALSE;
}
tmp = crm_element_value(login, PCMK_XA_OP);
if (!pcmk__str_eq(tmp, "authenticate", pcmk__str_casei)) {
crm_warn("Rejecting remote client: Unrecognizable message "
"(operation '%s' not 'authenticate')", tmp);
crm_log_xml_debug(login, "bad");
return FALSE;
}
user = crm_element_value(login, PCMK_XA_USER);
pass = crm_element_value(login, PCMK__XA_PASSWORD);
if (!user || !pass) {
crm_warn("Rejecting remote client: No %s given",
((user == NULL)? "username" : "password"));
crm_log_xml_debug(login, "bad");
return FALSE;
}
crm_log_xml_debug(login, "auth");
return check_group_membership(user, CRM_DAEMON_GROUP)
&& authenticate_user(user, pass);
}
static gboolean
remote_auth_timeout_cb(gpointer data)
{
pcmk__client_t *client = data;
client->remote->auth_timeout = 0;
if (pcmk_is_set(client->flags, pcmk__client_authenticated)) {
return FALSE;
}
mainloop_del_fd(client->remote->source);
crm_err("Remote client authentication timed out");
return FALSE;
}
static int
cib_remote_listen(gpointer data)
{
int csock = 0;
unsigned laddr;
struct sockaddr_storage addr;
char ipstr[INET6_ADDRSTRLEN];
int ssock = *(int *)data;
int rc;
pcmk__client_t *new_client = NULL;
static struct mainloop_fd_callbacks remote_client_fd_callbacks = {
.dispatch = cib_remote_msg,
.destroy = cib_remote_connection_destroy,
};
/* accept the connection */
laddr = sizeof(addr);
memset(&addr, 0, sizeof(addr));
csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
if (csock == -1) {
crm_warn("Could not accept remote connection: %s", pcmk_rc_str(errno));
return TRUE;
}
pcmk__sockaddr2str(&addr, ipstr);
rc = pcmk__set_nonblocking(csock);
if (rc != pcmk_rc_ok) {
crm_warn("Dropping remote connection from %s because "
"it could not be set to non-blocking: %s",
ipstr, pcmk_rc_str(rc));
close(csock);
return TRUE;
}
num_clients++;
new_client = pcmk__new_unauth_client(NULL);
new_client->remote = pcmk__assert_alloc(1, sizeof(pcmk__remote_t));
if (ssock == remote_tls_fd) {
pcmk__set_client_flags(new_client, pcmk__client_tls);
/* create gnutls session for the server socket */
- new_client->remote->tls_session = pcmk__new_tls_session(csock,
- GNUTLS_SERVER,
- GNUTLS_CRD_ANON,
- anon_cred_s);
+ new_client->remote->tls_session = pcmk__new_tls_session(tls, csock);
if (new_client->remote->tls_session == NULL) {
close(csock);
return TRUE;
}
} else {
pcmk__set_client_flags(new_client, pcmk__client_tcp);
new_client->remote->tcp_socket = csock;
}
// Require the client to authenticate within this time
new_client->remote->auth_timeout = pcmk__create_timer(REMOTE_AUTH_TIMEOUT,
remote_auth_timeout_cb,
new_client);
crm_info("%s connection from %s pending authentication for client %s",
((ssock == remote_tls_fd)? "Encrypted" : "Clear-text"),
ipstr, new_client->id);
new_client->remote->source =
mainloop_add_fd("cib-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
&remote_client_fd_callbacks);
return TRUE;
}
void
cib_remote_connection_destroy(gpointer user_data)
{
pcmk__client_t *client = user_data;
int csock = 0;
if (client == NULL) {
return;
}
crm_trace("Cleaning up after client %s disconnect",
pcmk__client_name(client));
num_clients--;
crm_trace("Num unfree'd clients: %d", num_clients);
switch (PCMK__CLIENT_TYPE(client)) {
case pcmk__client_tcp:
csock = client->remote->tcp_socket;
break;
case pcmk__client_tls:
if (client->remote->tls_session) {
- void *sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
+ void *sock_ptr = gnutls_transport_get_ptr(client->remote->tls_session);
csock = GPOINTER_TO_INT(sock_ptr);
if (pcmk_is_set(client->flags,
pcmk__client_tls_handshake_complete)) {
- gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_WR);
+ gnutls_bye(client->remote->tls_session, GNUTLS_SHUT_WR);
}
- gnutls_deinit(*client->remote->tls_session);
- gnutls_free(client->remote->tls_session);
+ gnutls_deinit(client->remote->tls_session);
client->remote->tls_session = NULL;
}
break;
default:
crm_warn("Unknown transport for client %s "
QB_XS " flags=%#016" PRIx64,
pcmk__client_name(client), client->flags);
}
if (csock > 0) {
close(csock);
}
pcmk__free_client(client);
crm_trace("Freed the cib client");
if (cib_shutdown_flag) {
cib_shutdown(0);
}
return;
}
static void
cib_handle_remote_msg(pcmk__client_t *client, xmlNode *command)
{
if (!pcmk__xe_is(command, PCMK__XE_CIB_COMMAND)) {
crm_log_xml_trace(command, "bad");
return;
}
if (client->name == NULL) {
client->name = pcmk__str_copy(client->id);
}
/* unset dangerous options */
pcmk__xe_remove_attr(command, PCMK__XA_SRC);
pcmk__xe_remove_attr(command, PCMK__XA_CIB_HOST);
pcmk__xe_remove_attr(command, PCMK__XA_CIB_UPDATE);
crm_xml_add(command, PCMK__XA_T, PCMK__VALUE_CIB);
crm_xml_add(command, PCMK__XA_CIB_CLIENTID, client->id);
crm_xml_add(command, PCMK__XA_CIB_CLIENTNAME, client->name);
crm_xml_add(command, PCMK__XA_CIB_USER, client->user);
if (crm_element_value(command, PCMK__XA_CIB_CALLID) == NULL) {
char *call_uuid = crm_generate_uuid();
/* fix the command */
crm_xml_add(command, PCMK__XA_CIB_CALLID, call_uuid);
free(call_uuid);
}
if (crm_element_value(command, PCMK__XA_CIB_CALLOPT) == NULL) {
crm_xml_add_int(command, PCMK__XA_CIB_CALLOPT, 0);
}
crm_log_xml_trace(command, "Remote command: ");
cib_common_callback_worker(0, 0, command, client, TRUE);
}
static int
cib_remote_msg(gpointer data)
{
xmlNode *command = NULL;
pcmk__client_t *client = data;
int rc;
const char *client_name = pcmk__client_name(client);
crm_trace("Remote %s message received for client %s",
pcmk__client_type_str(PCMK__CLIENT_TYPE(client)), client_name);
if ((PCMK__CLIENT_TYPE(client) == pcmk__client_tls)
&& !pcmk_is_set(client->flags, pcmk__client_tls_handshake_complete)) {
int rc = pcmk__read_handshake_data(client);
if (rc == EAGAIN) {
/* No more data is available at the moment. Just return for now;
* we'll get invoked again once the client sends more.
*/
return 0;
} else if (rc != pcmk_rc_ok) {
return -1;
}
crm_debug("Completed TLS handshake with remote client %s", client_name);
pcmk__set_client_flags(client, pcmk__client_tls_handshake_complete);
if (client->remote->auth_timeout) {
g_source_remove(client->remote->auth_timeout);
}
+ /* Now that the handshake is done, see if any client TLS certificate is
+ * close to its expiration date and log if so. If a TLS certificate is not
+ * in use, this function will just return so we don't need to check for the
+ * session type here.
+ */
+ pcmk__tls_check_cert_expiration(client->remote->tls_session);
+
// Require the client to authenticate within this time
client->remote->auth_timeout = pcmk__create_timer(REMOTE_AUTH_TIMEOUT,
remote_auth_timeout_cb,
client);
return 0;
}
rc = pcmk__read_available_remote_data(client->remote);
switch (rc) {
case pcmk_rc_ok:
break;
case EAGAIN:
/* We haven't read the whole message yet */
return 0;
default:
/* Error */
crm_trace("Error reading from remote client: %s", pcmk_rc_str(rc));
return -1;
}
/* must pass auth before we will process anything else */
if (!pcmk_is_set(client->flags, pcmk__client_authenticated)) {
xmlNode *reg;
const char *user = NULL;
command = pcmk__remote_message_xml(client->remote);
if (cib_remote_auth(command) == FALSE) {
pcmk__xml_free(command);
return -1;
}
pcmk__set_client_flags(client, pcmk__client_authenticated);
g_source_remove(client->remote->auth_timeout);
client->remote->auth_timeout = 0;
client->name = crm_element_value_copy(command, PCMK_XA_NAME);
user = crm_element_value(command, PCMK_XA_USER);
if (user) {
client->user = pcmk__str_copy(user);
}
crm_notice("Remote connection accepted for authenticated user %s "
QB_XS " client %s",
pcmk__s(user, ""), client_name);
/* send ACK */
reg = pcmk__xe_create(NULL, PCMK__XE_CIB_RESULT);
crm_xml_add(reg, PCMK__XA_CIB_OP, CRM_OP_REGISTER);
crm_xml_add(reg, PCMK__XA_CIB_CLIENTID, client->id);
pcmk__remote_send_xml(client->remote, reg);
pcmk__xml_free(reg);
pcmk__xml_free(command);
}
command = pcmk__remote_message_xml(client->remote);
if (command != NULL) {
crm_trace("Remote message received from client %s", client_name);
cib_handle_remote_msg(client, command);
pcmk__xml_free(command);
}
return 0;
}
#ifdef HAVE_PAM
/*!
* \internal
* \brief Pass remote user's password to PAM
*
* \param[in] num_msg Number of entries in \p msg
* \param[in] msg Array of PAM messages
* \param[out] response Where to set response to PAM
* \param[in] data User data (the password string)
*
* \return PAM return code (PAM_BUF_ERR for memory errors, PAM_CONV_ERR for all
* other errors, or PAM_SUCCESS on success)
* \note See pam_conv(3) for more explanation
*/
static int
construct_pam_passwd(int num_msg, const struct pam_message **msg,
struct pam_response **response, void *data)
{
/* In theory, multiple messages are allowed, but due to OS compatibility
* issues, PAM implementations are recommended to only send one message at a
* time. We can require that here for simplicity.
*/
CRM_CHECK((num_msg == 1) && (msg != NULL) && (response != NULL)
&& (data != NULL), return PAM_CONV_ERR);
switch (msg[0]->msg_style) {
case PAM_PROMPT_ECHO_OFF:
case PAM_PROMPT_ECHO_ON:
// Password requested
break;
case PAM_TEXT_INFO:
crm_info("PAM: %s", msg[0]->msg);
data = NULL;
break;
case PAM_ERROR_MSG:
/* In theory we should show msg[0]->msg, but that might
* contain the password, which we don't want in the logs
*/
crm_err("PAM reported an error");
data = NULL;
break;
default:
crm_warn("Ignoring PAM message of unrecognized type %d",
msg[0]->msg_style);
return PAM_CONV_ERR;
}
*response = calloc(1, sizeof(struct pam_response));
if (*response == NULL) {
return PAM_BUF_ERR;
}
(*response)->resp_retcode = 0;
(*response)->resp = pcmk__str_copy((const char *) data); // Caller will free
return PAM_SUCCESS;
}
#endif
/*!
* \internal
* \brief Verify the username and password passed for a remote CIB connection
*
* \param[in] user Username passed for remote CIB connection
* \param[in] passwd Password passed for remote CIB connection
*
* \return \c true if the username and password are accepted, otherwise \c false
* \note This function rejects all credentials when built without PAM support.
*/
static bool
authenticate_user(const char *user, const char *passwd)
{
#ifdef HAVE_PAM
int rc = 0;
bool pass = false;
const void *p_user = NULL;
struct pam_conv p_conv;
struct pam_handle *pam_h = NULL;
static const char *pam_name = NULL;
if (pam_name == NULL) {
pam_name = getenv("CIB_pam_service");
if (pam_name == NULL) {
pam_name = "login";
}
}
p_conv.conv = construct_pam_passwd;
p_conv.appdata_ptr = (void *) passwd;
rc = pam_start(pam_name, user, &p_conv, &pam_h);
if (rc != PAM_SUCCESS) {
crm_warn("Rejecting remote client for user %s "
"because PAM initialization failed: %s",
user, pam_strerror(pam_h, rc));
goto bail;
}
// Check user credentials
rc = pam_authenticate(pam_h, PAM_SILENT);
if (rc != PAM_SUCCESS) {
crm_notice("Access for remote user %s denied: %s",
user, pam_strerror(pam_h, rc));
goto bail;
}
/* Get the authenticated user name (PAM modules can map the original name to
* something else). Since the CIB manager runs as the daemon user (not
* root), that is the only user that can be successfully authenticated.
*/
rc = pam_get_item(pam_h, PAM_USER, &p_user);
if (rc != PAM_SUCCESS) {
crm_warn("Rejecting remote client for user %s "
"because PAM failed to return final user name: %s",
user, pam_strerror(pam_h, rc));
goto bail;
}
if (p_user == NULL) {
crm_warn("Rejecting remote client for user %s "
"because PAM returned no final user name", user);
goto bail;
}
// @TODO Why do we require these to match?
if (!pcmk__str_eq(p_user, user, pcmk__str_none)) {
crm_warn("Rejecting remote client for user %s "
"because PAM returned different final user name %s",
user, p_user);
goto bail;
}
// Check user account restrictions (expiration, etc.)
rc = pam_acct_mgmt(pam_h, PAM_SILENT);
if (rc != PAM_SUCCESS) {
crm_notice("Access for remote user %s denied: %s",
user, pam_strerror(pam_h, rc));
goto bail;
}
pass = true;
bail:
pam_end(pam_h, rc);
return pass;
#else
// @TODO Implement for non-PAM environments
crm_warn("Rejecting remote user %s because this build does not have "
"PAM support", user);
return false;
#endif
}
diff --git a/daemons/execd/remoted_tls.c b/daemons/execd/remoted_tls.c
index a89109d060..7b41d63943 100644
--- a/daemons/execd/remoted_tls.c
+++ b/daemons/execd/remoted_tls.c
@@ -1,440 +1,432 @@
/*
* Copyright 2012-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <glib.h>
#include <unistd.h>
#include <crm/crm.h>
#include <crm/common/mainloop.h>
#include <crm/common/xml.h>
#include <crm/common/remote_internal.h>
+#include <crm/common/tls_internal.h>
#include <crm/lrmd_internal.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include "pacemaker-execd.h"
#include <gnutls/gnutls.h>
#define LRMD_REMOTE_AUTH_TIMEOUT 10000
-gnutls_psk_server_credentials_t psk_cred_s;
-gnutls_dh_params_t dh_params;
+
+static pcmk__tls_t *tls = NULL;
static int ssock = -1;
extern int lrmd_call_id;
-static void
-debug_log(int level, const char *str)
-{
- fputs(str, stderr);
-}
-
/*!
* \internal
* \brief Read (more) TLS handshake data from client
*
* \param[in,out] client IPC client doing handshake
*
* \return 0 on success or more data needed, -1 on error
*/
static int
remoted__read_handshake_data(pcmk__client_t *client)
{
int rc = pcmk__read_handshake_data(client);
if (rc == EAGAIN) {
/* No more data is available at the moment. Just return for now;
* we'll get invoked again once the client sends more.
*/
return 0;
} else if (rc != pcmk_rc_ok) {
return -1;
}
if (client->remote->auth_timeout) {
g_source_remove(client->remote->auth_timeout);
}
client->remote->auth_timeout = 0;
pcmk__set_client_flags(client, pcmk__client_tls_handshake_complete);
crm_notice("Remote client connection accepted");
/* Only a client with access to the TLS key can connect, so we can treat
* it as privileged.
*/
pcmk__set_client_flags(client, pcmk__client_privileged);
// Alert other clients of the new connection
notify_of_new_client(client);
return 0;
}
static int
lrmd_remote_client_msg(gpointer data)
{
int id = 0;
int rc = pcmk_rc_ok;
xmlNode *request = NULL;
pcmk__client_t *client = data;
if (!pcmk_is_set(client->flags, pcmk__client_tls_handshake_complete)) {
return remoted__read_handshake_data(client);
}
rc = pcmk__remote_ready(client->remote, 0);
switch (rc) {
case pcmk_rc_ok:
break;
case ETIME:
/* No message available to read */
return 0;
default:
/* Error */
crm_info("Error polling remote client: %s", pcmk_rc_str(rc));
return -1;
}
rc = pcmk__read_available_remote_data(client->remote);
switch (rc) {
case pcmk_rc_ok:
break;
case EAGAIN:
/* We haven't read the whole message yet */
return 0;
default:
/* Error */
crm_info("Error reading from remote client: %s", pcmk_rc_str(rc));
return -1;
}
request = pcmk__remote_message_xml(client->remote);
if (request == NULL) {
return 0;
}
crm_element_value_int(request, PCMK__XA_LRMD_REMOTE_MSG_ID, &id);
crm_trace("Processing remote client request %d", id);
if (!client->name) {
client->name = crm_element_value_copy(request,
PCMK__XA_LRMD_CLIENTNAME);
}
lrmd_call_id++;
if (lrmd_call_id < 1) {
lrmd_call_id = 1;
}
crm_xml_add(request, PCMK__XA_LRMD_CLIENTID, client->id);
crm_xml_add(request, PCMK__XA_LRMD_CLIENTNAME, client->name);
crm_xml_add_int(request, PCMK__XA_LRMD_CALLID, lrmd_call_id);
process_lrmd_message(client, id, request);
pcmk__xml_free(request);
return 0;
}
static void
lrmd_remote_client_destroy(gpointer user_data)
{
pcmk__client_t *client = user_data;
if (client == NULL) {
return;
}
crm_notice("Cleaning up after remote client %s disconnected",
pcmk__client_name(client));
ipc_proxy_remove_provider(client);
/* if this is the last remote connection, stop recurring
* operations */
if (pcmk__ipc_client_count() == 1) {
client_disconnect_cleanup(NULL);
}
if (client->remote->tls_session) {
void *sock_ptr;
int csock;
- sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session);
+ sock_ptr = gnutls_transport_get_ptr(client->remote->tls_session);
csock = GPOINTER_TO_INT(sock_ptr);
- gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR);
- gnutls_deinit(*client->remote->tls_session);
- gnutls_free(client->remote->tls_session);
+ gnutls_bye(client->remote->tls_session, GNUTLS_SHUT_RDWR);
+ gnutls_deinit(client->remote->tls_session);
client->remote->tls_session = NULL;
close(csock);
}
lrmd_client_destroy(client);
return;
}
static gboolean
lrmd_auth_timeout_cb(gpointer data)
{
pcmk__client_t *client = data;
client->remote->auth_timeout = 0;
if (pcmk_is_set(client->flags,
pcmk__client_tls_handshake_complete)) {
return FALSE;
}
mainloop_del_fd(client->remote->source);
client->remote->source = NULL;
crm_err("Remote client authentication timed out");
return FALSE;
}
// Dispatch callback for remote server socket
static int
lrmd_remote_listen(gpointer data)
{
int csock = -1;
- gnutls_session_t *session = NULL;
+ gnutls_session_t session = NULL;
pcmk__client_t *new_client = NULL;
// For client socket
static struct mainloop_fd_callbacks lrmd_remote_fd_cb = {
.dispatch = lrmd_remote_client_msg,
.destroy = lrmd_remote_client_destroy,
};
CRM_CHECK(ssock >= 0, return TRUE);
if (pcmk__accept_remote_connection(ssock, &csock) != pcmk_rc_ok) {
return TRUE;
}
- session = pcmk__new_tls_session(csock, GNUTLS_SERVER, GNUTLS_CRD_PSK,
- psk_cred_s);
+ session = pcmk__new_tls_session(tls, csock);
if (session == NULL) {
close(csock);
return TRUE;
}
new_client = pcmk__new_unauth_client(NULL);
new_client->remote = pcmk__assert_alloc(1, sizeof(pcmk__remote_t));
pcmk__set_client_flags(new_client, pcmk__client_tls);
new_client->remote->tls_session = session;
// Require the client to authenticate within this time
new_client->remote->auth_timeout = pcmk__create_timer(LRMD_REMOTE_AUTH_TIMEOUT,
lrmd_auth_timeout_cb,
new_client);
crm_info("Remote client pending authentication "
QB_XS " %p id: %s", new_client, new_client->id);
new_client->remote->source =
mainloop_add_fd("pacemaker-remote-client", G_PRIORITY_DEFAULT, csock,
new_client, &lrmd_remote_fd_cb);
return TRUE;
}
static void
tls_server_dropped(gpointer user_data)
{
crm_notice("TLS server session ended");
return;
}
// \return 0 on success, -1 on error (gnutls_psk_server_credentials_function)
static int
lrmd_tls_server_key_cb(gnutls_session_t session, const char *username, gnutls_datum_t * key)
{
return (lrmd__init_remote_key(key) == pcmk_rc_ok)? 0 : -1;
}
static int
bind_and_listen(struct addrinfo *addr)
{
int optval;
int fd;
int rc;
char buffer[INET6_ADDRSTRLEN] = { 0, };
pcmk__sockaddr2str(addr->ai_addr, buffer);
crm_trace("Attempting to bind to address %s", buffer);
fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
if (fd < 0) {
rc = errno;
crm_err("Listener socket creation failed: %", pcmk_rc_str(rc));
return -rc;
}
/* reuse address */
optval = 1;
rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval));
if (rc < 0) {
rc = errno;
crm_err("Local address reuse not allowed on %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
if (addr->ai_family == AF_INET6) {
optval = 0;
rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval));
if (rc < 0) {
rc = errno;
crm_err("Couldn't disable IPV6-only on %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
}
if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) {
rc = errno;
crm_err("Cannot bind to %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
if (listen(fd, 10) == -1) {
rc = errno;
crm_err("Cannot listen on %s: %s", buffer, pcmk_rc_str(rc));
close(fd);
return -rc;
}
return fd;
}
static int
get_address_info(const char *bind_name, int port, struct addrinfo **res)
{
int rc;
char port_str[6]; // at most "65535"
struct addrinfo hints;
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_flags = AI_PASSIVE;
hints.ai_family = AF_UNSPEC; // IPv6 or IPv4
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
snprintf(port_str, sizeof(port_str), "%d", port);
rc = getaddrinfo(bind_name, port_str, &hints, res);
rc = pcmk__gaierror2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Unable to get IP address(es) for %s: %s",
(bind_name? bind_name : "local node"), pcmk_rc_str(rc));
return rc;
}
return pcmk_rc_ok;
}
int
lrmd_init_remote_tls_server(void)
{
+ int rc = pcmk_rc_ok;
int filter;
int port = crm_default_remote_port();
struct addrinfo *res = NULL, *iter;
gnutls_datum_t psk_key = { NULL, 0 };
const char *bind_name = pcmk__env_option(PCMK__ENV_REMOTE_ADDRESS);
static struct mainloop_fd_callbacks remote_listen_fd_callbacks = {
.dispatch = lrmd_remote_listen,
.destroy = tls_server_dropped,
};
CRM_CHECK(ssock == -1, return ssock);
crm_debug("Starting TLS listener on %s port %d",
(bind_name? bind_name : "all addresses on"), port);
- crm_gnutls_global_init();
- gnutls_global_set_log_function(debug_log);
- if (pcmk__init_tls_dh(&dh_params) != pcmk_rc_ok) {
+ rc = pcmk__init_tls(&tls, true, GNUTLS_CRD_PSK);
+ if (rc != pcmk_rc_ok) {
return -1;
}
- gnutls_psk_allocate_server_credentials(&psk_cred_s);
- gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb);
- gnutls_psk_set_server_dh_params(psk_cred_s, dh_params);
+
+ pcmk__tls_add_psk_callback(tls, lrmd_tls_server_key_cb);
/* The key callback won't get called until the first client connection
* attempt. Do it once here, so we can warn the user at start-up if we can't
* read the key. We don't error out, though, because it's fine if the key is
* going to be added later.
*/
if (lrmd__init_remote_key(&psk_key) != pcmk_rc_ok) {
crm_warn("A cluster connection will not be possible until the key is available");
}
gnutls_free(psk_key.data);
if (get_address_info(bind_name, port, &res) != pcmk_rc_ok) {
return -1;
}
/* Currently we listen on only one address from the resulting list (the
* first IPv6 address we can bind to if possible, otherwise the first IPv4
* address we can bind to). When bind_name is NULL, this should be the
* respective wildcard address.
*
* @TODO If there is demand for specifying more than one address, allow
* bind_name to be a space-separated list, call getaddrinfo() for each,
* and create a socket for each result (set IPV6_V6ONLY on IPv6 sockets
* since IPv4 listeners will have their own sockets).
*/
iter = res;
filter = AF_INET6;
while (iter) {
if (iter->ai_family == filter) {
ssock = bind_and_listen(iter);
}
if (ssock >= 0) {
break;
}
iter = iter->ai_next;
if (iter == NULL && filter == AF_INET6) {
iter = res;
filter = AF_INET;
}
}
if (ssock >= 0) {
mainloop_add_fd("pacemaker-remote-server", G_PRIORITY_DEFAULT, ssock,
NULL, &remote_listen_fd_callbacks);
crm_debug("Started TLS listener on %s port %d",
(bind_name? bind_name : "all addresses on"), port);
}
freeaddrinfo(res);
return ssock;
}
void
execd_stop_tls_server(void)
{
- if (psk_cred_s) {
- gnutls_psk_free_server_credentials(psk_cred_s);
- psk_cred_s = 0;
+ if (tls != NULL) {
+ pcmk__free_tls(tls);
+ tls = NULL;
}
if (ssock >= 0) {
close(ssock);
ssock = -1;
}
}
diff --git a/doc/sphinx/Pacemaker_Administration/configuring.rst b/doc/sphinx/Pacemaker_Administration/configuring.rst
index 5052f6ad50..70ce349d8a 100644
--- a/doc/sphinx/Pacemaker_Administration/configuring.rst
+++ b/doc/sphinx/Pacemaker_Administration/configuring.rst
@@ -1,228 +1,260 @@
.. index::
single: configuration
single: CIB
Configuring Pacemaker
---------------------
Pacemaker's configuration, the CIB, is stored in XML format. Cluster
administrators have multiple options for modifying the configuration either via
the XML, or at a more abstract (and easier for humans to understand) level.
Pacemaker reacts to configuration changes as soon as they are saved.
Pacemaker's command-line tools and most higher-level tools provide the ability
to batch changes together and commit them at once, rather than make a series of
small changes, which could cause avoid unnecessary actions as Pacemaker
responds to each change individually.
Pacemaker tracks revisions to the configuration and will reject any update
older than the current revision. Thus, it is a good idea to serialize all
changes to the configuration. Avoid attempting simultaneous changes, whether on
the same node or different nodes, and whether manually or using some automated
configuration tool.
.. note::
It is not necessary to update the configuration on all cluster nodes.
Pacemaker immediately synchronizes changes to all active members of the
cluster. To reduce bandwidth, the cluster only broadcasts the incremental
updates that result from your changes and uses checksums to ensure that each
copy is consistent.
Configuration Using Higher-level Tools
######################################
Most users will benefit from using higher-level tools provided by projects
separate from Pacemaker. Popular ones include the crm shell and pcs. [#]_
See those projects' documentation for details on how to configure Pacemaker
using them.
Configuration Using Pacemaker's Command-Line Tools
##################################################
Pacemaker provides lower-level, command-line tools to manage the cluster. Most
configuration tasks can be performed with these tools, without needing any XML
knowledge.
To enable STONITH for example, one could run:
.. code-block:: none
# crm_attribute --name stonith-enabled --update 1
Or, to check whether **node1** is allowed to run resources, there is:
.. code-block:: none
# crm_standby --query --node node1
Or, to change the failure threshold of **my-test-rsc**, one can use:
.. code-block:: none
# crm_resource -r my-test-rsc --set-parameter migration-threshold --parameter-value 3 --meta
Examples of using these tools for specific cases will be given throughout this
document where appropriate. See the man pages for further details.
See :ref:`cibadmin` for how to edit the CIB using XML.
See :ref:`crm_shadow` for a way to make a series of changes, then commit them
all at once to the live cluster.
.. index::
single: configuration; CIB properties
single: CIB; properties
single: CIB property
Working with CIB Properties
___________________________
Although these fields can be written to by the user, in
most cases the cluster will overwrite any values specified by the
user with the "correct" ones.
To change the ones that can be specified by the user, for example
``admin_epoch``, one should use:
.. code-block:: none
# cibadmin --modify --xml-text '<cib admin_epoch="42"/>'
A complete set of CIB properties will look something like this:
.. topic:: XML attributes set for a cib element
.. code-block:: xml
<cib crm_feature_set="3.0.7" validate-with="pacemaker-1.2"
admin_epoch="42" epoch="116" num_updates="1"
cib-last-written="Mon Jan 12 15:46:39 2015" update-origin="rhel7-1"
update-client="crm_attribute" have-quorum="1" dc-uuid="1">
.. index::
single: configuration; cluster options
Querying and Setting Cluster Options
____________________________________
Cluster options can be queried and modified using the ``crm_attribute`` tool.
To get the current value of ``cluster-delay``, you can run:
.. code-block:: none
# crm_attribute --query --name cluster-delay
which is more simply written as
.. code-block:: none
# crm_attribute -G -n cluster-delay
If a value is found, you'll see a result like this:
.. code-block:: none
# crm_attribute -G -n cluster-delay
scope=crm_config name=cluster-delay value=60s
If no value is found, the tool will display an error:
.. code-block:: none
# crm_attribute -G -n clusta-deway
scope=crm_config name=clusta-deway value=(null)
Error performing operation: No such device or address
To use a different value (for example, 30 seconds), simply run:
.. code-block:: none
# crm_attribute --name cluster-delay --update 30s
To go back to the cluster's default value, you can delete the value, for example:
.. code-block:: none
# crm_attribute --name cluster-delay --delete
Deleted crm_config option: id=cib-bootstrap-options-cluster-delay name=cluster-delay
When Options are Listed More Than Once
______________________________________
If you ever see something like the following, it means that the option you're
modifying is present more than once.
.. topic:: Deleting an option that is listed twice
.. code-block:: none
# crm_attribute --name batch-limit --delete
Please choose from one of the matches below and supply the 'id' with --id
Multiple attributes match name=batch-limit in crm_config:
Value: 50 (set=cib-bootstrap-options, id=cib-bootstrap-options-batch-limit)
Value: 100 (set=custom, id=custom-batch-limit)
In such cases, follow the on-screen instructions to perform the requested
action. To determine which value is currently being used by the cluster, refer
to the "Rules" chapter of *Pacemaker Explained*.
.. index::
single: configuration; remote
.. _remote_connection:
Connecting from a Remote Machine
################################
It is possible to run configuration commands from a machine that is not part of
the cluster.
For security reasons, this capability is disabled by default. If you wish to
allow remote access, set the ``remote-tls-port`` (encrypted) or
``remote-clear-port`` (unencrypted) CIB properties (attributes of the ``cib``
-element). Encrypted communication is keyless, which makes it subject to
-man-in-the-middle attacks, so either option should be used only on protected
-networks.
+element). Encrypted communication can be performed keyless (which makes it
+subject to man-in-the-middle attacks), but a better option is to also use
+TLS certificates.
+
+To enable TLS certificates, it is recommended to first set up your own
+Certificate Authority (CA) and generate a root CA certificate. Then create a
+public/private key pair and certificate signing request (CSR) for your server.
+Use the CA to sign this CSR.
+
+Then, create a public/private key pair and CSR for each remote system that you
+wish to have remote access. Use the CA to sign the CSRs. It is recommended to
+use a unique certificate for each remote system so they can be revoked if
+necessary.
+
+The server's public/private key pair and signed certificate should be installed
+to the |PCMK_CONFIG_DIR| directory and owned by ``CIB_user``. Remember that
+private keys should not be readable by anyone other than their owner. Finally,
+edit the |PCMK_CONFIG_FILE| file to refer to these credentials:
+
+.. code-block:: none
+
+ PCMK_ca_file="/etc/pacemaker/ca.cert.pem"
+ PCMK_cert_file="/etc/pacemaker/server.cert.pem"
+ PCMK_key_file="/etc/pacemaker/server.key.pem"
The administrator's machine simply needs Pacemaker installed. To connect to the
cluster, set the following environment variables:
* :ref:`CIB_port <CIB_port>` (required)
* :ref:`CIB_server <CIB_server>`
* :ref:`CIB_user <CIB_user>`
* :ref:`CIB_passwd <CIB_passwd>`
* :ref:`CIB_encrypted <CIB_encrypted>`
Only the Pacemaker daemon user (|CRM_DAEMON_USER|) may be used as ``CIB_user``.
+To use TLS certificates, the administrator's machine also needs their
+public/private key pair, signed client certificate, and root CA certificate.
+Those must additionally be specified with the following environment variables:
+
+* :ref:`CIB_ca_file <file>`
+* :ref:`CIB_cert_file <file>`
+* :ref:`CIB_key_file <file>`
+
As an example, if **node1** is a cluster node, and the CIB is configured with
``remote-tls-port`` set to 1234, the administrator could read the current
cluster configuration using the following commands, and would be prompted for
the daemon user's password:
.. code-block:: none
# export CIB_server=node1; export CIB_port=1234; export CIB_encrypted=true
+ # export CIB_ca_file=/etc/pacemaker/ca.cert.pem
+ # export CIB_cert_file=/etc/pacemaker/admin.cert.pem
+ # export CIB_key_file=/etc/pacemaker/admin.key.pem
# cibadmin -Q
.. note::
Pacemaker must have been built with PAM support for remote access to work.
You can check by running ``pacemakerd --features``. If the output contains
**pam**, remote access is supported. *(since 3.0.0; before 3.0.0, in a build
without PAM support, all remote connections are accepted without any
authentication)*
.. rubric:: Footnotes
.. [#] For a list, see "Configuration Tools" at
https://clusterlabs.org/components.html
diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index 8c185ac41c..0a25112082 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -1,331 +1,332 @@
""" Sphinx configuration for Pacemaker documentation
"""
-__copyright__ = "Copyright 2020-2023 the Pacemaker project contributors"
+__copyright__ = "Copyright 2020-2024 the Pacemaker project contributors"
__license__ = "GNU General Public License version 2 or later (GPLv2+) WITHOUT ANY WARRANTY"
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import datetime
import os
import sys
# Variables that can be used later in this file
authors = "the Pacemaker project contributors"
year = datetime.datetime.now().year
doc_license = "Creative Commons Attribution-ShareAlike International Public License"
doc_license += " version 4.0 or later (CC-BY-SA v4.0+)"
# rST markup to insert at beginning of every document; mainly used for
#
# .. |<abbr>| replace:: <Full text>
#
# where occurrences of |<abbr>| in the rST will be substituted with <Full text>
rst_prolog="""
.. |CFS_DISTRO| replace:: AlmaLinux
.. |CFS_DISTRO_VER| replace:: 9
.. |CRM_BLACKBOX_DIR| replace:: ``%CRM_BLACKBOX_DIR%``
.. |CRM_CONFIG_DIR| replace:: ``%CRM_CONFIG_DIR%``
.. |CRM_DAEMON_GROUP| replace:: ``%CRM_DAEMON_GROUP%``
.. |CRM_DAEMON_USER| replace:: ``%CRM_DAEMON_USER%``
.. |PCMK_SCHEMA_DIR| replace:: %PCMK_SCHEMA_DIR%
+.. |PCMK_CONFIG_DIR| replace:: ``%PACEMAKER_CONFIG_DIR%``
.. |PCMK_AUTHKEY_FILE| replace:: %PACEMAKER_CONFIG_DIR%/authkey
.. |PCMK_CONFIG_FILE| replace:: ``%CONFIGDIR%/pacemaker``
.. |PCMK__GNUTLS_PRIORITIES| replace:: %PCMK__GNUTLS_PRIORITIES%
.. |PCMK_INIT_ENV_FILE| replace:: ``%PACEMAKER_CONFIG_DIR%/pcmk-init.env``
.. |PCMK_LOG_FILE| replace:: %CRM_LOG_DIR%/pacemaker.log
.. |PCMK_CONTAINER_LOG_FILE| replace:: ``/var/log/pcmk-init.log``
.. |PCMK__REMOTE_SCHEMA_DIR| replace:: %PCMK__REMOTE_SCHEMA_DIR%
.. |REMOTE_DISTRO| replace:: AlmaLinux
.. |REMOTE_DISTRO_VER| replace:: 9
"""
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('%ABS_TOP_SRCDIR%/python'))
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
'sphinx.ext.autosummary']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = '%BOOK_ID%'
copyright = "2009-%s %s. Released under the terms of the %s" % (year, authors, doc_license)
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The full version, including alpha/beta/rc tags.
release = '%VERSION%'
# The short X.Y version.
version = release.rsplit('.', 1)[0]
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'vs'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'pyramid'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
html_style = 'pacemaker.css'
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
html_title = "%BOOK_TITLE%"
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = [ '%SRC_DIR%/_static' ]
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'Pacemakerdoc'
# -- Options for LaTeX output --------------------------------------------------
latex_engine = "xelatex"
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', '%BOOK_ID%.tex', '%BOOK_TITLE%', authors, 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', '%BOOK_ID%', 'Part of the Pacemaker documentation set', [authors], 8)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# -- Options for Texinfo output ------------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', '%BOOK_ID%', '%BOOK_TITLE%', authors, '%BOOK_TITLE%',
'Pacemaker is an advanced, scalable high-availability cluster resource manager.',
'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# -- Options for Epub output ---------------------------------------------------
# Bibliographic Dublin Core info.
epub_title = '%BOOK_TITLE%'
epub_author = authors
epub_publisher = 'ClusterLabs.org'
epub_copyright = copyright
# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''
# The scheme of the identifier. Typical schemes are ISBN or URL.
epub_scheme = 'URL'
# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
epub_identifier = 'https://www.clusterlabs.org/pacemaker/doc/%PACKAGE_SERIES%/%BOOK_ID%/epub/%BOOK_ID%.epub'
# A unique identification for the text.
epub_uid = 'ClusterLabs.org-Pacemaker-%BOOK_ID%'
# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()
# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []
# HTML files that should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []
# A list of files that should not be packed into the epub file.
epub_exclude_files = [
'_static/doctools.js',
'_static/jquery.js',
'_static/searchtools.js',
'_static/underscore.js',
'_static/basic.css',
'_static/websupport.js',
'search.html',
]
# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3
# Allow duplicate toc entries.
#epub_tocdup = True
autosummary_generate = True
diff --git a/etc/sysconfig/pacemaker.in b/etc/sysconfig/pacemaker.in
index 37ece9c85c..e4cc22a1df 100644
--- a/etc/sysconfig/pacemaker.in
+++ b/etc/sysconfig/pacemaker.in
@@ -1,385 +1,418 @@
#
# Pacemaker start-up configuration
#
# This file contains environment variables that affect Pacemaker behavior.
# They are not options stored in the Cluster Information Base (CIB) because
# they may be needed before the CIB is available.
#
## Logging
# PCMK_logfacility
#
# Enable logging via the system log or journal, using the specified log
# facility. Messages sent here are of value to all Pacemaker administrators.
# This can be disabled using "none", but that is not recommended. Allowed
# values:
#
# none
# daemon
# user
# local0
# local1
# local2
# local3
# local4
# local5
# local6
# local7
#
# Default: PCMK_logfacility="daemon"
# PCMK_logpriority
#
# Unless system logging is disabled using PCMK_logfacility=none, messages of
# the specified log severity and higher will be sent to the system log. The
# default is appropriate for most installations. Allowed values:
#
# emerg
# alert
# crit
# error
# warning
# notice
# info
# debug
#
# Default: PCMK_logpriority="notice"
# PCMK_logfile
#
# Unless set to "none", more detailed log messages will be sent to the
# specified file (in addition to the system log, if enabled). These messages
# may have extended information, and will include messages of info severity.
# This log is of more use to developers and advanced system administrators, and
# when reporting problems.
#
# Default: PCMK_logfile="@CRM_LOG_DIR@/pacemaker.log"
# PCMK_logfile_mode
#
# Pacemaker will set the permissions on the detail log to this value (see
# chmod(1)).
#
# Default: PCMK_logfile_mode="0660"
# PCMK_debug (Advanced Use Only)
#
# Whether to send debug severity messages to the detail log.
# This may be set for all subsystems (yes or no) or for specific
# (comma-separated) subsystems. Allowed subsystems are:
#
# pacemakerd
# pacemaker-attrd
# pacemaker-based
# pacemaker-controld
# pacemaker-execd
# pacemaker-fenced
# pacemaker-schedulerd
#
# Default: PCMK_debug="no"
# Example: PCMK_debug="pacemakerd,pacemaker-execd"
# PCMK_stderr (Advanced Use Only)
#
# Whether to send daemon log messages to stderr. This would be useful only
# during troubleshooting, when starting Pacemaker manually on the command line.
#
# Setting this option in this file is pointless, since this file is not read
# when starting Pacemaker manually. However, it can be set directly as an
# environment variable on the command line.
#
# Default: PCMK_stderr="no"
# PCMK_trace_functions (Advanced Use Only)
#
# Send debug and trace severity messages from these (comma-separated)
# source code functions to the detail log.
#
# Default: PCMK_trace_functions=""
# Example: PCMK_trace_functions="unpack_colocation_set,pcmk__cmp_instance"
# PCMK_trace_files (Advanced Use Only)
#
# Send debug and trace severity messages from all functions in these
# (comma-separated) source file names to the detail log.
#
# Default: PCMK_trace_files=""
# Example: PCMK_trace_files="remote.c,watchdog.c"
# PCMK_trace_formats (Advanced Use Only)
#
# Send trace severity messages that are generated by these (comma-separated)
# format strings in the source code to the detail log.
#
# Default: PCMK_trace_formats=""
# Example: PCMK_trace_formats="TLS handshake failed: %s (%d)"
# PCMK_trace_tags (Advanced Use Only)
#
# Send debug and trace severity messages related to these (comma-separated)
# resource IDs to the detail log.
#
# Default: PCMK_trace_tags=""
# Example: PCMK_trace_tags="client-ip,dbfs"
# PCMK_blackbox (Advanced Use Only)
#
# Enable blackbox logging globally (yes or no) or by subsystem. A blackbox
# contains a rolling buffer of all logs (of all severities). Blackboxes are
# stored under @CRM_BLACKBOX_DIR@ by default, and their contents can
# be viewed using the qb-blackbox(8) command.
#
# The blackbox recorder can be enabled at start using this variable, or at
# runtime by sending a Pacemaker subsystem daemon process a SIGUSR1 or SIGTRAP
# signal, and disabled by sending SIGUSR2 (see kill(1)). The blackbox will be
# written after a crash, assertion failure, or SIGTRAP signal.
#
# Default: PCMK_blackbox="no"
# Example: PCMK_blackbox="pacemaker-controld,pacemaker-fenced"
# PCMK_trace_blackbox (Advanced Use Only)
#
# Write a blackbox whenever the message at the specified function and line is
# logged. Multiple entries may be comma-separated.
#
# Default: PCMK_trace_blackbox=""
# Example: PCMK_trace_blackbox="remote.c:144,remote.c:149"
## Option overrides
# PCMK_node_start_state
#
# By default, the local host will join the cluster in an online or standby
# state when Pacemaker first starts depending on whether it was previously put
# into standby mode. If this variable is set to "standby" or "online", it will
# force the local host to join in the specified state.
#
# Default: PCMK_node_start_state="default"
# PCMK_node_action_limit
#
# If set, this overrides the node-action-limit cluster option for this node to
# specify the maximum number of jobs that can be scheduled on this node (or 0
# to use twice the number of CPU cores).
#
# Default: unset
# Example: PCMK_node_action_limit="1"
## Crash Handling
# PCMK_fail_fast
#
# By default, if a Pacemaker subsystem crashes, the main pacemakerd process
# will attempt to restart it. If this variable is set to "yes", pacemakerd
# will panic the local host instead.
#
# Default: PCMK_fail_fast="no"
# PCMK_panic_action
#
# Pacemaker panics the local node under certain conditions (for example, losing
# quorum when no-quorum-policy is "suicide", or being notified of the local
# node's own fencing when fence-reaction is "panic"). This variable determines
# the panic behavior. Allowed values:
#
# reboot Immediately reboot the host (not a clean reboot)
# off Immediately kill power to the host (not a clean shutdown)
# crash Trigger a kernel crash if possible, otherwise like reboot
# sync-reboot, sync-off, sync-crash
# "sync-" can be put in front of any of the above values to synchronize
# filesystems before panicking (making log messages more likely to be
# preserved, but with the risk that the host may be left active if the
# synchronization hangs)
#
# Default: PCMK_panic_action="reboot"
-## Pacemaker Remote
+## Pacemaker Remote and remote CIB administration
# PCMK_authkey_location
#
# Use the contents of this file as the authorization key to use with Pacemaker
# Remote connections. This file must be readable by Pacemaker daemons (that is,
# it must allow read permissions to either the @CRM_DAEMON_USER@ user or the
# @CRM_DAEMON_GROUP@ group), and its contents must be identical on all nodes.
#
# Default: PCMK_authkey_location="@PACEMAKER_CONFIG_DIR@/authkey"
# PCMK_remote_address
#
# By default, if the Pacemaker Remote service is run on the local node, it will
# listen for connections on all IP addresses. This may be set to one address to
# listen on instead, as a resolvable hostname or as a numeric IPv4 or IPv6
# address. When resolving names or listening on all addresses, IPv6 will be
# preferred if available. When listening on an IPv6 address, IPv4 clients will
# be supported via IPv4-mapped IPv6 addresses.
#
# Default: PCMK_remote_address=""
# Example: PCMK_remote_address="192.0.2.1"
# PCMK_remote_port
#
# Use this TCP port number for Pacemaker Remote node connections. This value
# must be the same on all nodes.
#
# Default: PCMK_remote_port="3121"
+# PCMK_ca_file
+#
+# The location of a file containing trusted Certificate Authorities, used to
+# verify client or server certificates. This file should be in PEM format.
+# If set, along with PCMK_key_file and PCMK_cert_file, X509 authentication
+# will be enabled for remote CIB connections.
+#
+# Default: PCMK_ca_file=""
+
+# PCMK_cert_file
+#
+# The location of a file containing the signed certificate for the server
+# (CIB manager) side of the connection, in PEM format. If set, along with
+# PCMK_ca_file and PCMK_key_file, X509 authentication will be enabled for
+# remote CIB connections.
+#
+# Default: PCMK_cert_file=""
+
+# PCMK_crl_file
+#
+# The location of a Certificate Revocation List file, in PEM format. This
+# setting is optional for X509 authentication.
+#
+# Default: PCMK_crl_file=""
+
+# PCMK_key_file
+#
+# The location of a file containing the private key for the matching PCMK_cert_file,
+# in PEM format. If set, along with PCMK_ca_file and PCMK_cert_file, X509
+# authentication will be enabled for remote CIB connections.
+#
+# Default: PCMK_key_file=""
+
# PCMK_remote_pid1 (Advanced Use Only)
#
# When a bundle resource's "run-command" option is left to default, Pacemaker
# Remote runs as PID 1 in the bundle's containers. When it does so, it loads
# environment variables from the container's
# @PACEMAKER_CONFIG_DIR@/pcmk-init.env and performs the PID 1 responsibility of
# reaping dead subprocesses.
#
# This option controls whether those actions are performed when Pacemaker
# Remote is not running as PID 1. It is intended primarily for developer testing
# but can be useful when "run-command" is set to a separate, custom PID 1
# process that launches Pacemaker Remote.
#
# * If set to "full", Pacemaker Remote loads environment variables from
# @PACEMAKER_CONFIG_DIR@/pcmk-init.env and reaps dead subprocesses.
# * If set to "vars", Pacemaker Remote loads environment variables from
# @PACEMAKER_CONFIG_DIR@/pcmk-init.env but does not reap dead subprocesses.
# * If set to "default", Pacemaker Remote performs neither action.
#
# If Pacemaker Remote is running as PID 1, this option is ignored, and the
# behavior is the same as for "full".
#
# Default: PCMK_remote_pid1="default"
# PCMK_tls_priorities (Advanced Use Only)
#
# These GnuTLS cipher priorities will be used for TLS connections (whether for
# Pacemaker Remote connections or remote CIB access, when enabled). See:
#
# https://gnutls.org/manual/html_node/Priority-Strings.html
#
# Pacemaker will append ":+ANON-DH" for remote CIB access and ":+DHE-PSK:+PSK"
# for Pacemaker Remote connections, as they are required for the respective
# functionality.
#
# Default: PCMK_tls_priorities="@PCMK__GNUTLS_PRIORITIES@"
# Example: PCMK_tls_priorities="SECURE128:+SECURE192:-VERS-ALL:+VERS-TLS1.2"
# PCMK_dh_max_bits (Advanced Use Only)
#
# Set an upper bound on the bit length of the prime number generated for
# Diffie-Hellman parameters needed by TLS connections. The default is no
# maximum.
#
# The server (Pacemaker Remote daemon, or CIB manager configured to accept
# remote clients) will use this value to provide a ceiling for the value
# recommended by the GnuTLS library. The library will only accept a limited
# number of specific values, which vary by library version, so setting these is
# recommended only when required for compatibility with specific client
# versions.
#
# Clients do not use PCMK_dh_max_bits.
#
# Default: PCMK_dh_max_bits="0" (no maximum)
## Inter-process Communication
# PCMK_ipc_type (Advanced Use Only)
#
# Force use of a particular IPC method. Allowed values:
#
# shared-mem
# socket
# posix
# sysv
#
# Default: PCMK_ipc_type="shared-mem"
# PCMK_ipc_buffer (Advanced Use Only)
#
# Specify an IPC buffer size in bytes. This can be useful when connecting to
# large clusters that result in messages exceeding the default size (which will
# also result in log messages referencing this variable).
#
# Default: PCMK_ipc_buffer="131072"
## Cluster type
# PCMK_cluster_type (Advanced Use Only)
#
# Specify the cluster layer to be used. If unset, Pacemaker will detect and use
# a supported cluster layer, if available. Currently, "corosync" is the only
# supported cluster layer. If multiple layers are supported in the future, this
# will allow overriding Pacemaker's automatic detection to select a specific
# one.
#
# Default: PCMK_cluster_type=""
## Developer Options
# PCMK_schema_directory (Advanced Use Only)
#
# Specify an alternate location for RNG schemas and XSL transforms.
#
# Default: PCMK_schema_directory="@PCMK_SCHEMA_DIR@"
# PCMK_remote_schema_directory (Advanced Use Only)
#
# Specify an alternate location on Pacemaker Remote nodes for storing newer
# RNG schemas and XSL transforms fetched from the cluster.
#
# Default: PCMK_remote_schema_directory="@PCMK__REMOTE_SCHEMA_DIR@"
# G_SLICE (Advanced Use Only)
#
# Affect the behavior of glib's memory allocator. Setting to "always-malloc"
# when running under valgrind will help valgrind track malloc/free better;
# setting to "debug-blocks" when not running under valgrind will perform
# (somewhat expensive) memory checks.
#
# Default: G_SLICE=""
# Example: G_SLICE="always-malloc"
# MALLOC_PERTURB_ (Advanced Use Only)
#
# Setting this to a decimal byte value will make malloc() initialize newly
# allocated memory and free() wipe it, to help catch uninitialized-memory and
# use-after-free bugs.
#
# Default: MALLOC_PERTURB_=""
# Example: MALLOC_PERTURB_="221"
# MALLOC_CHECK_ (Advanced Use Only)
#
# Setting this to 3 will make malloc() and friends print to stderr and abort
# for some (inexpensive) memory checks.
#
# Default: MALLOC_CHECK_=""
# Example: MALLOC_CHECK_="3"
# PCMK_valgrind_enabled (Advanced Use Only)
#
# Whether subsystem daemons should be run under valgrind. Allowed values are
# the same as for PCMK_debug.
#
# Default: PCMK_valgrind_enabled="no"
# PCMK_callgrind_enabled
#
# Whether subsystem daemons should be run under valgrind with the callgrind
# tool enabled. Allowed values are the same as for PCMK_debug.
#
# Default: PCMK_callgrind_enabled="no"
# VALGRIND_OPTS
#
# Pass these options to valgrind, when enabled (see valgrind(1)). "--vgdb=no"
# is specified because pacemaker-execd can lower privileges when executing
# commands, which would otherwise leave a bunch of unremovable files in /tmp.
#
# Default: VALGRIND_OPTS=""
VALGRIND_OPTS="--leak-check=full --trace-children=no --vgdb=no --num-callers=25 --log-file=@PCMK__PERSISTENT_DATA_DIR@/valgrind-%p --suppressions=@datadir@/pacemaker/tests/valgrind-pcmk.suppressions --gen-suppressions=all"
diff --git a/include/crm/common/ipc_internal.h b/include/crm/common/ipc_internal.h
index a405a5a142..9fe96609ef 100644
--- a/include/crm/common/ipc_internal.h
+++ b/include/crm/common/ipc_internal.h
@@ -1,284 +1,284 @@
/*
* Copyright 2013-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_COMMON_IPC_INTERNAL__H
#define PCMK__CRM_COMMON_IPC_INTERNAL__H
#include <stdbool.h> // bool
#include <stdint.h> // uint32_t, uint64_t, UINT64_C()
#include <sys/uio.h> // struct iovec
#include <sys/types.h> // uid_t, gid_t, pid_t, size_t
#include <gnutls/gnutls.h> // gnutls_session_t
#include <glib.h> // guint, gpointer, GQueue, ...
#include <libxml/tree.h> // xmlNode
#include <qb/qbipcs.h> // qb_ipcs_connection_t, ...
#include <crm_config.h> // HAVE_GETPEEREID
#include <crm/common/ipc.h>
#include <crm/common/ipc_controld.h> // pcmk_controld_api_reply
#include <crm/common/ipc_pacemakerd.h> // pcmk_pacemakerd_{api_reply,state}
#include <crm/common/mainloop.h> // mainloop_io_t
#ifdef __cplusplus
extern "C" {
#endif
/* denotes "non yieldable PID" on FreeBSD, or actual PID1 in scenarios that
require a delicate handling anyway (socket-based activation with systemd);
we can be reasonably sure that this PID is never possessed by the actual
child daemon, as it gets taken either by the proper init, or by pacemakerd
itself (i.e. this precludes anything else); note that value of zero
is meant to carry "unset" meaning, and better not to bet on/conditionalize
over signedness of pid_t */
#define PCMK__SPECIAL_PID 1
// Timeout (in seconds) to use for IPC client sends, reply waits, etc.
#define PCMK__IPC_TIMEOUT 120
#if defined(HAVE_GETPEEREID)
/* on FreeBSD, we don't want to expose "non-yieldable PID" (leading to
"IPC liveness check only") as its nominal representation, which could
cause confusion -- this is unambiguous as long as there's no
socket-based activation like with systemd (very improbable) */
#define PCMK__SPECIAL_PID_AS_0(p) (((p) == PCMK__SPECIAL_PID) ? 0 : (p))
#else
#define PCMK__SPECIAL_PID_AS_0(p) (p)
#endif
/*!
* \internal
* \brief Check the authenticity and liveness of the process via IPC end-point
*
* When IPC daemon under given IPC end-point (name) detected, its authenticity
* is verified by the means of comparing against provided referential UID and
* GID, and the result of this check can be deduced from the return value.
* As an exception, referential UID of 0 (~ root) satisfies arbitrary
* detected daemon's credentials.
*
* \param[in] name IPC name to base the search on
* \param[in] refuid referential UID to check against
* \param[in] refgid referential GID to check against
* \param[out] gotpid to optionally store obtained PID of the found process
* upon returning 1 or -2
* (not available on FreeBSD, special value of 1,
* see PCMK__SPECIAL_PID, used instead, and the caller
* is required to special case this value respectively)
*
* \return Standard Pacemaker return code
*
* \note Return codes of particular interest include pcmk_rc_ipc_unresponsive
* indicating that no trace of IPC liveness was detected, and
* pcmk_rc_ipc_unauthorized indicating that the IPC endpoint is blocked by
* an unauthorized process.
* \note This function emits a log message for return codes other than
* pcmk_rc_ok and pcmk_rc_ipc_unresponsive, and when there isn't a perfect
* match in respect to \p reguid and/or \p refgid, for a possible
* least privilege principle violation.
*
* \see crm_ipc_is_authentic_process
*/
int pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid,
gid_t refgid, pid_t *gotpid);
int pcmk__connect_generic_ipc(crm_ipc_t *ipc);
int pcmk__ipc_fd(crm_ipc_t *ipc, int *fd);
int pcmk__connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type,
int attempts);
/*
* Server-related
*/
typedef struct pcmk__client_s pcmk__client_t;
struct pcmk__remote_s {
/* Shared */
char *buffer;
size_t buffer_size;
size_t buffer_offset;
int auth_timeout;
int tcp_socket;
mainloop_io_t *source;
time_t uptime;
char *start_state;
/* CIB-only */
char *token;
/* TLS only */
- gnutls_session_t *tls_session;
+ gnutls_session_t tls_session;
};
enum pcmk__client_flags {
// Lower 32 bits are reserved for server (not library) use
// Next 8 bits are reserved for client type (sort of a cheap enum)
//! Client uses plain IPC
pcmk__client_ipc = (UINT64_C(1) << 32),
//! Client uses TCP connection
pcmk__client_tcp = (UINT64_C(1) << 33),
//! Client uses TCP with TLS
pcmk__client_tls = (UINT64_C(1) << 34),
// The rest are client attributes
//! Client IPC is proxied
pcmk__client_proxied = (UINT64_C(1) << 40),
//! Client is run by root or cluster user
pcmk__client_privileged = (UINT64_C(1) << 41),
//! Local client to be proxied
pcmk__client_to_proxy = (UINT64_C(1) << 42),
/*!
* \brief Client IPC connection accepted
*
* Used only for remote CIB connections via \c PCMK_XA_REMOTE_TLS_PORT.
*/
pcmk__client_authenticated = (UINT64_C(1) << 43),
//! Client TLS handshake is complete
pcmk__client_tls_handshake_complete = (UINT64_C(1) << 44),
};
#define PCMK__CLIENT_TYPE(client) ((client)->flags & UINT64_C(0xff00000000))
struct pcmk__client_s {
unsigned int pid;
char *id;
char *name;
char *user;
uint64_t flags; // Group of pcmk__client_flags
int request_id;
void *userdata;
int event_timer;
GQueue *event_queue;
/* Depending on the client type, only some of the following will be
* populated/valid. @TODO Maybe convert to a union.
*/
qb_ipcs_connection_t *ipcs; /* IPC */
struct pcmk__remote_s *remote; /* TCP/TLS */
unsigned int queue_backlog; /* IPC queue length after last flush */
unsigned int queue_max; /* Evict client whose queue grows this big */
};
#define pcmk__set_client_flags(client, flags_to_set) do { \
(client)->flags = pcmk__set_flags_as(__func__, __LINE__, \
LOG_TRACE, \
"Client", pcmk__client_name(client), \
(client)->flags, (flags_to_set), #flags_to_set); \
} while (0)
#define pcmk__clear_client_flags(client, flags_to_clear) do { \
(client)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
LOG_TRACE, \
"Client", pcmk__client_name(client), \
(client)->flags, (flags_to_clear), #flags_to_clear); \
} while (0)
#define pcmk__set_ipc_flags(ipc_flags, ipc_name, flags_to_set) do { \
ipc_flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
"IPC", (ipc_name), \
(ipc_flags), (flags_to_set), \
#flags_to_set); \
} while (0)
#define pcmk__clear_ipc_flags(ipc_flags, ipc_name, flags_to_clear) do { \
ipc_flags = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, \
"IPC", (ipc_name), \
(ipc_flags), (flags_to_clear), \
#flags_to_clear); \
} while (0)
guint pcmk__ipc_client_count(void);
void pcmk__foreach_ipc_client(GHFunc func, gpointer user_data);
void pcmk__client_cleanup(void);
pcmk__client_t *pcmk__find_client(const qb_ipcs_connection_t *c);
pcmk__client_t *pcmk__find_client_by_id(const char *id);
const char *pcmk__client_name(const pcmk__client_t *c);
const char *pcmk__client_type_str(uint64_t client_type);
pcmk__client_t *pcmk__new_unauth_client(void *key);
pcmk__client_t *pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid);
void pcmk__free_client(pcmk__client_t *c);
void pcmk__drop_all_clients(qb_ipcs_service_t *s);
void pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax);
xmlNode *pcmk__ipc_create_ack_as(const char *function, int line, uint32_t flags,
const char *tag, const char *ver, crm_exit_t status);
#define pcmk__ipc_create_ack(flags, tag, ver, st) \
pcmk__ipc_create_ack_as(__func__, __LINE__, (flags), (tag), (ver), (st))
int pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c,
uint32_t request, uint32_t flags, const char *tag,
const char *ver, crm_exit_t status);
#define pcmk__ipc_send_ack(c, req, flags, tag, ver, st) \
pcmk__ipc_send_ack_as(__func__, __LINE__, (c), (req), (flags), (tag), (ver), (st))
int pcmk__ipc_prepare_iov(uint32_t request, const xmlNode *message,
uint32_t max_send_size,
struct iovec **result, ssize_t *bytes);
int pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request,
const xmlNode *message, uint32_t flags);
int pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags);
xmlNode *pcmk__client_data2xml(pcmk__client_t *c, void *data,
uint32_t *id, uint32_t *flags);
int pcmk__client_pid(qb_ipcs_connection_t *c);
void pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs,
struct qb_ipcs_service_handlers *cb);
void pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs,
struct qb_ipcs_service_handlers *cb);
void pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs,
struct qb_ipcs_service_handlers *cb);
qb_ipcs_service_t *pcmk__serve_schedulerd_ipc(struct qb_ipcs_service_handlers *cb);
qb_ipcs_service_t *pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb);
void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro,
qb_ipcs_service_t **ipcs_rw,
qb_ipcs_service_t **ipcs_shm,
struct qb_ipcs_service_handlers *ro_cb,
struct qb_ipcs_service_handlers *rw_cb);
void pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro,
qb_ipcs_service_t *ipcs_rw,
qb_ipcs_service_t *ipcs_shm);
static inline const char *
pcmk__ipc_sys_name(const char *ipc_name, const char *fallback)
{
return ipc_name ? ipc_name : ((crm_system_name ? crm_system_name : fallback));
}
const char *pcmk__pcmkd_state_enum2friendly(enum pcmk_pacemakerd_state state);
const char *pcmk__controld_api_reply2str(enum pcmk_controld_api_reply reply);
const char *pcmk__pcmkd_api_reply2str(enum pcmk_pacemakerd_api_reply reply);
#ifdef __cplusplus
}
#endif
#endif // PCMK__CRM_COMMON_IPC_INTERNAL__H
diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h
index 39441508f3..4ef15b4b8d 100644
--- a/include/crm/common/options_internal.h
+++ b/include/crm/common/options_internal.h
@@ -1,247 +1,251 @@
/*
* Copyright 2006-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_COMMON_OPTIONS_INTERNAL__H
#define PCMK__CRM_COMMON_OPTIONS_INTERNAL__H
#ifndef PCMK__CONFIG_H
#define PCMK__CONFIG_H
#include <config.h> // _Noreturn
#endif
#include <glib.h> // GHashTable
#include <stdbool.h> // bool
#include <crm/common/strings.h> // pcmk_parse_interval_spec()
#include <crm/common/output_internal.h> // pcmk__output_t
#ifdef __cplusplus
extern "C" {
#endif
_Noreturn void pcmk__cli_help(char cmd);
/*
* Environment variable option handling
*/
const char *pcmk__env_option(const char *option);
void pcmk__set_env_option(const char *option, const char *value, bool compat);
bool pcmk__env_option_enabled(const char *daemon, const char *option);
/*
* Cluster option handling
*/
/*!
* \internal
* \enum pcmk__opt_flags
* \brief Option flags
*/
enum pcmk__opt_flags {
pcmk__opt_none = 0U, //!< No additional information
/*!
* \brief In CIB manager metadata
*
* \deprecated This flag will be removed with CIB manager metadata
*/
pcmk__opt_based = (1U << 0),
/*!
* \brief In controller metadata
*
* \deprecated This flag will be removed with controller metadata
*/
pcmk__opt_controld = (1U << 1),
/*!
* \brief In scheduler metadata
*
* \deprecated This flag will be removed with scheduler metadata
*/
pcmk__opt_schedulerd = (1U << 2),
pcmk__opt_advanced = (1U << 3), //!< Advanced use only
pcmk__opt_generated = (1U << 4), //!< Generated by Pacemaker
pcmk__opt_deprecated = (1U << 5), //!< Option is deprecated
pcmk__opt_fencing = (1U << 6), //!< Common fencing resource parameter
pcmk__opt_primitive = (1U << 7), //!< Primitive resource meta-attribute
};
typedef struct pcmk__cluster_option_s {
const char *name;
const char *alt_name;
const char *type;
const char *values;
const char *default_value;
bool (*is_valid)(const char *);
uint32_t flags; //!< Group of <tt>enum pcmk__opt_flags</tt>
const char *description_short;
const char *description_long;
} pcmk__cluster_option_t;
const char *pcmk__cluster_option(GHashTable *options, const char *name);
int pcmk__output_cluster_options(pcmk__output_t *out, const char *name,
const char *desc_short, const char *desc_long,
uint32_t filter, bool all);
int pcmk__output_fencing_params(pcmk__output_t *out, const char *name,
const char *desc_short, const char *desc_long,
bool all);
int pcmk__output_primitive_meta(pcmk__output_t *out, const char *name,
const char *desc_short, const char *desc_long,
bool all);
int pcmk__daemon_metadata(pcmk__output_t *out, const char *name,
const char *short_desc, const char *long_desc,
enum pcmk__opt_flags filter);
void pcmk__validate_cluster_options(GHashTable *options);
bool pcmk__valid_interval_spec(const char *value);
bool pcmk__valid_boolean(const char *value);
bool pcmk__valid_int(const char *value);
bool pcmk__valid_positive_int(const char *value);
bool pcmk__valid_no_quorum_policy(const char *value);
bool pcmk__valid_percentage(const char *value);
bool pcmk__valid_placement_strategy(const char *value);
// from watchdog.c
long pcmk__get_sbd_watchdog_timeout(void);
bool pcmk__get_sbd_sync_resource_startup(void);
long pcmk__auto_stonith_watchdog_timeout(void);
bool pcmk__valid_stonith_watchdog_timeout(const char *value);
// Constants for environment variable names
#define PCMK__ENV_AUTHKEY_LOCATION "authkey_location"
#define PCMK__ENV_BLACKBOX "blackbox"
+#define PCMK__ENV_CA_FILE "ca_file"
#define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled"
+#define PCMK__ENV_CERT_FILE "cert_file"
#define PCMK__ENV_CLUSTER_TYPE "cluster_type"
+#define PCMK__ENV_CRL_FILE "crl_file"
#define PCMK__ENV_DEBUG "debug"
#define PCMK__ENV_DH_MAX_BITS "dh_max_bits"
#define PCMK__ENV_FAIL_FAST "fail_fast"
#define PCMK__ENV_IPC_BUFFER "ipc_buffer"
#define PCMK__ENV_IPC_TYPE "ipc_type"
+#define PCMK__ENV_KEY_FILE "key_file"
#define PCMK__ENV_LOGFACILITY "logfacility"
#define PCMK__ENV_LOGFILE "logfile"
#define PCMK__ENV_LOGFILE_MODE "logfile_mode"
#define PCMK__ENV_LOGPRIORITY "logpriority"
#define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit"
#define PCMK__ENV_NODE_START_STATE "node_start_state"
#define PCMK__ENV_PANIC_ACTION "panic_action"
#define PCMK__ENV_REMOTE_ADDRESS "remote_address"
#define PCMK__ENV_REMOTE_SCHEMA_DIRECTORY "remote_schema_directory"
#define PCMK__ENV_REMOTE_PID1 "remote_pid1"
#define PCMK__ENV_REMOTE_PORT "remote_port"
#define PCMK__ENV_RESPAWNED "respawned"
#define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory"
#define PCMK__ENV_SERVICE "service"
#define PCMK__ENV_STDERR "stderr"
#define PCMK__ENV_TLS_PRIORITIES "tls_priorities"
#define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox"
#define PCMK__ENV_TRACE_FILES "trace_files"
#define PCMK__ENV_TRACE_FORMATS "trace_formats"
#define PCMK__ENV_TRACE_FUNCTIONS "trace_functions"
#define PCMK__ENV_TRACE_TAGS "trace_tags"
#define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled"
// Constants for meta-attribute names
#define PCMK__META_CLONE "clone"
#define PCMK__META_CONTAINER "container"
#define PCMK__META_DIGESTS_ALL "digests-all"
#define PCMK__META_DIGESTS_SECURE "digests-secure"
#define PCMK__META_INTERNAL_RSC "internal_rsc"
#define PCMK__META_MIGRATE_SOURCE "migrate_source"
#define PCMK__META_MIGRATE_TARGET "migrate_target"
#define PCMK__META_ON_NODE "on_node"
#define PCMK__META_ON_NODE_UUID "on_node_uuid"
#define PCMK__META_OP_NO_WAIT "op_no_wait"
#define PCMK__META_OP_TARGET_RC "op_target_rc"
#define PCMK__META_PHYSICAL_HOST "physical-host"
#define PCMK__META_STONITH_ACTION "stonith_action"
/* @TODO Plug these in. Currently, they're never set. These are op attrs for use
* with https://projects.clusterlabs.org/T382.
*/
#define PCMK__META_CLEAR_FAILURE_OP "clear_failure_op"
#define PCMK__META_CLEAR_FAILURE_INTERVAL "clear_failure_interval"
// @COMPAT Deprecated alias for PCMK__META_PROMOTED_MAX since 2.0.0
#define PCMK__META_PROMOTED_MAX_LEGACY "master-max"
// @COMPAT Deprecated alias for PCMK__META_PROMOTED_NODE_MAX since 2.0.0
#define PCMK__META_PROMOTED_NODE_MAX_LEGACY "master-node-max"
// Constants for enumerated values
#define PCMK__VALUE_ATTRD "attrd"
#define PCMK__VALUE_BOLD "bold"
#define PCMK__VALUE_BROADCAST "broadcast"
#define PCMK__VALUE_CIB "cib"
#define PCMK__VALUE_CIB_DIFF_NOTIFY "cib_diff_notify"
#define PCMK__VALUE_CIB_NOTIFY "cib_notify"
#define PCMK__VALUE_CIB_POST_NOTIFY "cib_post_notify"
#define PCMK__VALUE_CIB_PRE_NOTIFY "cib_pre_notify"
#define PCMK__VALUE_CIB_UPDATE_CONFIRMATION "cib_update_confirmation"
#define PCMK__VALUE_CLUSTER "cluster"
#define PCMK__VALUE_CRMD "crmd"
#define PCMK__VALUE_EN "en"
#define PCMK__VALUE_EPOCH "epoch"
#define PCMK__VALUE_HEALTH_RED "health_red"
#define PCMK__VALUE_HEALTH_YELLOW "health_yellow"
#define PCMK__VALUE_INIT "init"
#define PCMK__VALUE_LOCAL "local"
#define PCMK__VALUE_LOST "lost"
#define PCMK__VALUE_LRMD "lrmd"
#define PCMK__VALUE_MAINT "maint"
#define PCMK__VALUE_OUTPUT "output"
#define PCMK__VALUE_PASSWORD "password"
#define PCMK__VALUE_PRIMITIVE "primitive"
#define PCMK__VALUE_REFRESH "refresh"
#define PCMK__VALUE_REQUEST "request"
#define PCMK__VALUE_RESPONSE "response"
#define PCMK__VALUE_RSC_FAILED "rsc-failed"
#define PCMK__VALUE_RSC_FAILURE_IGNORED "rsc-failure-ignored"
#define PCMK__VALUE_RSC_MANAGED "rsc-managed"
#define PCMK__VALUE_RSC_MULTIPLE "rsc-multiple"
#define PCMK__VALUE_RSC_OK "rsc-ok"
#define PCMK__VALUE_RUNNING "running"
#define PCMK__VALUE_SCHEDULER "scheduler"
#define PCMK__VALUE_SHUTDOWN_COMPLETE "shutdown_complete"
#define PCMK__VALUE_SHUTTING_DOWN "shutting_down"
#define PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE "st-async-timeout-value"
#define PCMK__VALUE_ST_NOTIFY "st_notify"
#define PCMK__VALUE_ST_NOTIFY_DISCONNECT "st_notify_disconnect"
#define PCMK__VALUE_ST_NOTIFY_FENCE "st_notify_fence"
#define PCMK__VALUE_ST_NOTIFY_HISTORY "st_notify_history"
#define PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED "st_notify_history_synced"
#define PCMK__VALUE_STARTING_DAEMONS "starting_daemons"
#define PCMK__VALUE_STONITH_NG "stonith-ng"
#define PCMK__VALUE_WAIT_FOR_PING "wait_for_ping"
#define PCMK__VALUE_WARNING "warning"
/* @COMPAT Deprecated since 2.1.7 (used with PCMK__XA_ORDERING attribute of
* resource sets)
*/
#define PCMK__VALUE_GROUP "group"
// @COMPAT Drop when daemon metadata commands are dropped
#define PCMK__VALUE_TIME "time"
#ifdef __cplusplus
}
#endif
#endif // PCMK__OPTIONS_INTERNAL__H
diff --git a/include/crm/common/remote_internal.h b/include/crm/common/remote_internal.h
index cf3064cdc1..2f00183d62 100644
--- a/include/crm/common/remote_internal.h
+++ b/include/crm/common/remote_internal.h
@@ -1,128 +1,92 @@
/*
* Copyright 2008-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_COMMON_REMOTE_INTERNAL__H
#define PCMK__CRM_COMMON_REMOTE_INTERNAL__H
#include <stdio.h> // NULL
#include <stdbool.h> // bool
-#include <gnutls/gnutls.h> // gnutls_session_t, etc.
#include <libxml/tree.h> // xmlNode
-#include <gnutls/gnutls.h> // gnutls_session_t, gnutls_dh_params_t, etc.
#include <crm/common/ipc_internal.h> // pcmk__client_t
#include <crm/common/nodes_internal.h> // pcmk__node_variant_remote, etc.
#include <crm/common/resources_internal.h> // struct pcmk__remote_private
#include <crm/common/scheduler_types.h> // pcmk_node_t
#ifdef __cplusplus
extern "C" {
#endif
// internal functions from remote.c
typedef struct pcmk__remote_s pcmk__remote_t;
int pcmk__remote_send_xml(pcmk__remote_t *remote, const xmlNode *msg);
int pcmk__remote_ready(const pcmk__remote_t *remote, int timeout_ms);
int pcmk__read_available_remote_data(pcmk__remote_t *remote);
int pcmk__read_remote_message(pcmk__remote_t *remote, int timeout_ms);
xmlNode *pcmk__remote_message_xml(pcmk__remote_t *remote);
int pcmk__connect_remote(const char *host, int port, int timeout_ms,
int *timer_id, int *sock_fd, void *userdata,
void (*callback) (void *userdata, int rc, int sock));
int pcmk__accept_remote_connection(int ssock, int *csock);
void pcmk__sockaddr2str(const void *sa, char *s);
/*!
* \internal
* \brief Check whether a node is a Pacemaker Remote node of any kind
*
* \param[in] node Node to check
*
* \return true if \p node is a remote, guest, or bundle node, otherwise false
*/
static inline bool
pcmk__is_pacemaker_remote_node(const pcmk_node_t *node)
{
return (node != NULL)
&& (node->priv->variant == pcmk__node_variant_remote);
}
/*!
* \internal
* \brief Check whether a node is a remote node
*
* \param[in] node Node to check
*
* \return true if \p node is a remote node, otherwise false
*/
static inline bool
pcmk__is_remote_node(const pcmk_node_t *node)
{
return pcmk__is_pacemaker_remote_node(node)
&& ((node->priv->remote == NULL)
|| (node->priv->remote->priv->launcher == NULL));
}
/*!
* \internal
* \brief Check whether a node is a guest or bundle node
*
* \param[in] node Node to check
*
* \return true if \p node is a guest or bundle node, otherwise false
*/
static inline bool
pcmk__is_guest_or_bundle_node(const pcmk_node_t *node)
{
return pcmk__is_pacemaker_remote_node(node)
&& (node->priv->remote != NULL)
&& (node->priv->remote->priv->launcher != NULL);
}
-gnutls_session_t *pcmk__new_tls_session(int csock, unsigned int conn_type,
- gnutls_credentials_type_t cred_type,
- void *credentials);
-int pcmk__init_tls_dh(gnutls_dh_params_t *dh_params);
-int pcmk__read_handshake_data(const pcmk__client_t *client);
-
-/*!
- * \internal
- * \brief Make a single attempt to perform the client TLS handshake
- *
- * \param[in,out] remote Newly established remote connection
- * \param[out] gnutls_rc If this is non-NULL, it will be set to the GnuTLS
- * rc (for logging) if this function returns EPROTO,
- * otherwise GNUTLS_E_SUCCESS
- *
- * \return Standard Pacemaker return code
- */
-int pcmk__tls_client_try_handshake(pcmk__remote_t *remote, int *gnutls_rc);
-
-/*!
- * \internal
- * \brief Perform client TLS handshake after establishing TCP socket
- *
- * \param[in,out] remote Newly established remote connection
- * \param[in] timeout_sec Abort handshake if not completed within this time
- * \param[out] gnutls_rc If this is non-NULL, it will be set to the GnuTLS
- * rc (for logging) if this function returns EPROTO,
- * otherwise GNUTLS_E_SUCCESS
- *
- * \return Standard Pacemaker return code
- */
-int pcmk__tls_client_handshake(pcmk__remote_t *remote, int timeout_sec,
- int *gnutls_rc);
-
#ifdef __cplusplus
}
#endif
#endif // PCMK__CRM_COMMON_REMOTE_INTERNAL__H
diff --git a/include/crm/common/tls_internal.h b/include/crm/common/tls_internal.h
new file mode 100644
index 0000000000..68abccd5ff
--- /dev/null
+++ b/include/crm/common/tls_internal.h
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2024 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PCMK__CRM_COMMON_TLS_INTERNAL__H
+#define PCMK__CRM_COMMON_TLS_INTERNAL__H
+
+#include <gnutls/gnutls.h> // gnutls_session_t, gnutls_dh_params_t, etc.
+
+#include <crm/common/ipc_internal.h> // pcmk__client_t
+#include <crm/common/remote_internal.h> // pcmk__remote_t
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+ bool server;
+ gnutls_dh_params_t dh_params;
+ gnutls_credentials_type_t cred_type;
+
+ const char *ca_file;
+ const char *cert_file;
+ const char *crl_file;
+ const char *key_file;
+
+ union {
+ gnutls_anon_server_credentials_t anon_s;
+ gnutls_anon_client_credentials_t anon_c;
+ gnutls_certificate_credentials_t cert;
+ gnutls_psk_server_credentials_t psk_s;
+ gnutls_psk_client_credentials_t psk_c;
+ } credentials;
+} pcmk__tls_t;
+
+/*!
+ * \internal
+ * \brief Free a previously allocated \p pcmk__tls_t object
+ *
+ * \param[in,out] tls The object to free
+ */
+void pcmk__free_tls(pcmk__tls_t *tls);
+
+/*!
+ * \internal
+ * \brief Initialize a new TLS object
+ *
+ * Unlike \p pcmk__new_tls_session, this function is used for creating the
+ * global environment for TLS connections.
+ *
+ * \param[in,out] tls The object to be allocated and initialized
+ * \param[in] server Is this a server or not?
+ * \param[in] cred_type What type of gnutls credentials are in use?
+ * (GNUTLS_CRD_* constants)
+ *
+ * \returns Standard Pacemaker return code
+ */
+int pcmk__init_tls(pcmk__tls_t **tls, bool server,
+ gnutls_credentials_type_t cred_type);
+
+/*!
+ * \internal
+ * \brief Initialize Diffie-Hellman parameters for a TLS server
+ *
+ * \param[out] dh_params Parameter object to initialize
+ *
+ * \return Standard Pacemaker return code
+ * \todo The current best practice is to allow the client and server to
+ * negotiate the Diffie-Hellman parameters via a TLS extension (RFC 7919).
+ * However, we have to support both older versions of GnuTLS (<3.6) that
+ * don't support the extension on our side, and older Pacemaker versions
+ * that don't support the extension on the other side. The next best
+ * practice would be to use a known good prime (see RFC 5114 section 2.2),
+ * possibly stored in a file distributed with Pacemaker.
+ */
+int pcmk__init_tls_dh(gnutls_dh_params_t *dh_params);
+
+/*!
+ * \internal
+ * \brief Initialize a new TLS session
+ *
+ * \param[in] tls A TLS environment object
+ * \param[in] csock Connected socket for TLS session
+ *
+ * \return Pointer to newly created session object, or NULL on error
+ */
+gnutls_session_t pcmk__new_tls_session(pcmk__tls_t *tls, int csock);
+
+/*!
+ * \internal
+ * \brief Add the client PSK key to the TLS environment
+ *
+ * This function must be called for all TLS clients that are using PSK for
+ * authentication.
+ *
+ * \param[in,out] tls The TLS environment
+ * \param[in] key The client's PSK key
+ */
+void pcmk__tls_add_psk_key(pcmk__tls_t *tls, gnutls_datum_t *key);
+
+/*!
+ * \internal
+ * \brief Register the server's PSK credential fetching callback
+ *
+ * This function must be called for all TLS servers that are using PSK for
+ * authentication.
+ *
+ * \param[in,out] tls The TLS environment
+ * \param[in] cb The server's PSK credential fetching callback
+ */
+void pcmk__tls_add_psk_callback(pcmk__tls_t *tls,
+ gnutls_psk_server_credentials_function *cb);
+
+/*!
+ * \internal
+ * \brief Process handshake data from TLS client
+ *
+ * Read as much TLS handshake data as is available.
+ *
+ * \param[in] client Client connection
+ *
+ * \return Standard Pacemaker return code (of particular interest, EAGAIN
+ * if some data was successfully read but more data is needed)
+ */
+int pcmk__read_handshake_data(const pcmk__client_t *client);
+
+/*!
+ * \internal
+ * \brief Log if a TLS certificate is near its expiration date
+ *
+ * \param[in] session The gnutls session object after handshaking is
+ * complete
+ */
+void pcmk__tls_check_cert_expiration(gnutls_session_t session);
+
+/*!
+ * \internal
+ * \brief Perform client TLS handshake after establishing TCP socket
+ *
+ * \param[in,out] remote Newly established remote connection
+ * \param[in] timeout_sec Abort handshake if not completed within this time
+ * \param[out] gnutls_rc If this is non-NULL, it will be set to the GnuTLS
+ * rc (for logging) if this function returns EPROTO,
+ * otherwise GNUTLS_E_SUCCESS
+ *
+ * \return Standard Pacemaker return code
+ */
+int pcmk__tls_client_handshake(pcmk__remote_t *remote, int timeout_sec,
+ int *gnutls_rc);
+
+/*!
+ * \internal
+ * \brief Make a single attempt to perform the client TLS handshake
+ *
+ * \param[in,out] remote Newly established remote connection
+ * \param[out] gnutls_rc If this is non-NULL, it will be set to the GnuTLS
+ * rc (for logging) if this function returns EPROTO,
+ * otherwise GNUTLS_E_SUCCESS
+ *
+ * \return Standard Pacemaker return code
+ */
+int pcmk__tls_client_try_handshake(pcmk__remote_t *remote, int *gnutls_rc);
+
+/*!
+ * \internal
+ * \brief Is X509 authentication supported by the environment?
+ *
+ * \param[in] server Is this a server?
+ *
+ * \return true if the appropriate environment variables are set (see
+ * etc/sysconfig/pacemaker.in), otherwise false
+ */
+bool pcmk__x509_enabled(bool server);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // PCMK__CRM_COMMON_TLS_INTERNAL__H
diff --git a/include/crm/common/util.h b/include/crm/common/util.h
index 868ab96d0f..d656c54b67 100644
--- a/include/crm/common/util.h
+++ b/include/crm/common/util.h
@@ -1,97 +1,96 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_COMMON_UTIL__H
#define PCMK__CRM_COMMON_UTIL__H
#include <sys/types.h> // gid_t, mode_t, size_t, time_t, uid_t
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h> // uint32_t
#include <limits.h>
#include <signal.h>
#include <glib.h>
#include <crm/common/acl.h>
#include <crm/common/actions.h>
#include <crm/common/agents.h>
#include <crm/common/results.h>
#include <crm/common/scores.h>
#include <crm/common/strings.h>
#include <crm/common/nvpair.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief Utility functions
* \ingroup core
*/
/* public node attribute functions (from attrs.c) */
char *pcmk_promotion_score_name(const char *rsc_id);
/* public Pacemaker Remote functions (from remote.c) */
int crm_default_remote_port(void);
int compare_version(const char *version1, const char *version2);
/*!
* \brief Check whether any of specified flags are set in a flag group
*
* \param[in] flag_group The flag group being examined
* \param[in] flags_to_check Which flags in flag_group should be checked
*
* \return true if \p flags_to_check is nonzero and any of its flags are set in
* \p flag_group, or false otherwise
*/
static inline bool
pcmk_any_flags_set(uint64_t flag_group, uint64_t flags_to_check)
{
return (flag_group & flags_to_check) != 0;
}
/*!
* \brief Check whether all of specified flags are set in a flag group
*
* \param[in] flag_group The flag group being examined
* \param[in] flags_to_check Which flags in flag_group should be checked
*
* \return true if \p flags_to_check is zero or all of its flags are set in
* \p flag_group, or false otherwise
*/
static inline bool
pcmk_all_flags_set(uint64_t flag_group, uint64_t flags_to_check)
{
return (flag_group & flags_to_check) == flags_to_check;
}
/*!
* \brief Convenience alias for pcmk_all_flags_set(), to check single flag
*/
#define pcmk_is_set(g, f) pcmk_all_flags_set((g), (f))
void pcmk_common_cleanup(void);
char *crm_md5sum(const char *buffer);
char *crm_generate_uuid(void);
int crm_user_lookup(const char *name, uid_t * uid, gid_t * gid);
int pcmk_daemon_user(uid_t *uid, gid_t *gid);
-void crm_gnutls_global_init(void);
#ifdef __cplusplus
}
#endif
#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
#include <crm/common/util_compat.h>
#endif
#endif
diff --git a/include/crm/common/util_compat.h b/include/crm/common/util_compat.h
index 8576977257..ebcefcb9e9 100644
--- a/include/crm/common/util_compat.h
+++ b/include/crm/common/util_compat.h
@@ -1,44 +1,47 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_COMMON_UTIL_COMPAT__H
#define PCMK__CRM_COMMON_UTIL_COMPAT__H
#include <stdbool.h> // bool
#include <glib.h> // gboolean
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief Deprecated Pacemaker utilities
* \ingroup core
* \deprecated Do not include this header directly. The utilities in this
* header, and the header itself, will be removed in a future
* release.
*/
+//! \deprecated Use gnutls_global_init() instead
+void crm_gnutls_global_init(void);
+
//! \deprecated Do not use (will be dropped in a future release)
bool crm_is_daemon_name(const char *name);
// NOTE: sbd (as of at least 1.5.2) uses this
//! \deprecated Use pcmk_is_set() or pcmk_all_flags_set() instead
static inline gboolean
is_set(long long word, long long bit)
{
return ((word & bit) == bit);
}
#ifdef __cplusplus
}
#endif
#endif // PCMK__CRM_COMMON_UTIL_COMPAT__H
diff --git a/lib/cib/cib_client.c b/lib/cib/cib_client.c
index 1322fb5c0b..5cbbb141c0 100644
--- a/lib/cib/cib_client.c
+++ b/lib/cib/cib_client.c
@@ -1,769 +1,771 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <pwd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <glib.h>
#include <crm/crm.h>
#include <crm/cib/internal.h>
#include <crm/common/xml.h>
static GHashTable *cib_op_callback_table = NULL;
static gint
ciblib_GCompareFunc(gconstpointer a, gconstpointer b)
{
int rc = 0;
const cib_notify_client_t *a_client = a;
const cib_notify_client_t *b_client = b;
CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
rc = strcmp(a_client->event, b_client->event);
if (rc == 0) {
if (a_client->callback == b_client->callback) {
return 0;
} else if (((long)a_client->callback) < ((long)b_client->callback)) {
crm_trace("callbacks for %s are not equal: %p < %p",
a_client->event, a_client->callback, b_client->callback);
return -1;
}
crm_trace("callbacks for %s are not equal: %p > %p",
a_client->event, a_client->callback, b_client->callback);
return 1;
}
return rc;
}
static int
cib_client_add_notify_callback(cib_t * cib, const char *event,
void (*callback) (const char *event,
xmlNode * msg))
{
GList *list_item = NULL;
cib_notify_client_t *new_client = NULL;
if ((cib->variant != cib_native) && (cib->variant != cib_remote)) {
return -EPROTONOSUPPORT;
}
crm_trace("Adding callback for %s events (%d)",
event, g_list_length(cib->notify_list));
new_client = pcmk__assert_alloc(1, sizeof(cib_notify_client_t));
new_client->event = event;
new_client->callback = callback;
list_item = g_list_find_custom(cib->notify_list, new_client,
ciblib_GCompareFunc);
if (list_item != NULL) {
crm_warn("Callback already present");
free(new_client);
return -EINVAL;
} else {
cib->notify_list = g_list_append(cib->notify_list, new_client);
cib->cmds->register_notification(cib, event, 1);
crm_trace("Callback added (%d)", g_list_length(cib->notify_list));
}
return pcmk_ok;
}
static int
get_notify_list_event_count(cib_t *cib, const char *event)
{
int count = 0;
for (GList *iter = g_list_first(cib->notify_list); iter != NULL;
iter = iter->next) {
cib_notify_client_t *client = (cib_notify_client_t *) iter->data;
if (strcmp(client->event, event) == 0) {
count++;
}
}
crm_trace("event(%s) count : %d", event, count);
return count;
}
static int
cib_client_del_notify_callback(cib_t *cib, const char *event,
void (*callback) (const char *event,
xmlNode *msg))
{
GList *list_item = NULL;
cib_notify_client_t *new_client = NULL;
if (cib->variant != cib_native && cib->variant != cib_remote) {
return -EPROTONOSUPPORT;
}
if (get_notify_list_event_count(cib, event) == 0) {
crm_debug("The callback of the event does not exist(%s)", event);
return pcmk_ok;
}
crm_debug("Removing callback for %s events", event);
new_client = pcmk__assert_alloc(1, sizeof(cib_notify_client_t));
new_client->event = event;
new_client->callback = callback;
list_item = g_list_find_custom(cib->notify_list, new_client, ciblib_GCompareFunc);
if (list_item != NULL) {
cib_notify_client_t *list_client = list_item->data;
cib->notify_list = g_list_remove(cib->notify_list, list_client);
free(list_client);
crm_trace("Removed callback");
} else {
crm_trace("Callback not present");
}
if (get_notify_list_event_count(cib, event) == 0) {
/* When there is not the registration of the event, the processing turns off a notice. */
cib->cmds->register_notification(cib, event, 0);
}
free(new_client);
return pcmk_ok;
}
static gboolean
cib_async_timeout_handler(gpointer data)
{
struct timer_rec_s *timer = data;
crm_debug("Async call %d timed out after %ds",
timer->call_id, timer->timeout);
cib_native_callback(timer->cib, NULL, timer->call_id, -ETIME);
// We remove the handler in remove_cib_op_callback()
return G_SOURCE_CONTINUE;
}
static gboolean
cib_client_register_callback_full(cib_t *cib, int call_id, int timeout,
gboolean only_success, void *user_data,
const char *callback_name,
void (*callback)(xmlNode *, int, int,
xmlNode *, void *),
void (*free_func)(void *))
{
cib_callback_client_t *blob = NULL;
if (call_id < 0) {
if (only_success == FALSE) {
callback(NULL, call_id, call_id, NULL, user_data);
} else {
crm_warn("CIB call failed: %s", pcmk_strerror(call_id));
}
if (user_data && free_func) {
free_func(user_data);
}
return FALSE;
}
blob = pcmk__assert_alloc(1, sizeof(cib_callback_client_t));
blob->id = callback_name;
blob->only_success = only_success;
blob->user_data = user_data;
blob->callback = callback;
blob->free_func = free_func;
if (timeout > 0) {
struct timer_rec_s *async_timer =
pcmk__assert_alloc(1, sizeof(struct timer_rec_s));
blob->timer = async_timer;
async_timer->cib = cib;
async_timer->call_id = call_id;
async_timer->timeout = timeout * 1000;
async_timer->ref = pcmk__create_timer(async_timer->timeout,
cib_async_timeout_handler,
async_timer);
}
crm_trace("Adding callback %s for call %d", callback_name, call_id);
pcmk__intkey_table_insert(cib_op_callback_table, call_id, blob);
return TRUE;
}
static gboolean
cib_client_register_callback(cib_t *cib, int call_id, int timeout,
gboolean only_success, void *user_data,
const char *callback_name,
void (*callback) (xmlNode *, int, int, xmlNode *,
void *))
{
return cib_client_register_callback_full(cib, call_id, timeout,
only_success, user_data,
callback_name, callback, NULL);
}
static int
cib_client_noop(cib_t * cib, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_NOOP, NULL, NULL, NULL, NULL,
call_options, cib->user);
}
static int
cib_client_ping(cib_t * cib, xmlNode ** output_data, int call_options)
{
return cib_internal_op(cib, CRM_OP_PING, NULL, NULL, NULL, output_data,
call_options, cib->user);
}
static int
cib_client_query(cib_t * cib, const char *section, xmlNode ** output_data, int call_options)
{
return cib->cmds->query_from(cib, NULL, section, output_data, call_options);
}
static int
cib_client_query_from(cib_t * cib, const char *host, const char *section,
xmlNode ** output_data, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_QUERY, host, section, NULL,
output_data, call_options, cib->user);
}
static int
set_secondary(cib_t *cib, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_SECONDARY, NULL, NULL, NULL,
NULL, call_options, cib->user);
}
static int
set_primary(cib_t *cib, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_PRIMARY, NULL, NULL, NULL,
NULL, call_options, cib->user);
}
static int
cib_client_bump_epoch(cib_t * cib, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_BUMP, NULL, NULL, NULL, NULL,
call_options, cib->user);
}
static int
cib_client_upgrade(cib_t * cib, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_UPGRADE, NULL, NULL, NULL,
NULL, call_options, cib->user);
}
static int
cib_client_sync(cib_t * cib, const char *section, int call_options)
{
return cib->cmds->sync_from(cib, NULL, section, call_options);
}
static int
cib_client_sync_from(cib_t * cib, const char *host, const char *section, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_SYNC_TO_ALL, host, section,
NULL, NULL, call_options, cib->user);
}
static int
cib_client_create(cib_t * cib, const char *section, xmlNode * data, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_CREATE, NULL, section, data,
NULL, call_options, cib->user);
}
static int
cib_client_modify(cib_t * cib, const char *section, xmlNode * data, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_MODIFY, NULL, section, data,
NULL, call_options, cib->user);
}
static int
cib_client_replace(cib_t * cib, const char *section, xmlNode * data, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_REPLACE, NULL, section, data,
NULL, call_options, cib->user);
}
static int
cib_client_delete(cib_t * cib, const char *section, xmlNode * data, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_DELETE, NULL, section, data,
NULL, call_options, cib->user);
}
static int
cib_client_erase(cib_t * cib, xmlNode ** output_data, int call_options)
{
return cib_internal_op(cib, PCMK__CIB_REQUEST_ERASE, NULL, NULL, NULL,
output_data, call_options, cib->user);
}
static int
cib_client_init_transaction(cib_t *cib)
{
int rc = pcmk_rc_ok;
if (cib == NULL) {
return -EINVAL;
}
if (cib->transaction != NULL) {
// A client can have at most one transaction at a time
rc = pcmk_rc_already;
}
if (rc == pcmk_rc_ok) {
cib->transaction = pcmk__xe_create(NULL, PCMK__XE_CIB_TRANSACTION);
}
if (rc != pcmk_rc_ok) {
const char *client_id = NULL;
cib->cmds->client_id(cib, NULL, &client_id);
crm_err("Failed to initialize CIB transaction for client %s: %s",
client_id, pcmk_rc_str(rc));
}
return pcmk_rc2legacy(rc);
}
static int
cib_client_end_transaction(cib_t *cib, bool commit, int call_options)
{
const char *client_id = NULL;
int rc = pcmk_ok;
if (cib == NULL) {
return -EINVAL;
}
cib->cmds->client_id(cib, NULL, &client_id);
client_id = pcmk__s(client_id, "(unidentified)");
if (commit) {
if (cib->transaction == NULL) {
rc = pcmk_rc_no_transaction;
crm_err("Failed to commit transaction for CIB client %s: %s",
client_id, pcmk_rc_str(rc));
return pcmk_rc2legacy(rc);
}
rc = cib_internal_op(cib, PCMK__CIB_REQUEST_COMMIT_TRANSACT, NULL, NULL,
cib->transaction, NULL, call_options, cib->user);
} else {
// Discard always succeeds
if (cib->transaction != NULL) {
crm_trace("Discarded transaction for CIB client %s", client_id);
} else {
crm_trace("No transaction found for CIB client %s", client_id);
}
}
pcmk__xml_free(cib->transaction);
cib->transaction = NULL;
return rc;
}
static int
cib_client_fetch_schemas(cib_t *cib, xmlNode **output_data, const char *after_ver,
int call_options)
{
xmlNode *data = pcmk__xe_create(NULL, PCMK__XA_SCHEMA);
int rc = pcmk_ok;
crm_xml_add(data, PCMK_XA_VERSION, after_ver);
rc = cib_internal_op(cib, PCMK__CIB_REQUEST_SCHEMAS, NULL, NULL, data,
output_data, call_options, NULL);
pcmk__xml_free(data);
return rc;
}
static void
cib_client_set_user(cib_t *cib, const char *user)
{
pcmk__str_update(&(cib->user), user);
}
static void
cib_destroy_op_callback(gpointer data)
{
cib_callback_client_t *blob = data;
if (blob->timer && blob->timer->ref > 0) {
g_source_remove(blob->timer->ref);
}
free(blob->timer);
if (blob->user_data && blob->free_func) {
blob->free_func(blob->user_data);
}
free(blob);
}
static void
destroy_op_callback_table(void)
{
if (cib_op_callback_table != NULL) {
g_hash_table_destroy(cib_op_callback_table);
cib_op_callback_table = NULL;
}
}
char *
get_shadow_file(const char *suffix)
{
char *cib_home = NULL;
char *fullname = NULL;
char *name = crm_strdup_printf("shadow.%s", suffix);
const char *dir = getenv("CIB_shadow_dir");
if (dir == NULL) {
uid_t uid = geteuid();
struct passwd *pwent = getpwuid(uid);
const char *user = NULL;
if (pwent) {
user = pwent->pw_name;
} else {
user = getenv("USER");
crm_perror(LOG_ERR,
"Assuming %s because cannot get user details for user ID %d",
(user? user : "unprivileged user"), uid);
}
if (pcmk__strcase_any_of(user, "root", CRM_DAEMON_USER, NULL)) {
dir = CRM_CONFIG_DIR;
} else {
const char *home = NULL;
if ((home = getenv("HOME")) == NULL) {
if (pwent) {
home = pwent->pw_dir;
}
}
dir = pcmk__get_tmpdir();
if (home && home[0] == '/') {
int rc = 0;
cib_home = crm_strdup_printf("%s/.cib", home);
rc = mkdir(cib_home, 0700);
if (rc < 0 && errno != EEXIST) {
crm_perror(LOG_ERR, "Couldn't create user-specific shadow directory: %s",
cib_home);
errno = 0;
} else {
dir = cib_home;
}
}
}
}
fullname = crm_strdup_printf("%s/%s", dir, name);
free(cib_home);
free(name);
return fullname;
}
cib_t *
cib_shadow_new(const char *shadow)
{
cib_t *new_cib = NULL;
char *shadow_file = NULL;
CRM_CHECK(shadow != NULL, return NULL);
shadow_file = get_shadow_file(shadow);
new_cib = cib_file_new(shadow_file);
free(shadow_file);
return new_cib;
}
/*!
* \brief Create a new CIB connection object, ignoring any active shadow CIB
*
* Create a new live, file, or remote CIB connection object based on the values
* of CIB-related environment variables (CIB_file, CIB_port, CIB_server,
* CIB_user, and CIB_passwd). The object will not be connected.
*
* \return Newly allocated CIB connection object
* \note The CIB API does not fully support opening multiple CIB connection
* objects simultaneously, so the returned object should be treated as a
* singleton.
*/
cib_t *
cib_new_no_shadow(void)
{
const char *shadow = getenv("CIB_shadow");
cib_t *cib = NULL;
unsetenv("CIB_shadow");
cib = cib_new();
if (shadow != NULL) {
setenv("CIB_shadow", shadow, 1);
}
return cib;
}
/*!
* \brief Create a new CIB connection object
*
* Create a new live, remote, file, or shadow file CIB connection object based
* on the values of CIB-related environment variables (CIB_shadow, CIB_file,
* CIB_port, CIB_server, CIB_user, and CIB_passwd). The object will not be
* connected.
*
* \return Newly allocated CIB connection object
* \note The CIB API does not fully support opening multiple CIB connection
* objects simultaneously, so the returned object should be treated as a
* singleton.
*/
/* @TODO Ensure all APIs support multiple simultaneous CIB connection objects
* (at least cib_free_callbacks() currently does not).
*/
cib_t *
cib_new(void)
{
const char *value = getenv("CIB_shadow");
+ const char *server = NULL;
+ const char *user = NULL;
+ const char *pass = NULL;
+ gboolean encrypted = TRUE;
int port;
if (!pcmk__str_empty(value)) {
return cib_shadow_new(value);
}
value = getenv("CIB_file");
if (!pcmk__str_empty(value)) {
return cib_file_new(value);
}
value = getenv("CIB_port");
- if (!pcmk__str_empty(value)) {
- gboolean encrypted = TRUE;
- const char *server = getenv("CIB_server");
- const char *user = getenv("CIB_user");
- const char *pass = getenv("CIB_passwd");
-
- /* We don't ensure port is valid (>= 0) because cib_new() currently
- * can't return NULL in practice, and introducing a NULL return here
- * could cause core dumps that would previously just cause signon()
- * failures.
- */
- pcmk__scan_port(value, &port);
-
- if (!crm_is_true(getenv("CIB_encrypted"))) {
- encrypted = FALSE;
- }
+ if (pcmk__str_empty(value)) {
+ return cib_native_new();
+ }
- if (pcmk__str_empty(user)) {
- user = CRM_DAEMON_USER;
- }
+ /* We don't ensure port is valid (>= 0) because cib_new() currently can't
+ * return NULL in practice, and introducing a NULL return here could cause
+ * core dumps that would previously just cause signon() failures.
+ */
+ pcmk__scan_port(value, &port);
- if (pcmk__str_empty(server)) {
- server = "localhost";
- }
+ if (!crm_is_true(getenv("CIB_encrypted"))) {
+ encrypted = FALSE;
+ }
+
+ server = getenv("CIB_server");
+ user = getenv("CIB_user");
+ pass = getenv("CIB_passwd");
+
+ if (pcmk__str_empty(user)) {
+ user = CRM_DAEMON_USER;
+ }
- crm_debug("Initializing %s remote CIB access to %s:%d as user %s",
- (encrypted? "encrypted" : "plain-text"), server, port, user);
- return cib_remote_new(server, user, pass, port, encrypted);
+ if (pcmk__str_empty(server)) {
+ server = "localhost";
}
- return cib_native_new();
+ crm_debug("Initializing %s remote CIB access to %s:%d as user %s",
+ (encrypted? "encrypted" : "plain-text"), server, port, user);
+ return cib_remote_new(server, user, pass, port, encrypted);
}
/*!
* \internal
* \brief Create a generic CIB connection instance
*
* \return Newly allocated and initialized cib_t instance
*
* \note This is called by each variant's cib_*_new() function before setting
* variant-specific values.
*/
cib_t *
cib_new_variant(void)
{
cib_t *new_cib = NULL;
new_cib = calloc(1, sizeof(cib_t));
if (new_cib == NULL) {
return NULL;
}
remove_cib_op_callback(0, TRUE); /* remove all */
new_cib->call_id = 1;
new_cib->variant = cib_undefined;
new_cib->type = cib_no_connection;
new_cib->state = cib_disconnected;
new_cib->variant_opaque = NULL;
new_cib->notify_list = NULL;
/* the rest will get filled in by the variant constructor */
new_cib->cmds = calloc(1, sizeof(cib_api_operations_t));
if (new_cib->cmds == NULL) {
free(new_cib);
return NULL;
}
new_cib->cmds->add_notify_callback = cib_client_add_notify_callback;
new_cib->cmds->del_notify_callback = cib_client_del_notify_callback;
new_cib->cmds->register_callback = cib_client_register_callback;
new_cib->cmds->register_callback_full = cib_client_register_callback_full;
new_cib->cmds->noop = cib_client_noop; // Deprecated method
new_cib->cmds->ping = cib_client_ping;
new_cib->cmds->query = cib_client_query;
new_cib->cmds->sync = cib_client_sync;
new_cib->cmds->query_from = cib_client_query_from;
new_cib->cmds->sync_from = cib_client_sync_from;
new_cib->cmds->set_primary = set_primary;
new_cib->cmds->set_secondary = set_secondary;
new_cib->cmds->upgrade = cib_client_upgrade;
new_cib->cmds->bump_epoch = cib_client_bump_epoch;
new_cib->cmds->create = cib_client_create;
new_cib->cmds->modify = cib_client_modify;
new_cib->cmds->replace = cib_client_replace;
new_cib->cmds->remove = cib_client_delete;
new_cib->cmds->erase = cib_client_erase;
new_cib->cmds->init_transaction = cib_client_init_transaction;
new_cib->cmds->end_transaction = cib_client_end_transaction;
new_cib->cmds->set_user = cib_client_set_user;
new_cib->cmds->fetch_schemas = cib_client_fetch_schemas;
return new_cib;
}
void
cib_free_notify(cib_t *cib)
{
if (cib) {
GList *list = cib->notify_list;
while (list != NULL) {
cib_notify_client_t *client = g_list_nth_data(list, 0);
list = g_list_remove(list, client);
free(client);
}
cib->notify_list = NULL;
}
}
/*!
* \brief Free all callbacks for a CIB connection
*
* \param[in,out] cib CIB connection to clean up
*/
void
cib_free_callbacks(cib_t *cib)
{
cib_free_notify(cib);
destroy_op_callback_table();
}
/*!
* \brief Free all memory used by CIB connection
*
* \param[in,out] cib CIB connection to delete
*/
void
cib_delete(cib_t *cib)
{
cib_free_callbacks(cib);
if (cib) {
cib->cmds->free(cib);
}
}
void
remove_cib_op_callback(int call_id, gboolean all_callbacks)
{
if (all_callbacks) {
destroy_op_callback_table();
cib_op_callback_table = pcmk__intkey_table(cib_destroy_op_callback);
} else {
pcmk__intkey_table_remove(cib_op_callback_table, call_id);
}
}
int
num_cib_op_callbacks(void)
{
if (cib_op_callback_table == NULL) {
return 0;
}
return g_hash_table_size(cib_op_callback_table);
}
static void
cib_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
{
int call = GPOINTER_TO_INT(key);
cib_callback_client_t *blob = value;
crm_debug("Call %d (%s): pending", call, pcmk__s(blob->id, "without ID"));
}
void
cib_dump_pending_callbacks(void)
{
if (cib_op_callback_table == NULL) {
return;
}
return g_hash_table_foreach(cib_op_callback_table, cib_dump_pending_op, NULL);
}
cib_callback_client_t*
cib__lookup_id (int call_id)
{
return pcmk__intkey_table_lookup(cib_op_callback_table, call_id);
}
diff --git a/lib/cib/cib_remote.c b/lib/cib/cib_remote.c
index 42869e8559..70b53cea57 100644
--- a/lib/cib/cib_remote.c
+++ b/lib/cib/cib_remote.c
@@ -1,672 +1,673 @@
/*
* Copyright 2008-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <netdb.h>
#include <termios.h>
#include <sys/socket.h>
#include <glib.h>
#include <crm/crm.h>
#include <crm/cib/internal.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/mainloop.h>
#include <crm/common/xml.h>
#include <crm/common/remote_internal.h>
+#include <crm/common/tls_internal.h>
#include <crm/common/output_internal.h>
#include <gnutls/gnutls.h>
// GnuTLS handshake timeout in seconds
#define TLS_HANDSHAKE_TIMEOUT 5
-static gnutls_anon_client_credentials_t anon_cred_c;
-static gboolean remote_gnutls_credentials_init = FALSE;
+static pcmk__tls_t *tls = NULL;
#include <arpa/inet.h>
typedef struct cib_remote_opaque_s {
int port;
char *server;
char *user;
char *passwd;
gboolean encrypted;
pcmk__remote_t command;
pcmk__remote_t callback;
pcmk__output_t *out;
time_t start_time;
int timeout_sec;
} cib_remote_opaque_t;
static int
cib_remote_perform_op(cib_t *cib, const char *op, const char *host,
const char *section, xmlNode *data,
xmlNode **output_data, int call_options,
const char *user_name)
{
int rc;
int remaining_time = 0;
time_t start_time;
xmlNode *op_msg = NULL;
xmlNode *op_reply = NULL;
cib_remote_opaque_t *private = cib->variant_opaque;
if (cib->state == cib_disconnected) {
return -ENOTCONN;
}
if (output_data != NULL) {
*output_data = NULL;
}
if (op == NULL) {
crm_err("No operation specified");
return -EINVAL;
}
rc = cib__create_op(cib, op, host, section, data, call_options, user_name,
NULL, &op_msg);
if (rc != pcmk_ok) {
return rc;
}
if (pcmk_is_set(call_options, cib_transaction)) {
rc = cib__extend_transaction(cib, op_msg);
pcmk__xml_free(op_msg);
return rc;
}
crm_trace("Sending %s message to the CIB manager", op);
if (!(call_options & cib_sync_call)) {
pcmk__remote_send_xml(&private->callback, op_msg);
} else {
pcmk__remote_send_xml(&private->command, op_msg);
}
pcmk__xml_free(op_msg);
if ((call_options & cib_discard_reply)) {
crm_trace("Discarding reply");
return pcmk_ok;
} else if (!(call_options & cib_sync_call)) {
return cib->call_id;
}
crm_trace("Waiting for a synchronous reply");
start_time = time(NULL);
remaining_time = cib->call_timeout ? cib->call_timeout : 60;
rc = pcmk_rc_ok;
while (remaining_time > 0 && (rc != ENOTCONN)) {
int reply_id = -1;
int msg_id = cib->call_id;
rc = pcmk__read_remote_message(&private->command,
remaining_time * 1000);
op_reply = pcmk__remote_message_xml(&private->command);
if (!op_reply) {
break;
}
crm_element_value_int(op_reply, PCMK__XA_CIB_CALLID, &reply_id);
if (reply_id == msg_id) {
break;
} else if (reply_id < msg_id) {
crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
crm_log_xml_trace(op_reply, "Old reply");
} else if ((reply_id - 10000) > msg_id) {
/* wrap-around case */
crm_debug("Received old reply: %d (wanted %d)", reply_id, msg_id);
crm_log_xml_trace(op_reply, "Old reply");
} else {
crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id);
}
pcmk__xml_free(op_reply);
op_reply = NULL;
/* wasn't the right reply, try and read some more */
remaining_time = time(NULL) - start_time;
}
if (rc == ENOTCONN) {
crm_err("Disconnected while waiting for reply.");
return -ENOTCONN;
} else if (op_reply == NULL) {
crm_err("No reply message - empty");
return -ENOMSG;
}
crm_trace("Synchronous reply received");
/* Start processing the reply... */
if (crm_element_value_int(op_reply, PCMK__XA_CIB_RC, &rc) != 0) {
rc = -EPROTO;
}
if (rc == -pcmk_err_diff_resync) {
/* This is an internal value that clients do not and should not care about */
rc = pcmk_ok;
}
if (rc == pcmk_ok || rc == -EPERM) {
crm_log_xml_debug(op_reply, "passed");
} else {
crm_err("Call failed: %s", pcmk_strerror(rc));
crm_log_xml_warn(op_reply, "failed");
}
if (output_data == NULL) {
/* do nothing more */
} else if (!(call_options & cib_discard_reply)) {
xmlNode *wrapper = pcmk__xe_first_child(op_reply, PCMK__XE_CIB_CALLDATA,
NULL, NULL);
xmlNode *tmp = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
if (tmp == NULL) {
crm_trace("No output in reply to \"%s\" command %d", op, cib->call_id - 1);
} else {
*output_data = pcmk__xml_copy(NULL, tmp);
}
}
pcmk__xml_free(op_reply);
return rc;
}
static int
cib_remote_callback_dispatch(gpointer user_data)
{
int rc;
cib_t *cib = user_data;
cib_remote_opaque_t *private = cib->variant_opaque;
xmlNode *msg = NULL;
const char *type = NULL;
/* If start time is 0, we've previously handled a complete message and this
* connection is being reused for a new message. Reset the start_time,
* giving this new message timeout_sec from now to complete.
*/
if (private->start_time == 0) {
private->start_time = time(NULL);
}
rc = pcmk__read_available_remote_data(&private->callback);
switch (rc) {
case pcmk_rc_ok:
/* We have the whole message so process it */
break;
case EAGAIN:
/* Have we timed out? */
if (time(NULL) >= private->start_time + private->timeout_sec) {
crm_info("Error reading from CIB manager connection: %s",
pcmk_rc_str(ETIME));
return -1;
}
/* We haven't read the whole message yet */
return 0;
default:
/* Error */
crm_info("Error reading from CIB manager connection: %s",
pcmk_rc_str(rc));
return -1;
}
msg = pcmk__remote_message_xml(&private->callback);
if (msg == NULL) {
private->start_time = 0;
return 0;
}
type = crm_element_value(msg, PCMK__XA_T);
crm_trace("Activating %s callbacks...", type);
if (pcmk__str_eq(type, PCMK__VALUE_CIB, pcmk__str_none)) {
cib_native_callback(cib, msg, 0, 0);
} else if (pcmk__str_eq(type, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) {
g_list_foreach(cib->notify_list, cib_native_notify, msg);
} else {
crm_err("Unknown message type: %s", type);
}
pcmk__xml_free(msg);
private->start_time = 0;
return 0;
}
static int
cib_remote_command_dispatch(gpointer user_data)
{
int rc;
cib_t *cib = user_data;
cib_remote_opaque_t *private = cib->variant_opaque;
/* See cib_remote_callback_dispatch */
if (private->start_time == 0) {
private->start_time = time(NULL);
}
rc = pcmk__read_available_remote_data(&private->command);
if (rc == EAGAIN) {
/* Have we timed out? */
if (time(NULL) >= private->start_time + private->timeout_sec) {
crm_info("Error reading from CIB manager connection: %s",
pcmk_rc_str(ETIME));
return -1;
}
/* We haven't read the whole message yet */
return 0;
}
free(private->command.buffer);
private->command.buffer = NULL;
crm_err("received late reply for remote cib connection, discarding");
if (rc != pcmk_rc_ok) {
crm_info("Error reading from CIB manager connection: %s",
pcmk_rc_str(rc));
return -1;
}
private->start_time = 0;
return 0;
}
static int
cib_tls_close(cib_t *cib)
{
cib_remote_opaque_t *private = cib->variant_opaque;
if (private->encrypted) {
if (private->command.tls_session) {
- gnutls_bye(*(private->command.tls_session), GNUTLS_SHUT_RDWR);
- gnutls_deinit(*(private->command.tls_session));
- gnutls_free(private->command.tls_session);
+ gnutls_bye(private->command.tls_session, GNUTLS_SHUT_RDWR);
+ gnutls_deinit(private->command.tls_session);
}
if (private->callback.tls_session) {
- gnutls_bye(*(private->callback.tls_session), GNUTLS_SHUT_RDWR);
- gnutls_deinit(*(private->callback.tls_session));
- gnutls_free(private->callback.tls_session);
+ gnutls_bye(private->callback.tls_session, GNUTLS_SHUT_RDWR);
+ gnutls_deinit(private->callback.tls_session);
}
+
private->command.tls_session = NULL;
private->callback.tls_session = NULL;
- if (remote_gnutls_credentials_init) {
- gnutls_anon_free_client_credentials(anon_cred_c);
- gnutls_global_deinit();
- remote_gnutls_credentials_init = FALSE;
- }
+ pcmk__free_tls(tls);
+ tls = NULL;
}
if (private->command.tcp_socket) {
shutdown(private->command.tcp_socket, SHUT_RDWR); /* no more receptions */
close(private->command.tcp_socket);
}
if (private->callback.tcp_socket) {
shutdown(private->callback.tcp_socket, SHUT_RDWR); /* no more receptions */
close(private->callback.tcp_socket);
}
private->command.tcp_socket = 0;
private->callback.tcp_socket = 0;
free(private->command.buffer);
free(private->callback.buffer);
private->command.buffer = NULL;
private->callback.buffer = NULL;
return 0;
}
static void
cib_remote_connection_destroy(gpointer user_data)
{
crm_err("Connection destroyed");
cib_tls_close(user_data);
}
static int
cib_tls_signon(cib_t *cib, pcmk__remote_t *connection, gboolean event_channel)
{
cib_remote_opaque_t *private = cib->variant_opaque;
int rc;
xmlNode *answer = NULL;
xmlNode *login = NULL;
static struct mainloop_fd_callbacks cib_fd_callbacks = { 0, };
cib_fd_callbacks.dispatch =
event_channel ? cib_remote_callback_dispatch : cib_remote_command_dispatch;
cib_fd_callbacks.destroy = cib_remote_connection_destroy;
connection->tcp_socket = -1;
connection->tls_session = NULL;
rc = pcmk__connect_remote(private->server, private->port, 0, NULL,
&(connection->tcp_socket), NULL, NULL);
if (rc != pcmk_rc_ok) {
crm_info("Remote connection to %s:%d failed: %s " QB_XS " rc=%d",
private->server, private->port, pcmk_rc_str(rc), rc);
return -ENOTCONN;
}
if (private->encrypted) {
+ bool use_cert = pcmk__x509_enabled(false);
int tls_rc = GNUTLS_E_SUCCESS;
- /* initialize GnuTls lib */
- if (remote_gnutls_credentials_init == FALSE) {
- crm_gnutls_global_init();
- gnutls_anon_allocate_client_credentials(&anon_cred_c);
- remote_gnutls_credentials_init = TRUE;
+ rc = pcmk__init_tls(&tls, false, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_ANON);
+ if (rc != pcmk_rc_ok) {
+ return -1;
}
/* bind the socket to GnuTls lib */
- connection->tls_session = pcmk__new_tls_session(connection->tcp_socket,
- GNUTLS_CLIENT,
- GNUTLS_CRD_ANON,
- anon_cred_c);
+ connection->tls_session = pcmk__new_tls_session(tls, connection->tcp_socket);
if (connection->tls_session == NULL) {
cib_tls_close(cib);
return -1;
}
rc = pcmk__tls_client_handshake(connection, TLS_HANDSHAKE_TIMEOUT,
&tls_rc);
if (rc != pcmk_rc_ok) {
crm_err("Remote CIB session creation for %s:%d failed: %s",
private->server, private->port,
(rc == EPROTO)? gnutls_strerror(tls_rc) : pcmk_rc_str(rc));
- gnutls_deinit(*connection->tls_session);
- gnutls_free(connection->tls_session);
+ gnutls_deinit(connection->tls_session);
connection->tls_session = NULL;
cib_tls_close(cib);
return -1;
}
}
+ /* Now that the handshake is done, see if any client TLS certificate is
+ * close to its expiration date and log if so. If a TLS certificate is not
+ * in use, this function will just return so we don't need to check for the
+ * session type here.
+ */
+ pcmk__tls_check_cert_expiration(connection->tls_session);
+
/* login to server */
login = pcmk__xe_create(NULL, PCMK__XE_CIB_COMMAND);
crm_xml_add(login, PCMK_XA_OP, "authenticate");
crm_xml_add(login, PCMK_XA_USER, private->user);
crm_xml_add(login, PCMK__XA_PASSWORD, private->passwd);
crm_xml_add(login, PCMK__XA_HIDDEN, PCMK__VALUE_PASSWORD);
pcmk__remote_send_xml(connection, login);
pcmk__xml_free(login);
rc = pcmk_ok;
if (pcmk__read_remote_message(connection, -1) == ENOTCONN) {
rc = -ENOTCONN;
}
answer = pcmk__remote_message_xml(connection);
crm_log_xml_trace(answer, "Reply");
if (answer == NULL) {
rc = -EPROTO;
} else {
/* grab the token */
const char *msg_type = crm_element_value(answer, PCMK__XA_CIB_OP);
const char *tmp_ticket = crm_element_value(answer,
PCMK__XA_CIB_CLIENTID);
if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
crm_err("Invalid registration message: %s", msg_type);
rc = -EPROTO;
} else if (tmp_ticket == NULL) {
rc = -EPROTO;
} else {
connection->token = strdup(tmp_ticket);
}
}
pcmk__xml_free(answer);
answer = NULL;
if (rc != 0) {
cib_tls_close(cib);
return rc;
}
crm_trace("remote client connection established");
private->timeout_sec = 60;
connection->source = mainloop_add_fd("cib-remote", G_PRIORITY_HIGH,
connection->tcp_socket, cib,
&cib_fd_callbacks);
return rc;
}
static int
cib_remote_signon(cib_t *cib, const char *name, enum cib_conn_type type)
{
int rc = pcmk_ok;
cib_remote_opaque_t *private = cib->variant_opaque;
xmlNode *hello = NULL;
if (name == NULL) {
name = pcmk__s(crm_system_name, "client");
}
if (private->passwd == NULL) {
if (private->out == NULL) {
/* If no pcmk__output_t is set, just assume that a text prompt
* is good enough.
*/
pcmk__text_prompt("Password", false, &(private->passwd));
} else {
private->out->prompt("Password", false, &(private->passwd));
}
}
if (private->server == NULL || private->user == NULL) {
rc = -EINVAL;
+ goto done;
}
- if (rc == pcmk_ok) {
- rc = cib_tls_signon(cib, &(private->command), FALSE);
+ rc = cib_tls_signon(cib, &(private->command), FALSE);
+ if (rc != pcmk_ok) {
+ goto done;
}
- if (rc == pcmk_ok) {
- rc = cib_tls_signon(cib, &(private->callback), TRUE);
+ rc = cib_tls_signon(cib, &(private->callback), TRUE);
+ if (rc != pcmk_ok) {
+ goto done;
}
- if (rc == pcmk_ok) {
- rc = cib__create_op(cib, CRM_OP_REGISTER, NULL, NULL, NULL, cib_none,
- NULL, name, &hello);
+ rc = cib__create_op(cib, CRM_OP_REGISTER, NULL, NULL, NULL, cib_none, NULL,
+ name, &hello);
+ if (rc != pcmk_ok) {
+ goto done;
}
- if (rc == pcmk_ok) {
- rc = pcmk__remote_send_xml(&private->command, hello);
- rc = pcmk_rc2legacy(rc);
- pcmk__xml_free(hello);
- }
+ rc = pcmk__remote_send_xml(&private->command, hello);
+ rc = pcmk_rc2legacy(rc);
+ pcmk__xml_free(hello);
+done:
if (rc == pcmk_ok) {
crm_info("Opened connection to %s:%d for %s",
private->server, private->port, name);
cib->state = cib_connected_command;
cib->type = cib_command;
} else {
crm_info("Connection to %s:%d for %s failed: %s\n",
private->server, private->port, name, pcmk_strerror(rc));
}
return rc;
}
static int
cib_remote_signoff(cib_t *cib)
{
int rc = pcmk_ok;
crm_debug("Disconnecting from the CIB manager");
cib_tls_close(cib);
cib->cmds->end_transaction(cib, false, cib_none);
cib->state = cib_disconnected;
cib->type = cib_no_connection;
return rc;
}
static int
cib_remote_free(cib_t *cib)
{
int rc = pcmk_ok;
crm_warn("Freeing CIB");
if (cib->state != cib_disconnected) {
rc = cib_remote_signoff(cib);
if (rc == pcmk_ok) {
cib_remote_opaque_t *private = cib->variant_opaque;
free(private->server);
free(private->user);
free(private->passwd);
free(cib->cmds);
free(cib->user);
free(private);
free(cib);
}
}
return rc;
}
static int
cib_remote_register_notification(cib_t * cib, const char *callback, int enabled)
{
xmlNode *notify_msg = pcmk__xe_create(NULL, PCMK__XE_CIB_COMMAND);
cib_remote_opaque_t *private = cib->variant_opaque;
crm_xml_add(notify_msg, PCMK__XA_CIB_OP, PCMK__VALUE_CIB_NOTIFY);
crm_xml_add(notify_msg, PCMK__XA_CIB_NOTIFY_TYPE, callback);
crm_xml_add_int(notify_msg, PCMK__XA_CIB_NOTIFY_ACTIVATE, enabled);
pcmk__remote_send_xml(&private->callback, notify_msg);
pcmk__xml_free(notify_msg);
return pcmk_ok;
}
static int
cib_remote_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
{
return -EPROTONOSUPPORT;
}
/*!
* \internal
* \brief Get the given CIB connection's unique client identifiers
*
* These can be used to check whether this client requested the action that
* triggered a CIB notification.
*
* \param[in] cib CIB connection
* \param[out] async_id If not \p NULL, where to store asynchronous client ID
* \param[out] sync_id If not \p NULL, where to store synchronous client ID
*
* \return Legacy Pacemaker return code (specifically, \p pcmk_ok)
*
* \note This is the \p cib_remote variant implementation of
* \p cib_api_operations_t:client_id().
* \note The client IDs are assigned during CIB sign-on.
*/
static int
cib_remote_client_id(const cib_t *cib, const char **async_id,
const char **sync_id)
{
cib_remote_opaque_t *private = cib->variant_opaque;
if (async_id != NULL) {
// private->callback is the channel for async requests
*async_id = private->callback.token;
}
if (sync_id != NULL) {
// private->command is the channel for sync requests
*sync_id = private->command.token;
}
return pcmk_ok;
}
cib_t *
cib_remote_new(const char *server, const char *user, const char *passwd, int port,
gboolean encrypted)
{
cib_remote_opaque_t *private = NULL;
cib_t *cib = cib_new_variant();
if (cib == NULL) {
return NULL;
}
private = calloc(1, sizeof(cib_remote_opaque_t));
if (private == NULL) {
free(cib);
return NULL;
}
cib->variant = cib_remote;
cib->variant_opaque = private;
private->server = pcmk__str_copy(server);
private->user = pcmk__str_copy(user);
private->passwd = pcmk__str_copy(passwd);
private->port = port;
private->encrypted = encrypted;
/* assign variant specific ops */
cib->delegate_fn = cib_remote_perform_op;
cib->cmds->signon = cib_remote_signon;
cib->cmds->signoff = cib_remote_signoff;
cib->cmds->free = cib_remote_free;
cib->cmds->register_notification = cib_remote_register_notification;
cib->cmds->set_connection_dnotify = cib_remote_set_connection_dnotify;
cib->cmds->client_id = cib_remote_client_id;
return cib;
}
void
cib__set_output(cib_t *cib, pcmk__output_t *out)
{
cib_remote_opaque_t *private;
if (cib->variant != cib_remote) {
return;
}
private = cib->variant_opaque;
private->out = out;
}
diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am
index 753211ea67..45fd0a0265 100644
--- a/lib/common/Makefile.am
+++ b/lib/common/Makefile.am
@@ -1,141 +1,142 @@
#
# Copyright 2004-2024 the Pacemaker project contributors
#
# The version control history for this file may have further details.
#
# This source code is licensed under the GNU General Public License version 2
# or later (GPLv2+) WITHOUT ANY WARRANTY.
#
include $(top_srcdir)/mk/common.mk
## libraries
lib_LTLIBRARIES = libcrmcommon.la
check_LTLIBRARIES = libcrmcommon_test.la
# Disable -Wcast-qual if used, because we do some hacky casting,
# and because libxml2 has some signatures that should be const but aren't
# for backward compatibility reasons.
# s390 needs -fPIC
# s390-suse-linux/bin/ld: .libs/ipc.o: relocation R_390_PC32DBL against `__stack_chk_fail@@GLIBC_2.4' can not be used when making a shared object; recompile with -fPIC
CFLAGS = $(CFLAGS_COPY:-Wcast-qual=) -fPIC
# Without "." here, check-recursive will run through the subdirectories first
# and then run "make check" here. This will fail, because there's things in
# the subdirectories that need check_LTLIBRARIES built first. Adding "." here
# changes the order so the subdirectories are processed afterwards.
SUBDIRS = . tests
noinst_HEADERS = crmcommon_private.h \
mock_private.h
libcrmcommon_la_LDFLAGS = -version-info 68:0:0
libcrmcommon_la_CFLAGS = $(CFLAGS_HARDENED_LIB)
libcrmcommon_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB)
# If configured with --with-profiling or --with-coverage, BUILD_PROFILING will
# be set and -fno-builtin will be added to the CFLAGS. However, libcrmcommon
# uses the fabs() function which is normally supplied by gcc as one of its
# builtins. Therefore we need to explicitly link against libm here or the
# tests won't link.
if BUILD_PROFILING
libcrmcommon_la_LIBADD = -lm
endif
## Library sources (*must* use += format for bumplibs)
libcrmcommon_la_SOURCES =
libcrmcommon_la_SOURCES += acl.c
libcrmcommon_la_SOURCES += actions.c
libcrmcommon_la_SOURCES += agents.c
libcrmcommon_la_SOURCES += alerts.c
libcrmcommon_la_SOURCES += attrs.c
libcrmcommon_la_SOURCES += cib.c
if BUILD_CIBSECRETS
libcrmcommon_la_SOURCES += cib_secrets.c
endif
libcrmcommon_la_SOURCES += cmdline.c
libcrmcommon_la_SOURCES += digest.c
libcrmcommon_la_SOURCES += health.c
libcrmcommon_la_SOURCES += io.c
libcrmcommon_la_SOURCES += ipc_attrd.c
libcrmcommon_la_SOURCES += ipc_client.c
libcrmcommon_la_SOURCES += ipc_common.c
libcrmcommon_la_SOURCES += ipc_controld.c
libcrmcommon_la_SOURCES += ipc_pacemakerd.c
libcrmcommon_la_SOURCES += ipc_schedulerd.c
libcrmcommon_la_SOURCES += ipc_server.c
libcrmcommon_la_SOURCES += iso8601.c
libcrmcommon_la_SOURCES += lists.c
libcrmcommon_la_SOURCES += logging.c
libcrmcommon_la_SOURCES += mainloop.c
libcrmcommon_la_SOURCES += messages.c
libcrmcommon_la_SOURCES += nodes.c
libcrmcommon_la_SOURCES += nvpair.c
libcrmcommon_la_SOURCES += options.c
libcrmcommon_la_SOURCES += options_display.c
libcrmcommon_la_SOURCES += output.c
libcrmcommon_la_SOURCES += output_html.c
libcrmcommon_la_SOURCES += output_log.c
libcrmcommon_la_SOURCES += output_none.c
libcrmcommon_la_SOURCES += output_text.c
libcrmcommon_la_SOURCES += output_xml.c
libcrmcommon_la_SOURCES += patchset.c
libcrmcommon_la_SOURCES += patchset_display.c
libcrmcommon_la_SOURCES += pid.c
libcrmcommon_la_SOURCES += probes.c
libcrmcommon_la_SOURCES += procfs.c
libcrmcommon_la_SOURCES += remote.c
libcrmcommon_la_SOURCES += resources.c
libcrmcommon_la_SOURCES += results.c
libcrmcommon_la_SOURCES += roles.c
libcrmcommon_la_SOURCES += rules.c
libcrmcommon_la_SOURCES += scheduler.c
libcrmcommon_la_SOURCES += schemas.c
libcrmcommon_la_SOURCES += scores.c
libcrmcommon_la_SOURCES += servers.c
libcrmcommon_la_SOURCES += strings.c
+libcrmcommon_la_SOURCES += tls.c
libcrmcommon_la_SOURCES += utils.c
libcrmcommon_la_SOURCES += watchdog.c
libcrmcommon_la_SOURCES += xml.c
libcrmcommon_la_SOURCES += xml_attr.c
libcrmcommon_la_SOURCES += xml_comment.c
libcrmcommon_la_SOURCES += xml_display.c
libcrmcommon_la_SOURCES += xml_element.c
libcrmcommon_la_SOURCES += xml_idref.c
libcrmcommon_la_SOURCES += xml_io.c
libcrmcommon_la_SOURCES += xpath.c
#
# libcrmcommon_test is used only with unit tests, so we can mock system calls.
# See mock.c for details.
#
include $(top_srcdir)/mk/tap.mk
libcrmcommon_test_la_SOURCES = $(libcrmcommon_la_SOURCES)
libcrmcommon_test_la_SOURCES += mock.c
libcrmcommon_test_la_SOURCES += unittest.c
libcrmcommon_test_la_LDFLAGS = $(libcrmcommon_la_LDFLAGS) \
-rpath $(libdir) \
$(LDFLAGS_WRAP)
# If GCC emits a builtin function in place of something we've mocked up, that will
# get used instead of the mocked version which leads to unexpected test results. So
# disable all builtins. Older versions of GCC (at least, on RHEL7) will still emit
# replacement code for strdup (and possibly other functions) unless -fno-inline is
# also added.
libcrmcommon_test_la_CFLAGS = $(libcrmcommon_la_CFLAGS) \
-DPCMK__UNIT_TESTING \
-fno-builtin \
-fno-inline
# If -fno-builtin is used, -lm also needs to be added. See the comment at
# BUILD_PROFILING above.
libcrmcommon_test_la_LIBADD = $(libcrmcommon_la_LIBADD)
if BUILD_COVERAGE
libcrmcommon_test_la_LIBADD += -lgcov
endif
libcrmcommon_test_la_LIBADD += -lcmocka
libcrmcommon_test_la_LIBADD += -lm
nodist_libcrmcommon_test_la_SOURCES = $(nodist_libcrmcommon_la_SOURCES)
diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c
index bffd980f76..1912fadd93 100644
--- a/lib/common/ipc_server.c
+++ b/lib/common/ipc_server.c
@@ -1,1010 +1,1010 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <errno.h>
#include <bzlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include "crmcommon_private.h"
/* Evict clients whose event queue grows this large (by default) */
#define PCMK_IPC_DEFAULT_QUEUE_MAX 500
static GHashTable *client_connections = NULL;
/*!
* \internal
* \brief Count IPC clients
*
* \return Number of active IPC client connections
*/
guint
pcmk__ipc_client_count(void)
{
return client_connections? g_hash_table_size(client_connections) : 0;
}
/*!
* \internal
* \brief Execute a function for each active IPC client connection
*
* \param[in] func Function to call
* \param[in,out] user_data Pointer to pass to function
*
* \note The parameters are the same as for g_hash_table_foreach().
*/
void
pcmk__foreach_ipc_client(GHFunc func, gpointer user_data)
{
if ((func != NULL) && (client_connections != NULL)) {
g_hash_table_foreach(client_connections, func, user_data);
}
}
pcmk__client_t *
pcmk__find_client(const qb_ipcs_connection_t *c)
{
if (client_connections) {
return g_hash_table_lookup(client_connections, c);
}
crm_trace("No client found for %p", c);
return NULL;
}
pcmk__client_t *
pcmk__find_client_by_id(const char *id)
{
if ((client_connections != NULL) && (id != NULL)) {
gpointer key;
pcmk__client_t *client = NULL;
GHashTableIter iter;
g_hash_table_iter_init(&iter, client_connections);
while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) {
if (strcmp(client->id, id) == 0) {
return client;
}
}
}
crm_trace("No client found with id='%s'", pcmk__s(id, ""));
return NULL;
}
/*!
* \internal
* \brief Get a client identifier for use in log messages
*
* \param[in] c Client
*
* \return Client's name, client's ID, or a string literal, as available
* \note This is intended to be used in format strings like "client %s".
*/
const char *
pcmk__client_name(const pcmk__client_t *c)
{
if (c == NULL) {
return "(unspecified)";
} else if (c->name != NULL) {
return c->name;
} else if (c->id != NULL) {
return c->id;
} else {
return "(unidentified)";
}
}
void
pcmk__client_cleanup(void)
{
if (client_connections != NULL) {
int active = g_hash_table_size(client_connections);
if (active > 0) {
crm_warn("Exiting with %d active IPC client%s",
active, pcmk__plural_s(active));
}
g_hash_table_destroy(client_connections);
client_connections = NULL;
}
}
void
pcmk__drop_all_clients(qb_ipcs_service_t *service)
{
qb_ipcs_connection_t *c = NULL;
if (service == NULL) {
return;
}
c = qb_ipcs_connection_first_get(service);
while (c != NULL) {
qb_ipcs_connection_t *last = c;
c = qb_ipcs_connection_next_get(service, last);
/* There really shouldn't be anyone connected at this point */
crm_notice("Disconnecting client %p, pid=%d...",
last, pcmk__client_pid(last));
qb_ipcs_disconnect(last);
qb_ipcs_connection_unref(last);
}
}
/*!
* \internal
* \brief Allocate a new pcmk__client_t object based on an IPC connection
*
* \param[in] c IPC connection (NULL to allocate generic client)
* \param[in] key Connection table key (NULL to use sane default)
* \param[in] uid_client UID corresponding to c (ignored if c is NULL)
*
* \return Pointer to new pcmk__client_t (guaranteed not to be \c NULL)
*/
static pcmk__client_t *
client_from_connection(qb_ipcs_connection_t *c, void *key, uid_t uid_client)
{
pcmk__client_t *client = pcmk__assert_alloc(1, sizeof(pcmk__client_t));
if (c) {
client->user = pcmk__uid2username(uid_client);
if (client->user == NULL) {
client->user = pcmk__str_copy("#unprivileged");
crm_err("Unable to enforce ACLs for user ID %d, assuming unprivileged",
uid_client);
}
client->ipcs = c;
pcmk__set_client_flags(client, pcmk__client_ipc);
client->pid = pcmk__client_pid(c);
if (key == NULL) {
key = c;
}
}
client->id = crm_generate_uuid();
if (key == NULL) {
key = client->id;
}
if (client_connections == NULL) {
crm_trace("Creating IPC client table");
client_connections = g_hash_table_new(g_direct_hash, g_direct_equal);
}
g_hash_table_insert(client_connections, key, client);
return client;
}
/*!
* \brief Allocate a new pcmk__client_t object and generate its ID
*
* \param[in] key What to use as connections hash table key (NULL to use ID)
*
* \return Pointer to new pcmk__client_t (asserts on failure)
*/
pcmk__client_t *
pcmk__new_unauth_client(void *key)
{
return client_from_connection(NULL, key, 0);
}
pcmk__client_t *
pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid_client, gid_t gid_client)
{
gid_t uid_cluster = 0;
gid_t gid_cluster = 0;
pcmk__client_t *client = NULL;
CRM_CHECK(c != NULL, return NULL);
if (pcmk_daemon_user(&uid_cluster, &gid_cluster) < 0) {
static bool need_log = TRUE;
if (need_log) {
crm_warn("Could not find user and group IDs for user %s",
CRM_DAEMON_USER);
need_log = FALSE;
}
}
if (uid_client != 0) {
crm_trace("Giving group %u access to new IPC connection", gid_cluster);
/* Passing -1 to chown(2) means don't change */
qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP);
}
/* TODO: Do our own auth checking, return NULL if unauthorized */
client = client_from_connection(c, NULL, uid_client);
if ((uid_client == 0) || (uid_client == uid_cluster)) {
/* Remember when a connection came from root or hacluster */
pcmk__set_client_flags(client, pcmk__client_privileged);
}
crm_debug("New IPC client %s for PID %u with uid %d and gid %d",
client->id, client->pid, uid_client, gid_client);
return client;
}
static struct iovec *
pcmk__new_ipc_event(void)
{
return (struct iovec *) pcmk__assert_alloc(2, sizeof(struct iovec));
}
/*!
* \brief Free an I/O vector created by pcmk__ipc_prepare_iov()
*
* \param[in,out] event I/O vector to free
*/
void
pcmk_free_ipc_event(struct iovec *event)
{
if (event != NULL) {
free(event[0].iov_base);
free(event[1].iov_base);
free(event);
}
}
static void
free_event(gpointer data)
{
pcmk_free_ipc_event((struct iovec *) data);
}
static void
add_event(pcmk__client_t *c, struct iovec *iov)
{
if (c->event_queue == NULL) {
c->event_queue = g_queue_new();
}
g_queue_push_tail(c->event_queue, iov);
}
void
pcmk__free_client(pcmk__client_t *c)
{
if (c == NULL) {
return;
}
if (client_connections) {
if (c->ipcs) {
crm_trace("Destroying %p/%p (%d remaining)",
c, c->ipcs, g_hash_table_size(client_connections) - 1);
g_hash_table_remove(client_connections, c->ipcs);
} else {
crm_trace("Destroying remote connection %p (%d remaining)",
c, g_hash_table_size(client_connections) - 1);
g_hash_table_remove(client_connections, c->id);
}
}
if (c->event_timer) {
g_source_remove(c->event_timer);
}
if (c->event_queue) {
crm_debug("Destroying %d events", g_queue_get_length(c->event_queue));
g_queue_free_full(c->event_queue, free_event);
}
free(c->id);
free(c->name);
free(c->user);
if (c->remote) {
if (c->remote->auth_timeout) {
g_source_remove(c->remote->auth_timeout);
}
if (c->remote->tls_session != NULL) {
/* @TODO Reduce duplication at callers. Put here everything
* necessary to tear down and free tls_session.
*/
- gnutls_free(c->remote->tls_session);
+ gnutls_deinit(c->remote->tls_session);
}
free(c->remote->buffer);
free(c->remote);
}
free(c);
}
/*!
* \internal
* \brief Raise IPC eviction threshold for a client, if allowed
*
* \param[in,out] client Client to modify
* \param[in] qmax New threshold
*/
void
pcmk__set_client_queue_max(pcmk__client_t *client, const char *qmax)
{
int rc = pcmk_rc_ok;
long long qmax_ll = 0LL;
unsigned int orig_value = 0U;
CRM_CHECK(client != NULL, return);
orig_value = client->queue_max;
if (pcmk_is_set(client->flags, pcmk__client_privileged)) {
rc = pcmk__scan_ll(qmax, &qmax_ll, 0LL);
if (rc == pcmk_rc_ok) {
if ((qmax_ll <= 0LL) || (qmax_ll > UINT_MAX)) {
rc = ERANGE;
} else {
client->queue_max = (unsigned int) qmax_ll;
}
}
} else {
rc = EACCES;
}
if (rc != pcmk_rc_ok) {
crm_info("Could not set IPC threshold for client %s[%u] to %s: %s",
pcmk__client_name(client), client->pid,
pcmk__s(qmax, "default"), pcmk_rc_str(rc));
} else if (client->queue_max != orig_value) {
crm_debug("IPC threshold for client %s[%u] is now %u (was %u)",
pcmk__client_name(client), client->pid,
client->queue_max, orig_value);
}
}
int
pcmk__client_pid(qb_ipcs_connection_t *c)
{
struct qb_ipcs_connection_stats stats;
stats.client_pid = 0;
qb_ipcs_connection_stats_get(c, &stats, 0);
return stats.client_pid;
}
/*!
* \internal
* \brief Retrieve message XML from data read from client IPC
*
* \param[in,out] c IPC client connection
* \param[in] data Data read from client connection
* \param[out] id Where to store message ID from libqb header
* \param[out] flags Where to store flags from libqb header
*
* \return Message XML on success, NULL otherwise
*/
xmlNode *
pcmk__client_data2xml(pcmk__client_t *c, void *data, uint32_t *id,
uint32_t *flags)
{
xmlNode *xml = NULL;
char *uncompressed = NULL;
char *text = ((char *)data) + sizeof(pcmk__ipc_header_t);
pcmk__ipc_header_t *header = data;
if (!pcmk__valid_ipc_header(header)) {
return NULL;
}
if (id) {
*id = ((struct qb_ipc_response_header *)data)->id;
}
if (flags) {
*flags = header->flags;
}
if (pcmk_is_set(header->flags, crm_ipc_proxied)) {
/* Mark this client as being the endpoint of a proxy connection.
* Proxy connections responses are sent on the event channel, to avoid
* blocking the controller serving as proxy.
*/
pcmk__set_client_flags(c, pcmk__client_proxied);
}
if (header->size_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->size_uncompressed;
uncompressed = pcmk__assert_alloc(1, size_u);
crm_trace("Decompressing message data %u bytes into %u bytes",
header->size_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0);
text = uncompressed;
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Decompression failed: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
free(uncompressed);
return NULL;
}
}
pcmk__assert(text[header->size_uncompressed - 1] == 0);
xml = pcmk__xml_parse(text);
crm_log_xml_trace(xml, "[IPC received]");
free(uncompressed);
return xml;
}
static int crm_ipcs_flush_events(pcmk__client_t *c);
static gboolean
crm_ipcs_flush_events_cb(gpointer data)
{
pcmk__client_t *c = data;
c->event_timer = 0;
crm_ipcs_flush_events(c);
return FALSE;
}
/*!
* \internal
* \brief Add progressive delay before next event queue flush
*
* \param[in,out] c Client connection to add delay to
* \param[in] queue_len Current event queue length
*/
static inline void
delay_next_flush(pcmk__client_t *c, unsigned int queue_len)
{
/* Delay a maximum of 1.5 seconds */
guint delay = (queue_len < 5)? (1000 + 100 * queue_len) : 1500;
c->event_timer = pcmk__create_timer(delay, crm_ipcs_flush_events_cb, c);
}
/*!
* \internal
* \brief Send client any messages in its queue
*
* \param[in,out] c Client to flush
*
* \return Standard Pacemaker return value
*/
static int
crm_ipcs_flush_events(pcmk__client_t *c)
{
int rc = pcmk_rc_ok;
ssize_t qb_rc = 0;
unsigned int sent = 0;
unsigned int queue_len = 0;
if (c == NULL) {
return rc;
} else if (c->event_timer) {
/* There is already a timer, wait until it goes off */
crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer);
return rc;
}
if (c->event_queue) {
queue_len = g_queue_get_length(c->event_queue);
}
while (sent < 100) {
pcmk__ipc_header_t *header = NULL;
struct iovec *event = NULL;
if (c->event_queue) {
// We don't pop unless send is successful
event = g_queue_peek_head(c->event_queue);
}
if (event == NULL) { // Queue is empty
break;
}
qb_rc = qb_ipcs_event_sendv(c->ipcs, event, 2);
if (qb_rc < 0) {
rc = (int) -qb_rc;
break;
}
event = g_queue_pop_head(c->event_queue);
sent++;
header = event[0].iov_base;
if (header->size_compressed) {
crm_trace("Event %d to %p[%d] (%lld compressed bytes) sent",
header->qb.id, c->ipcs, c->pid, (long long) qb_rc);
} else {
crm_trace("Event %d to %p[%d] (%lld bytes) sent: %.120s",
header->qb.id, c->ipcs, c->pid, (long long) qb_rc,
(char *) (event[1].iov_base));
}
pcmk_free_ipc_event(event);
}
queue_len -= sent;
if (sent > 0 || queue_len) {
crm_trace("Sent %d events (%d remaining) for %p[%d]: %s (%lld)",
sent, queue_len, c->ipcs, c->pid,
pcmk_rc_str(rc), (long long) qb_rc);
}
if (queue_len) {
/* Allow clients to briefly fall behind on processing incoming messages,
* but drop completely unresponsive clients so the connection doesn't
* consume resources indefinitely.
*/
if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) {
if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) {
/* Don't evict for a new or shrinking backlog */
crm_warn("Client with process ID %u has a backlog of %u messages "
QB_XS " %p", c->pid, queue_len, c->ipcs);
} else {
crm_err("Evicting client with process ID %u due to backlog of %u messages "
QB_XS " %p", c->pid, queue_len, c->ipcs);
c->queue_backlog = 0;
qb_ipcs_disconnect(c->ipcs);
return rc;
}
}
c->queue_backlog = queue_len;
delay_next_flush(c, queue_len);
} else {
/* Event queue is empty, there is no backlog */
c->queue_backlog = 0;
}
return rc;
}
/*!
* \internal
* \brief Create an I/O vector for sending an IPC XML message
*
* \param[in] request Identifier for libqb response header
* \param[in] message XML message to send
* \param[in] max_send_size If 0, default IPC buffer size is used
* \param[out] result Where to store prepared I/O vector
* \param[out] bytes Size of prepared data in bytes
*
* \return Standard Pacemaker return code
*/
int
pcmk__ipc_prepare_iov(uint32_t request, const xmlNode *message,
uint32_t max_send_size, struct iovec **result,
ssize_t *bytes)
{
struct iovec *iov;
unsigned int total = 0;
GString *buffer = NULL;
pcmk__ipc_header_t *header = NULL;
int rc = pcmk_rc_ok;
if ((message == NULL) || (result == NULL)) {
rc = EINVAL;
goto done;
}
header = calloc(1, sizeof(pcmk__ipc_header_t));
if (header == NULL) {
rc = ENOMEM;
goto done;
}
buffer = g_string_sized_new(1024);
pcmk__xml_string(message, 0, buffer, 0);
if (max_send_size == 0) {
max_send_size = crm_ipc_default_buffer_size();
}
CRM_LOG_ASSERT(max_send_size != 0);
*result = NULL;
iov = pcmk__new_ipc_event();
iov[0].iov_len = sizeof(pcmk__ipc_header_t);
iov[0].iov_base = header;
header->version = PCMK__IPC_VERSION;
header->size_uncompressed = 1 + buffer->len;
total = iov[0].iov_len + header->size_uncompressed;
if (total < max_send_size) {
iov[1].iov_base = pcmk__str_copy(buffer->str);
iov[1].iov_len = header->size_uncompressed;
} else {
static unsigned int biggest = 0;
char *compressed = NULL;
unsigned int new_size = 0;
if (pcmk__compress(buffer->str,
(unsigned int) header->size_uncompressed,
(unsigned int) max_send_size, &compressed,
&new_size) == pcmk_rc_ok) {
pcmk__set_ipc_flags(header->flags, "send data", crm_ipc_compressed);
header->size_compressed = new_size;
iov[1].iov_len = header->size_compressed;
iov[1].iov_base = compressed;
biggest = QB_MAX(header->size_compressed, biggest);
} else {
crm_log_xml_trace(message, "EMSGSIZE");
biggest = QB_MAX(header->size_uncompressed, biggest);
crm_err("Could not compress %u-byte message into less than IPC "
"limit of %u bytes; set PCMK_ipc_buffer to higher value "
"(%u bytes suggested)",
header->size_uncompressed, max_send_size, 4 * biggest);
free(compressed);
pcmk_free_ipc_event(iov);
rc = EMSGSIZE;
goto done;
}
}
header->qb.size = iov[0].iov_len + iov[1].iov_len;
header->qb.id = (int32_t)request; /* Replying to a specific request */
*result = iov;
pcmk__assert(header->qb.size > 0);
if (bytes != NULL) {
*bytes = header->qb.size;
}
done:
if (buffer != NULL) {
g_string_free(buffer, TRUE);
}
return rc;
}
int
pcmk__ipc_send_iov(pcmk__client_t *c, struct iovec *iov, uint32_t flags)
{
int rc = pcmk_rc_ok;
static uint32_t id = 1;
pcmk__ipc_header_t *header = iov[0].iov_base;
if (c->flags & pcmk__client_proxied) {
/* _ALL_ replies to proxied connections need to be sent as events */
if (!pcmk_is_set(flags, crm_ipc_server_event)) {
/* The proxied flag lets us know this was originally meant to be a
* response, even though we're sending it over the event channel.
*/
pcmk__set_ipc_flags(flags, "server event",
crm_ipc_server_event
|crm_ipc_proxied_relay_response);
}
}
pcmk__set_ipc_flags(header->flags, "server event", flags);
if (flags & crm_ipc_server_event) {
header->qb.id = id++; /* We don't really use it, but doesn't hurt to set one */
if (flags & crm_ipc_server_free) {
crm_trace("Sending the original to %p[%d]", c->ipcs, c->pid);
add_event(c, iov);
} else {
struct iovec *iov_copy = pcmk__new_ipc_event();
crm_trace("Sending a copy to %p[%d]", c->ipcs, c->pid);
iov_copy[0].iov_len = iov[0].iov_len;
iov_copy[0].iov_base = malloc(iov[0].iov_len);
memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len);
iov_copy[1].iov_len = iov[1].iov_len;
iov_copy[1].iov_base = malloc(iov[1].iov_len);
memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len);
add_event(c, iov_copy);
}
} else {
ssize_t qb_rc;
CRM_LOG_ASSERT(header->qb.id != 0); /* Replying to a specific request */
qb_rc = qb_ipcs_response_sendv(c->ipcs, iov, 2);
if (qb_rc < header->qb.size) {
if (qb_rc < 0) {
rc = (int) -qb_rc;
}
crm_notice("Response %d to pid %d failed: %s "
QB_XS " bytes=%u rc=%lld ipcs=%p",
header->qb.id, c->pid, pcmk_rc_str(rc),
header->qb.size, (long long) qb_rc, c->ipcs);
} else {
crm_trace("Response %d sent, %lld bytes to %p[%d]",
header->qb.id, (long long) qb_rc, c->ipcs, c->pid);
}
if (flags & crm_ipc_server_free) {
pcmk_free_ipc_event(iov);
}
}
if (flags & crm_ipc_server_event) {
rc = crm_ipcs_flush_events(c);
} else {
crm_ipcs_flush_events(c);
}
if ((rc == EPIPE) || (rc == ENOTCONN)) {
crm_trace("Client %p disconnected", c->ipcs);
}
return rc;
}
int
pcmk__ipc_send_xml(pcmk__client_t *c, uint32_t request, const xmlNode *message,
uint32_t flags)
{
struct iovec *iov = NULL;
int rc = pcmk_rc_ok;
if (c == NULL) {
return EINVAL;
}
rc = pcmk__ipc_prepare_iov(request, message, crm_ipc_default_buffer_size(),
&iov, NULL);
if (rc == pcmk_rc_ok) {
pcmk__set_ipc_flags(flags, "send data", crm_ipc_server_free);
rc = pcmk__ipc_send_iov(c, iov, flags);
} else {
pcmk_free_ipc_event(iov);
crm_notice("IPC message to pid %d failed: %s " QB_XS " rc=%d",
c->pid, pcmk_rc_str(rc), rc);
}
return rc;
}
/*!
* \internal
* \brief Create an acknowledgement with a status code to send to a client
*
* \param[in] function Calling function
* \param[in] line Source file line within calling function
* \param[in] flags IPC flags to use when sending
* \param[in] tag Element name to use for acknowledgement
* \param[in] ver IPC protocol version (can be NULL)
* \param[in] status Exit status code to add to ack
*
* \return Newly created XML for ack
*
* \note The caller is responsible for freeing the return value with
* \c pcmk__xml_free().
*/
xmlNode *
pcmk__ipc_create_ack_as(const char *function, int line, uint32_t flags,
const char *tag, const char *ver, crm_exit_t status)
{
xmlNode *ack = NULL;
if (pcmk_is_set(flags, crm_ipc_client_response)) {
ack = pcmk__xe_create(NULL, tag);
crm_xml_add(ack, PCMK_XA_FUNCTION, function);
crm_xml_add_int(ack, PCMK__XA_LINE, line);
crm_xml_add_int(ack, PCMK_XA_STATUS, (int) status);
crm_xml_add(ack, PCMK__XA_IPC_PROTO_VERSION, ver);
}
return ack;
}
/*!
* \internal
* \brief Send an acknowledgement with a status code to a client
*
* \param[in] function Calling function
* \param[in] line Source file line within calling function
* \param[in] c Client to send ack to
* \param[in] request Request ID being replied to
* \param[in] flags IPC flags to use when sending
* \param[in] tag Element name to use for acknowledgement
* \param[in] ver IPC protocol version (can be NULL)
* \param[in] status Status code to send with acknowledgement
*
* \return Standard Pacemaker return code
*/
int
pcmk__ipc_send_ack_as(const char *function, int line, pcmk__client_t *c,
uint32_t request, uint32_t flags, const char *tag,
const char *ver, crm_exit_t status)
{
int rc = pcmk_rc_ok;
xmlNode *ack = pcmk__ipc_create_ack_as(function, line, flags, tag, ver, status);
if (ack != NULL) {
crm_trace("Ack'ing IPC message from client %s as <%s status=%d>",
pcmk__client_name(c), tag, status);
crm_log_xml_trace(ack, "sent-ack");
c->request_id = 0;
rc = pcmk__ipc_send_xml(c, request, ack, flags);
pcmk__xml_free(ack);
}
return rc;
}
/*!
* \internal
* \brief Add an IPC server to the main loop for the CIB manager API
*
* \param[out] ipcs_ro New IPC server for read-only CIB manager API
* \param[out] ipcs_rw New IPC server for read/write CIB manager API
* \param[out] ipcs_shm New IPC server for shared-memory CIB manager API
* \param[in] ro_cb IPC callbacks for read-only API
* \param[in] rw_cb IPC callbacks for read/write and shared-memory APIs
*
* \note This function exits fatally if unable to create the servers.
* \note There is no actual difference between the three IPC endpoints other
* than their names.
*/
void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro,
qb_ipcs_service_t **ipcs_rw,
qb_ipcs_service_t **ipcs_shm,
struct qb_ipcs_service_handlers *ro_cb,
struct qb_ipcs_service_handlers *rw_cb)
{
*ipcs_ro = mainloop_add_ipc_server(PCMK__SERVER_BASED_RO,
QB_IPC_NATIVE, ro_cb);
*ipcs_rw = mainloop_add_ipc_server(PCMK__SERVER_BASED_RW,
QB_IPC_NATIVE, rw_cb);
*ipcs_shm = mainloop_add_ipc_server(PCMK__SERVER_BASED_SHM,
QB_IPC_SHM, rw_cb);
if (*ipcs_ro == NULL || *ipcs_rw == NULL || *ipcs_shm == NULL) {
crm_err("Failed to create the CIB manager: exiting and inhibiting respawn");
crm_warn("Verify pacemaker and pacemaker_remote are not both enabled");
crm_exit(CRM_EX_FATAL);
}
}
/*!
* \internal
* \brief Destroy IPC servers for the CIB manager API
*
* \param[out] ipcs_ro IPC server for read-only the CIB manager API
* \param[out] ipcs_rw IPC server for read/write the CIB manager API
* \param[out] ipcs_shm IPC server for shared-memory the CIB manager API
*
* \note This is a convenience function for calling qb_ipcs_destroy() for each
* argument.
*/
void
pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro,
qb_ipcs_service_t *ipcs_rw,
qb_ipcs_service_t *ipcs_shm)
{
qb_ipcs_destroy(ipcs_ro);
qb_ipcs_destroy(ipcs_rw);
qb_ipcs_destroy(ipcs_shm);
}
/*!
* \internal
* \brief Add an IPC server to the main loop for the controller API
*
* \param[in] cb IPC callbacks
*
* \return Newly created IPC server
*/
qb_ipcs_service_t *
pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb)
{
return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb);
}
/*!
* \internal
* \brief Add an IPC server to the main loop for the attribute manager API
*
* \param[out] ipcs Where to store newly created IPC server
* \param[in] cb IPC callbacks
*
* \note This function exits fatally if unable to create the servers.
*/
void
pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs,
struct qb_ipcs_service_handlers *cb)
{
*ipcs = mainloop_add_ipc_server(PCMK__VALUE_ATTRD, QB_IPC_NATIVE, cb);
if (*ipcs == NULL) {
crm_crit("Exiting fatally because unable to serve " PCMK__SERVER_ATTRD
" IPC (verify pacemaker and pacemaker_remote are not both "
"enabled)");
crm_exit(CRM_EX_FATAL);
}
}
/*!
* \internal
* \brief Add an IPC server to the main loop for the fencer API
*
* \param[out] ipcs Where to store newly created IPC server
* \param[in] cb IPC callbacks
*
* \note This function exits fatally if unable to create the servers.
*/
void
pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs,
struct qb_ipcs_service_handlers *cb)
{
*ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb,
QB_LOOP_HIGH);
if (*ipcs == NULL) {
crm_err("Failed to create fencer: exiting and inhibiting respawn.");
crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
crm_exit(CRM_EX_FATAL);
}
}
/*!
* \internal
* \brief Add an IPC server to the main loop for the pacemakerd API
*
* \param[out] ipcs Where to store newly created IPC server
* \param[in] cb IPC callbacks
*
* \note This function exits with CRM_EX_OSERR if unable to create the servers.
*/
void
pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs,
struct qb_ipcs_service_handlers *cb)
{
*ipcs = mainloop_add_ipc_server(CRM_SYSTEM_MCP, QB_IPC_NATIVE, cb);
if (*ipcs == NULL) {
crm_err("Couldn't start pacemakerd IPC server");
crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
/* sub-daemons are observed by pacemakerd. Thus we exit CRM_EX_FATAL
* if we want to prevent pacemakerd from restarting them.
* With pacemakerd we leave the exit-code shown to e.g. systemd
* to what it was prior to moving the code here from pacemakerd.c
*/
crm_exit(CRM_EX_OSERR);
}
}
/*!
* \internal
* \brief Add an IPC server to the main loop for the scheduler API
*
* \param[in] cb IPC callbacks
*
* \return Newly created IPC server
* \note This function exits fatally if unable to create the servers.
*/
qb_ipcs_service_t *
pcmk__serve_schedulerd_ipc(struct qb_ipcs_service_handlers *cb)
{
return mainloop_add_ipc_server(CRM_SYSTEM_PENGINE, QB_IPC_NATIVE, cb);
}
diff --git a/lib/common/remote.c b/lib/common/remote.c
index 1f03988081..87146f226e 100644
--- a/lib/common/remote.c
+++ b/lib/common/remote.c
@@ -1,1246 +1,1029 @@
/*
* Copyright 2008-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netdb.h>
#include <stdlib.h>
#include <errno.h>
#include <inttypes.h> // PRIx32
#include <glib.h>
#include <bzlib.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/xml.h>
#include <crm/common/mainloop.h>
#include <crm/common/remote_internal.h>
+#include <crm/common/tls_internal.h>
#include <gnutls/gnutls.h>
/* Swab macros from linux/swab.h */
#ifdef HAVE_LINUX_SWAB_H
# include <linux/swab.h>
#else
/*
* casts are necessary for constants, because we never know how for sure
* how U/UL/ULL map to __u16, __u32, __u64. At least not in a portable way.
*/
#define __swab16(x) ((uint16_t)( \
(((uint16_t)(x) & (uint16_t)0x00ffU) << 8) | \
(((uint16_t)(x) & (uint16_t)0xff00U) >> 8)))
#define __swab32(x) ((uint32_t)( \
(((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) | \
(((uint32_t)(x) & (uint32_t)0x0000ff00UL) << 8) | \
(((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) | \
(((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24)))
#define __swab64(x) ((uint64_t)( \
(((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) | \
(((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) | \
(((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \
(((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) << 8) | \
(((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >> 8) | \
(((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \
(((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \
(((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56)))
#endif
#define REMOTE_MSG_VERSION 1
#define ENDIAN_LOCAL 0xBADADBBD
struct remote_header_v0 {
uint32_t endian; /* Detect messages from hosts with different endian-ness */
uint32_t version;
uint64_t id;
uint64_t flags;
uint32_t size_total;
uint32_t payload_offset;
uint32_t payload_compressed;
uint32_t payload_uncompressed;
/* New fields get added here */
} __attribute__ ((packed));
/*!
* \internal
* \brief Retrieve remote message header, in local endianness
*
* Return a pointer to the header portion of a remote connection's message
* buffer, converting the header to local endianness if needed.
*
* \param[in,out] remote Remote connection with new message
*
* \return Pointer to message header, localized if necessary
*/
static struct remote_header_v0 *
localized_remote_header(pcmk__remote_t *remote)
{
struct remote_header_v0 *header = (struct remote_header_v0 *)remote->buffer;
if(remote->buffer_offset < sizeof(struct remote_header_v0)) {
return NULL;
} else if(header->endian != ENDIAN_LOCAL) {
uint32_t endian = __swab32(header->endian);
CRM_LOG_ASSERT(endian == ENDIAN_LOCAL);
if(endian != ENDIAN_LOCAL) {
crm_err("Invalid message detected, endian mismatch: %" PRIx32
" is neither %" PRIx32 " nor the swab'd %" PRIx32,
ENDIAN_LOCAL, header->endian, endian);
return NULL;
}
header->id = __swab64(header->id);
header->flags = __swab64(header->flags);
header->endian = __swab32(header->endian);
header->version = __swab32(header->version);
header->size_total = __swab32(header->size_total);
header->payload_offset = __swab32(header->payload_offset);
header->payload_compressed = __swab32(header->payload_compressed);
header->payload_uncompressed = __swab32(header->payload_uncompressed);
}
return header;
}
-int
-pcmk__tls_client_try_handshake(pcmk__remote_t *remote, int *gnutls_rc)
-{
- int rc = pcmk_rc_ok;
-
- if (gnutls_rc != NULL) {
- *gnutls_rc = GNUTLS_E_SUCCESS;
- }
-
- rc = gnutls_handshake(*remote->tls_session);
-
- switch (rc) {
- case GNUTLS_E_SUCCESS:
- rc = pcmk_rc_ok;
- break;
-
- case GNUTLS_E_INTERRUPTED:
- case GNUTLS_E_AGAIN:
- rc = EAGAIN;
- break;
-
- default:
- if (gnutls_rc != NULL) {
- *gnutls_rc = rc;
- }
-
- rc = EPROTO;
- break;
- }
-
- return rc;
-}
-
-int
-pcmk__tls_client_handshake(pcmk__remote_t *remote, int timeout_sec,
- int *gnutls_rc)
-{
- const time_t time_limit = time(NULL) + timeout_sec;
-
- do {
- int rc = pcmk__tls_client_try_handshake(remote, gnutls_rc);
-
- if (rc != EAGAIN) {
- return rc;
- }
- } while (time(NULL) < time_limit);
-
- return ETIME;
-}
-
-/*!
- * \internal
- * \brief Initialize a new TLS session
- *
- * \param[in] csock Connected socket for TLS session
- * \param[in] conn_type GNUTLS_SERVER or GNUTLS_CLIENT
- * \param[in] cred_type GNUTLS_CRD_ANON or GNUTLS_CRD_PSK
- * \param[in] credentials TLS session credentials
- *
- * \return Pointer to newly created session object, or NULL on error
- */
-gnutls_session_t *
-pcmk__new_tls_session(int csock, unsigned int conn_type,
- gnutls_credentials_type_t cred_type, void *credentials)
-{
- int rc = GNUTLS_E_SUCCESS;
- const char *prio_base = NULL;
- char *prio = NULL;
- gnutls_session_t *session = NULL;
-
- /* Determine list of acceptable ciphers, etc. Pacemaker always adds the
- * values required for its functionality.
- *
- * For an example of anonymous authentication, see:
- * http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication
- */
-
- prio_base = pcmk__env_option(PCMK__ENV_TLS_PRIORITIES);
- if (prio_base == NULL) {
- prio_base = PCMK__GNUTLS_PRIORITIES;
- }
- prio = crm_strdup_printf("%s:%s", prio_base,
- (cred_type == GNUTLS_CRD_ANON)? "+ANON-DH" : "+DHE-PSK:+PSK");
-
- session = gnutls_malloc(sizeof(gnutls_session_t));
- if (session == NULL) {
- rc = GNUTLS_E_MEMORY_ERROR;
- goto error;
- }
-
- rc = gnutls_init(session, conn_type);
- if (rc != GNUTLS_E_SUCCESS) {
- goto error;
- }
-
- /* @TODO On the server side, it would be more efficient to cache the
- * priority with gnutls_priority_init2() and set it with
- * gnutls_priority_set() for all sessions.
- */
- rc = gnutls_priority_set_direct(*session, prio, NULL);
- if (rc != GNUTLS_E_SUCCESS) {
- goto error;
- }
-
- gnutls_transport_set_ptr(*session,
- (gnutls_transport_ptr_t) GINT_TO_POINTER(csock));
-
- rc = gnutls_credentials_set(*session, cred_type, credentials);
- if (rc != GNUTLS_E_SUCCESS) {
- goto error;
- }
- free(prio);
- return session;
-
-error:
- crm_err("Could not initialize %s TLS %s session: %s "
- QB_XS " rc=%d priority='%s'",
- (cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK",
- (conn_type == GNUTLS_SERVER)? "server" : "client",
- gnutls_strerror(rc), rc, prio);
- free(prio);
- if (session != NULL) {
- gnutls_free(session);
- }
- return NULL;
-}
-
-/*!
- * \internal
- * \brief Initialize Diffie-Hellman parameters for a TLS server
- *
- * \param[out] dh_params Parameter object to initialize
- *
- * \return Standard Pacemaker return code
- * \todo The current best practice is to allow the client and server to
- * negotiate the Diffie-Hellman parameters via a TLS extension (RFC 7919).
- * However, we have to support both older versions of GnuTLS (<3.6) that
- * don't support the extension on our side, and older Pacemaker versions
- * that don't support the extension on the other side. The next best
- * practice would be to use a known good prime (see RFC 5114 section 2.2),
- * possibly stored in a file distributed with Pacemaker.
- */
-int
-pcmk__init_tls_dh(gnutls_dh_params_t *dh_params)
-{
- int rc = GNUTLS_E_SUCCESS;
- unsigned int dh_bits = 0;
- int dh_max_bits = 0;
-
- rc = gnutls_dh_params_init(dh_params);
- if (rc != GNUTLS_E_SUCCESS) {
- goto error;
- }
-
- dh_bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
- GNUTLS_SEC_PARAM_NORMAL);
- if (dh_bits == 0) {
- rc = GNUTLS_E_DH_PRIME_UNACCEPTABLE;
- goto error;
- }
-
- pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MAX_BITS), &dh_max_bits,
- 0);
- if ((dh_max_bits > 0) && (dh_bits > dh_max_bits)) {
- dh_bits = dh_max_bits;
- }
-
- crm_info("Generating Diffie-Hellman parameters with %u-bit prime for TLS",
- dh_bits);
- rc = gnutls_dh_params_generate2(*dh_params, dh_bits);
- if (rc != GNUTLS_E_SUCCESS) {
- goto error;
- }
-
- return pcmk_rc_ok;
-
-error:
- crm_err("Could not initialize Diffie-Hellman parameters for TLS: %s "
- QB_XS " rc=%d", gnutls_strerror(rc), rc);
- return EPROTO;
-}
-
-/*!
- * \internal
- * \brief Process handshake data from TLS client
- *
- * Read as much TLS handshake data as is available.
- *
- * \param[in] client Client connection
- *
- * \return Standard Pacemaker return code (of particular interest, EAGAIN
- * if some data was successfully read but more data is needed)
- */
-int
-pcmk__read_handshake_data(const pcmk__client_t *client)
-{
- int rc = 0;
-
- pcmk__assert((client != NULL) && (client->remote != NULL)
- && (client->remote->tls_session != NULL));
-
- do {
- rc = gnutls_handshake(*client->remote->tls_session);
- } while (rc == GNUTLS_E_INTERRUPTED);
-
- if (rc == GNUTLS_E_AGAIN) {
- /* No more data is available at the moment. This function should be
- * invoked again once the client sends more.
- */
- return EAGAIN;
- } else if (rc != GNUTLS_E_SUCCESS) {
- crm_err("TLS handshake with remote client failed: %s "
- QB_XS " rc=%d", gnutls_strerror(rc), rc);
- return EPROTO;
- }
- return pcmk_rc_ok;
-}
-
// \return Standard Pacemaker return code
static int
-send_tls(gnutls_session_t *session, struct iovec *iov)
+send_tls(gnutls_session_t session, struct iovec *iov)
{
const char *unsent = iov->iov_base;
size_t unsent_len = iov->iov_len;
ssize_t gnutls_rc;
if (unsent == NULL) {
return EINVAL;
}
crm_trace("Sending TLS message of %llu bytes",
(unsigned long long) unsent_len);
while (true) {
- gnutls_rc = gnutls_record_send(*session, unsent, unsent_len);
+ gnutls_rc = gnutls_record_send(session, unsent, unsent_len);
if (gnutls_rc == GNUTLS_E_INTERRUPTED || gnutls_rc == GNUTLS_E_AGAIN) {
crm_trace("Retrying to send %llu bytes remaining",
(unsigned long long) unsent_len);
} else if (gnutls_rc < 0) {
// Caller can log as error if necessary
crm_info("TLS connection terminated: %s " QB_XS " rc=%lld",
gnutls_strerror((int) gnutls_rc),
(long long) gnutls_rc);
return ECONNABORTED;
} else if (gnutls_rc < unsent_len) {
crm_trace("Sent %lld of %llu bytes remaining",
(long long) gnutls_rc, (unsigned long long) unsent_len);
unsent_len -= gnutls_rc;
unsent += gnutls_rc;
} else {
crm_trace("Sent all %lld bytes remaining", (long long) gnutls_rc);
break;
}
}
return pcmk_rc_ok;
}
// \return Standard Pacemaker return code
static int
send_plaintext(int sock, struct iovec *iov)
{
const char *unsent = iov->iov_base;
size_t unsent_len = iov->iov_len;
ssize_t write_rc;
if (unsent == NULL) {
return EINVAL;
}
crm_debug("Sending plaintext message of %llu bytes to socket %d",
(unsigned long long) unsent_len, sock);
while (true) {
write_rc = write(sock, unsent, unsent_len);
if (write_rc < 0) {
int rc = errno;
if ((errno == EINTR) || (errno == EAGAIN)) {
crm_trace("Retrying to send %llu bytes remaining to socket %d",
(unsigned long long) unsent_len, sock);
continue;
}
// Caller can log as error if necessary
crm_info("Could not send message: %s " QB_XS " rc=%d socket=%d",
pcmk_rc_str(rc), rc, sock);
return rc;
} else if (write_rc < unsent_len) {
crm_trace("Sent %lld of %llu bytes remaining",
(long long) write_rc, (unsigned long long) unsent_len);
unsent += write_rc;
unsent_len -= write_rc;
continue;
} else {
crm_trace("Sent all %lld bytes remaining: %.100s",
(long long) write_rc, (char *) (iov->iov_base));
break;
}
}
return pcmk_rc_ok;
}
// \return Standard Pacemaker return code
static int
remote_send_iovs(pcmk__remote_t *remote, struct iovec *iov, int iovs)
{
int rc = pcmk_rc_ok;
for (int lpc = 0; (lpc < iovs) && (rc == pcmk_rc_ok); lpc++) {
if (remote->tls_session) {
rc = send_tls(remote->tls_session, &(iov[lpc]));
continue;
}
if (remote->tcp_socket) {
rc = send_plaintext(remote->tcp_socket, &(iov[lpc]));
} else {
rc = ESOCKTNOSUPPORT;
}
}
return rc;
}
/*!
* \internal
* \brief Send an XML message over a Pacemaker Remote connection
*
* \param[in,out] remote Pacemaker Remote connection to use
* \param[in] msg XML to send
*
* \return Standard Pacemaker return code
*/
int
pcmk__remote_send_xml(pcmk__remote_t *remote, const xmlNode *msg)
{
int rc = pcmk_rc_ok;
static uint64_t id = 0;
GString *xml_text = NULL;
struct iovec iov[2];
struct remote_header_v0 *header;
CRM_CHECK((remote != NULL) && (msg != NULL), return EINVAL);
xml_text = g_string_sized_new(1024);
pcmk__xml_string(msg, 0, xml_text, 0);
CRM_CHECK(xml_text->len > 0,
g_string_free(xml_text, TRUE); return EINVAL);
header = pcmk__assert_alloc(1, sizeof(struct remote_header_v0));
iov[0].iov_base = header;
iov[0].iov_len = sizeof(struct remote_header_v0);
iov[1].iov_len = 1 + xml_text->len;
iov[1].iov_base = g_string_free(xml_text, FALSE);
id++;
header->id = id;
header->endian = ENDIAN_LOCAL;
header->version = REMOTE_MSG_VERSION;
header->payload_offset = iov[0].iov_len;
header->payload_uncompressed = iov[1].iov_len;
header->size_total = iov[0].iov_len + iov[1].iov_len;
rc = remote_send_iovs(remote, iov, 2);
if (rc != pcmk_rc_ok) {
crm_err("Could not send remote message: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
}
free(iov[0].iov_base);
g_free((gchar *) iov[1].iov_base);
return rc;
}
/*!
* \internal
* \brief Obtain the XML from the currently buffered remote connection message
*
* \param[in,out] remote Remote connection possibly with message available
*
* \return Newly allocated XML object corresponding to message data, or NULL
* \note This effectively removes the message from the connection buffer.
*/
xmlNode *
pcmk__remote_message_xml(pcmk__remote_t *remote)
{
xmlNode *xml = NULL;
struct remote_header_v0 *header = localized_remote_header(remote);
if (header == NULL) {
return NULL;
}
/* Support compression on the receiving end now, in case we ever want to add it later */
if (header->payload_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->payload_uncompressed;
char *uncompressed =
pcmk__assert_alloc(1, header->payload_offset + size_u);
crm_trace("Decompressing message data %d bytes into %d bytes",
header->payload_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset, &size_u,
remote->buffer + header->payload_offset,
header->payload_compressed, 1, 0);
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok && header->version > REMOTE_MSG_VERSION) {
crm_warn("Couldn't decompress v%d message, we only understand v%d",
header->version, REMOTE_MSG_VERSION);
free(uncompressed);
return NULL;
} else if (rc != pcmk_rc_ok) {
crm_err("Decompression failed: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
free(uncompressed);
return NULL;
}
pcmk__assert(size_u == header->payload_uncompressed);
memcpy(uncompressed, remote->buffer, header->payload_offset); /* Preserve the header */
remote->buffer_size = header->payload_offset + size_u;
free(remote->buffer);
remote->buffer = uncompressed;
header = localized_remote_header(remote);
}
/* take ownership of the buffer */
remote->buffer_offset = 0;
CRM_LOG_ASSERT(remote->buffer[sizeof(struct remote_header_v0) + header->payload_uncompressed - 1] == 0);
xml = pcmk__xml_parse(remote->buffer + header->payload_offset);
if (xml == NULL && header->version > REMOTE_MSG_VERSION) {
crm_warn("Couldn't parse v%d message, we only understand v%d",
header->version, REMOTE_MSG_VERSION);
} else if (xml == NULL) {
crm_err("Couldn't parse: '%.120s'", remote->buffer + header->payload_offset);
}
crm_log_xml_trace(xml, "[remote msg]");
return xml;
}
static int
get_remote_socket(const pcmk__remote_t *remote)
{
if (remote->tls_session) {
- void *sock_ptr = gnutls_transport_get_ptr(*remote->tls_session);
+ void *sock_ptr = gnutls_transport_get_ptr(remote->tls_session);
return GPOINTER_TO_INT(sock_ptr);
}
if (remote->tcp_socket) {
return remote->tcp_socket;
}
crm_err("Remote connection type undetermined (bug?)");
return -1;
}
/*!
* \internal
* \brief Wait for a remote session to have data to read
*
* \param[in] remote Connection to check
* \param[in] timeout_ms Maximum time (in ms) to wait
*
* \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
* there is data ready to be read, and ETIME if there is no data within
* the specified timeout)
*/
int
pcmk__remote_ready(const pcmk__remote_t *remote, int timeout_ms)
{
struct pollfd fds = { 0, };
int sock = 0;
int rc = 0;
time_t start;
int timeout = timeout_ms;
sock = get_remote_socket(remote);
if (sock <= 0) {
crm_trace("No longer connected");
return ENOTCONN;
}
start = time(NULL);
errno = 0;
do {
fds.fd = sock;
fds.events = POLLIN;
/* If we got an EINTR while polling, and we have a
* specific timeout we are trying to honor, attempt
* to adjust the timeout to the closest second. */
if (errno == EINTR && (timeout > 0)) {
timeout = timeout_ms - ((time(NULL) - start) * 1000);
if (timeout < 1000) {
timeout = 1000;
}
}
rc = poll(&fds, 1, timeout);
} while (rc < 0 && errno == EINTR);
if (rc < 0) {
return errno;
}
return (rc == 0)? ETIME : pcmk_rc_ok;
}
/*!
* \internal
* \brief Read bytes from non-blocking remote connection
*
* \param[in,out] remote Remote connection to read
*
* \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
* a full message has been received, or EAGAIN for a partial message)
* \note Use only with non-blocking sockets after polling the socket.
* \note This function will return when the socket read buffer is empty or an
* error is encountered.
*/
int
pcmk__read_available_remote_data(pcmk__remote_t *remote)
{
int rc = pcmk_rc_ok;
size_t read_len = sizeof(struct remote_header_v0);
struct remote_header_v0 *header = localized_remote_header(remote);
ssize_t read_rc;
if(header) {
/* Stop at the end of the current message */
read_len = header->size_total;
}
/* automatically grow the buffer when needed */
if(remote->buffer_size < read_len) {
remote->buffer_size = 2 * read_len;
crm_trace("Expanding buffer to %llu bytes",
(unsigned long long) remote->buffer_size);
remote->buffer = pcmk__realloc(remote->buffer, remote->buffer_size + 1);
}
if (remote->tls_session) {
- read_rc = gnutls_record_recv(*(remote->tls_session),
+ read_rc = gnutls_record_recv(remote->tls_session,
remote->buffer + remote->buffer_offset,
remote->buffer_size - remote->buffer_offset);
if (read_rc == GNUTLS_E_INTERRUPTED) {
rc = EINTR;
} else if (read_rc == GNUTLS_E_AGAIN) {
rc = EAGAIN;
} else if (read_rc < 0) {
crm_debug("TLS receive failed: %s (%lld)",
gnutls_strerror(read_rc), (long long) read_rc);
rc = EIO;
}
} else if (remote->tcp_socket) {
read_rc = read(remote->tcp_socket,
remote->buffer + remote->buffer_offset,
remote->buffer_size - remote->buffer_offset);
if (read_rc < 0) {
rc = errno;
}
} else {
crm_err("Remote connection type undetermined (bug?)");
return ESOCKTNOSUPPORT;
}
/* process any errors. */
if (read_rc > 0) {
remote->buffer_offset += read_rc;
/* always null terminate buffer, the +1 to alloc always allows for this. */
remote->buffer[remote->buffer_offset] = '\0';
crm_trace("Received %lld more bytes (%llu total)",
(long long) read_rc,
(unsigned long long) remote->buffer_offset);
} else if ((rc == EINTR) || (rc == EAGAIN)) {
crm_trace("No data available for non-blocking remote read: %s (%d)",
pcmk_rc_str(rc), rc);
} else if (read_rc == 0) {
crm_debug("End of remote data encountered after %llu bytes",
(unsigned long long) remote->buffer_offset);
return ENOTCONN;
} else {
crm_debug("Error receiving remote data after %llu bytes: %s (%d)",
(unsigned long long) remote->buffer_offset,
pcmk_rc_str(rc), rc);
return ENOTCONN;
}
header = localized_remote_header(remote);
if(header) {
if(remote->buffer_offset < header->size_total) {
crm_trace("Read partial remote message (%llu of %u bytes)",
(unsigned long long) remote->buffer_offset,
header->size_total);
} else {
crm_trace("Read full remote message of %llu bytes",
(unsigned long long) remote->buffer_offset);
return pcmk_rc_ok;
}
}
return EAGAIN;
}
/*!
* \internal
* \brief Read one message from a remote connection
*
* \param[in,out] remote Remote connection to read
* \param[in] timeout_ms Fail if message not read in this many milliseconds
* (10s will be used if 0, and 60s if negative)
*
* \return Standard Pacemaker return code
*/
int
pcmk__read_remote_message(pcmk__remote_t *remote, int timeout_ms)
{
int rc = pcmk_rc_ok;
time_t start = time(NULL);
int remaining_timeout = 0;
if (timeout_ms == 0) {
timeout_ms = 10000;
} else if (timeout_ms < 0) {
timeout_ms = 60000;
}
remaining_timeout = timeout_ms;
while (remaining_timeout > 0) {
crm_trace("Waiting for remote data (%d ms of %d ms timeout remaining)",
remaining_timeout, timeout_ms);
rc = pcmk__remote_ready(remote, remaining_timeout);
if (rc == ETIME) {
crm_err("Timed out (%d ms) while waiting for remote data",
remaining_timeout);
return rc;
} else if (rc != pcmk_rc_ok) {
crm_debug("Wait for remote data aborted (will retry): %s "
QB_XS " rc=%d", pcmk_rc_str(rc), rc);
} else {
rc = pcmk__read_available_remote_data(remote);
if (rc == pcmk_rc_ok) {
return rc;
} else if (rc == EAGAIN) {
crm_trace("Waiting for more remote data");
} else {
crm_debug("Could not receive remote data: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
}
}
// Don't waste time retrying after fatal errors
if ((rc == ENOTCONN) || (rc == ESOCKTNOSUPPORT)) {
return rc;
}
remaining_timeout = timeout_ms - ((time(NULL) - start) * 1000);
}
return ETIME;
}
struct tcp_async_cb_data {
int sock;
int timeout_ms;
time_t start;
void *userdata;
void (*callback) (void *userdata, int rc, int sock);
};
// \return TRUE if timer should be rescheduled, FALSE otherwise
static gboolean
check_connect_finished(gpointer userdata)
{
struct tcp_async_cb_data *cb_data = userdata;
int rc;
fd_set rset, wset;
struct timeval ts = { 0, };
if (cb_data->start == 0) {
// Last connect() returned success immediately
rc = pcmk_rc_ok;
goto dispatch_done;
}
// If the socket is ready for reading or writing, the connect succeeded
FD_ZERO(&rset);
FD_SET(cb_data->sock, &rset);
wset = rset;
rc = select(cb_data->sock + 1, &rset, &wset, NULL, &ts);
if (rc < 0) { // select() error
rc = errno;
if ((rc == EINPROGRESS) || (rc == EAGAIN)) {
if ((time(NULL) - cb_data->start) < pcmk__timeout_ms2s(cb_data->timeout_ms)) {
return TRUE; // There is time left, so reschedule timer
} else {
rc = ETIMEDOUT;
}
}
crm_trace("Could not check socket %d for connection success: %s (%d)",
cb_data->sock, pcmk_rc_str(rc), rc);
} else if (rc == 0) { // select() timeout
if ((time(NULL) - cb_data->start) < pcmk__timeout_ms2s(cb_data->timeout_ms)) {
return TRUE; // There is time left, so reschedule timer
}
crm_debug("Timed out while waiting for socket %d connection success",
cb_data->sock);
rc = ETIMEDOUT;
// select() returned number of file descriptors that are ready
} else if (FD_ISSET(cb_data->sock, &rset)
|| FD_ISSET(cb_data->sock, &wset)) {
// The socket is ready; check it for connection errors
int error = 0;
socklen_t len = sizeof(error);
if (getsockopt(cb_data->sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) {
rc = errno;
crm_trace("Couldn't check socket %d for connection errors: %s (%d)",
cb_data->sock, pcmk_rc_str(rc), rc);
} else if (error != 0) {
rc = error;
crm_trace("Socket %d connected with error: %s (%d)",
cb_data->sock, pcmk_rc_str(rc), rc);
} else {
rc = pcmk_rc_ok;
}
} else { // Should not be possible
crm_trace("select() succeeded, but socket %d not in resulting "
"read/write sets", cb_data->sock);
rc = EAGAIN;
}
dispatch_done:
if (rc == pcmk_rc_ok) {
crm_trace("Socket %d is connected", cb_data->sock);
} else {
close(cb_data->sock);
cb_data->sock = -1;
}
if (cb_data->callback) {
cb_data->callback(cb_data->userdata, rc, cb_data->sock);
}
free(cb_data);
return FALSE; // Do not reschedule timer
}
/*!
* \internal
* \brief Attempt to connect socket, calling callback when done
*
* Set a given socket non-blocking, then attempt to connect to it,
* retrying periodically until success or a timeout is reached.
* Call a caller-supplied callback function when completed.
*
* \param[in] sock Newly created socket
* \param[in] addr Socket address information for connect
* \param[in] addrlen Size of socket address information in bytes
* \param[in] timeout_ms Fail if not connected within this much time
* \param[out] timer_id If not NULL, store retry timer ID here
* \param[in] userdata User data to pass to callback
* \param[in] callback Function to call when connection attempt completes
*
* \return Standard Pacemaker return code
*/
static int
connect_socket_retry(int sock, const struct sockaddr *addr, socklen_t addrlen,
int timeout_ms, int *timer_id, void *userdata,
void (*callback) (void *userdata, int rc, int sock))
{
int rc = 0;
int interval = 500;
int timer;
struct tcp_async_cb_data *cb_data = NULL;
rc = pcmk__set_nonblocking(sock);
if (rc != pcmk_rc_ok) {
crm_warn("Could not set socket non-blocking: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
rc = connect(sock, addr, addrlen);
if (rc < 0 && (errno != EINPROGRESS) && (errno != EAGAIN)) {
rc = errno;
crm_warn("Could not connect socket: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
cb_data = pcmk__assert_alloc(1, sizeof(struct tcp_async_cb_data));
cb_data->userdata = userdata;
cb_data->callback = callback;
cb_data->sock = sock;
cb_data->timeout_ms = timeout_ms;
if (rc == 0) {
/* The connect was successful immediately, we still return to mainloop
* and let this callback get called later. This avoids the user of this api
* to have to account for the fact the callback could be invoked within this
* function before returning. */
cb_data->start = 0;
interval = 1;
} else {
cb_data->start = time(NULL);
}
/* This timer function does a non-blocking poll on the socket to see if we
* can use it. Once we can, the connect has completed. This method allows us
* to connect without blocking the mainloop.
*
* @TODO Use a mainloop fd callback for this instead of polling. Something
* about the way mainloop is currently polling prevents this from
* working at the moment though. (See connect(2) regarding EINPROGRESS
* for possible new handling needed.)
*/
crm_trace("Scheduling check in %dms for whether connect to fd %d finished",
interval, sock);
timer = pcmk__create_timer(interval, check_connect_finished, cb_data);
if (timer_id) {
*timer_id = timer;
}
// timer callback should be taking care of cb_data
// cppcheck-suppress memleak
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Attempt once to connect socket and set it non-blocking
*
* \param[in] sock Newly created socket
* \param[in] addr Socket address information for connect
* \param[in] addrlen Size of socket address information in bytes
*
* \return Standard Pacemaker return code
*/
static int
connect_socket_once(int sock, const struct sockaddr *addr, socklen_t addrlen)
{
int rc = connect(sock, addr, addrlen);
if (rc < 0) {
rc = errno;
crm_warn("Could not connect socket: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
rc = pcmk__set_nonblocking(sock);
if (rc != pcmk_rc_ok) {
crm_warn("Could not set socket non-blocking: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
return rc;
}
return pcmk_ok;
}
/*!
* \internal
* \brief Connect to server at specified TCP port
*
* \param[in] host Name of server to connect to
* \param[in] port Server port to connect to
* \param[in] timeout_ms If asynchronous, fail if not connected in this time
* \param[out] timer_id If asynchronous and this is non-NULL, retry timer ID
* will be put here (for ease of cancelling by caller)
* \param[out] sock_fd Where to store socket file descriptor
* \param[in] userdata If asynchronous, data to pass to callback
* \param[in] callback If NULL, attempt a single synchronous connection,
* otherwise retry asynchronously then call this
*
* \return Standard Pacemaker return code
*/
int
pcmk__connect_remote(const char *host, int port, int timeout, int *timer_id,
int *sock_fd, void *userdata,
void (*callback) (void *userdata, int rc, int sock))
{
char buffer[INET6_ADDRSTRLEN];
struct addrinfo *res = NULL;
struct addrinfo *rp = NULL;
struct addrinfo hints;
const char *server = host;
int rc;
int sock = -1;
CRM_CHECK((host != NULL) && (sock_fd != NULL), return EINVAL);
// Get host's IP address(es)
memset(&hints, 0, sizeof(struct addrinfo));
hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_CANONNAME;
rc = getaddrinfo(server, NULL, &hints, &res);
rc = pcmk__gaierror2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Unable to get IP address info for %s: %s",
server, pcmk_rc_str(rc));
goto async_cleanup;
}
if (!res || !res->ai_addr) {
crm_err("Unable to get IP address info for %s: no result", server);
rc = ENOTCONN;
goto async_cleanup;
}
// getaddrinfo() returns a list of host's addresses, try them in order
for (rp = res; rp != NULL; rp = rp->ai_next) {
struct sockaddr *addr = rp->ai_addr;
if (!addr) {
continue;
}
if (rp->ai_canonname) {
server = res->ai_canonname;
}
crm_debug("Got canonical name %s for %s", server, host);
sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
if (sock == -1) {
rc = errno;
crm_warn("Could not create socket for remote connection to %s:%d: "
"%s " QB_XS " rc=%d", server, port, pcmk_rc_str(rc), rc);
continue;
}
/* Set port appropriately for address family */
/* (void*) casts avoid false-positive compiler alignment warnings */
if (addr->sa_family == AF_INET6) {
((struct sockaddr_in6 *)(void*)addr)->sin6_port = htons(port);
} else {
((struct sockaddr_in *)(void*)addr)->sin_port = htons(port);
}
memset(buffer, 0, PCMK__NELEM(buffer));
pcmk__sockaddr2str(addr, buffer);
crm_info("Attempting remote connection to %s:%d", buffer, port);
if (callback) {
if (connect_socket_retry(sock, rp->ai_addr, rp->ai_addrlen, timeout,
timer_id, userdata, callback) == pcmk_rc_ok) {
goto async_cleanup; /* Success for now, we'll hear back later in the callback */
}
} else if (connect_socket_once(sock, rp->ai_addr,
rp->ai_addrlen) == pcmk_rc_ok) {
break; /* Success */
}
// Connect failed
close(sock);
sock = -1;
rc = ENOTCONN;
}
async_cleanup:
if (res) {
freeaddrinfo(res);
}
*sock_fd = sock;
return rc;
}
/*!
* \internal
* \brief Convert an IP address (IPv4 or IPv6) to a string for logging
*
* \param[in] sa Socket address for IP
* \param[out] s Storage for at least INET6_ADDRSTRLEN bytes
*
* \note sa The socket address can be a pointer to struct sockaddr_in (IPv4),
* struct sockaddr_in6 (IPv6) or struct sockaddr_storage (either),
* as long as its sa_family member is set correctly.
*/
void
pcmk__sockaddr2str(const void *sa, char *s)
{
switch (((const struct sockaddr *) sa)->sa_family) {
case AF_INET:
inet_ntop(AF_INET, &(((const struct sockaddr_in *) sa)->sin_addr),
s, INET6_ADDRSTRLEN);
break;
case AF_INET6:
inet_ntop(AF_INET6,
&(((const struct sockaddr_in6 *) sa)->sin6_addr),
s, INET6_ADDRSTRLEN);
break;
default:
strcpy(s, "<invalid>");
}
}
/*!
* \internal
* \brief Accept a client connection on a remote server socket
*
* \param[in] ssock Server socket file descriptor being listened on
* \param[out] csock Where to put new client socket's file descriptor
*
* \return Standard Pacemaker return code
*/
int
pcmk__accept_remote_connection(int ssock, int *csock)
{
int rc;
struct sockaddr_storage addr;
socklen_t laddr = sizeof(addr);
char addr_str[INET6_ADDRSTRLEN];
#ifdef TCP_USER_TIMEOUT
long sbd_timeout = 0;
#endif
/* accept the connection */
memset(&addr, 0, sizeof(addr));
*csock = accept(ssock, (struct sockaddr *)&addr, &laddr);
if (*csock == -1) {
rc = errno;
crm_err("Could not accept remote client connection: %s "
QB_XS " rc=%d", pcmk_rc_str(rc), rc);
return rc;
}
pcmk__sockaddr2str(&addr, addr_str);
crm_info("Accepted new remote client connection from %s", addr_str);
rc = pcmk__set_nonblocking(*csock);
if (rc != pcmk_rc_ok) {
crm_err("Could not set socket non-blocking: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
close(*csock);
*csock = -1;
return rc;
}
#ifdef TCP_USER_TIMEOUT
sbd_timeout = pcmk__get_sbd_watchdog_timeout();
if (sbd_timeout > 0) {
// Time to fail and retry before watchdog
long half = sbd_timeout / 2;
unsigned int optval = (half <= UINT_MAX)? half : UINT_MAX;
rc = setsockopt(*csock, SOL_TCP, TCP_USER_TIMEOUT,
&optval, sizeof(optval));
if (rc < 0) {
rc = errno;
crm_err("Could not set TCP timeout to %d ms on remote connection: "
"%s " QB_XS " rc=%d", optval, pcmk_rc_str(rc), rc);
close(*csock);
*csock = -1;
return rc;
}
}
#endif
return rc;
}
/*!
* \brief Get the default remote connection TCP port on this host
*
* \return Remote connection TCP port number
*/
int
crm_default_remote_port(void)
{
static int port = 0;
if (port == 0) {
const char *env = pcmk__env_option(PCMK__ENV_REMOTE_PORT);
if (env) {
errno = 0;
port = strtol(env, NULL, 10);
if (errno || (port < 1) || (port > 65535)) {
crm_warn("Environment variable PCMK_" PCMK__ENV_REMOTE_PORT
" has invalid value '%s', using %d instead",
env, DEFAULT_REMOTE_PORT);
port = DEFAULT_REMOTE_PORT;
}
} else {
port = DEFAULT_REMOTE_PORT;
}
}
return port;
}
diff --git a/lib/common/tls.c b/lib/common/tls.c
new file mode 100644
index 0000000000..c3d99129aa
--- /dev/null
+++ b/lib/common/tls.c
@@ -0,0 +1,529 @@
+/*
+ * Copyright 2024 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#include <crm_internal.h>
+
+#include <errno.h>
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+#include <stdlib.h>
+
+#include <crm/common/tls_internal.h>
+
+static char *
+get_gnutls_priorities(gnutls_credentials_type_t cred_type)
+{
+ const char *prio_base = pcmk__env_option(PCMK__ENV_TLS_PRIORITIES);
+
+ if (prio_base == NULL) {
+ prio_base = PCMK__GNUTLS_PRIORITIES;
+ }
+
+ return crm_strdup_printf("%s:%s", prio_base,
+ (cred_type == GNUTLS_CRD_ANON)? "+ANON-DH" : "+DHE-PSK:+PSK");
+}
+
+static const char *
+tls_cred_str(gnutls_credentials_type_t cred_type)
+{
+ if (cred_type == GNUTLS_CRD_ANON) {
+ return "unauthenticated";
+ } else if (cred_type == GNUTLS_CRD_PSK) {
+ return "shared-key-authenticated";
+ } else if (cred_type == GNUTLS_CRD_CERTIFICATE) {
+ return "certificate-authenticated";
+ } else {
+ return "unknown";
+ }
+}
+
+static int
+tls_load_x509_data(pcmk__tls_t *tls)
+{
+ int rc;
+
+ CRM_CHECK(tls->cred_type == GNUTLS_CRD_CERTIFICATE, return EINVAL);
+
+ /* Load a trusted CA to be used to verify client certificates. Use
+ * of this function instead of gnutls_certificate_set_x509_system_trust
+ * means we do not look at the system-wide authorities installed in
+ * /etc/pki somewhere. This requires the cluster admin to set up their
+ * own CA.
+ */
+ rc = gnutls_certificate_set_x509_trust_file(tls->credentials.cert,
+ tls->ca_file,
+ GNUTLS_X509_FMT_PEM);
+ if (rc <= 0) {
+ crm_err("Failed to set X509 CA file: %s", gnutls_strerror(rc));
+ return ENODATA;
+ }
+
+ /* If a Certificate Revocation List (CRL) file was given in the environment,
+ * load that now so we know which clients have been banned.
+ */
+ if (tls->crl_file != NULL) {
+ rc = gnutls_certificate_set_x509_crl_file(tls->credentials.cert,
+ tls->crl_file,
+ GNUTLS_X509_FMT_PEM);
+ if (rc < 0) {
+ crm_err("Failed to set X509 CRL file: %s",
+ gnutls_strerror(rc));
+ return ENODATA;
+ }
+ }
+
+ /* NULL = no password for the key, GNUTLS_PKCS_PLAIN = unencrypted key
+ * file
+ */
+ rc = gnutls_certificate_set_x509_key_file2(tls->credentials.cert,
+ tls->cert_file, tls->key_file,
+ GNUTLS_X509_FMT_PEM, NULL,
+ GNUTLS_PKCS_PLAIN);
+ if (rc < 0) {
+ crm_err("Failed to set X509 cert/key pair: %s",
+ gnutls_strerror(rc));
+ return ENODATA;
+ }
+
+ return pcmk_rc_ok;
+}
+
+/*!
+ * \internal
+ * \brief Verify a peer's certificate
+ *
+ * \return 0 if the certificate is trusted and the gnutls handshake should
+ * continue, -1 otherwise
+ */
+static int
+verify_peer_cert(gnutls_session_t session)
+{
+ int rc;
+ int type;
+ unsigned int status;
+ gnutls_datum_t out;
+
+ /* NULL = no hostname comparison will be performed */
+ rc = gnutls_certificate_verify_peers3(session, NULL, &status);
+
+ /* Success means it was able to perform the verification. We still have
+ * to check status to see whether the cert is valid or not.
+ */
+ if (rc != GNUTLS_E_SUCCESS) {
+ crm_err("Failed to verify peer certificate: %s", gnutls_strerror(rc));
+ return -1;
+ }
+
+ if (status == 0) {
+ /* The certificate is trusted. */
+ return 0;
+ }
+
+ type = gnutls_certificate_type_get(session);
+ gnutls_certificate_verification_status_print(status, type, &out, 0);
+ crm_err("Peer certificate invalid: %s", out.data);
+ gnutls_free(out.data);
+ return GNUTLS_E_CERTIFICATE_VERIFICATION_ERROR;
+}
+
+static void
+_gnutls_log_func(int level, const char *msg)
+{
+ crm_trace("%s", msg);
+}
+
+void
+pcmk__free_tls(pcmk__tls_t *tls)
+{
+ if (tls == NULL) {
+ return;
+ }
+
+ /* This is only set on the server side. */
+ if (tls->server) {
+ gnutls_dh_params_deinit(tls->dh_params);
+ }
+
+ if (tls->cred_type == GNUTLS_CRD_ANON) {
+ if (tls->server) {
+ gnutls_anon_free_server_credentials(tls->credentials.anon_s);
+ } else {
+ gnutls_anon_free_client_credentials(tls->credentials.anon_c);
+ }
+ } else if (tls->cred_type == GNUTLS_CRD_CERTIFICATE) {
+ gnutls_certificate_free_credentials(tls->credentials.cert);
+ } else if (tls->cred_type == GNUTLS_CRD_PSK) {
+ if (tls->server) {
+ gnutls_psk_free_server_credentials(tls->credentials.psk_s);
+ } else {
+ gnutls_psk_free_client_credentials(tls->credentials.psk_c);
+ }
+ }
+
+ free(tls);
+ tls = NULL;
+
+ gnutls_global_deinit();
+}
+
+int
+pcmk__init_tls(pcmk__tls_t **tls, bool server, gnutls_credentials_type_t cred_type)
+{
+ int rc = pcmk_rc_ok;
+
+ if (*tls != NULL) {
+ return rc;
+ }
+
+ *tls = pcmk__assert_alloc(1, sizeof(pcmk__tls_t));
+
+ signal(SIGPIPE, SIG_IGN);
+
+ /* gnutls_global_init is safe to call multiple times, but we have to call
+ * gnutls_global_deinit the same number of times for that function to do
+ * anything.
+ *
+ * FIXME: When we can use gnutls >= 3.3.0, we don't have to call
+ * gnutls_global_init anymore.
+ */
+ gnutls_global_init();
+ gnutls_global_set_log_level(8);
+ gnutls_global_set_log_function(_gnutls_log_func);
+
+ if (server) {
+ rc = pcmk__init_tls_dh(&(*tls)->dh_params);
+ if (rc != pcmk_rc_ok) {
+ pcmk__free_tls(*tls);
+ return rc;
+ }
+ }
+
+ (*tls)->cred_type = cred_type;
+ (*tls)->server = server;
+
+ if (cred_type == GNUTLS_CRD_ANON) {
+ if (server) {
+ gnutls_anon_allocate_server_credentials(&(*tls)->credentials.anon_s);
+ gnutls_anon_set_server_dh_params((*tls)->credentials.anon_s,
+ (*tls)->dh_params);
+ } else {
+ gnutls_anon_allocate_client_credentials(&(*tls)->credentials.anon_c);
+ }
+ } else if (cred_type == GNUTLS_CRD_CERTIFICATE) {
+ /* Grab these environment variables before doing anything else. */
+ if (server) {
+ (*tls)->ca_file = pcmk__env_option(PCMK__ENV_CA_FILE);
+ (*tls)->cert_file = pcmk__env_option(PCMK__ENV_CERT_FILE);
+ (*tls)->crl_file = pcmk__env_option(PCMK__ENV_CRL_FILE);
+ (*tls)->key_file = pcmk__env_option(PCMK__ENV_KEY_FILE);
+ } else {
+ (*tls)->ca_file = getenv("CIB_ca_file");
+ (*tls)->cert_file = getenv("CIB_cert_file");
+ (*tls)->crl_file = getenv("CIB_crl_file");
+ (*tls)->key_file = getenv("CIB_key_file");
+ }
+
+ gnutls_certificate_allocate_credentials(&(*tls)->credentials.cert);
+
+ if (server) {
+ gnutls_certificate_set_dh_params((*tls)->credentials.cert,
+ (*tls)->dh_params);
+
+ }
+
+ rc = tls_load_x509_data(*tls);
+ if (rc != pcmk_rc_ok) {
+ pcmk__free_tls(*tls);
+ return rc;
+ }
+ } else if (cred_type == GNUTLS_CRD_PSK) {
+ if (server) {
+ gnutls_psk_allocate_server_credentials(&(*tls)->credentials.psk_s);
+ gnutls_psk_set_server_dh_params((*tls)->credentials.psk_s,
+ (*tls)->dh_params);
+ } else {
+ gnutls_psk_allocate_client_credentials(&(*tls)->credentials.psk_c);
+ }
+ }
+
+ return rc;
+}
+
+int
+pcmk__init_tls_dh(gnutls_dh_params_t *dh_params)
+{
+ int rc = GNUTLS_E_SUCCESS;
+ unsigned int dh_bits = 0;
+ int dh_max_bits = 0;
+
+ rc = gnutls_dh_params_init(dh_params);
+ if (rc != GNUTLS_E_SUCCESS) {
+ goto error;
+ }
+
+ dh_bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
+ GNUTLS_SEC_PARAM_NORMAL);
+ if (dh_bits == 0) {
+ rc = GNUTLS_E_DH_PRIME_UNACCEPTABLE;
+ goto error;
+ }
+
+ pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MAX_BITS), &dh_max_bits, 0);
+ if ((dh_max_bits > 0) && (dh_bits > dh_max_bits)) {
+ dh_bits = dh_max_bits;
+ }
+
+ crm_info("Generating Diffie-Hellman parameters with %u-bit prime for TLS",
+ dh_bits);
+ rc = gnutls_dh_params_generate2(*dh_params, dh_bits);
+ if (rc != GNUTLS_E_SUCCESS) {
+ goto error;
+ }
+
+ return pcmk_rc_ok;
+
+error:
+ crm_err("Could not initialize Diffie-Hellman parameters for TLS: %s "
+ QB_XS " rc=%d", gnutls_strerror(rc), rc);
+ return EPROTO;
+}
+
+gnutls_session_t
+pcmk__new_tls_session(pcmk__tls_t *tls, int csock)
+{
+ unsigned int conn_type = tls->server ? GNUTLS_SERVER : GNUTLS_CLIENT;
+ int rc = GNUTLS_E_SUCCESS;
+ char *prio = NULL;
+ gnutls_session_t session = NULL;
+
+ rc = gnutls_init(&session, conn_type);
+ if (rc != GNUTLS_E_SUCCESS) {
+ goto error;
+ }
+
+ /* Determine list of acceptable ciphers, etc. Pacemaker always adds the
+ * values required for its functionality.
+ *
+ * For an example of anonymous authentication, see:
+ * http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication
+ */
+ prio = get_gnutls_priorities(tls->cred_type);
+
+ /* @TODO On the server side, it would be more efficient to cache the
+ * priority with gnutls_priority_init2() and set it with
+ * gnutls_priority_set() for all sessions.
+ */
+ rc = gnutls_priority_set_direct(session, prio, NULL);
+ if (rc != GNUTLS_E_SUCCESS) {
+ goto error;
+ }
+
+ gnutls_transport_set_ptr(session,
+ (gnutls_transport_ptr_t) GINT_TO_POINTER(csock));
+
+ /* gnutls does not make this easy */
+ if (tls->cred_type == GNUTLS_CRD_ANON && tls->server) {
+ rc = gnutls_credentials_set(session, tls->cred_type, tls->credentials.anon_s);
+ } else if (tls->cred_type == GNUTLS_CRD_ANON) {
+ rc = gnutls_credentials_set(session, tls->cred_type, tls->credentials.anon_c);
+ } else if (tls->cred_type == GNUTLS_CRD_CERTIFICATE) {
+ rc = gnutls_credentials_set(session, tls->cred_type, tls->credentials.cert);
+ } else if (tls->cred_type == GNUTLS_CRD_PSK && tls->server) {
+ rc = gnutls_credentials_set(session, tls->cred_type, tls->credentials.psk_s);
+ } else if (tls->cred_type == GNUTLS_CRD_PSK) {
+ rc = gnutls_credentials_set(session, tls->cred_type, tls->credentials.psk_c);
+ } else {
+ crm_err("Unknown credential type: %d", tls->cred_type);
+ rc = EINVAL;
+ goto error;
+ }
+
+ if (rc != GNUTLS_E_SUCCESS) {
+ goto error;
+ }
+
+ free(prio);
+
+ if (tls->cred_type == GNUTLS_CRD_CERTIFICATE) {
+ if (conn_type == GNUTLS_SERVER) {
+ /* Require the client to send a certificate for the server to verify. */
+ gnutls_certificate_server_set_request(session, GNUTLS_CERT_REQUIRE);
+ }
+
+ /* Register a function to verify the peer's certificate.
+ *
+ * FIXME: When we can require gnutls >= 3.4.6, remove verify_peer_cert
+ * and use gnutls_session_set_verify_cert instead.
+ */
+ gnutls_certificate_set_verify_function(tls->credentials.cert, verify_peer_cert);
+ }
+
+ return session;
+
+error:
+ crm_err("Could not initialize %s TLS %s session: %s " QB_XS " rc=%d priority='%s'",
+ tls_cred_str(tls->cred_type),
+ (conn_type == GNUTLS_SERVER)? "server" : "client",
+ gnutls_strerror(rc), rc, prio);
+ free(prio);
+ if (session != NULL) {
+ gnutls_deinit(session);
+ }
+ return NULL;
+}
+
+int
+pcmk__read_handshake_data(const pcmk__client_t *client)
+{
+ int rc = 0;
+
+ pcmk__assert((client != NULL) && (client->remote != NULL)
+ && (client->remote->tls_session != NULL));
+
+ do {
+ rc = gnutls_handshake(client->remote->tls_session);
+ } while (rc == GNUTLS_E_INTERRUPTED);
+
+ if (rc == GNUTLS_E_AGAIN) {
+ /* No more data is available at the moment. This function should be
+ * invoked again once the client sends more.
+ */
+ return EAGAIN;
+ } else if (rc != GNUTLS_E_SUCCESS) {
+ crm_err("TLS handshake with remote client failed: %s "
+ QB_XS " rc=%d", gnutls_strerror(rc), rc);
+ return EPROTO;
+ }
+ return pcmk_rc_ok;
+}
+
+void
+pcmk__tls_add_psk_key(pcmk__tls_t *tls, gnutls_datum_t *key)
+{
+ gnutls_psk_set_client_credentials(tls->credentials.psk_c,
+ DEFAULT_REMOTE_USERNAME, key,
+ GNUTLS_PSK_KEY_RAW);
+}
+
+void
+pcmk__tls_add_psk_callback(pcmk__tls_t *tls,
+ gnutls_psk_server_credentials_function *cb)
+{
+ gnutls_psk_set_server_credentials_function(tls->credentials.psk_s, cb);
+}
+
+void
+pcmk__tls_check_cert_expiration(gnutls_session_t session)
+{
+ gnutls_x509_crt_t cert;
+ const gnutls_datum_t *datum = NULL;
+ time_t expiry;
+
+ if (session == NULL) {
+ return;
+ }
+
+ if (gnutls_certificate_type_get(session) != GNUTLS_CRT_X509) {
+ return;
+ }
+
+ datum = gnutls_certificate_get_ours(session);
+ if (datum == NULL) {
+ return;
+ }
+
+ gnutls_x509_crt_init(&cert);
+ gnutls_x509_crt_import(cert, datum, GNUTLS_X509_FMT_DER);
+
+ expiry = gnutls_x509_crt_get_expiration_time(cert);
+
+ if (expiry != -1) {
+ time_t now = time(NULL);
+
+ /* If the cert is going to expire within ~ one month (30 days), log it */
+ if (expiry - now <= 60 * 60 * 24 * 30) {
+ crm_time_t *expiry_t = pcmk__copy_timet(expiry);
+
+ crm_time_log(LOG_WARNING, "TLS certificate will expire on",
+ expiry_t, crm_time_log_date | crm_time_log_timeofday);
+ crm_time_free(expiry_t);
+ }
+ }
+
+ gnutls_x509_crt_deinit(cert);
+}
+
+int
+pcmk__tls_client_try_handshake(pcmk__remote_t *remote, int *gnutls_rc)
+{
+ int rc = pcmk_rc_ok;
+
+ if (gnutls_rc != NULL) {
+ *gnutls_rc = GNUTLS_E_SUCCESS;
+ }
+
+ rc = gnutls_handshake(remote->tls_session);
+
+ switch (rc) {
+ case GNUTLS_E_SUCCESS:
+ rc = pcmk_rc_ok;
+ break;
+
+ case GNUTLS_E_INTERRUPTED:
+ case GNUTLS_E_AGAIN:
+ rc = EAGAIN;
+ break;
+
+ default:
+ if (gnutls_rc != NULL) {
+ *gnutls_rc = rc;
+ }
+
+ rc = EPROTO;
+ break;
+ }
+
+ return rc;
+}
+
+int
+pcmk__tls_client_handshake(pcmk__remote_t *remote, int timeout_sec,
+ int *gnutls_rc)
+{
+ const time_t time_limit = time(NULL) + timeout_sec;
+
+ do {
+ int rc = pcmk__tls_client_try_handshake(remote, gnutls_rc);
+
+ if (rc != EAGAIN) {
+ return rc;
+ }
+ } while (time(NULL) < time_limit);
+
+ return ETIME;
+}
+
+bool
+pcmk__x509_enabled(bool server)
+{
+ /* Environment variables for servers come through the sysconfig file, and
+ * have names like PCMK_<whatever>. Environment variables for clients come
+ * from the environment and have names like CIB_<whatever>. This function
+ * is used for both, so we need to check both.
+ */
+ if (server) {
+ return !pcmk__str_empty(pcmk__env_option(PCMK__ENV_CERT_FILE)) &&
+ !pcmk__str_empty(pcmk__env_option(PCMK__ENV_CA_FILE)) &&
+ !pcmk__str_empty(pcmk__env_option(PCMK__ENV_KEY_FILE));
+ } else {
+ return !pcmk__str_empty(getenv("CIB_cert_file")) &&
+ !pcmk__str_empty(getenv("CIB_ca_file")) &&
+ !pcmk__str_empty(getenv("CIB_key_file"));
+ }
+}
diff --git a/lib/common/utils.c b/lib/common/utils.c
index c137330a95..f49f5c0b1b 100644
--- a/lib/common/utils.c
+++ b/lib/common/utils.c
@@ -1,492 +1,500 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/stat.h>
#include <sys/utsname.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>
#include <pwd.h>
#include <time.h>
#include <libgen.h>
#include <signal.h>
#include <grp.h>
#include <qb/qbdefs.h>
#include <crm/crm.h>
#include <crm/services.h>
#include <crm/cib/internal.h>
#include <crm/common/xml.h>
#include <crm/common/util.h>
#include <crm/common/ipc.h>
#include <crm/common/iso8601.h>
#include <crm/common/mainloop.h>
#include <libxml2/libxml/relaxng.h>
#include "crmcommon_private.h"
CRM_TRACE_INIT_DATA(common);
bool pcmk__config_has_error = false;
bool pcmk__config_has_warning = false;
char *crm_system_name = NULL;
/*!
* \brief Free all memory used by libcrmcommon
*
* Free all global memory allocated by the libcrmcommon library. This should be
* called before exiting a process that uses the library, and the process should
* not call any libcrmcommon or libxml2 APIs after calling this one.
*/
void
pcmk_common_cleanup(void)
{
// @TODO This isn't really everything, move all cleanup here
mainloop_cleanup();
pcmk__xml_cleanup();
pcmk__free_common_logger();
qb_log_fini(); // Don't log anything after this point
free(crm_system_name);
crm_system_name = NULL;
}
bool
pcmk__is_user_in_group(const char *user, const char *group)
{
struct group *grent;
char **gr_mem;
if (user == NULL || group == NULL) {
return false;
}
setgrent();
while ((grent = getgrent()) != NULL) {
if (grent->gr_mem == NULL) {
continue;
}
if(strcmp(group, grent->gr_name) != 0) {
continue;
}
gr_mem = grent->gr_mem;
while (*gr_mem != NULL) {
if (!strcmp(user, *gr_mem++)) {
endgrent();
return true;
}
}
}
endgrent();
return false;
}
int
crm_user_lookup(const char *name, uid_t * uid, gid_t * gid)
{
int rc = pcmk_ok;
char *buffer = NULL;
struct passwd pwd;
struct passwd *pwentry = NULL;
buffer = calloc(1, PCMK__PW_BUFFER_LEN);
if (buffer == NULL) {
return -ENOMEM;
}
rc = getpwnam_r(name, &pwd, buffer, PCMK__PW_BUFFER_LEN, &pwentry);
if (pwentry) {
if (uid) {
*uid = pwentry->pw_uid;
}
if (gid) {
*gid = pwentry->pw_gid;
}
crm_trace("User %s has uid=%d gid=%d", name, pwentry->pw_uid, pwentry->pw_gid);
} else {
rc = rc? -rc : -EINVAL;
crm_info("User %s lookup: %s", name, pcmk_strerror(rc));
}
free(buffer);
return rc;
}
/*!
* \brief Get user and group IDs of pacemaker daemon user
*
* \param[out] uid If non-NULL, where to store daemon user ID
* \param[out] gid If non-NULL, where to store daemon group ID
*
* \return pcmk_ok on success, -errno otherwise
*/
int
pcmk_daemon_user(uid_t *uid, gid_t *gid)
{
static uid_t daemon_uid;
static gid_t daemon_gid;
static bool found = false;
int rc = pcmk_ok;
if (!found) {
rc = crm_user_lookup(CRM_DAEMON_USER, &daemon_uid, &daemon_gid);
if (rc == pcmk_ok) {
found = true;
}
}
if (found) {
if (uid) {
*uid = daemon_uid;
}
if (gid) {
*gid = daemon_gid;
}
}
return rc;
}
/*!
* \internal
* \brief Return the integer equivalent of a portion of a string
*
* \param[in] text Pointer to beginning of string portion
* \param[out] end_text This will point to next character after integer
*/
static int
version_helper(const char *text, const char **end_text)
{
int atoi_result = -1;
pcmk__assert(end_text != NULL);
errno = 0;
if (text != NULL && text[0] != 0) {
/* seemingly sacrificing const-correctness -- because while strtol
doesn't modify the input, it doesn't want to artificially taint the
"end_text" pointer-to-pointer-to-first-char-in-string with constness
in case the input wasn't actually constant -- by semantic definition
not a single character will get modified so it shall be perfectly
safe to make compiler happy with dropping "const" qualifier here */
atoi_result = (int) strtol(text, (char **) end_text, 10);
if (errno == EINVAL) {
crm_err("Conversion of '%s' %c failed", text, text[0]);
atoi_result = -1;
}
}
return atoi_result;
}
/*
* version1 < version2 : -1
* version1 = version2 : 0
* version1 > version2 : 1
*/
int
compare_version(const char *version1, const char *version2)
{
int rc = 0;
int lpc = 0;
const char *ver1_iter, *ver2_iter;
if (version1 == NULL && version2 == NULL) {
return 0;
} else if (version1 == NULL) {
return -1;
} else if (version2 == NULL) {
return 1;
}
ver1_iter = version1;
ver2_iter = version2;
while (1) {
int digit1 = 0;
int digit2 = 0;
lpc++;
if (ver1_iter == ver2_iter) {
break;
}
if (ver1_iter != NULL) {
digit1 = version_helper(ver1_iter, &ver1_iter);
}
if (ver2_iter != NULL) {
digit2 = version_helper(ver2_iter, &ver2_iter);
}
if (digit1 < digit2) {
rc = -1;
break;
} else if (digit1 > digit2) {
rc = 1;
break;
}
if (ver1_iter != NULL && *ver1_iter == '.') {
ver1_iter++;
}
if (ver1_iter != NULL && *ver1_iter == '\0') {
ver1_iter = NULL;
}
if (ver2_iter != NULL && *ver2_iter == '.') {
ver2_iter++;
}
if (ver2_iter != NULL && *ver2_iter == 0) {
ver2_iter = NULL;
}
}
if (rc == 0) {
crm_trace("%s == %s (%d)", version1, version2, lpc);
} else if (rc < 0) {
crm_trace("%s < %s (%d)", version1, version2, lpc);
} else if (rc > 0) {
crm_trace("%s > %s (%d)", version1, version2, lpc);
}
return rc;
}
/*!
* \internal
* \brief Convert the current process to a daemon process
*
* Fork a child process, exit the parent, create a PID file with the current
* process ID, and close the standard input/output/error file descriptors.
* Exit instead if a daemon is already running and using the PID file.
*
* \param[in] name Daemon executable name
* \param[in] pidfile File name to use as PID file
*/
void
pcmk__daemonize(const char *name, const char *pidfile)
{
int rc;
pid_t pid;
/* Check before we even try... */
rc = pcmk__pidfile_matches(pidfile, 1, name, &pid);
if ((rc != pcmk_rc_ok) && (rc != ENOENT)) {
crm_err("%s: already running [pid %lld in %s]",
name, (long long) pid, pidfile);
printf("%s: already running [pid %lld in %s]\n",
name, (long long) pid, pidfile);
crm_exit(CRM_EX_ERROR);
}
pid = fork();
if (pid < 0) {
fprintf(stderr, "%s: could not start daemon\n", name);
crm_perror(LOG_ERR, "fork");
crm_exit(CRM_EX_OSERR);
} else if (pid > 0) {
crm_exit(CRM_EX_OK);
}
rc = pcmk__lock_pidfile(pidfile, name);
if (rc != pcmk_rc_ok) {
crm_err("Could not lock '%s' for %s: %s " QB_XS " rc=%d",
pidfile, name, pcmk_rc_str(rc), rc);
printf("Could not lock '%s' for %s: %s (%d)\n",
pidfile, name, pcmk_rc_str(rc), rc);
crm_exit(CRM_EX_ERROR);
}
umask(S_IWGRP | S_IWOTH | S_IROTH);
close(STDIN_FILENO);
pcmk__open_devnull(O_RDONLY); // stdin (fd 0)
close(STDOUT_FILENO);
pcmk__open_devnull(O_WRONLY); // stdout (fd 1)
close(STDERR_FILENO);
pcmk__open_devnull(O_WRONLY); // stderr (fd 2)
}
#ifdef HAVE_UUID_UUID_H
# include <uuid/uuid.h>
#endif
char *
crm_generate_uuid(void)
{
unsigned char uuid[16];
char *buffer = malloc(37); /* Including NUL byte */
pcmk__mem_assert(buffer);
uuid_generate(uuid);
uuid_unparse(uuid, buffer);
return buffer;
}
-void
-crm_gnutls_global_init(void)
-{
- signal(SIGPIPE, SIG_IGN);
- gnutls_global_init();
-}
-
/*!
* \internal
* \brief Sleep for given milliseconds
*
* \param[in] ms Time to sleep
*
* \note The full time might not be slept if a signal is received.
*/
void
pcmk__sleep_ms(unsigned int ms)
{
// @TODO Impose a sane maximum sleep to avoid hanging a process for long
//CRM_CHECK(ms <= MAX_SLEEP, ms = MAX_SLEEP);
// Use sleep() for any whole seconds
if (ms >= 1000) {
sleep(ms / 1000);
ms -= ms / 1000;
}
if (ms == 0) {
return;
}
#if defined(HAVE_NANOSLEEP)
// nanosleep() is POSIX-2008, so prefer that
{
struct timespec req = { .tv_sec = 0, .tv_nsec = (long) (ms * 1000000) };
nanosleep(&req, NULL);
}
#elif defined(HAVE_USLEEP)
// usleep() is widely available, though considered obsolete
usleep((useconds_t) ms);
#else
// Otherwise use a trick with select() timeout
{
struct timeval tv = { .tv_sec = 0, .tv_usec = (suseconds_t) ms };
select(0, NULL, NULL, NULL, &tv);
}
#endif
}
/*!
* \internal
* \brief Add a timer
*
* \param[in] interval_ms The interval for the function to be called, in ms
* \param[in] fn The function to be called
* \param[in] data Data to be passed to fn (can be NULL)
*
* \return The ID of the event source
*/
guint
pcmk__create_timer(guint interval_ms, GSourceFunc fn, gpointer data)
{
pcmk__assert(interval_ms != 0 && fn != NULL);
if (interval_ms % 1000 == 0) {
/* In case interval_ms is 0, the call to pcmk__timeout_ms2s ensures
* an interval of one second.
*/
return g_timeout_add_seconds(pcmk__timeout_ms2s(interval_ms), fn, data);
} else {
return g_timeout_add(interval_ms, fn, data);
}
}
/*!
* \internal
* \brief Convert milliseconds to seconds
*
* \param[in] timeout_ms The interval, in ms
*
* \return If \p timeout_ms is 0, return 0. Otherwise, return the number of
* seconds, rounded to the nearest integer, with a minimum of 1.
*/
guint
pcmk__timeout_ms2s(guint timeout_ms)
{
guint quot, rem;
if (timeout_ms == 0) {
return 0;
} else if (timeout_ms < 1000) {
return 1;
}
quot = timeout_ms / 1000;
rem = timeout_ms % 1000;
if (rem >= 500) {
quot += 1;
}
return quot;
}
// Deprecated functions kept only for backward API compatibility
// LCOV_EXCL_START
#include <crm/common/util_compat.h>
+static void
+_gnutls_log_func(int level, const char *msg)
+{
+ crm_trace("%s", msg);
+}
+
+void
+crm_gnutls_global_init(void)
+{
+ signal(SIGPIPE, SIG_IGN);
+ gnutls_global_init();
+ gnutls_global_set_log_level(8);
+ gnutls_global_set_log_function(_gnutls_log_func);
+}
+
/*!
* \brief Check whether string represents a client name used by cluster daemons
*
* \param[in] name String to check
*
* \return true if name is standard client name used by daemons, false otherwise
*
* \note This is provided by the client, and so cannot be used by itself as a
* secure means of authentication.
*/
bool
crm_is_daemon_name(const char *name)
{
return pcmk__str_any_of(name,
"attrd",
CRM_SYSTEM_CIB,
CRM_SYSTEM_CRMD,
CRM_SYSTEM_DC,
CRM_SYSTEM_LRMD,
CRM_SYSTEM_MCP,
CRM_SYSTEM_PENGINE,
CRM_SYSTEM_TENGINE,
"pacemaker-attrd",
"pacemaker-based",
"pacemaker-controld",
"pacemaker-execd",
"pacemaker-fenced",
"pacemaker-remoted",
"pacemaker-schedulerd",
"stonith-ng",
"stonithd",
NULL);
}
// LCOV_EXCL_STOP
// End deprecated API
diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
index 07f9f3a602..07e765fdfd 100644
--- a/lib/lrmd/lrmd_client.c
+++ b/lib/lrmd/lrmd_client.c
@@ -1,2674 +1,2669 @@
/*
* Copyright 2012-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <stdint.h> // uint32_t, uint64_t
#include <stdarg.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <glib.h>
#include <dirent.h>
#include <crm/crm.h>
#include <crm/lrmd.h>
#include <crm/lrmd_internal.h>
#include <crm/services.h>
#include <crm/services_internal.h>
#include <crm/common/mainloop.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/remote_internal.h>
+#include <crm/common/tls_internal.h>
#include <crm/common/xml.h>
#include <crm/stonith-ng.h>
#include <crm/fencing/internal.h> // stonith__*
#include <gnutls/gnutls.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <arpa/inet.h>
#include <netdb.h>
#define MAX_TLS_RECV_WAIT 10000
CRM_TRACE_INIT_DATA(lrmd);
static int lrmd_api_disconnect(lrmd_t * lrmd);
static int lrmd_api_is_connected(lrmd_t * lrmd);
/* IPC proxy functions */
int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
// GnuTLS client handshake timeout in seconds
#define TLS_HANDSHAKE_TIMEOUT 5
-gnutls_psk_client_credentials_t psk_cred_s;
static void lrmd_tls_disconnect(lrmd_t * lrmd);
static int global_remote_msg_id = 0;
static void lrmd_tls_connection_destroy(gpointer userdata);
static int add_tls_to_mainloop(lrmd_t *lrmd, bool do_api_handshake);
typedef struct lrmd_private_s {
uint64_t type;
char *token;
mainloop_io_t *source;
/* IPC parameters */
crm_ipc_t *ipc;
pcmk__remote_t *remote;
/* Extra TLS parameters */
char *remote_nodename;
char *server;
int port;
- gnutls_psk_client_credentials_t psk_cred_c;
+ pcmk__tls_t *tls;
/* while the async connection is occurring, this is the id
* of the connection timeout timer. */
int async_timer;
int sock;
/* since tls requires a round trip across the network for a
* request/reply, there are times where we just want to be able
* to send a request from the client and not wait around (or even care
* about) what the reply is. */
int expected_late_replies;
GList *pending_notify;
crm_trigger_t *process_notify;
crm_trigger_t *handshake_trigger;
lrmd_event_callback callback;
/* Internal IPC proxy msg passing for remote guests */
void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
void *proxy_callback_userdata;
char *peer_version;
} lrmd_private_t;
static int process_lrmd_handshake_reply(xmlNode *reply, lrmd_private_t *native);
static void report_async_connection_result(lrmd_t * lrmd, int rc);
static lrmd_list_t *
lrmd_list_add(lrmd_list_t * head, const char *value)
{
lrmd_list_t *p, *end;
p = pcmk__assert_alloc(1, sizeof(lrmd_list_t));
p->val = strdup(value);
end = head;
while (end && end->next) {
end = end->next;
}
if (end) {
end->next = p;
} else {
head = p;
}
return head;
}
void
lrmd_list_freeall(lrmd_list_t * head)
{
lrmd_list_t *p;
while (head) {
char *val = (char *)head->val;
p = head->next;
free(val);
free(head);
head = p;
}
}
lrmd_key_value_t *
lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
{
lrmd_key_value_t *p, *end;
p = pcmk__assert_alloc(1, sizeof(lrmd_key_value_t));
p->key = strdup(key);
p->value = strdup(value);
end = head;
while (end && end->next) {
end = end->next;
}
if (end) {
end->next = p;
} else {
head = p;
}
return head;
}
void
lrmd_key_value_freeall(lrmd_key_value_t * head)
{
lrmd_key_value_t *p;
while (head) {
p = head->next;
free(head->key);
free(head->value);
free(head);
head = p;
}
}
/*!
* \brief Create a new lrmd_event_data_t object
*
* \param[in] rsc_id ID of resource involved in event
* \param[in] task Action name
* \param[in] interval_ms Action interval
*
* \return Newly allocated and initialized lrmd_event_data_t
* \note This functions asserts on memory errors, so the return value is
* guaranteed to be non-NULL. The caller is responsible for freeing the
* result with lrmd_free_event().
*/
lrmd_event_data_t *
lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
{
lrmd_event_data_t *event = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
// lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
event->rsc_id = pcmk__str_copy(rsc_id);
event->op_type = pcmk__str_copy(task);
event->interval_ms = interval_ms;
return event;
}
lrmd_event_data_t *
lrmd_copy_event(lrmd_event_data_t * event)
{
lrmd_event_data_t *copy = NULL;
copy = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
copy->type = event->type;
// lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
copy->rsc_id = pcmk__str_copy(event->rsc_id);
copy->op_type = pcmk__str_copy(event->op_type);
copy->user_data = pcmk__str_copy(event->user_data);
copy->output = pcmk__str_copy(event->output);
copy->remote_nodename = pcmk__str_copy(event->remote_nodename);
copy->exit_reason = pcmk__str_copy(event->exit_reason);
copy->call_id = event->call_id;
copy->timeout = event->timeout;
copy->interval_ms = event->interval_ms;
copy->start_delay = event->start_delay;
copy->rsc_deleted = event->rsc_deleted;
copy->rc = event->rc;
copy->op_status = event->op_status;
copy->t_run = event->t_run;
copy->t_rcchange = event->t_rcchange;
copy->exec_time = event->exec_time;
copy->queue_time = event->queue_time;
copy->connection_rc = event->connection_rc;
copy->params = pcmk__str_table_dup(event->params);
return copy;
}
/*!
* \brief Free an executor event
*
* \param[in,out] Executor event object to free
*/
void
lrmd_free_event(lrmd_event_data_t *event)
{
if (event == NULL) {
return;
}
// @TODO Why are these const char *?
free((void *) event->rsc_id);
free((void *) event->op_type);
free((void *) event->user_data);
free((void *) event->remote_nodename);
lrmd__reset_result(event);
if (event->params != NULL) {
g_hash_table_destroy(event->params);
}
free(event);
}
static void
lrmd_dispatch_internal(gpointer data, gpointer user_data)
{
xmlNode *msg = data;
lrmd_t *lrmd = user_data;
const char *type;
const char *proxy_session = crm_element_value(msg,
PCMK__XA_LRMD_IPC_SESSION);
lrmd_private_t *native = lrmd->lrmd_private;
lrmd_event_data_t event = { 0, };
if (proxy_session != NULL) {
/* this is proxy business */
lrmd_internal_proxy_dispatch(lrmd, msg);
return;
} else if (!native->callback) {
/* no callback set */
crm_trace("notify event received but client has not set callback");
return;
}
event.remote_nodename = native->remote_nodename;
type = crm_element_value(msg, PCMK__XA_LRMD_OP);
crm_element_value_int(msg, PCMK__XA_LRMD_CALLID, &event.call_id);
event.rsc_id = crm_element_value(msg, PCMK__XA_LRMD_RSC_ID);
if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
event.type = lrmd_event_register;
} else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
event.type = lrmd_event_unregister;
} else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
int rc = 0;
int exec_time = 0;
int queue_time = 0;
time_t epoch = 0;
crm_element_value_int(msg, PCMK__XA_LRMD_TIMEOUT, &event.timeout);
crm_element_value_ms(msg, PCMK__XA_LRMD_RSC_INTERVAL,
&event.interval_ms);
crm_element_value_int(msg, PCMK__XA_LRMD_RSC_START_DELAY,
&event.start_delay);
crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_RC, &rc);
event.rc = (enum ocf_exitcode) rc;
crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_OP_STATUS,
&event.op_status);
crm_element_value_int(msg, PCMK__XA_LRMD_RSC_DELETED,
&event.rsc_deleted);
crm_element_value_epoch(msg, PCMK__XA_LRMD_RUN_TIME, &epoch);
event.t_run = epoch;
crm_element_value_epoch(msg, PCMK__XA_LRMD_RCCHANGE_TIME, &epoch);
event.t_rcchange = epoch;
crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_TIME, &exec_time);
CRM_LOG_ASSERT(exec_time >= 0);
event.exec_time = QB_MAX(0, exec_time);
crm_element_value_int(msg, PCMK__XA_LRMD_QUEUE_TIME, &queue_time);
CRM_LOG_ASSERT(queue_time >= 0);
event.queue_time = QB_MAX(0, queue_time);
event.op_type = crm_element_value(msg, PCMK__XA_LRMD_RSC_ACTION);
event.user_data = crm_element_value(msg,
PCMK__XA_LRMD_RSC_USERDATA_STR);
event.type = lrmd_event_exec_complete;
/* output and exit_reason may be freed by a callback */
event.output = crm_element_value_copy(msg, PCMK__XA_LRMD_RSC_OUTPUT);
lrmd__set_result(&event, event.rc, event.op_status,
crm_element_value(msg, PCMK__XA_LRMD_RSC_EXIT_REASON));
event.params = xml2list(msg);
} else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
event.type = lrmd_event_new_client;
} else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
event.type = lrmd_event_poke;
} else {
return;
}
crm_trace("op %s notify event received", type);
native->callback(&event);
if (event.params) {
g_hash_table_destroy(event.params);
}
lrmd__reset_result(&event);
}
// \return Always 0, to indicate that IPC mainloop source should be kept
static int
lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
if (native->callback != NULL) {
xmlNode *msg = pcmk__xml_parse(buffer);
lrmd_dispatch_internal(msg, lrmd);
pcmk__xml_free(msg);
}
return 0;
}
static void
lrmd_free_xml(gpointer userdata)
{
pcmk__xml_free((xmlNode *) userdata);
}
static bool
remote_executor_connected(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
return (native->remote->tls_session != NULL);
}
static void
handle_remote_msg(xmlNode *xml, lrmd_t *lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
const char *msg_type = NULL;
msg_type = crm_element_value(xml, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
lrmd_dispatch_internal(xml, lrmd);
} else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
const char *op = crm_element_value(xml, PCMK__XA_LRMD_OP);
if (native->expected_late_replies > 0) {
native->expected_late_replies--;
/* The register op message we get as a response to lrmd_handshake_async
* is a reply, so we have to handle that here.
*/
if (pcmk__str_eq(op, "register", pcmk__str_casei)) {
int rc = process_lrmd_handshake_reply(xml, native);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
}
} else {
int reply_id = 0;
crm_element_value_int(xml, PCMK__XA_LRMD_CALLID, &reply_id);
/* if this happens, we want to know about it */
crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
}
}
}
/*!
* \internal
* \brief Notify trigger handler
*
* \param[in,out] userdata API connection
*
* \return Always return G_SOURCE_CONTINUE to leave this trigger handler in the
* mainloop
*/
static int
process_pending_notifies(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
if (native->pending_notify == NULL) {
return G_SOURCE_CONTINUE;
}
crm_trace("Processing pending notifies");
g_list_foreach(native->pending_notify, lrmd_dispatch_internal, lrmd);
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
return G_SOURCE_CONTINUE;
}
/*!
* \internal
* \brief TLS dispatch function for file descriptor sources
*
* \param[in,out] userdata API connection
*
* \return -1 on error to remove the source from the mainloop, or 0 otherwise
* to leave it in the mainloop
*/
static int
lrmd_tls_dispatch(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *xml = NULL;
int rc = pcmk_rc_ok;
if (!remote_executor_connected(lrmd)) {
crm_trace("TLS dispatch triggered after disconnect");
return -1;
}
crm_trace("TLS dispatch triggered");
rc = pcmk__remote_ready(native->remote, 0);
if (rc == pcmk_rc_ok) {
rc = pcmk__read_remote_message(native->remote, -1);
}
if (rc != pcmk_rc_ok && rc != ETIME) {
crm_info("Lost %s executor connection while reading data",
(native->remote_nodename? native->remote_nodename : "local"));
lrmd_tls_disconnect(lrmd);
return -1;
}
/* If rc is ETIME, there was nothing to read but we may already have a
* full message in the buffer
*/
xml = pcmk__remote_message_xml(native->remote);
if (xml == NULL) {
return 0;
}
handle_remote_msg(xml, lrmd);
pcmk__xml_free(xml);
return 0;
}
/* Not used with mainloop */
int
lrmd_poll(lrmd_t * lrmd, int timeout)
{
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
return crm_ipc_ready(native->ipc);
case pcmk__client_tls:
if (native->pending_notify) {
return 1;
} else {
int rc = pcmk__remote_ready(native->remote, 0);
switch (rc) {
case pcmk_rc_ok:
return 1;
case ETIME:
return 0;
default:
return pcmk_rc2legacy(rc);
}
}
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
return -EPROTONOSUPPORT;
}
}
/* Not used with mainloop */
bool
lrmd_dispatch(lrmd_t * lrmd)
{
lrmd_private_t *private = NULL;
pcmk__assert(lrmd != NULL);
private = lrmd->lrmd_private;
switch (private->type) {
case pcmk__client_ipc:
while (crm_ipc_ready(private->ipc)) {
if (crm_ipc_read(private->ipc) > 0) {
const char *msg = crm_ipc_buffer(private->ipc);
lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
}
}
break;
case pcmk__client_tls:
lrmd_tls_dispatch(lrmd);
break;
default:
crm_err("Unsupported executor connection type (bug?): %d",
private->type);
}
if (lrmd_api_is_connected(lrmd) == FALSE) {
crm_err("Connection closed");
return FALSE;
}
return TRUE;
}
static xmlNode *
lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
enum lrmd_call_options options)
{
xmlNode *op_msg = NULL;
CRM_CHECK(token != NULL, return NULL);
op_msg = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
crm_xml_add(op_msg, PCMK__XA_T, PCMK__VALUE_LRMD);
crm_xml_add(op_msg, PCMK__XA_LRMD_OP, op);
crm_xml_add_int(op_msg, PCMK__XA_LRMD_TIMEOUT, timeout);
crm_xml_add_int(op_msg, PCMK__XA_LRMD_CALLOPT, options);
if (data != NULL) {
xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_LRMD_CALLDATA);
pcmk__xml_copy(wrapper, data);
}
crm_trace("Created executor %s command with call options %.8lx (%d)",
op, (long)options, options);
return op_msg;
}
static void
lrmd_ipc_connection_destroy(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
crm_info("Disconnected from local executor");
break;
case pcmk__client_tls:
crm_info("Disconnected from remote executor on %s",
native->remote_nodename);
break;
default:
crm_err("Unsupported executor connection type %d (bug?)",
native->type);
}
/* Prevent these from being cleaned up in lrmd_api_disconnect() */
native->ipc = NULL;
native->source = NULL;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.type = lrmd_event_disconnect;
event.remote_nodename = native->remote_nodename;
native->callback(&event);
}
}
static void
lrmd_tls_connection_destroy(gpointer userdata)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
crm_info("TLS connection destroyed");
if (native->remote->tls_session) {
- gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
- gnutls_deinit(*native->remote->tls_session);
- gnutls_free(native->remote->tls_session);
+ gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR);
+ gnutls_deinit(native->remote->tls_session);
native->remote->tls_session = NULL;
}
- if (native->psk_cred_c) {
- gnutls_psk_free_client_credentials(native->psk_cred_c);
+ if (native->tls) {
+ pcmk__free_tls(native->tls);
+ native->tls = NULL;
}
if (native->sock) {
close(native->sock);
}
if (native->process_notify) {
mainloop_destroy_trigger(native->process_notify);
native->process_notify = NULL;
}
if (native->pending_notify) {
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
if (native->handshake_trigger != NULL) {
mainloop_destroy_trigger(native->handshake_trigger);
native->handshake_trigger = NULL;
}
free(native->remote->buffer);
free(native->remote->start_state);
native->remote->buffer = NULL;
native->remote->start_state = NULL;
native->source = 0;
native->sock = 0;
- native->psk_cred_c = NULL;
- native->sock = 0;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.remote_nodename = native->remote_nodename;
event.type = lrmd_event_disconnect;
native->callback(&event);
}
return;
}
// \return Standard Pacemaker return code
int
lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
const char *msg_type)
{
crm_xml_add_int(msg, PCMK__XA_LRMD_REMOTE_MSG_ID, id);
crm_xml_add(msg, PCMK__XA_LRMD_REMOTE_MSG_TYPE, msg_type);
return pcmk__remote_send_xml(session, msg);
}
// \return Standard Pacemaker return code
static int
read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
xmlNode **reply)
{
lrmd_private_t *native = lrmd->lrmd_private;
time_t start = time(NULL);
const char *msg_type = NULL;
int reply_id = 0;
int remaining_timeout = 0;
int rc = pcmk_rc_ok;
/* A timeout of 0 here makes no sense. We have to wait a period of time
* for the response to come back. If -1 or 0, default to 10 seconds. */
if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
total_timeout = MAX_TLS_RECV_WAIT;
}
for (*reply = NULL; *reply == NULL; ) {
*reply = pcmk__remote_message_xml(native->remote);
if (*reply == NULL) {
/* read some more off the tls buffer if we still have time left. */
if (remaining_timeout) {
remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
} else {
remaining_timeout = total_timeout;
}
if (remaining_timeout <= 0) {
return ETIME;
}
rc = pcmk__read_remote_message(native->remote, remaining_timeout);
if (rc != pcmk_rc_ok) {
return rc;
}
*reply = pcmk__remote_message_xml(native->remote);
if (*reply == NULL) {
return ENOMSG;
}
}
crm_element_value_int(*reply, PCMK__XA_LRMD_REMOTE_MSG_ID, &reply_id);
msg_type = crm_element_value(*reply, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
if (!msg_type) {
crm_err("Empty msg type received while waiting for reply");
pcmk__xml_free(*reply);
*reply = NULL;
} else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
/* got a notify while waiting for reply, trigger the notify to be processed later */
crm_info("queueing notify");
native->pending_notify = g_list_append(native->pending_notify, *reply);
if (native->process_notify) {
crm_info("notify trigger set.");
mainloop_set_trigger(native->process_notify);
}
*reply = NULL;
} else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
/* msg isn't a reply, make some noise */
crm_err("Expected a reply, got %s", msg_type);
pcmk__xml_free(*reply);
*reply = NULL;
} else if (reply_id != expected_reply_id) {
if (native->expected_late_replies > 0) {
native->expected_late_replies--;
} else {
crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
}
pcmk__xml_free(*reply);
*reply = NULL;
}
}
if (native->remote->buffer && native->process_notify) {
mainloop_set_trigger(native->process_notify);
}
return rc;
}
// \return Standard Pacemaker return code
static int
send_remote_message(lrmd_t *lrmd, xmlNode *msg)
{
int rc = pcmk_rc_ok;
lrmd_private_t *native = lrmd->lrmd_private;
global_remote_msg_id++;
if (global_remote_msg_id <= 0) {
global_remote_msg_id = 1;
}
rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
"request");
if (rc != pcmk_rc_ok) {
crm_err("Disconnecting because TLS message could not be sent to "
"Pacemaker Remote: %s", pcmk_rc_str(rc));
lrmd_tls_disconnect(lrmd);
}
return rc;
}
static int
lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
{
int rc = 0;
xmlNode *xml = NULL;
if (!remote_executor_connected(lrmd)) {
return -ENOTCONN;
}
rc = send_remote_message(lrmd, msg);
if (rc != pcmk_rc_ok) {
return pcmk_rc2legacy(rc);
}
rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
if (rc != pcmk_rc_ok) {
crm_err("Disconnecting remote after request %d reply not received: %s "
QB_XS " rc=%d timeout=%dms",
global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
lrmd_tls_disconnect(lrmd);
}
if (reply) {
*reply = xml;
} else {
pcmk__xml_free(xml);
}
return pcmk_rc2legacy(rc);
}
static int
lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
break;
case pcmk__client_tls:
rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
break;
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static int
lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
break;
case pcmk__client_tls:
rc = send_remote_message(lrmd, msg);
if (rc == pcmk_rc_ok) {
/* we don't want to wait around for the reply, but
* since the request/reply protocol needs to behave the same
* as libqb, a reply will eventually come later anyway. */
native->expected_late_replies++;
}
rc = pcmk_rc2legacy(rc);
break;
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static int
lrmd_api_is_connected(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
return crm_ipc_connected(native->ipc);
case pcmk__client_tls:
return remote_executor_connected(lrmd);
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
return 0;
}
}
/*!
* \internal
* \brief Send a prepared API command to the executor
*
* \param[in,out] lrmd Existing connection to the executor
* \param[in] op Name of API command to send
* \param[in] data Command data XML to add to the sent command
* \param[out] output_data If expecting a reply, it will be stored here
* \param[in] timeout Timeout in milliseconds (if 0, defaults to
* a sensible value per the type of connection,
* standard vs. pacemaker remote);
* also propagated to the command XML
* \param[in] call_options Call options to pass to server when sending
* \param[in] expect_reply If true, wait for a reply from the server;
* must be true for IPC (as opposed to TLS) clients
*
* \return pcmk_ok on success, -errno on error
*/
static int
lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
xmlNode **output_data, int timeout,
enum lrmd_call_options options, bool expect_reply)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *op_msg = NULL;
xmlNode *op_reply = NULL;
if (!lrmd_api_is_connected(lrmd)) {
return -ENOTCONN;
}
if (op == NULL) {
crm_err("No operation specified");
return -EINVAL;
}
CRM_LOG_ASSERT(native->token != NULL);
crm_trace("Sending %s op to executor", op);
op_msg = lrmd_create_op(native->token, op, data, timeout, options);
if (op_msg == NULL) {
return -EINVAL;
}
if (expect_reply) {
rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
} else {
rc = lrmd_send_xml_no_reply(lrmd, op_msg);
goto done;
}
if (rc < 0) {
crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
goto done;
} else if (op_reply == NULL) {
rc = -ENOMSG;
goto done;
}
rc = pcmk_ok;
crm_trace("%s op reply received", op);
if (crm_element_value_int(op_reply, PCMK__XA_LRMD_RC, &rc) != 0) {
rc = -ENOMSG;
goto done;
}
crm_log_xml_trace(op_reply, "Reply");
if (output_data) {
*output_data = op_reply;
op_reply = NULL; /* Prevent subsequent free */
}
done:
if (lrmd_api_is_connected(lrmd) == FALSE) {
crm_err("Executor disconnected");
}
pcmk__xml_free(op_msg);
pcmk__xml_free(op_reply);
return rc;
}
static int
lrmd_api_poke_connection(lrmd_t * lrmd)
{
int rc;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
(native->type == pcmk__client_ipc));
pcmk__xml_free(data);
return rc < 0 ? rc : pcmk_ok;
}
// \return Standard Pacemaker return code
int
lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
{
int rc = pcmk_rc_ok;
const char *value;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *data = pcmk__xe_create(NULL, PCMK__XA_LRMD_OP);
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
value = g_hash_table_lookup(hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
if ((value) &&
(stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
crm_xml_add(data, PCMK__XA_LRMD_WATCHDOG, value);
}
rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
(native->type == pcmk__client_ipc));
pcmk__xml_free(data);
return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
}
static xmlNode *
lrmd_handshake_hello_msg(const char *name, bool is_proxy)
{
xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
crm_xml_add(hello, PCMK__XA_T, PCMK__VALUE_LRMD);
crm_xml_add(hello, PCMK__XA_LRMD_OP, CRM_OP_REGISTER);
crm_xml_add(hello, PCMK__XA_LRMD_CLIENTNAME, name);
crm_xml_add(hello, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
/* advertise that we are a proxy provider */
if (is_proxy) {
pcmk__xe_set_bool_attr(hello, PCMK__XA_LRMD_IS_IPC_PROVIDER, true);
}
return hello;
}
static int
process_lrmd_handshake_reply(xmlNode *reply, lrmd_private_t *native)
{
int rc = pcmk_rc_ok;
const char *version = crm_element_value(reply, PCMK__XA_LRMD_PROTOCOL_VERSION);
const char *msg_type = crm_element_value(reply, PCMK__XA_LRMD_OP);
const char *tmp_ticket = crm_element_value(reply, PCMK__XA_LRMD_CLIENTID);
const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE);
long long uptime = -1;
crm_element_value_int(reply, PCMK__XA_LRMD_RC, &rc);
rc = pcmk_legacy2rc(rc);
/* The remote executor may add its uptime to the XML reply, which is useful
* in handling transient attributes when the connection to the remote node
* unexpectedly drops. If no parameter is given, just default to -1.
*/
crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
native->remote->uptime = uptime;
if (start_state) {
native->remote->start_state = strdup(start_state);
}
if (rc == EPROTO) {
crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
LRMD_PROTOCOL_VERSION, version);
crm_log_xml_err(reply, "Protocol Error");
} else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
crm_err("Invalid registration message: %s", msg_type);
crm_log_xml_err(reply, "Bad reply");
rc = EPROTO;
} else if (tmp_ticket == NULL) {
crm_err("No registration token provided");
crm_log_xml_err(reply, "Bad reply");
rc = EPROTO;
} else {
crm_trace("Obtained registration token: %s", tmp_ticket);
native->token = strdup(tmp_ticket);
native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
rc = pcmk_rc_ok;
}
return rc;
}
static int
lrmd_handshake(lrmd_t * lrmd, const char *name)
{
int rc = pcmk_rc_ok;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *reply = NULL;
xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL);
rc = lrmd_send_xml(lrmd, hello, -1, &reply);
if (rc < 0) {
crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
rc = ECOMM;
} else if (reply == NULL) {
crm_err("Did not receive registration reply");
rc = EPROTO;
} else {
rc = process_lrmd_handshake_reply(reply, native);
}
pcmk__xml_free(reply);
pcmk__xml_free(hello);
if (rc != pcmk_rc_ok) {
lrmd_api_disconnect(lrmd);
}
return rc;
}
static int
lrmd_handshake_async(lrmd_t * lrmd, const char *name)
{
int rc = pcmk_rc_ok;
lrmd_private_t *native = lrmd->lrmd_private;
xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL);
rc = send_remote_message(lrmd, hello);
if (rc == pcmk_rc_ok) {
native->expected_late_replies++;
} else {
lrmd_api_disconnect(lrmd);
}
pcmk__xml_free(hello);
return rc;
}
static int
lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
struct ipc_client_callbacks lrmd_callbacks = {
.dispatch = lrmd_ipc_dispatch,
.destroy = lrmd_ipc_connection_destroy
};
crm_info("Connecting to executor");
if (fd) {
/* No mainloop */
native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
if (native->ipc != NULL) {
rc = pcmk__connect_generic_ipc(native->ipc);
if (rc == pcmk_rc_ok) {
rc = pcmk__ipc_fd(native->ipc, fd);
}
if (rc != pcmk_rc_ok) {
crm_err("Connection to executor failed: %s", pcmk_rc_str(rc));
rc = -ENOTCONN;
}
}
} else {
native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
native->ipc = mainloop_get_ipc_client(native->source);
}
if (native->ipc == NULL) {
crm_debug("Could not connect to the executor API");
rc = -ENOTCONN;
}
return rc;
}
static void
copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
{
pcmk__assert((dest != NULL) && (source != NULL) && (source->data != NULL));
dest->data = gnutls_malloc(source->size);
pcmk__mem_assert(dest->data);
memcpy(dest->data, source->data, source->size);
dest->size = source->size;
}
static void
clear_gnutls_datum(gnutls_datum_t *datum)
{
gnutls_free(datum->data);
datum->data = NULL;
datum->size = 0;
}
#define KEY_READ_LEN 256 // Chunk size for reading key from file
// \return Standard Pacemaker return code
static int
read_gnutls_key(const char *location, gnutls_datum_t *key)
{
FILE *stream = NULL;
size_t buf_len = KEY_READ_LEN;
if ((location == NULL) || (key == NULL)) {
return EINVAL;
}
stream = fopen(location, "r");
if (stream == NULL) {
return errno;
}
key->data = gnutls_malloc(buf_len);
key->size = 0;
while (!feof(stream)) {
int next = fgetc(stream);
if (next == EOF) {
if (!feof(stream)) {
crm_warn("Pacemaker Remote key read was partially successful "
"(copy in memory may be corrupted)");
}
break;
}
if (key->size == buf_len) {
buf_len = key->size + KEY_READ_LEN;
key->data = gnutls_realloc(key->data, buf_len);
pcmk__assert(key->data);
}
key->data[key->size++] = (unsigned char) next;
}
fclose(stream);
if (key->size == 0) {
clear_gnutls_datum(key);
return ENOKEY;
}
return pcmk_rc_ok;
}
// Cache the most recently used Pacemaker Remote authentication key
struct key_cache_s {
time_t updated; // When cached key was read (valid for 1 minute)
const char *location; // Where cached key was read from
gnutls_datum_t key; // Cached key
};
static bool
key_is_cached(struct key_cache_s *key_cache)
{
return key_cache->updated != 0;
}
static bool
key_cache_expired(struct key_cache_s *key_cache)
{
return (time(NULL) - key_cache->updated) >= 60;
}
static void
clear_key_cache(struct key_cache_s *key_cache)
{
clear_gnutls_datum(&(key_cache->key));
if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
key_cache->updated = 0;
key_cache->location = NULL;
crm_debug("Cleared Pacemaker Remote key cache");
}
}
static void
get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
{
copy_gnutls_datum(key, &(key_cache->key));
crm_debug("Using cached Pacemaker Remote key from %s",
pcmk__s(key_cache->location, "unknown location"));
}
static void
cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
const char *location)
{
key_cache->updated = time(NULL);
key_cache->location = location;
copy_gnutls_datum(&(key_cache->key), key);
crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
pcmk__s(location, "unknown location"));
}
/*!
* \internal
* \brief Get Pacemaker Remote authentication key from file or cache
*
* \param[in] location Path to key file to try (this memory must
* persist across all calls of this function)
* \param[out] key Key from location or cache
*
* \return Standard Pacemaker return code
*/
static int
get_remote_key(const char *location, gnutls_datum_t *key)
{
static struct key_cache_s key_cache = { 0, };
int rc = pcmk_rc_ok;
if ((location == NULL) || (key == NULL)) {
return EINVAL;
}
if (key_is_cached(&key_cache)) {
if (key_cache_expired(&key_cache)) {
clear_key_cache(&key_cache);
} else {
get_cached_key(&key_cache, key);
return pcmk_rc_ok;
}
}
rc = read_gnutls_key(location, key);
if (rc != pcmk_rc_ok) {
return rc;
}
cache_key(&key_cache, key, location);
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Initialize the Pacemaker Remote authentication key
*
* Try loading the Pacemaker Remote authentication key from cache if available,
* otherwise from these locations, in order of preference:
*
* - The value of the PCMK_authkey_location environment variable, if set
* - The Pacemaker default key file location
*
* \param[out] key Where to store key
*
* \return Standard Pacemaker return code
*/
int
lrmd__init_remote_key(gnutls_datum_t *key)
{
static const char *env_location = NULL;
static bool need_env = true;
int rc = pcmk_rc_ok;
if (need_env) {
env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
need_env = false;
}
// Try location in environment variable, if set
if (env_location != NULL) {
rc = get_remote_key(env_location, key);
if (rc == pcmk_rc_ok) {
return pcmk_rc_ok;
}
crm_warn("Could not read Pacemaker Remote key from %s: %s",
env_location, pcmk_rc_str(rc));
return ENOKEY;
}
// Try default location, if environment wasn't explicitly set to it
rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
if (rc == pcmk_rc_ok) {
return pcmk_rc_ok;
}
crm_warn("Could not read Pacemaker Remote key from default location %s: %s",
DEFAULT_REMOTE_KEY_LOCATION, pcmk_rc_str(rc));
return ENOKEY;
}
-static void
-lrmd_gnutls_global_init(void)
-{
- static int gnutls_init = 0;
-
- if (!gnutls_init) {
- crm_gnutls_global_init();
- }
- gnutls_init = 1;
-}
-
static void
report_async_connection_result(lrmd_t * lrmd, int rc)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->callback) {
lrmd_event_data_t event = { 0, };
event.type = lrmd_event_connect;
event.remote_nodename = native->remote_nodename;
event.connection_rc = rc;
native->callback(&event);
}
}
static void
tls_handshake_failed(lrmd_t *lrmd, int tls_rc, int rc)
{
lrmd_private_t *native = lrmd->lrmd_private;
crm_warn("Disconnecting after TLS handshake with "
"Pacemaker Remote server %s:%d failed: %s",
native->server, native->port,
(rc == EPROTO)? gnutls_strerror(tls_rc) : pcmk_rc_str(rc));
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
- gnutls_deinit(*native->remote->tls_session);
- gnutls_free(native->remote->tls_session);
+ gnutls_deinit(native->remote->tls_session);
native->remote->tls_session = NULL;
lrmd_tls_connection_destroy(lrmd);
}
static void
tls_handshake_succeeded(lrmd_t *lrmd)
{
int rc = pcmk_rc_ok;
lrmd_private_t *native = lrmd->lrmd_private;
crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
native->server, native->port);
rc = add_tls_to_mainloop(lrmd, true);
/* If add_tls_to_mainloop failed, report that right now. Otherwise, we have
* to wait until we read the async reply to report anything.
*/
if (rc != pcmk_rc_ok) {
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
}
}
/*!
* \internal
* \brief Perform a TLS client handshake with a Pacemaker Remote server
*
* \param[in] lrmd Newly established Pacemaker Remote executor connection
*
* \return Standard Pacemaker return code
*/
static int
tls_client_handshake(lrmd_t *lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
int tls_rc = GNUTLS_E_SUCCESS;
int rc = pcmk__tls_client_handshake(native->remote, TLS_HANDSHAKE_TIMEOUT,
&tls_rc);
if (rc != pcmk_rc_ok) {
tls_handshake_failed(lrmd, tls_rc, rc);
}
return rc;
}
/*!
* \internal
* \brief Add trigger and file descriptor mainloop sources for TLS
*
* \param[in,out] lrmd API connection with established TLS session
* \param[in] do_api_handshake Whether to perform executor handshake
*
* \return Standard Pacemaker return code
*/
static int
add_tls_to_mainloop(lrmd_t *lrmd, bool do_api_handshake)
{
lrmd_private_t *native = lrmd->lrmd_private;
int rc = pcmk_rc_ok;
char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
native->server, native->port);
struct mainloop_fd_callbacks tls_fd_callbacks = {
.dispatch = lrmd_tls_dispatch,
.destroy = lrmd_tls_connection_destroy,
};
native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
process_pending_notifies, lrmd);
native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
&tls_fd_callbacks);
/* Async connections lose the client name provided by the API caller, so we
* have to use our generated name here to perform the executor handshake.
*
* @TODO Keep track of the caller-provided name. Perhaps we should be using
* that name in this function instead of generating one anyway.
*/
if (do_api_handshake) {
rc = lrmd_handshake_async(lrmd, name);
}
free(name);
return rc;
}
struct handshake_data_s {
lrmd_t *lrmd;
time_t start_time;
int timeout_sec;
};
static gboolean
try_handshake_cb(gpointer user_data)
{
struct handshake_data_s *hs = user_data;
lrmd_t *lrmd = hs->lrmd;
lrmd_private_t *native = lrmd->lrmd_private;
pcmk__remote_t *remote = native->remote;
int rc = pcmk_rc_ok;
int tls_rc = GNUTLS_E_SUCCESS;
if (time(NULL) >= hs->start_time + hs->timeout_sec) {
rc = ETIME;
tls_handshake_failed(lrmd, GNUTLS_E_TIMEDOUT, rc);
free(hs);
return 0;
}
rc = pcmk__tls_client_try_handshake(remote, &tls_rc);
if (rc == pcmk_rc_ok) {
tls_handshake_succeeded(lrmd);
free(hs);
return 0;
} else if (rc == EAGAIN) {
mainloop_set_trigger(native->handshake_trigger);
return 1;
} else {
rc = EKEYREJECTED;
tls_handshake_failed(lrmd, tls_rc, rc);
free(hs);
return 0;
}
}
static void
lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
{
lrmd_t *lrmd = userdata;
lrmd_private_t *native = lrmd->lrmd_private;
gnutls_datum_t psk_key = { NULL, 0 };
int tls_rc = GNUTLS_E_SUCCESS;
native->async_timer = 0;
if (rc != pcmk_rc_ok) {
lrmd_tls_connection_destroy(lrmd);
crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
QB_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
return;
}
/* The TCP connection was successful, so establish the TLS connection. */
native->sock = sock;
+ if (native->tls == NULL) {
+ rc = pcmk__init_tls(&native->tls, false, GNUTLS_CRD_PSK);
+
+ if (rc != pcmk_rc_ok) {
+ lrmd_tls_connection_destroy(lrmd);
+ report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
+ return;
+ }
+ }
+
rc = lrmd__init_remote_key(&psk_key);
if (rc != pcmk_rc_ok) {
crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
QB_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
return;
}
- gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
- gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
+ pcmk__tls_add_psk_key(native->tls, &psk_key);
gnutls_free(psk_key.data);
- native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
- GNUTLS_CRD_PSK,
- native->psk_cred_c);
+ native->remote->tls_session = pcmk__new_tls_session(native->tls, sock);
if (native->remote->tls_session == NULL) {
lrmd_tls_connection_destroy(lrmd);
report_async_connection_result(lrmd, -EPROTO);
return;
}
/* If the TLS handshake immediately succeeds or fails, we can handle that
* now without having to deal with mainloops and retries. Otherwise, add a
* trigger to keep trying until we get a result (or it times out).
*/
rc = pcmk__tls_client_try_handshake(native->remote, &tls_rc);
if (rc == EAGAIN) {
struct handshake_data_s *hs = NULL;
if (native->handshake_trigger != NULL) {
return;
}
hs = pcmk__assert_alloc(1, sizeof(struct handshake_data_s));
hs->lrmd = lrmd;
hs->start_time = time(NULL);
hs->timeout_sec = TLS_HANDSHAKE_TIMEOUT;
native->handshake_trigger = mainloop_add_trigger(G_PRIORITY_LOW, try_handshake_cb, hs);
mainloop_set_trigger(native->handshake_trigger);
} else if (rc == pcmk_rc_ok) {
tls_handshake_succeeded(lrmd);
} else {
tls_handshake_failed(lrmd, tls_rc, rc);
}
}
static int
lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
{
int rc = pcmk_rc_ok;
int timer_id = 0;
lrmd_private_t *native = lrmd->lrmd_private;
- lrmd_gnutls_global_init();
native->sock = -1;
rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
&(native->sock), lrmd, lrmd_tcp_connect_cb);
if (rc != pcmk_rc_ok) {
crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
QB_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
return rc;
}
native->async_timer = timer_id;
return rc;
}
static int
lrmd_tls_connect(lrmd_t * lrmd, int *fd)
{
int rc = pcmk_rc_ok;
lrmd_private_t *native = lrmd->lrmd_private;
gnutls_datum_t psk_key = { NULL, 0 };
- lrmd_gnutls_global_init();
-
native->sock = -1;
rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
&(native->sock), NULL, NULL);
if (rc != pcmk_rc_ok) {
crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
QB_XS " rc=%d",
native->server, native->port, pcmk_rc_str(rc), rc);
lrmd_tls_connection_destroy(lrmd);
return ENOTCONN;
}
+ if (native->tls == NULL) {
+ rc = pcmk__init_tls(&native->tls, false, GNUTLS_CRD_PSK);
+
+ if (rc != pcmk_rc_ok) {
+ lrmd_tls_connection_destroy(lrmd);
+ return rc;
+ }
+ }
+
rc = lrmd__init_remote_key(&psk_key);
if (rc != pcmk_rc_ok) {
lrmd_tls_connection_destroy(lrmd);
return rc;
}
- gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
- gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
+ pcmk__tls_add_psk_key(native->tls, &psk_key);
gnutls_free(psk_key.data);
- native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
- GNUTLS_CRD_PSK,
- native->psk_cred_c);
+ native->remote->tls_session = pcmk__new_tls_session(native->tls, native->sock);
if (native->remote->tls_session == NULL) {
lrmd_tls_connection_destroy(lrmd);
return EPROTO;
}
if (tls_client_handshake(lrmd) != pcmk_rc_ok) {
return EKEYREJECTED;
}
crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
native->port);
if (fd) {
*fd = native->sock;
} else {
rc = add_tls_to_mainloop(lrmd, false);
}
return rc;
}
static int
lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
{
int rc = -ENOTCONN;
lrmd_private_t *native = lrmd->lrmd_private;
switch (native->type) {
case pcmk__client_ipc:
rc = lrmd_ipc_connect(lrmd, fd);
break;
case pcmk__client_tls:
rc = lrmd_tls_connect(lrmd, fd);
rc = pcmk_rc2legacy(rc);
break;
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
if (rc == pcmk_ok) {
rc = lrmd_handshake(lrmd, name);
rc = pcmk_rc2legacy(rc);
}
return rc;
}
static int
lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
{
int rc = pcmk_ok;
lrmd_private_t *native = lrmd->lrmd_private;
CRM_CHECK(native && native->callback, return -EINVAL);
switch (native->type) {
case pcmk__client_ipc:
/* fake async connection with ipc. it should be fast
* enough that we gain very little from async */
rc = lrmd_api_connect(lrmd, name, NULL);
if (!rc) {
report_async_connection_result(lrmd, rc);
}
break;
case pcmk__client_tls:
rc = lrmd_tls_connect_async(lrmd, timeout);
rc = pcmk_rc2legacy(rc);
break;
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static void
lrmd_ipc_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->source != NULL) {
/* Attached to mainloop */
mainloop_del_ipc_client(native->source);
native->source = NULL;
native->ipc = NULL;
} else if (native->ipc) {
/* Not attached to mainloop */
crm_ipc_t *ipc = native->ipc;
native->ipc = NULL;
crm_ipc_close(ipc);
crm_ipc_destroy(ipc);
}
}
static void
lrmd_tls_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->remote->tls_session) {
- gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
- gnutls_deinit(*native->remote->tls_session);
- gnutls_free(native->remote->tls_session);
+ gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR);
+ gnutls_deinit(native->remote->tls_session);
native->remote->tls_session = NULL;
}
if (native->async_timer) {
g_source_remove(native->async_timer);
native->async_timer = 0;
}
if (native->source != NULL) {
/* Attached to mainloop */
mainloop_del_ipc_client(native->source);
native->source = NULL;
} else if (native->sock) {
close(native->sock);
native->sock = 0;
}
if (native->pending_notify) {
g_list_free_full(native->pending_notify, lrmd_free_xml);
native->pending_notify = NULL;
}
}
static int
lrmd_api_disconnect(lrmd_t * lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
int rc = pcmk_ok;
switch (native->type) {
case pcmk__client_ipc:
crm_debug("Disconnecting from local executor");
lrmd_ipc_disconnect(lrmd);
break;
case pcmk__client_tls:
crm_debug("Disconnecting from remote executor on %s",
native->remote_nodename);
lrmd_tls_disconnect(lrmd);
break;
default:
crm_err("Unsupported executor connection type (bug?): %d",
native->type);
rc = -EPROTONOSUPPORT;
}
free(native->token);
native->token = NULL;
free(native->peer_version);
native->peer_version = NULL;
return rc;
}
static int
lrmd_api_register_rsc(lrmd_t * lrmd,
const char *rsc_id,
const char *class,
const char *provider, const char *type, enum lrmd_call_options options)
{
int rc = pcmk_ok;
xmlNode *data = NULL;
if (!class || !type || !rsc_id) {
return -EINVAL;
}
if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
&& (provider == NULL)) {
return -EINVAL;
}
data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
crm_xml_add(data, PCMK__XA_LRMD_CLASS, class);
crm_xml_add(data, PCMK__XA_LRMD_PROVIDER, provider);
crm_xml_add(data, PCMK__XA_LRMD_TYPE, type);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, true);
pcmk__xml_free(data);
return rc;
}
static int
lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
{
int rc = pcmk_ok;
xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, true);
pcmk__xml_free(data);
return rc;
}
lrmd_rsc_info_t *
lrmd_new_rsc_info(const char *rsc_id, const char *standard,
const char *provider, const char *type)
{
lrmd_rsc_info_t *rsc_info = pcmk__assert_alloc(1, sizeof(lrmd_rsc_info_t));
rsc_info->id = pcmk__str_copy(rsc_id);
rsc_info->standard = pcmk__str_copy(standard);
rsc_info->provider = pcmk__str_copy(provider);
rsc_info->type = pcmk__str_copy(type);
return rsc_info;
}
lrmd_rsc_info_t *
lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
{
return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
rsc_info->provider, rsc_info->type);
}
void
lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
{
if (!rsc_info) {
return;
}
free(rsc_info->id);
free(rsc_info->type);
free(rsc_info->standard);
free(rsc_info->provider);
free(rsc_info);
}
static lrmd_rsc_info_t *
lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
{
lrmd_rsc_info_t *rsc_info = NULL;
xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
xmlNode *output = NULL;
const char *class = NULL;
const char *provider = NULL;
const char *type = NULL;
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, true);
pcmk__xml_free(data);
if (!output) {
return NULL;
}
class = crm_element_value(output, PCMK__XA_LRMD_CLASS);
provider = crm_element_value(output, PCMK__XA_LRMD_PROVIDER);
type = crm_element_value(output, PCMK__XA_LRMD_TYPE);
if (!class || !type) {
pcmk__xml_free(output);
return NULL;
} else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
&& !provider) {
pcmk__xml_free(output);
return NULL;
}
rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
pcmk__xml_free(output);
return rsc_info;
}
void
lrmd_free_op_info(lrmd_op_info_t *op_info)
{
if (op_info) {
free(op_info->rsc_id);
free(op_info->action);
free(op_info->interval_ms_s);
free(op_info->timeout_ms_s);
free(op_info);
}
}
static int
lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
enum lrmd_call_options options, GList **output)
{
xmlNode *data = NULL;
xmlNode *output_xml = NULL;
int rc = pcmk_ok;
if (output == NULL) {
return -EINVAL;
}
*output = NULL;
// Send request
if (rsc_id) {
data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
}
rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
timeout_ms, options, true);
if (data) {
pcmk__xml_free(data);
}
// Process reply
if ((rc != pcmk_ok) || (output_xml == NULL)) {
return rc;
}
for (const xmlNode *rsc_xml = pcmk__xe_first_child(output_xml,
PCMK__XE_LRMD_RSC, NULL,
NULL);
(rsc_xml != NULL) && (rc == pcmk_ok);
rsc_xml = pcmk__xe_next(rsc_xml, PCMK__XE_LRMD_RSC)) {
rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
if (rsc_id == NULL) {
crm_err("Could not parse recurring operation information from executor");
continue;
}
for (const xmlNode *op_xml = pcmk__xe_first_child(rsc_xml,
PCMK__XE_LRMD_RSC_OP,
NULL, NULL);
op_xml != NULL;
op_xml = pcmk__xe_next(op_xml, PCMK__XE_LRMD_RSC_OP)) {
lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
if (op_info == NULL) {
rc = -ENOMEM;
break;
}
op_info->rsc_id = strdup(rsc_id);
op_info->action = crm_element_value_copy(op_xml,
PCMK__XA_LRMD_RSC_ACTION);
op_info->interval_ms_s =
crm_element_value_copy(op_xml, PCMK__XA_LRMD_RSC_INTERVAL);
op_info->timeout_ms_s =
crm_element_value_copy(op_xml, PCMK__XA_LRMD_TIMEOUT);
*output = g_list_prepend(*output, op_info);
}
}
pcmk__xml_free(output_xml);
return rc;
}
static void
lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
{
lrmd_private_t *native = lrmd->lrmd_private;
native->callback = callback;
}
void
lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
{
lrmd_private_t *native = lrmd->lrmd_private;
native->proxy_callback = callback;
native->proxy_callback_userdata = userdata;
}
void
lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->proxy_callback) {
crm_log_xml_trace(msg, "PROXY_INBOUND");
native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
}
}
int
lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
{
if (lrmd == NULL) {
return -ENOTCONN;
}
crm_xml_add(msg, PCMK__XA_LRMD_OP, CRM_OP_IPC_FWD);
crm_log_xml_trace(msg, "PROXY_OUTBOUND");
return lrmd_send_xml_no_reply(lrmd, msg);
}
static int
stonith_get_metadata(const char *provider, const char *type, char **output)
{
int rc = pcmk_ok;
stonith_t *stonith_api = stonith_api_new();
if (stonith_api == NULL) {
crm_err("Could not get fence agent meta-data: API memory allocation failed");
return -ENOMEM;
}
rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
provider, output, 0);
if ((rc == pcmk_ok) && (*output == NULL)) {
rc = -EIO;
}
stonith_api->cmds->free(stonith_api);
return rc;
}
static int
lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
const char *type, char **output,
enum lrmd_call_options options)
{
return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
output, options, NULL);
}
static int
lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
const char *provider, const char *type,
char **output, enum lrmd_call_options options,
lrmd_key_value_t *params)
{
svc_action_t *action = NULL;
GHashTable *params_table = NULL;
if (!standard || !type) {
lrmd_key_value_freeall(params);
return -EINVAL;
}
if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
lrmd_key_value_freeall(params);
return stonith_get_metadata(provider, type, output);
}
params_table = pcmk__strkey_table(free, free);
for (const lrmd_key_value_t *param = params; param; param = param->next) {
pcmk__insert_dup(params_table, param->key, param->value);
}
action = services__create_resource_action(type, standard, provider, type,
PCMK_ACTION_META_DATA, 0,
PCMK_DEFAULT_ACTION_TIMEOUT_MS,
params_table, 0);
lrmd_key_value_freeall(params);
if (action == NULL) {
return -ENOMEM;
}
if (action->rc != PCMK_OCF_UNKNOWN) {
services_action_free(action);
return -EINVAL;
}
if (!services_action_sync(action)) {
crm_err("Failed to retrieve meta-data for %s:%s:%s",
standard, provider, type);
services_action_free(action);
return -EIO;
}
if (!action->stdout_data) {
crm_err("Failed to receive meta-data for %s:%s:%s",
standard, provider, type);
services_action_free(action);
return -EIO;
}
*output = strdup(action->stdout_data);
services_action_free(action);
return pcmk_ok;
}
static int
lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
const char *userdata, guint interval_ms,
int timeout, /* ms */
int start_delay, /* ms */
enum lrmd_call_options options, lrmd_key_value_t * params)
{
int rc = pcmk_ok;
xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
lrmd_key_value_t *tmp = NULL;
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
crm_xml_add(data, PCMK__XA_LRMD_RSC_USERDATA_STR, userdata);
crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
crm_xml_add_int(data, PCMK__XA_LRMD_RSC_START_DELAY, start_delay);
for (tmp = params; tmp; tmp = tmp->next) {
hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
}
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, true);
pcmk__xml_free(data);
lrmd_key_value_freeall(params);
return rc;
}
/* timeout is in ms */
static int
lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
int timeout, lrmd_key_value_t *params)
{
int rc = pcmk_ok;
xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_ALERT);
xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
lrmd_key_value_t *tmp = NULL;
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
crm_xml_add(data, PCMK__XA_LRMD_ALERT_ID, alert_id);
crm_xml_add(data, PCMK__XA_LRMD_ALERT_PATH, alert_path);
crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
for (tmp = params; tmp; tmp = tmp->next) {
hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
}
rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
lrmd_opt_notify_orig_only, true);
pcmk__xml_free(data);
lrmd_key_value_freeall(params);
return rc;
}
static int
lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
guint interval_ms)
{
int rc = pcmk_ok;
xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, true);
pcmk__xml_free(data);
return rc;
}
static int
list_stonith_agents(lrmd_list_t ** resources)
{
int rc = 0;
stonith_t *stonith_api = stonith_api_new();
stonith_key_value_t *stonith_resources = NULL;
stonith_key_value_t *dIter = NULL;
if (stonith_api == NULL) {
crm_err("Could not list fence agents: API memory allocation failed");
return -ENOMEM;
}
stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
&stonith_resources, 0);
stonith_api->cmds->free(stonith_api);
for (dIter = stonith_resources; dIter; dIter = dIter->next) {
rc++;
if (resources) {
*resources = lrmd_list_add(*resources, dIter->value);
}
}
stonith_key_value_freeall(stonith_resources, 1, 0);
return rc;
}
static int
lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
const char *provider)
{
int rc = 0;
int stonith_count = 0; // Initially, whether to include stonith devices
if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
stonith_count = 1;
} else {
GList *gIter = NULL;
GList *agents = resources_list_agents(class, provider);
for (gIter = agents; gIter != NULL; gIter = gIter->next) {
*resources = lrmd_list_add(*resources, (const char *)gIter->data);
rc++;
}
g_list_free_full(agents, free);
if (!class) {
stonith_count = 1;
}
}
if (stonith_count) {
// Now, if stonith devices are included, how many there are
stonith_count = list_stonith_agents(resources);
if (stonith_count > 0) {
rc += stonith_count;
}
}
if (rc == 0) {
crm_notice("No agents found for class %s", class);
rc = -EPROTONOSUPPORT;
}
return rc;
}
static bool
does_provider_have_agent(const char *agent, const char *provider, const char *class)
{
bool found = false;
GList *agents = NULL;
GList *gIter2 = NULL;
agents = resources_list_agents(class, provider);
for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
found = true;
}
}
g_list_free_full(agents, free);
return found;
}
static int
lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
{
int rc = pcmk_ok;
char *provider = NULL;
GList *ocf_providers = NULL;
GList *gIter = NULL;
ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
provider = gIter->data;
if (!agent || does_provider_have_agent(agent, provider,
PCMK_RESOURCE_CLASS_OCF)) {
*providers = lrmd_list_add(*providers, (const char *)gIter->data);
rc++;
}
}
g_list_free_full(ocf_providers, free);
return rc;
}
static int
lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
{
int rc = 0;
GList *standards = NULL;
GList *gIter = NULL;
standards = resources_list_standards();
for (gIter = standards; gIter != NULL; gIter = gIter->next) {
*supported = lrmd_list_add(*supported, (const char *)gIter->data);
rc++;
}
if (list_stonith_agents(NULL) > 0) {
*supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
rc++;
}
g_list_free_full(standards, free);
return rc;
}
/*!
* \internal
* \brief Create an executor API object
*
* \param[out] api Will be set to newly created API object (it is the
* caller's responsibility to free this value with
* lrmd_api_delete() if this function succeeds)
* \param[in] nodename If the object will be used for a remote connection,
* the node name to use in cluster for remote executor
* \param[in] server If the object will be used for a remote connection,
* the resolvable host name to connect to
* \param[in] port If the object will be used for a remote connection,
* port number on \p server to connect to
*
* \return Standard Pacemaker return code
* \note If the caller leaves one of \p nodename or \p server NULL, the other's
* value will be used for both. If the caller leaves both NULL, an API
* object will be created for a local executor connection.
*/
int
lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
{
lrmd_private_t *pvt = NULL;
if (api == NULL) {
return EINVAL;
}
*api = NULL;
// Allocate all memory needed
*api = calloc(1, sizeof(lrmd_t));
if (*api == NULL) {
return ENOMEM;
}
pvt = calloc(1, sizeof(lrmd_private_t));
if (pvt == NULL) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
(*api)->lrmd_private = pvt;
// @TODO Do we need to do this for local connections?
pvt->remote = calloc(1, sizeof(pcmk__remote_t));
(*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
// Set methods
(*api)->cmds->connect = lrmd_api_connect;
(*api)->cmds->connect_async = lrmd_api_connect_async;
(*api)->cmds->is_connected = lrmd_api_is_connected;
(*api)->cmds->poke_connection = lrmd_api_poke_connection;
(*api)->cmds->disconnect = lrmd_api_disconnect;
(*api)->cmds->register_rsc = lrmd_api_register_rsc;
(*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
(*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
(*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
(*api)->cmds->set_callback = lrmd_api_set_callback;
(*api)->cmds->get_metadata = lrmd_api_get_metadata;
(*api)->cmds->exec = lrmd_api_exec;
(*api)->cmds->cancel = lrmd_api_cancel;
(*api)->cmds->list_agents = lrmd_api_list_agents;
(*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
(*api)->cmds->list_standards = lrmd_api_list_standards;
(*api)->cmds->exec_alert = lrmd_api_exec_alert;
(*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
if ((nodename == NULL) && (server == NULL)) {
pvt->type = pcmk__client_ipc;
} else {
if (nodename == NULL) {
nodename = server;
} else if (server == NULL) {
server = nodename;
}
pvt->type = pcmk__client_tls;
pvt->remote_nodename = strdup(nodename);
pvt->server = strdup(server);
if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
lrmd_api_delete(*api);
*api = NULL;
return ENOMEM;
}
pvt->port = port;
if (pvt->port == 0) {
pvt->port = crm_default_remote_port();
}
}
return pcmk_rc_ok;
}
lrmd_t *
lrmd_api_new(void)
{
lrmd_t *api = NULL;
pcmk__assert(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
return api;
}
lrmd_t *
lrmd_remote_api_new(const char *nodename, const char *server, int port)
{
lrmd_t *api = NULL;
pcmk__assert(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
return api;
}
void
lrmd_api_delete(lrmd_t * lrmd)
{
if (lrmd == NULL) {
return;
}
if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
}
free(lrmd->cmds);
}
if (lrmd->lrmd_private != NULL) {
lrmd_private_t *native = lrmd->lrmd_private;
free(native->server);
free(native->remote_nodename);
free(native->remote);
free(native->token);
free(native->peer_version);
free(lrmd->lrmd_private);
}
free(lrmd);
}
struct metadata_cb {
void (*callback)(int pid, const pcmk__action_result_t *result,
void *user_data);
void *user_data;
};
/*!
* \internal
* \brief Process asynchronous metadata completion
*
* \param[in,out] action Metadata action that completed
*/
static void
metadata_complete(svc_action_t *action)
{
struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
pcmk__set_result(&result, action->rc, action->status,
services__exit_reason(action));
pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
metadata_cb->callback(0, &result, metadata_cb->user_data);
result.action_stdout = NULL; // Prevent free, because action owns it
result.action_stderr = NULL; // Prevent free, because action owns it
pcmk__reset_result(&result);
free(metadata_cb);
}
/*!
* \internal
* \brief Retrieve agent metadata asynchronously
*
* \param[in] rsc Resource agent specification
* \param[in] callback Function to call with result (this will always be
* called, whether by this function directly or later
* via the main loop, and on success the metadata will
* be in its result argument's action_stdout)
* \param[in,out] user_data User data to pass to callback
*
* \return Standard Pacemaker return code
* \note This function is not a lrmd_api_operations_t method because it does not
* need an lrmd_t object and does not go through the executor, but
* executes the agent directly.
*/
int
lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
void (*callback)(int pid,
const pcmk__action_result_t *result,
void *user_data),
void *user_data)
{
svc_action_t *action = NULL;
struct metadata_cb *metadata_cb = NULL;
pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
CRM_CHECK(callback != NULL, return EINVAL);
if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
PCMK_EXEC_ERROR_FATAL,
"Invalid resource specification");
callback(0, &result, user_data);
pcmk__reset_result(&result);
return EINVAL;
}
if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
return stonith__metadata_async(rsc->type,
pcmk__timeout_ms2s(PCMK_DEFAULT_ACTION_TIMEOUT_MS),
callback, user_data);
}
action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
rsc->standard, rsc->provider,
rsc->type,
PCMK_ACTION_META_DATA, 0,
PCMK_DEFAULT_ACTION_TIMEOUT_MS,
NULL, 0);
if (action == NULL) {
pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
"Out of memory");
callback(0, &result, user_data);
pcmk__reset_result(&result);
return ENOMEM;
}
if (action->rc != PCMK_OCF_UNKNOWN) {
pcmk__set_result(&result, action->rc, action->status,
services__exit_reason(action));
callback(0, &result, user_data);
pcmk__reset_result(&result);
services_action_free(action);
return EINVAL;
}
action->cb_data = calloc(1, sizeof(struct metadata_cb));
if (action->cb_data == NULL) {
services_action_free(action);
pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
"Out of memory");
callback(0, &result, user_data);
pcmk__reset_result(&result);
return ENOMEM;
}
metadata_cb = (struct metadata_cb *) action->cb_data;
metadata_cb->callback = callback;
metadata_cb->user_data = user_data;
if (!services_action_async(action, metadata_complete)) {
services_action_free(action);
return pcmk_rc_error; // @TODO Derive from action->rc and ->status
}
// The services library has taken responsibility for action
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Set the result of an executor event
*
* \param[in,out] event Executor event to set
* \param[in] rc OCF exit status of event
* \param[in] op_status Executor status of event
* \param[in] exit_reason Human-friendly description of event
*/
void
lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
const char *exit_reason)
{
if (event == NULL) {
return;
}
event->rc = rc;
event->op_status = op_status;
// lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
pcmk__str_update((char **) &event->exit_reason, exit_reason);
}
/*!
* \internal
* \brief Clear an executor event's exit reason, output, and error output
*
* \param[in,out] event Executor event to reset
*/
void
lrmd__reset_result(lrmd_event_data_t *event)
{
if (event == NULL) {
return;
}
free((void *) event->exit_reason);
event->exit_reason = NULL;
free((void *) event->output);
event->output = NULL;
}
/*!
* \internal
* \brief Get the uptime of a remote resource connection
*
* When the cluster connects to a remote resource, part of that resource's
* handshake includes the uptime of the remote resource's connection. This
* uptime is stored in the lrmd_t object.
*
* \return The connection's uptime, or -1 if unknown
*/
time_t
lrmd__uptime(lrmd_t *lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->remote == NULL) {
return -1;
} else {
return native->remote->uptime;
}
}
const char *
lrmd__node_start_state(lrmd_t *lrmd)
{
lrmd_private_t *native = lrmd->lrmd_private;
if (native->remote == NULL) {
return NULL;
} else {
return native->remote->start_state;
}
}