diff --git a/include/crm/cluster.h b/include/crm/cluster.h
index 117070225d..f7c0c169f7 100644
--- a/include/crm/cluster.h
+++ b/include/crm/cluster.h
@@ -1,242 +1,241 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_CLUSTER__H
# define PCMK__CRM_CLUSTER__H
# include <stdint.h> // uint32_t, uint64_t
# include <glib.h> // gboolean, GHashTable
# include <libxml/tree.h> // xmlNode
# include <crm/common/xml.h>
# include <crm/common/util.h>
#ifdef __cplusplus
extern "C" {
#endif
# if SUPPORT_COROSYNC
# include <corosync/cpg.h>
# endif
extern gboolean crm_have_quorum;
extern GHashTable *crm_peer_cache;
extern GHashTable *crm_remote_peer_cache;
extern unsigned long long crm_peer_seq;
#define CRM_NODE_LOST "lost"
#define CRM_NODE_MEMBER "member"
enum crm_join_phase {
/* @COMPAT: crm_join_nack_quiet can be replaced by crm_node_t:user_data
* at a compatibility break.
*/
//! Not allowed to join, but don't send a nack message
crm_join_nack_quiet = -2,
crm_join_nack = -1,
crm_join_none = 0,
crm_join_welcomed = 1,
crm_join_integrated = 2,
crm_join_finalized = 3,
crm_join_confirmed = 4,
};
enum crm_node_flags {
/* node is not a cluster node and should not be considered for cluster membership */
crm_remote_node = 0x0001,
/* node's cache entry is dirty */
crm_node_dirty = 0x0010,
};
typedef struct crm_peer_node_s {
char *uname; // Node name as known to cluster
/* @COMPAT This is less than ideal since the value is not a valid XML ID
* (for Corosync, it's the string equivalent of the node's numeric node ID,
* but XML IDs can't start with a number) and the three elements should have
* different IDs.
*
* Ideally, we would use something like node-NODEID, node_state-NODEID, and
* transient_attributes-NODEID as the element IDs. Unfortunately changing it
* would be impractical due to backward compatibility; older nodes in a
* rolling upgrade will always write and expect the value in the old format.
*
* This is also named poorly, since the value is not a UUID, but at least
* that can be changed at an API compatibility break.
*/
/*! Value of the PCMK_XA_ID XML attribute to use with the node's
* PCMK_XE_NODE, PCMK_XE_NODE_STATE, and PCMK_XE_TRANSIENT_ATTRIBUTES
* XML elements in the CIB
*/
char *uuid;
char *state; // @TODO change to enum
uint64_t flags; // Bitmask of crm_node_flags
uint64_t last_seen; // Only needed by cluster nodes
uint32_t processes; // @TODO most not needed, merge into flags
/* @TODO When we can break public API compatibility, we can make the rest of
* these members separate structs and use void *cluster_data and
* void *user_data here instead, to abstract the cluster layer further.
*/
// Currently only needed by corosync stack
uint32_t id; // Node ID
time_t when_lost; // When CPG membership was last lost
// Only used by controller
enum crm_join_phase join;
char *expected;
time_t peer_lost;
char *conn_host;
time_t when_member; // Since when node has been a cluster member
time_t when_online; // Since when peer has been online in CPG
} crm_node_t;
void crm_peer_init(void);
void crm_peer_destroy(void);
typedef struct crm_cluster_s {
char *uuid;
char *uname;
uint32_t nodeid;
void (*destroy) (gpointer);
# if SUPPORT_COROSYNC
/* @TODO When we can break public API compatibility, make these members a
* separate struct and use void *cluster_data here instead, to abstract the
* cluster layer further.
*/
struct cpg_name group;
cpg_callbacks_t cpg;
cpg_handle_t cpg_handle;
# endif
} crm_cluster_t;
int pcmk_cluster_connect(crm_cluster_t *cluster);
int pcmk_cluster_disconnect(crm_cluster_t *cluster);
-void crm_cluster_disconnect(crm_cluster_t *cluster);
crm_cluster_t *pcmk_cluster_new(void);
void pcmk_cluster_free(crm_cluster_t *cluster);
enum crm_ais_msg_class {
crm_class_cluster = 0,
};
enum crm_ais_msg_types {
crm_msg_none = 0,
crm_msg_ais = 1,
crm_msg_lrmd = 2,
crm_msg_cib = 3,
crm_msg_crmd = 4,
crm_msg_attrd = 5,
crm_msg_stonithd = 6,
crm_msg_te = 7,
crm_msg_pe = 8,
crm_msg_stonith_ng = 9,
};
gboolean send_cluster_message(const crm_node_t *node,
enum crm_ais_msg_types service,
const xmlNode *data, gboolean ordered);
int crm_remote_peer_cache_size(void);
/* Initialize and refresh the remote peer cache from a cib config */
void crm_remote_peer_cache_refresh(xmlNode *cib);
crm_node_t *crm_remote_peer_get(const char *node_name);
void crm_remote_peer_cache_remove(const char *node_name);
guint crm_active_peers(void);
gboolean crm_is_peer_active(const crm_node_t * node);
guint reap_crm_member(uint32_t id, const char *name);
# if SUPPORT_COROSYNC
uint32_t get_local_nodeid(cpg_handle_t handle);
void pcmk_cpg_membership(cpg_handle_t handle,
const struct cpg_name *groupName,
const struct cpg_address *member_list, size_t member_list_entries,
const struct cpg_address *left_list, size_t left_list_entries,
const struct cpg_address *joined_list, size_t joined_list_entries);
gboolean crm_is_corosync_peer_active(const crm_node_t * node);
gboolean send_cluster_text(enum crm_ais_msg_class msg_class, const char *data,
gboolean local, const crm_node_t *node,
enum crm_ais_msg_types dest);
char *pcmk_message_common_cs(cpg_handle_t handle, uint32_t nodeid, uint32_t pid, void *msg,
uint32_t *kind, const char **from);
# endif
const char *crm_peer_uuid(crm_node_t *node);
const char *crm_peer_uname(const char *uuid);
enum crm_status_type {
crm_status_uname,
crm_status_nstate,
crm_status_processes,
};
enum crm_ais_msg_types text2msg_type(const char *text);
void crm_set_status_callback(void (*dispatch) (enum crm_status_type, crm_node_t *, const void *));
void crm_set_autoreap(gboolean autoreap);
enum cluster_type_e {
pcmk_cluster_unknown = 0x0001,
pcmk_cluster_invalid = 0x0002,
// 0x0004 was heartbeat
// 0x0010 was corosync 1 with plugin
pcmk_cluster_corosync = 0x0020,
// 0x0040 was corosync 1 with CMAN
};
enum cluster_type_e get_cluster_type(void);
const char *name_for_cluster_type(enum cluster_type_e type);
gboolean is_corosync_cluster(void);
const char *get_local_node_name(void);
char *get_node_name(uint32_t nodeid);
/*!
* \brief Get log-friendly string equivalent of a join phase
*
* \param[in] phase Join phase
*
* \return Log-friendly string equivalent of \p phase
*/
static inline const char *
crm_join_phase_str(enum crm_join_phase phase)
{
switch (phase) {
case crm_join_nack_quiet: return "nack_quiet";
case crm_join_nack: return "nack";
case crm_join_none: return "none";
case crm_join_welcomed: return "welcomed";
case crm_join_integrated: return "integrated";
case crm_join_finalized: return "finalized";
case crm_join_confirmed: return "confirmed";
default: return "invalid";
}
}
#if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
#include <crm/cluster/compat.h>
#endif
#ifdef __cplusplus
}
#endif
#endif
diff --git a/include/crm/cluster/compat.h b/include/crm/cluster/compat.h
index 84238db5aa..a434b337b6 100644
--- a/include/crm/cluster/compat.h
+++ b/include/crm/cluster/compat.h
@@ -1,69 +1,72 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_CLUSTER_COMPAT__H
# define PCMK__CRM_CLUSTER_COMPAT__H
#include <libxml/tree.h> // xmlNode
#include <crm/cluster.h> // crm_node_t
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief Deprecated Pacemaker cluster API
* \ingroup cluster
* \deprecated Do not include this header directly. The cluster APIs in this
* header, and the header itself, will be removed in a future
* release.
*/
// \deprecated Do not use
enum crm_get_peer_flags {
CRM_GET_PEER_CLUSTER = 0x0001,
CRM_GET_PEER_REMOTE = 0x0002,
CRM_GET_PEER_ANY = CRM_GET_PEER_CLUSTER|CRM_GET_PEER_REMOTE,
};
// \deprecated Do not use Pacemaker for cluster node cacheing
crm_node_t *crm_get_peer(unsigned int id, const char *uname);
// \deprecated Do not use Pacemaker for cluster node cacheing
crm_node_t *crm_get_peer_full(unsigned int id, const char *uname, int flags);
// \deprecated Use stonith_api_kick() from libstonithd instead
int crm_terminate_member(int nodeid, const char *uname, void *unused);
// \deprecated Use stonith_api_kick() from libstonithd instead
int crm_terminate_member_no_mainloop(int nodeid, const char *uname,
int *connection);
// \deprecated Use crm_xml_add(xml, attr, crm_peer_uuid(node)) instead
void set_uuid(xmlNode *xml, const char *attr, crm_node_t *node);
#if SUPPORT_COROSYNC
// \deprecated Do not use
gboolean cluster_connect_cpg(crm_cluster_t *cluster);
// \deprecated Do not use
void cluster_disconnect_cpg(crm_cluster_t *cluster);
#endif // SUPPORT_COROSYNC
// \deprecated Use \c pcmk_cluster_connect() instead
gboolean crm_cluster_connect(crm_cluster_t *cluster);
+// \deprecated Use \c pcmk_cluster_disconnect() instead
+void crm_cluster_disconnect(crm_cluster_t *cluster);
+
#ifdef __cplusplus
}
#endif
#endif // PCMK_CLUSTER_COMPAT__H
diff --git a/lib/cluster/cluster.c b/lib/cluster/cluster.c
index 5e1407bab5..204154e0e2 100644
--- a/lib/cluster/cluster.c
+++ b/lib/cluster/cluster.c
@@ -1,435 +1,430 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <dlfcn.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <time.h>
#include <sys/param.h>
#include <sys/types.h>
#include <crm/crm.h>
#include <crm/common/ipc.h>
#include <crm/common/xml.h>
#include <crm/cluster/internal.h>
#include "crmcluster_private.h"
CRM_TRACE_INIT_DATA(cluster);
/*!
* \brief Get (and set if needed) a node's UUID
*
* \param[in,out] peer Node to check
*
* \return Node UUID of \p peer, or NULL if unknown
*/
const char *
crm_peer_uuid(crm_node_t *peer)
{
char *uuid = NULL;
// Check simple cases first, to avoid any calls that might block
if (peer == NULL) {
return NULL;
}
if (peer->uuid != NULL) {
return peer->uuid;
}
switch (get_cluster_type()) {
case pcmk_cluster_corosync:
#if SUPPORT_COROSYNC
uuid = pcmk__corosync_uuid(peer);
#endif
break;
case pcmk_cluster_unknown:
case pcmk_cluster_invalid:
crm_err("Unsupported cluster type");
break;
}
peer->uuid = uuid;
return peer->uuid;
}
/*!
* \internal
* \brief Connect to the cluster layer
*
* \param[in,out] cluster Initialized cluster object to connect
*
* \return Standard Pacemaker return code
*/
int
pcmk_cluster_connect(crm_cluster_t *cluster)
{
enum cluster_type_e type = get_cluster_type();
crm_notice("Connecting to %s cluster infrastructure",
name_for_cluster_type(type));
switch (type) {
case pcmk_cluster_corosync:
#if SUPPORT_COROSYNC
crm_peer_init();
return pcmk__corosync_connect(cluster);
#else
break;
#endif // SUPPORT_COROSYNC
default:
break;
}
crm_err("Failed to connect to unsupported cluster type %s",
name_for_cluster_type(type));
return EPROTONOSUPPORT;
}
/*!
* \brief Disconnect from the cluster layer
*
* \param[in,out] cluster Cluster object to disconnect
*
* \return Standard Pacemaker return code
*/
int
pcmk_cluster_disconnect(crm_cluster_t *cluster)
{
enum cluster_type_e type = get_cluster_type();
crm_info("Disconnecting from %s cluster infrastructure",
name_for_cluster_type(type));
switch (type) {
case pcmk_cluster_corosync:
#if SUPPORT_COROSYNC
crm_peer_destroy();
pcmk__corosync_disconnect(cluster);
return pcmk_rc_ok;
#else
break;
#endif // SUPPORT_COROSYNC
default:
break;
}
crm_err("Failed to disconnect from unsupported cluster type %s",
name_for_cluster_type(type));
return EPROTONOSUPPORT;
}
-/*!
- * \brief Disconnect from the cluster layer
- *
- * \param[in,out] cluster Cluster object to disconnect
- */
-void
-crm_cluster_disconnect(crm_cluster_t *cluster)
-{
- pcmk_cluster_disconnect(cluster);
-}
-
/*!
* \brief Allocate a new \p crm_cluster_t object
*
* \return A newly allocated \p crm_cluster_t object (guaranteed not \c NULL)
* \note The caller is responsible for freeing the return value using
* \p pcmk_cluster_free().
*/
crm_cluster_t *
pcmk_cluster_new(void)
{
return (crm_cluster_t *) pcmk__assert_alloc(1, sizeof(crm_cluster_t));
}
/*!
* \brief Free a \p crm_cluster_t object and its dynamically allocated members
*
* \param[in,out] cluster Cluster object to free
*/
void
pcmk_cluster_free(crm_cluster_t *cluster)
{
if (cluster == NULL) {
return;
}
free(cluster->uuid);
free(cluster->uname);
free(cluster);
}
/*!
* \brief Send an XML message via the cluster messaging layer
*
* \param[in] node Cluster node to send message to
* \param[in] service Message type to use in message host info
* \param[in] data XML message to send
* \param[in] ordered Ignored for currently supported messaging layers
*
* \return TRUE on success, otherwise FALSE
*/
gboolean
send_cluster_message(const crm_node_t *node, enum crm_ais_msg_types service,
const xmlNode *data, gboolean ordered)
{
switch (get_cluster_type()) {
case pcmk_cluster_corosync:
#if SUPPORT_COROSYNC
return pcmk__cpg_send_xml(data, node, service);
#endif
break;
default:
break;
}
return FALSE;
}
/*!
* \brief Get the local node's name
*
* \return Local node's name
* \note This will fatally exit if local node name cannot be known.
*/
const char *
get_local_node_name(void)
{
static char *name = NULL;
if (name == NULL) {
name = get_node_name(0);
}
return name;
}
/*!
* \brief Get the node name corresponding to a cluster node ID
*
* \param[in] nodeid Node ID to check (or 0 for local node)
*
* \return Node name corresponding to \p nodeid
* \note This will fatally exit if \p nodeid is 0 and local node name cannot be
* known.
*/
char *
get_node_name(uint32_t nodeid)
{
char *name = NULL;
enum cluster_type_e stack = get_cluster_type();
switch (stack) {
case pcmk_cluster_corosync:
#if SUPPORT_COROSYNC
name = pcmk__corosync_name(0, nodeid);
break;
#endif // SUPPORT_COROSYNC
default:
crm_err("Unknown cluster type: %s (%d)", name_for_cluster_type(stack), stack);
}
if ((name == NULL) && (nodeid == 0)) {
name = pcmk_hostname();
if (name == NULL) {
// @TODO Maybe let the caller decide what to do
crm_err("Could not obtain the local %s node name",
name_for_cluster_type(stack));
crm_exit(CRM_EX_FATAL);
}
crm_notice("Defaulting to uname -n for the local %s node name",
name_for_cluster_type(stack));
}
if (name == NULL) {
crm_notice("Could not obtain a node name for %s node with "
PCMK_XA_ID " %u", name_for_cluster_type(stack), nodeid);
}
return name;
}
/*!
* \brief Get the node name corresponding to a node UUID
*
* \param[in] uuid UUID of desired node
*
* \return name of desired node
*
* \note This relies on the remote peer cache being populated with all
* remote nodes in the cluster, so callers should maintain that cache.
*/
const char *
crm_peer_uname(const char *uuid)
{
GHashTableIter iter;
crm_node_t *node = NULL;
CRM_CHECK(uuid != NULL, return NULL);
/* remote nodes have the same uname and uuid */
if (g_hash_table_lookup(crm_remote_peer_cache, uuid)) {
return uuid;
}
/* avoid blocking calls where possible */
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
if (pcmk__str_eq(node->uuid, uuid, pcmk__str_casei)) {
if (node->uname != NULL) {
return node->uname;
}
break;
}
}
node = NULL;
if (is_corosync_cluster()) {
long long id;
if ((pcmk__scan_ll(uuid, &id, 0LL) != pcmk_rc_ok)
|| (id < 1LL) || (id > UINT32_MAX)) {
crm_err("Invalid Corosync node ID '%s'", uuid);
return NULL;
}
node = pcmk__search_node_caches((uint32_t) id, NULL,
pcmk__node_search_cluster);
if (node != NULL) {
crm_info("Setting uuid for node %s[%u] to %s",
node->uname, node->id, uuid);
node->uuid = strdup(uuid);
return node->uname;
}
return NULL;
}
return NULL;
}
/*!
* \brief Get a log-friendly string equivalent of a cluster type
*
* \param[in] type Cluster type
*
* \return Log-friendly string corresponding to \p type
*/
const char *
name_for_cluster_type(enum cluster_type_e type)
{
switch (type) {
case pcmk_cluster_corosync:
return "corosync";
case pcmk_cluster_unknown:
return "unknown";
case pcmk_cluster_invalid:
return "invalid";
}
crm_err("Invalid cluster type: %d", type);
return "invalid";
}
/*!
* \brief Get (and validate) the local cluster type
*
* \return Local cluster type
* \note This will fatally exit if the local cluster type is invalid.
*/
enum cluster_type_e
get_cluster_type(void)
{
bool detected = false;
const char *cluster = NULL;
static enum cluster_type_e cluster_type = pcmk_cluster_unknown;
/* Return the previous calculation, if any */
if (cluster_type != pcmk_cluster_unknown) {
return cluster_type;
}
cluster = pcmk__env_option(PCMK__ENV_CLUSTER_TYPE);
#if SUPPORT_COROSYNC
/* If nothing is defined in the environment, try corosync (if supported) */
if (cluster == NULL) {
crm_debug("Testing with Corosync");
cluster_type = pcmk__corosync_detect();
if (cluster_type != pcmk_cluster_unknown) {
detected = true;
goto done;
}
}
#endif
/* Something was defined in the environment, test it against what we support */
crm_info("Verifying cluster type: '%s'",
((cluster == NULL)? "-unspecified-" : cluster));
if (cluster == NULL) {
#if SUPPORT_COROSYNC
} else if (pcmk__str_eq(cluster, "corosync", pcmk__str_casei)) {
cluster_type = pcmk_cluster_corosync;
#endif
} else {
cluster_type = pcmk_cluster_invalid;
goto done; /* Keep the compiler happy when no stacks are supported */
}
done:
if (cluster_type == pcmk_cluster_unknown) {
crm_notice("Could not determine the current cluster type");
} else if (cluster_type == pcmk_cluster_invalid) {
crm_notice("This installation does not support the '%s' cluster infrastructure: terminating.",
cluster);
crm_exit(CRM_EX_FATAL);
} else {
crm_info("%s an active '%s' cluster",
(detected? "Detected" : "Assuming"),
name_for_cluster_type(cluster_type));
}
return cluster_type;
}
/*!
* \brief Check whether the local cluster is a Corosync cluster
*
* \return TRUE if the local cluster is a Corosync cluster, otherwise FALSE
*/
gboolean
is_corosync_cluster(void)
{
return get_cluster_type() == pcmk_cluster_corosync;
}
// Deprecated functions kept only for backward API compatibility
// LCOV_EXCL_START
#include <crm/cluster/compat.h>
void
set_uuid(xmlNode *xml, const char *attr, crm_node_t *node)
{
crm_xml_add(xml, attr, crm_peer_uuid(node));
}
gboolean
crm_cluster_connect(crm_cluster_t *cluster)
{
return pcmk_cluster_connect(cluster) == pcmk_rc_ok;
}
+void
+crm_cluster_disconnect(crm_cluster_t *cluster)
+{
+ pcmk_cluster_disconnect(cluster);
+}
+
// LCOV_EXCL_STOP
// End deprecated API