diff --git a/cib/main.c b/cib/main.c
index 2a480545d8..e20a2b6dff 100644
--- a/cib/main.c
+++ b/cib/main.c
@@ -1,576 +1,583 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <sys/utsname.h>
#include <stdlib.h>
#include <errno.h>
#include <fcntl.h>
#include <crm/crm.h>
#include <crm/cib/internal.h>
#include <crm/msg_xml.h>
#include <crm/cluster/internal.h>
#include <crm/common/xml.h>
#include <crm/common/mainloop.h>
#include <cibio.h>
#include <callbacks.h>
#include <pwd.h>
#include <grp.h>
#include "common.h"
#if HAVE_LIBXML2
# include <libxml/parser.h>
#endif
#ifdef HAVE_GETOPT_H
# include <getopt.h>
#endif
#if HAVE_BZLIB_H
# include <bzlib.h>
#endif
extern int init_remote_listener(int port, gboolean encrypted);
gboolean cib_shutdown_flag = FALSE;
int cib_status = pcmk_ok;
crm_cluster_t crm_cluster;
#if SUPPORT_HEARTBEAT
oc_ev_t *cib_ev_token;
ll_cluster_t *hb_conn = NULL;
extern void oc_ev_special(const oc_ev_t *, oc_ev_class_t, int);
gboolean cib_register_ha(ll_cluster_t * hb_cluster, const char *client_name);
#else
void *hb_conn = NULL;
#endif
extern void terminate_cib(const char *caller, gboolean fast);
GMainLoop *mainloop = NULL;
const char *cib_root = NULL;
char *cib_our_uname = NULL;
gboolean preserve_status = FALSE;
gboolean cib_writes_enabled = TRUE;
int remote_fd = 0;
int remote_tls_fd = 0;
int cib_init(void);
void cib_shutdown(int nsig);
gboolean startCib(const char *filename);
extern int write_cib_contents(gpointer p);
GHashTable *config_hash = NULL;
GHashTable *local_notify_queue = NULL;
char *channel1 = NULL;
char *channel2 = NULL;
char *channel3 = NULL;
char *channel4 = NULL;
char *channel5 = NULL;
#define OPTARGS "maswr:V?"
void cib_cleanup(void);
static void
cib_enable_writes(int nsig)
{
crm_info("(Re)enabling disk writes");
cib_writes_enabled = TRUE;
}
static void
log_cib_client(gpointer key, gpointer value, gpointer user_data)
{
crm_info("Client %s", crm_client_name(value));
}
/* *INDENT-OFF* */
static struct crm_option long_options[] = {
/* Top-level Options */
{"help", 0, 0, '?', "\tThis text"},
{"verbose", 0, 0, 'V', "\tIncrease debug output"},
{"per-action-cib", 0, 0, 'a', "\tAdvanced use only"},
{"stand-alone", 0, 0, 's', "\tAdvanced use only"},
{"disk-writes", 0, 0, 'w', "\tAdvanced use only"},
{"cib-root", 1, 0, 'r', "\tAdvanced use only"},
{0, 0, 0, 0}
};
/* *INDENT-ON* */
int
main(int argc, char **argv)
{
int flag;
int rc = 0;
int index = 0;
int argerr = 0;
struct passwd *pwentry = NULL;
crm_log_preinit(NULL, argc, argv);
crm_set_options(NULL, "[options]",
long_options, "Daemon for storing and replicating the cluster configuration");
crm_peer_init();
mainloop_add_signal(SIGTERM, cib_shutdown);
mainloop_add_signal(SIGPIPE, cib_enable_writes);
cib_writer = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_contents, NULL);
while (1) {
flag = crm_get_option(argc, argv, &index);
if (flag == -1)
break;
switch (flag) {
case 'V':
crm_bump_log_level(argc, argv);
break;
case 's':
stand_alone = TRUE;
preserve_status = TRUE;
cib_writes_enabled = FALSE;
pwentry = getpwnam(CRM_DAEMON_USER);
CRM_CHECK(pwentry != NULL,
crm_perror(LOG_ERR, "Invalid uid (%s) specified", CRM_DAEMON_USER);
return 100);
rc = setgid(pwentry->pw_gid);
if (rc < 0) {
crm_perror(LOG_ERR, "Could not set group to %d", pwentry->pw_gid);
return 100;
}
rc = initgroups(CRM_DAEMON_GROUP, pwentry->pw_gid);
if (rc < 0) {
crm_perror(LOG_ERR, "Could not setup groups for user %d", pwentry->pw_uid);
return 100;
}
rc = setuid(pwentry->pw_uid);
if (rc < 0) {
crm_perror(LOG_ERR, "Could not set user to %d", pwentry->pw_uid);
return 100;
}
break;
case '?': /* Help message */
crm_help(flag, EX_OK);
break;
case 'w':
cib_writes_enabled = TRUE;
break;
case 'r':
cib_root = optarg;
break;
case 'm':
cib_metadata();
return 0;
default:
++argerr;
break;
}
}
if (argc - optind == 1 && safe_str_eq("metadata", argv[optind])) {
cib_metadata();
return 0;
}
if (optind > argc) {
++argerr;
}
if (argerr) {
crm_help('?', EX_USAGE);
}
crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
if (cib_root == NULL) {
char *path = crm_strdup_printf("%s/cib.xml", CRM_CONFIG_DIR);
char *legacy = crm_strdup_printf("%s/cib.xml", CRM_LEGACY_CONFIG_DIR);
if (g_file_test(path, G_FILE_TEST_EXISTS)) {
cib_root = CRM_CONFIG_DIR;
} else if (g_file_test(legacy, G_FILE_TEST_EXISTS)) {
cib_root = CRM_LEGACY_CONFIG_DIR;
crm_notice("Using legacy config location: %s", cib_root);
} else {
cib_root = CRM_CONFIG_DIR;
crm_notice("Using new config location: %s", cib_root);
}
free(legacy);
free(path);
} else {
crm_notice("Using custom config location: %s", cib_root);
}
if (crm_is_writable(cib_root, NULL, CRM_DAEMON_USER, CRM_DAEMON_GROUP, FALSE) == FALSE) {
crm_err("Bad permissions on %s. Terminating", cib_root);
fprintf(stderr, "ERROR: Bad permissions on %s. See logs for details\n", cib_root);
fflush(stderr);
return 100;
}
/* read local config file */
rc = cib_init();
CRM_CHECK(crm_hash_table_size(client_connections) == 0,
crm_warn("Not all clients gone at exit"));
g_hash_table_foreach(client_connections, log_cib_client, NULL);
cib_cleanup();
#if SUPPORT_HEARTBEAT
if (hb_conn) {
hb_conn->llc_ops->delete(hb_conn);
}
#endif
crm_info("Done");
return rc;
}
void
cib_cleanup(void)
{
crm_peer_destroy();
if (local_notify_queue) {
g_hash_table_destroy(local_notify_queue);
}
crm_client_cleanup();
g_hash_table_destroy(config_hash);
free(cib_our_uname);
free(channel1);
free(channel2);
free(channel3);
free(channel4);
free(channel5);
}
unsigned long cib_num_ops = 0;
const char *cib_stat_interval = "10min";
unsigned long cib_num_local = 0, cib_num_updates = 0, cib_num_fail = 0;
unsigned long cib_bad_connects = 0, cib_num_timeouts = 0;
#if SUPPORT_HEARTBEAT
gboolean ccm_connect(void);
static void
ccm_connection_destroy(gpointer user_data)
{
crm_err("CCM connection failed... blocking while we reconnect");
CRM_ASSERT(ccm_connect());
return;
}
static void *ccm_library = NULL;
gboolean
ccm_connect(void)
{
gboolean did_fail = TRUE;
int num_ccm_fails = 0;
int max_ccm_fails = 30;
int ret;
int cib_ev_fd;
int (*ccm_api_register) (oc_ev_t ** token) =
find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_register", 1);
int (*ccm_api_set_callback) (const oc_ev_t * token,
oc_ev_class_t class,
oc_ev_callback_t * fn,
oc_ev_callback_t ** prev_fn) =
find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_set_callback", 1);
void (*ccm_api_special) (const oc_ev_t *, oc_ev_class_t, int) =
find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_special", 1);
int (*ccm_api_activate) (const oc_ev_t * token, int *fd) =
find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_activate", 1);
int (*ccm_api_unregister) (oc_ev_t * token) =
find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_unregister", 1);
static struct mainloop_fd_callbacks ccm_fd_callbacks = {
.dispatch = cib_ccm_dispatch,
.destroy = ccm_connection_destroy,
};
while (did_fail) {
did_fail = FALSE;
crm_info("Registering with CCM...");
ret = (*ccm_api_register) (&cib_ev_token);
if (ret != 0) {
did_fail = TRUE;
}
if (did_fail == FALSE) {
crm_trace("Setting up CCM callbacks");
ret = (*ccm_api_set_callback) (cib_ev_token, OC_EV_MEMB_CLASS,
cib_ccm_msg_callback, NULL);
if (ret != 0) {
crm_warn("CCM callback not set");
did_fail = TRUE;
}
}
if (did_fail == FALSE) {
(*ccm_api_special) (cib_ev_token, OC_EV_MEMB_CLASS, 0);
crm_trace("Activating CCM token");
ret = (*ccm_api_activate) (cib_ev_token, &cib_ev_fd);
if (ret != 0) {
crm_warn("CCM Activation failed");
did_fail = TRUE;
}
}
if (did_fail) {
num_ccm_fails++;
(*ccm_api_unregister) (cib_ev_token);
if (num_ccm_fails < max_ccm_fails) {
crm_warn("CCM Connection failed %d times (%d max)", num_ccm_fails, max_ccm_fails);
sleep(3);
} else {
crm_err("CCM Activation failed %d (max) times", num_ccm_fails);
return FALSE;
}
}
}
crm_debug("CCM Activation passed... all set to go!");
mainloop_add_fd("heartbeat-ccm", G_PRIORITY_MEDIUM, cib_ev_fd, cib_ev_token, &ccm_fd_callbacks);
return TRUE;
}
#endif
#if SUPPORT_COROSYNC
static void
cib_cs_dispatch(cpg_handle_t handle,
const struct cpg_name *groupName,
uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
{
uint32_t kind = 0;
xmlNode *xml = NULL;
const char *from = NULL;
char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
if(data == NULL) {
return;
}
if (kind == crm_class_cluster) {
xml = string2xml(data);
if (xml == NULL) {
crm_err("Invalid XML: '%.120s'", data);
free(data);
return;
}
crm_xml_add(xml, F_ORIG, from);
/* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
cib_peer_callback(xml, NULL);
}
free_xml(xml);
free(data);
}
static void
cib_cs_destroy(gpointer user_data)
{
if (cib_shutdown_flag) {
crm_info("Corosync disconnection complete");
} else {
crm_err("Corosync connection lost! Exiting.");
terminate_cib(__FUNCTION__, TRUE);
}
}
#endif
static void
cib_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
if ((type == crm_status_processes) && legacy_mode
&& is_not_set(node->processes, crm_get_cluster_proc())) {
uint32_t old = 0;
if (data) {
old = *(const uint32_t *)data;
}
if ((node->processes ^ old) & crm_proc_cpg) {
crm_info("Attempting to disable legacy mode after %s left the cluster", node->uname);
legacy_mode = FALSE;
}
}
if (cib_shutdown_flag && crm_active_peers() < 2 && crm_hash_table_size(client_connections) == 0) {
crm_info("No more peers");
+ /* @TODO
+ * terminate_cib() calls crm_cluster_disconnect() which calls
+ * crm_peer_destroy() which destroys the peer caches, which a peer
+ * status callback shouldn't do. For now, there is a workaround in
+ * crm_update_peer_proc(), but CIB should be refactored to avoid
+ * destroying the peer caches here.
+ */
terminate_cib(__FUNCTION__, FALSE);
}
}
#if SUPPORT_HEARTBEAT
static void
cib_ha_connection_destroy(gpointer user_data)
{
if (cib_shutdown_flag) {
crm_info("Heartbeat disconnection complete... exiting");
terminate_cib(__FUNCTION__, FALSE);
} else {
crm_err("Heartbeat connection lost! Exiting.");
terminate_cib(__FUNCTION__, TRUE);
}
}
#endif
int
cib_init(void)
{
if (is_openais_cluster()) {
#if SUPPORT_COROSYNC
crm_cluster.destroy = cib_cs_destroy;
crm_cluster.cpg.cpg_deliver_fn = cib_cs_dispatch;
crm_cluster.cpg.cpg_confchg_fn = pcmk_cpg_membership;
#endif
} else if (is_heartbeat_cluster()) {
#if SUPPORT_HEARTBEAT
crm_cluster.hb_dispatch = cib_ha_peer_callback;
crm_cluster.destroy = cib_ha_connection_destroy;
#endif
}
config_hash =
g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
if (startCib("cib.xml") == FALSE) {
crm_crit("Cannot start CIB... terminating");
crm_exit(ENODATA);
}
if (stand_alone == FALSE) {
if (crm_cluster_connect(&crm_cluster) == FALSE) {
crm_crit("Cannot sign in to the cluster... terminating");
crm_exit(DAEMON_RESPAWN_STOP);
}
cib_our_uname = crm_cluster.uname;
if (is_openais_cluster()) {
crm_set_status_callback(&cib_peer_update_callback);
}
#if SUPPORT_HEARTBEAT
if (is_heartbeat_cluster()) {
gboolean was_error = FALSE;
hb_conn = crm_cluster.hb_conn;
if (was_error == FALSE) {
if (HA_OK !=
hb_conn->llc_ops->set_cstatus_callback(hb_conn, cib_client_status_callback,
hb_conn)) {
crm_err("Cannot set cstatus callback: %s", hb_conn->llc_ops->errmsg(hb_conn));
was_error = TRUE;
}
}
if (was_error == FALSE) {
was_error = (ccm_connect() == FALSE);
}
if (was_error == FALSE) {
/* Async get client status information in the cluster */
crm_info("Requesting the list of configured nodes");
hb_conn->llc_ops->client_status(hb_conn, NULL, CRM_SYSTEM_CIB, -1);
}
}
#endif
} else {
cib_our_uname = strdup("localhost");
}
cib_ipc_servers_init(&ipcs_ro,
&ipcs_rw,
&ipcs_shm,
&ipc_ro_callbacks,
&ipc_rw_callbacks);
if (stand_alone) {
cib_is_master = TRUE;
}
/* Create the mainloop and run it... */
mainloop = g_main_new(FALSE);
crm_info("Starting %s mainloop", crm_system_name);
g_main_run(mainloop);
cib_ipc_servers_destroy(ipcs_ro, ipcs_rw, ipcs_shm);
return crm_exit(pcmk_ok);
}
gboolean
startCib(const char *filename)
{
gboolean active = FALSE;
xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status);
CRM_ASSERT(cib != NULL);
if (activateCibXml(cib, TRUE, "start") == 0) {
int port = 0;
const char *port_s = NULL;
active = TRUE;
cib_read_config(config_hash, cib);
port_s = crm_element_value(cib, "remote-tls-port");
if (port_s) {
port = crm_parse_int(port_s, "0");
remote_tls_fd = init_remote_listener(port, TRUE);
}
port_s = crm_element_value(cib, "remote-clear-port");
if (port_s) {
port = crm_parse_int(port_s, "0");
remote_fd = init_remote_listener(port, FALSE);
}
crm_info("CIB Initialization completed successfully");
}
return active;
}
diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c
index 28f41cb4b0..b7958ebcf3 100644
--- a/lib/cluster/membership.c
+++ b/lib/cluster/membership.c
@@ -1,887 +1,895 @@
/*
* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include <crm_internal.h>
#ifndef _GNU_SOURCE
# define _GNU_SOURCE
#endif
#include <sys/param.h>
#include <sys/types.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <glib.h>
#include <crm/common/ipc.h>
#include <crm/cluster/internal.h>
#include <crm/msg_xml.h>
#include <crm/stonith-ng.h>
GHashTable *crm_peer_cache = NULL;
GHashTable *crm_remote_peer_cache = NULL;
unsigned long long crm_peer_seq = 0;
gboolean crm_have_quorum = FALSE;
static gboolean crm_autoreap = TRUE;
int
crm_remote_peer_cache_size(void)
{
if (crm_remote_peer_cache == NULL) {
return 0;
}
return g_hash_table_size(crm_remote_peer_cache);
}
void
crm_remote_peer_cache_add(const char *node_name)
{
crm_node_t *node = g_hash_table_lookup(crm_remote_peer_cache, node_name);
if (node == NULL) {
crm_trace("added %s to remote cache", node_name);
node = calloc(1, sizeof(crm_node_t));
node->flags = crm_remote_node;
CRM_ASSERT(node);
node->uname = strdup(node_name);
node->uuid = strdup(node_name);
node->state = strdup(CRM_NODE_MEMBER);
g_hash_table_replace(crm_remote_peer_cache, node->uname, node);
}
}
void
crm_remote_peer_cache_remove(const char *node_name)
{
g_hash_table_remove(crm_remote_peer_cache, node_name);
}
static void
remote_cache_refresh_helper(xmlNode *cib, const char *xpath, const char *field, int flags)
{
const char *remote = NULL;
crm_node_t *node = NULL;
xmlXPathObjectPtr xpathObj = NULL;
int max = 0;
int lpc = 0;
xpathObj = xpath_search(cib, xpath);
max = numXpathResults(xpathObj);
for (lpc = 0; lpc < max; lpc++) {
xmlNode *xml = getXpathResult(xpathObj, lpc);
CRM_LOG_ASSERT(xml != NULL);
if(xml != NULL) {
remote = crm_element_value(xml, field);
}
if (remote) {
crm_trace("added %s to remote cache", remote);
node = calloc(1, sizeof(crm_node_t));
node->flags = flags;
CRM_ASSERT(node);
node->uname = strdup(remote);
node->uuid = strdup(remote);
node->state = strdup(CRM_NODE_MEMBER);
g_hash_table_replace(crm_remote_peer_cache, node->uname, node);
}
}
freeXpathObject(xpathObj);
}
void crm_remote_peer_cache_refresh(xmlNode *cib)
{
const char *xpath = NULL;
g_hash_table_remove_all(crm_remote_peer_cache);
/* remote nodes associated with a cluster resource */
xpath = "//" XML_TAG_CIB "//" XML_CIB_TAG_CONFIGURATION "//" XML_CIB_TAG_RESOURCE "//" XML_TAG_META_SETS "//" XML_CIB_TAG_NVPAIR "[@name='remote-node']";
remote_cache_refresh_helper(cib, xpath, "value", crm_remote_node | crm_remote_container);
/* baremetal nodes defined by connection resources*/
xpath = "//" XML_TAG_CIB "//" XML_CIB_TAG_CONFIGURATION "//" XML_CIB_TAG_RESOURCE "[@type='remote'][@provider='pacemaker']";
remote_cache_refresh_helper(cib, xpath, "id", crm_remote_node | crm_remote_baremetal);
/* baremetal nodes we have seen in the config that may or may not have connection
* resources associated with them anymore */
xpath = "//" XML_TAG_CIB "//" XML_CIB_TAG_STATUS "//" XML_CIB_TAG_STATE "[@remote_node='true']";
remote_cache_refresh_helper(cib, xpath, "id", crm_remote_node | crm_remote_baremetal);
}
gboolean
crm_is_peer_active(const crm_node_t * node)
{
if(node == NULL) {
return FALSE;
}
if (is_set(node->flags, crm_remote_node)) {
/* remote nodes are never considered active members. This
* guarantees they will never be considered for DC membership.*/
return FALSE;
}
#if SUPPORT_COROSYNC
if (is_openais_cluster()) {
return crm_is_corosync_peer_active(node);
}
#endif
#if SUPPORT_HEARTBEAT
if (is_heartbeat_cluster()) {
return crm_is_heartbeat_peer_active(node);
}
#endif
crm_err("Unhandled cluster type: %s", name_for_cluster_type(get_cluster_type()));
return FALSE;
}
static gboolean
crm_reap_dead_member(gpointer key, gpointer value, gpointer user_data)
{
crm_node_t *node = value;
crm_node_t *search = user_data;
if (search == NULL) {
return FALSE;
} else if (search->id && node->id != search->id) {
return FALSE;
} else if (search->id == 0 && safe_str_neq(node->uname, search->uname)) {
return FALSE;
} else if (crm_is_peer_active(value) == FALSE) {
crm_notice("Removing %s/%u from the membership list", node->uname, node->id);
return TRUE;
}
return FALSE;
}
/*!
* \brief Remove all peer cache entries matching a node ID and/or uname
*
* \param[in] id ID of node to remove (or 0 to ignore)
* \param[in] name Uname of node to remove (or NULL to ignore)
*
* \return Number of cache entries removed
*/
guint
reap_crm_member(uint32_t id, const char *name)
{
int matches = 0;
crm_node_t search;
if (crm_peer_cache == NULL) {
crm_trace("Nothing to do, cache not initialized");
return 0;
}
search.id = id;
search.uname = name ? strdup(name) : NULL;
matches = g_hash_table_foreach_remove(crm_peer_cache, crm_reap_dead_member, &search);
if(matches) {
crm_notice("Purged %d peers with id=%u and/or uname=%s from the membership cache",
matches, search.id, search.uname);
} else {
crm_info("No peers with id=%u and/or uname=%s exist", id, name);
}
free(search.uname);
return matches;
}
static void
crm_count_peer(gpointer key, gpointer value, gpointer user_data)
{
guint *count = user_data;
crm_node_t *node = value;
if (crm_is_peer_active(node)) {
*count = *count + 1;
}
}
guint
crm_active_peers(void)
{
guint count = 0;
if (crm_peer_cache) {
g_hash_table_foreach(crm_peer_cache, crm_count_peer, &count);
}
return count;
}
static void
destroy_crm_node(gpointer data)
{
crm_node_t *node = data;
crm_trace("Destroying entry for node %u: %s", node->id, node->uname);
free(node->addr);
free(node->uname);
free(node->state);
free(node->uuid);
free(node->expected);
free(node);
}
void
crm_peer_init(void)
{
if (crm_peer_cache == NULL) {
crm_peer_cache = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, free, destroy_crm_node);
}
if (crm_remote_peer_cache == NULL) {
crm_remote_peer_cache = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, destroy_crm_node);
}
}
void
crm_peer_destroy(void)
{
if (crm_peer_cache != NULL) {
crm_trace("Destroying peer cache with %d members", g_hash_table_size(crm_peer_cache));
g_hash_table_destroy(crm_peer_cache);
crm_peer_cache = NULL;
}
if (crm_remote_peer_cache != NULL) {
crm_trace("Destroying remote peer cache with %d members", g_hash_table_size(crm_remote_peer_cache));
g_hash_table_destroy(crm_remote_peer_cache);
crm_remote_peer_cache = NULL;
}
}
void (*crm_status_callback) (enum crm_status_type, crm_node_t *, const void *) = NULL;
/*!
* \brief Set a client function that will be called after peer status changes
*
* \param[in] dispatch Pointer to function to use as callback
*
* \note Previously, client callbacks were responsible for peer cache
* management. This is no longer the case, and client callbacks should do
* only client-specific handling. Callbacks MUST NOT add or remove entries
* in the peer caches.
*/
void
crm_set_status_callback(void (*dispatch) (enum crm_status_type, crm_node_t *, const void *))
{
crm_status_callback = dispatch;
}
/*!
* \brief Tell the library whether to automatically reap lost nodes
*
* If TRUE (the default), calling crm_update_peer_proc() will also update the
* peer state to CRM_NODE_MEMBER or CRM_NODE_LOST, and crm_update_peer_state()
* will reap peers whose state changes to anything other than CRM_NODE_MEMBER.
* Callers should leave this enabled unless they plan to manage the cache
* separately on their own.
*
* \param[in] autoreap TRUE to enable automatic reaping, FALSE to disable
*/
void
crm_set_autoreap(gboolean autoreap)
{
crm_autoreap = autoreap;
}
static void crm_dump_peer_hash(int level, const char *caller)
{
GHashTableIter iter;
const char *id = NULL;
crm_node_t *node = NULL;
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, (gpointer *) &id, (gpointer *) &node)) {
do_crm_log(level, "%s: Node %u/%s = %p - %s", caller, node->id, node->uname, node, id);
}
}
static gboolean crm_hash_find_by_data(gpointer key, gpointer value, gpointer user_data)
{
if(value == user_data) {
return TRUE;
}
return FALSE;
}
crm_node_t *
crm_find_peer_full(unsigned int id, const char *uname, int flags)
{
crm_node_t *node = NULL;
CRM_ASSERT(id > 0 || uname != NULL);
crm_peer_init();
if (flags & CRM_GET_PEER_REMOTE) {
node = g_hash_table_lookup(crm_remote_peer_cache, uname);
}
if (node == NULL && (flags & CRM_GET_PEER_CLUSTER)) {
node = crm_find_peer(id, uname);
}
return node;
}
crm_node_t *
crm_get_peer_full(unsigned int id, const char *uname, int flags)
{
crm_node_t *node = NULL;
CRM_ASSERT(id > 0 || uname != NULL);
crm_peer_init();
if (flags & CRM_GET_PEER_REMOTE) {
node = g_hash_table_lookup(crm_remote_peer_cache, uname);
}
if (node == NULL && (flags & CRM_GET_PEER_CLUSTER)) {
node = crm_get_peer(id, uname);
}
return node;
}
crm_node_t *
crm_find_peer(unsigned int id, const char *uname)
{
GHashTableIter iter;
crm_node_t *node = NULL;
crm_node_t *by_id = NULL;
crm_node_t *by_name = NULL;
CRM_ASSERT(id > 0 || uname != NULL);
crm_peer_init();
if (uname != NULL) {
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
if(node->uname && strcasecmp(node->uname, uname) == 0) {
crm_trace("Name match: %s = %p", node->uname, node);
by_name = node;
break;
}
}
}
if (id > 0) {
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
if(node->id == id) {
crm_trace("ID match: %u = %p", node->id, node);
by_id = node;
break;
}
}
}
node = by_id; /* Good default */
if(by_id == by_name) {
/* Nothing to do if they match (both NULL counts) */
crm_trace("Consistent: %p for %u/%s", by_id, id, uname);
} else if(by_id == NULL && by_name) {
crm_trace("Only one: %p for %u/%s", by_name, id, uname);
if(id && by_name->id) {
crm_dump_peer_hash(LOG_WARNING, __FUNCTION__);
crm_crit("Node %u and %u share the same name '%s'",
id, by_name->id, uname);
node = NULL; /* Create a new one */
} else {
node = by_name;
}
} else if(by_name == NULL && by_id) {
crm_trace("Only one: %p for %u/%s", by_id, id, uname);
if(uname && by_id->uname) {
crm_dump_peer_hash(LOG_WARNING, __FUNCTION__);
crm_crit("Node '%s' and '%s' share the same cluster nodeid %u: assuming '%s' is correct",
uname, by_id->uname, id, uname);
}
} else if(uname && by_id->uname) {
if(safe_str_eq(uname, by_id->uname)) {
crm_notice("Node '%s' has changed its ID from %u to %u", by_id->uname, by_name->id, by_id->id);
g_hash_table_foreach_remove(crm_peer_cache, crm_hash_find_by_data, by_name);
} else {
crm_warn("Node '%s' and '%s' share the same cluster nodeid: %u %s", by_id->uname, by_name->uname, id, uname);
crm_dump_peer_hash(LOG_INFO, __FUNCTION__);
crm_abort(__FILE__, __FUNCTION__, __LINE__, "member weirdness", TRUE, TRUE);
}
} else if(id && by_name->id) {
crm_warn("Node %u and %u share the same name: '%s'", by_id->id, by_name->id, uname);
} else {
/* Simple merge */
/* Only corosync based clusters use nodeid's
*
* The functions that call crm_update_peer_state() only know nodeid
* so 'by_id' is authorative when merging
*
* Same for crm_update_peer_proc()
*/
crm_dump_peer_hash(LOG_DEBUG, __FUNCTION__);
crm_info("Merging %p into %p", by_name, by_id);
g_hash_table_foreach_remove(crm_peer_cache, crm_hash_find_by_data, by_name);
}
return node;
}
#if SUPPORT_COROSYNC
static guint
crm_remove_conflicting_peer(crm_node_t *node)
{
int matches = 0;
GHashTableIter iter;
crm_node_t *existing_node = NULL;
if (node->id == 0 || node->uname == NULL) {
return 0;
}
# if !SUPPORT_PLUGIN
if (corosync_cmap_has_config("nodelist") != 0) {
return 0;
}
# endif
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &existing_node)) {
if (existing_node->id > 0
&& existing_node->id != node->id
&& existing_node->uname != NULL
&& strcasecmp(existing_node->uname, node->uname) == 0) {
if (crm_is_peer_active(existing_node)) {
continue;
}
crm_warn("Removing cached offline node %u/%s which has conflicting uname with %u",
existing_node->id, existing_node->uname, node->id);
g_hash_table_iter_remove(&iter);
matches++;
}
}
return matches;
}
#endif
/* coverity[-alloc] Memory is referenced in one or both hashtables */
crm_node_t *
crm_get_peer(unsigned int id, const char *uname)
{
crm_node_t *node = NULL;
char *uname_lookup = NULL;
CRM_ASSERT(id > 0 || uname != NULL);
crm_peer_init();
node = crm_find_peer(id, uname);
/* if uname wasn't provided, and find_peer did not turn up a uname based on id.
* we need to do a lookup of the node name using the id in the cluster membership. */
if ((node == NULL || node->uname == NULL) && (uname == NULL)) {
uname_lookup = get_node_name(id);
}
if (uname_lookup) {
uname = uname_lookup;
crm_trace("Inferred a name of '%s' for node %u", uname, id);
/* try to turn up the node one more time now that we know the uname. */
if (node == NULL) {
node = crm_find_peer(id, uname);
}
}
if (node == NULL) {
char *uniqueid = crm_generate_uuid();
node = calloc(1, sizeof(crm_node_t));
CRM_ASSERT(node);
crm_info("Created entry %s/%p for node %s/%u (%d total)",
uniqueid, node, uname, id, 1 + g_hash_table_size(crm_peer_cache));
g_hash_table_replace(crm_peer_cache, uniqueid, node);
}
if(id > 0 && uname && (node->id == 0 || node->uname == NULL)) {
crm_info("Node %u is now known as %s", id, uname);
}
if(id > 0 && node->id == 0) {
node->id = id;
}
if(uname && node->uname == NULL) {
int lpc, len = strlen(uname);
for (lpc = 0; lpc < len; lpc++) {
if (uname[lpc] >= 'A' && uname[lpc] <= 'Z') {
crm_warn("Node names with capitals are discouraged, consider changing '%s' to something else",
uname);
break;
}
}
node->uname = strdup(uname);
if (crm_status_callback) {
crm_status_callback(crm_status_uname, node, NULL);
}
#if SUPPORT_COROSYNC
if (is_openais_cluster()) {
crm_remove_conflicting_peer(node);
}
#endif
}
if(node->uuid == NULL) {
const char *uuid = crm_peer_uuid(node);
if (uuid) {
crm_info("Node %u has uuid %s", id, uuid);
} else {
crm_info("Cannot obtain a UUID for node %u/%s", id, node->uname);
}
}
free(uname_lookup);
return node;
}
/*!
* \internal
* \brief Update all of a node's information (process list, state, etc.)
*
* \param[in] source Caller's function name (for log messages)
*
* \return NULL if node was reaped from peer caches, pointer to node otherwise
*
* \note This function should not be called within a peer cache iteration,
* otherwise reaping could invalidate the iterator.
*/
crm_node_t *
crm_update_peer(const char *source, unsigned int id, uint64_t born, uint64_t seen, int32_t votes,
uint32_t children, const char *uuid, const char *uname, const char *addr,
const char *state)
{
#if SUPPORT_PLUGIN
gboolean addr_changed = FALSE;
gboolean votes_changed = FALSE;
#endif
crm_node_t *node = NULL;
id = get_corosync_id(id, uuid);
node = crm_get_peer(id, uname);
CRM_ASSERT(node != NULL);
if (node->uuid == NULL) {
if (is_openais_cluster()) {
/* Yes, overrule whatever was passed in */
crm_peer_uuid(node);
} else if (uuid != NULL) {
node->uuid = strdup(uuid);
}
}
if (children > 0) {
if (crm_update_peer_proc(source, node, children, state) == NULL) {
return NULL;
}
}
if (state != NULL) {
if (crm_update_peer_state(source, node, state, seen) == NULL) {
return NULL;
}
}
#if SUPPORT_HEARTBEAT
if (born != 0) {
node->born = born;
}
#endif
#if SUPPORT_PLUGIN
/* These were only used by the plugin */
if (born != 0) {
node->born = born;
}
if (votes > 0 && node->votes != votes) {
votes_changed = TRUE;
node->votes = votes;
}
if (addr != NULL) {
if (node->addr == NULL || crm_str_eq(node->addr, addr, FALSE) == FALSE) {
addr_changed = TRUE;
free(node->addr);
node->addr = strdup(addr);
}
}
if (addr_changed || votes_changed) {
crm_info("%s: Node %s: id=%u state=%s addr=%s%s votes=%d%s born=" U64T " seen=" U64T
" proc=%.32x", source, node->uname, node->id, node->state,
node->addr, addr_changed ? " (new)" : "", node->votes,
votes_changed ? " (new)" : "", node->born, node->last_seen, node->processes);
}
#endif
return node;
}
/*!
* \internal
* \brief Update a node's process information (and potentially state)
*
* \param[in] source Caller's function name (for log messages)
* \param[in] node Node object to update
* \param[in] flag Bitmask of new process information
* \param[in] status node status (online, offline, etc.)
*
* \return NULL if any node was reaped from peer caches, value of node otherwise
*
* \note If this function returns TRUE, the supplied node object was likely
* freed and should not be used again. This function should not be
* called within a cache iteration if reaping is possible, otherwise
* reaping could invalidate the iterator.
*/
crm_node_t *
crm_update_peer_proc(const char *source, crm_node_t * node, uint32_t flag, const char *status)
{
uint32_t last = 0;
gboolean changed = FALSE;
CRM_CHECK(node != NULL, crm_err("%s: Could not set %s to %s for NULL",
source, peer2text(flag), status); return NULL);
last = node->processes;
if (status == NULL) {
node->processes = flag;
if (node->processes != last) {
changed = TRUE;
}
} else if (safe_str_eq(status, ONLINESTATUS)) {
if ((node->processes & flag) != flag) {
set_bit(node->processes, flag);
changed = TRUE;
}
#if SUPPORT_PLUGIN
} else if (safe_str_eq(status, CRM_NODE_MEMBER)) {
if (flag > 0 && node->processes != flag) {
node->processes = flag;
changed = TRUE;
}
#endif
} else if (node->processes & flag) {
clear_bit(node->processes, flag);
changed = TRUE;
}
if (changed) {
if (status == NULL && flag <= crm_proc_none) {
crm_info("%s: Node %s[%u] - all processes are now offline", source, node->uname,
node->id);
} else {
crm_info("%s: Node %s[%u] - %s is now %s", source, node->uname, node->id,
peer2text(flag), status);
}
/* Call the client callback first, then update the peer state,
* in case the node will be reaped
*/
if (crm_status_callback) {
crm_status_callback(crm_status_processes, node, &last);
}
+
+ /* The client callback shouldn't touch the peer caches,
+ * but as a safety net, bail if the peer cache was destroyed.
+ */
+ if (crm_peer_cache == NULL) {
+ return NULL;
+ }
+
if (crm_autoreap) {
node = crm_update_peer_state(__FUNCTION__, node,
is_set(node->processes, crm_get_cluster_proc())?
CRM_NODE_MEMBER : CRM_NODE_LOST, 0);
}
} else {
crm_trace("%s: Node %s[%u] - %s is unchanged (%s)", source, node->uname, node->id,
peer2text(flag), status);
}
return node;
}
void
crm_update_peer_expected(const char *source, crm_node_t * node, const char *expected)
{
char *last = NULL;
gboolean changed = FALSE;
CRM_CHECK(node != NULL, crm_err("%s: Could not set 'expected' to %s", source, expected);
return);
last = node->expected;
if (expected != NULL && safe_str_neq(node->expected, expected)) {
node->expected = strdup(expected);
changed = TRUE;
}
if (changed) {
crm_info("%s: Node %s[%u] - expected state is now %s (was %s)", source, node->uname, node->id,
expected, last);
free(last);
} else {
crm_trace("%s: Node %s[%u] - expected state is unchanged (%s)", source, node->uname,
node->id, expected);
}
}
/*!
* \internal
* \brief Update a node's state and membership information
*
* \param[in] source Caller's function name (for log messages)
* \param[in] node Node object to update
* \param[in] state Node's new state
* \param[in] membership Node's new membership ID
*
* \return NULL if any node was reaped, value of node otherwise
*
* \note If this function returns NULL, the supplied node object was likely
* freed and should not be used again. This function should not be
* called within a cache iteration if reaping is possible,
* otherwise reaping could invalidate the iterator.
*/
crm_node_t *
crm_update_peer_state(const char *source, crm_node_t * node, const char *state, int membership)
{
gboolean is_member;
CRM_CHECK(node != NULL, crm_err("%s: Could not set 'state' to %s", source, state);
return NULL);
is_member = safe_str_eq(state, CRM_NODE_MEMBER);
if (membership && is_member) {
node->last_seen = membership;
}
if (state && safe_str_neq(node->state, state)) {
char *last = node->state;
enum crm_status_type status_type = is_set(node->flags, crm_remote_node)?
crm_status_rstate : crm_status_nstate;
node->state = strdup(state);
crm_notice("%s: Node %s[%u] - state is now %s (was %s)",
source, node->uname, node->id, state, last);
if (crm_status_callback) {
crm_status_callback(status_type, node, last);
}
free(last);
if (!is_member && crm_autoreap) {
if (status_type == crm_status_rstate) {
crm_remote_peer_cache_remove(node->uname);
} else {
reap_crm_member(node->id, node->uname);
}
node = NULL;
}
} else {
crm_trace("%s: Node %s[%u] - state is unchanged (%s)", source, node->uname, node->id,
state);
}
return node;
}
/*!
* \internal
* \brief Reap all nodes from cache whose membership information does not match
*
* \param[in] membership Membership ID of nodes to keep
*/
void
crm_reap_unseen_nodes(uint64_t membership)
{
GHashTableIter iter;
crm_node_t *node = NULL;
crm_trace("Reaping unseen nodes...");
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&node)) {
if (node->last_seen != membership) {
if (node->state) {
/* crm_update_peer_state() cannot be called here, because that
* might modify the peer cache, invalidating our iterator
*/
if (safe_str_eq(node->state, CRM_NODE_LOST)) {
crm_trace("Node %s[%u] - state is unchanged (%s)",
node->uname, node->id, CRM_NODE_LOST);
} else {
char *last = node->state;
node->state = strdup(CRM_NODE_LOST);
crm_notice("Node %s[%u] - state is now %s (was %s)",
node->uname, node->id, CRM_NODE_LOST, last);
if (crm_status_callback) {
crm_status_callback(crm_status_nstate, node, last);
}
if (crm_autoreap) {
g_hash_table_iter_remove(&iter);
}
free(last);
}
} else {
crm_info("State of node %s[%u] is still unknown",
node->uname, node->id);
}
}
}
}
int
crm_terminate_member(int nodeid, const char *uname, void *unused)
{
/* Always use the synchronous, non-mainloop version */
return stonith_api_kick(nodeid, uname, 120, TRUE);
}
int
crm_terminate_member_no_mainloop(int nodeid, const char *uname, int *connection)
{
return stonith_api_kick(nodeid, uname, 120, TRUE);
}