diff --git a/daemons/attrd/attrd_elections.c b/daemons/attrd/attrd_elections.c index dede86fcfa..281ec12c2f 100644 --- a/daemons/attrd/attrd_elections.c +++ b/daemons/attrd/attrd_elections.c @@ -1,186 +1,176 @@ /* * Copyright 2013-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include "pacemaker-attrd.h" static char *peer_writer = NULL; -static election_t *writer = NULL; -static gboolean -attrd_election_cb(gpointer user_data) +static void +attrd_election_cb(pcmk_cluster_t *cluster) { attrd_declare_winner(); /* Update the peers after an election */ attrd_peer_sync(NULL); /* After winning an election, update the CIB with the values of all * attributes as the winner knows them. */ attrd_write_attributes(attrd_write_all); - return G_SOURCE_REMOVE; } void attrd_election_init(void) { - writer = election_init(pcmk_ipc_attrd, PCMK__VALUE_ATTRD, - attrd_cluster->priv->node_name, 120000, - attrd_election_cb); -} - -void -attrd_election_fini(void) -{ - election_fini(writer); + election_init(attrd_cluster, attrd_election_cb); } void attrd_start_election_if_needed(void) { if ((peer_writer == NULL) - && (election_state(writer) != election_in_progress) + && (election_state(attrd_cluster) != election_in_progress) && !attrd_shutting_down(false)) { crm_info("Starting an election to determine the writer"); - election_vote(writer); + election_vote(attrd_cluster); } } bool attrd_election_won(void) { - return (election_state(writer) == election_won); + return (election_state(attrd_cluster) == election_won); } void attrd_handle_election_op(const pcmk__node_status_t *peer, xmlNode *xml) { enum election_result rc = 0; - enum election_result previous = election_state(writer); + enum election_result previous = election_state(attrd_cluster); crm_xml_add(xml, PCMK__XA_SRC, peer->name); // Don't become writer if we're shutting down - rc = election_count_vote(writer, xml, !attrd_shutting_down(false)); + rc = election_count_vote(attrd_cluster, xml, !attrd_shutting_down(false)); switch(rc) { case election_start: crm_debug("Unsetting writer (was %s) and starting new election", peer_writer? peer_writer : "unset"); free(peer_writer); peer_writer = NULL; - election_vote(writer); + election_vote(attrd_cluster); break; case election_lost: /* The election API should really distinguish between "we just lost * to this peer" and "we already lost previously, and we are * discarding this vote for some reason", but it doesn't. * * In the first case, we want to tentatively set the peer writer to * this peer, even though another peer may eventually win (which we * will learn via attrd_check_for_new_writer()), so * attrd_start_election_if_needed() doesn't start a new election. * * Approximate a test for that case as best as possible. */ if ((peer_writer == NULL) || (previous != election_lost)) { pcmk__str_update(&peer_writer, peer->name); crm_debug("Election lost, presuming %s is writer for now", peer_writer); } break; case election_in_progress: - election_check(writer); + election_check(attrd_cluster); break; default: crm_info("Ignoring election op from %s due to error", peer->name); break; } } bool attrd_check_for_new_writer(const pcmk__node_status_t *peer, const xmlNode *xml) { int peer_state = 0; crm_element_value_int(xml, PCMK__XA_ATTR_WRITER, &peer_state); if (peer_state == election_won) { - if ((election_state(writer) == election_won) + if ((election_state(attrd_cluster) == election_won) && !pcmk__str_eq(peer->name, attrd_cluster->priv->node_name, pcmk__str_casei)) { crm_notice("Detected another attribute writer (%s), starting new " "election", peer->name); - election_vote(writer); + election_vote(attrd_cluster); } else if (!pcmk__str_eq(peer->name, peer_writer, pcmk__str_casei)) { crm_notice("Recorded new attribute writer: %s (was %s)", peer->name, pcmk__s(peer_writer, "unset")); pcmk__str_update(&peer_writer, peer->name); } } return (peer_state == election_won); } void attrd_declare_winner(void) { crm_notice("Recorded local node as attribute writer (was %s)", (peer_writer? peer_writer : "unset")); pcmk__str_update(&peer_writer, attrd_cluster->priv->node_name); } void attrd_remove_voter(const pcmk__node_status_t *peer) { - election_remove(writer, peer->name); + election_remove(attrd_cluster, peer->name); if ((peer_writer != NULL) && pcmk__str_eq(peer->name, peer_writer, pcmk__str_casei)) { free(peer_writer); peer_writer = NULL; crm_notice("Lost attribute writer %s", peer->name); /* Clear any election dampening in effect. Otherwise, if the lost writer * had just won, the election could fizzle out with no new writer. */ - election_clear_dampening(writer); + election_clear_dampening(attrd_cluster); /* If the writer received attribute updates during its shutdown, it will * not have written them to the CIB. Ensure we get a new writer so they * are written out. This means that every node that sees the writer * leave will start a new election, but that's better than losing * attributes. */ attrd_start_election_if_needed(); /* If an election is in progress, we need to call election_check(), in case * this lost peer is the only one that hasn't voted, otherwise the election * would be pending until it's timed out. */ - } else if (election_state(writer) == election_in_progress) { + } else if (election_state(attrd_cluster) == election_in_progress) { crm_debug("Checking election status upon loss of voter %s", peer->name); - election_check(writer); + election_check(attrd_cluster); } } void attrd_xml_add_writer(xmlNode *xml) { - crm_xml_add_int(xml, PCMK__XA_ATTR_WRITER, election_state(writer)); + crm_xml_add_int(xml, PCMK__XA_ATTR_WRITER, election_state(attrd_cluster)); } diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c index c2acf83335..bae1bfd79b 100644 --- a/daemons/attrd/pacemaker-attrd.c +++ b/daemons/attrd/pacemaker-attrd.c @@ -1,225 +1,224 @@ /* * Copyright 2013-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pacemaker-attrd.h" #define SUMMARY "daemon for managing Pacemaker node attributes" gboolean stand_alone = FALSE; gchar **log_files = NULL; static GOptionEntry entries[] = { { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone, "(Advanced use only) Run in stand-alone mode", NULL }, { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY, &log_files, "Send logs to the additional named logfile", NULL }, { NULL } }; static pcmk__output_t *out = NULL; static pcmk__supported_format_t formats[] = { PCMK__SUPPORTED_FORMAT_NONE, PCMK__SUPPORTED_FORMAT_TEXT, PCMK__SUPPORTED_FORMAT_XML, { NULL, NULL, NULL } }; lrmd_t *the_lrmd = NULL; pcmk_cluster_t *attrd_cluster = NULL; crm_trigger_t *attrd_config_read = NULL; crm_exit_t attrd_exit_status = CRM_EX_OK; static bool ipc_already_running(void) { pcmk_ipc_api_t *old_instance = NULL; int rc = pcmk_rc_ok; rc = pcmk_new_ipc_api(&old_instance, pcmk_ipc_attrd); if (rc != pcmk_rc_ok) { return false; } rc = pcmk__connect_ipc(old_instance, pcmk_ipc_dispatch_sync, 2); if (rc != pcmk_rc_ok) { crm_debug("No existing %s manager instance found: %s", pcmk_ipc_name(old_instance, true), pcmk_rc_str(rc)); pcmk_free_ipc_api(old_instance); return false; } pcmk_disconnect_ipc(old_instance); pcmk_free_ipc_api(old_instance); return true; } static GOptionContext * build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) { GOptionContext *context = NULL; context = pcmk__build_arg_context(args, "text (default), xml", group, NULL); pcmk__add_main_args(context, entries); return context; } int main(int argc, char **argv) { int rc = pcmk_rc_ok; GError *error = NULL; bool initialized = false; GOptionGroup *output_group = NULL; pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY); gchar **processed_args = pcmk__cmdline_preproc(argv, NULL); GOptionContext *context = build_arg_context(args, &output_group); attrd_init_mainloop(); crm_log_preinit(NULL, argc, argv); mainloop_add_signal(SIGTERM, attrd_shutdown); pcmk__register_formats(output_group, formats); if (!g_option_context_parse_strv(context, &processed_args, &error)) { attrd_exit_status = CRM_EX_USAGE; goto done; } rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv); if ((rc != pcmk_rc_ok) || (out == NULL)) { attrd_exit_status = CRM_EX_ERROR; g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, "Error creating output format %s: %s", args->output_ty, pcmk_rc_str(rc)); goto done; } if (args->version) { out->version(out, false); goto done; } // Open additional log files pcmk__add_logfiles(log_files, out); crm_log_init(PCMK__VALUE_ATTRD, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); crm_notice("Starting Pacemaker node attribute manager%s", stand_alone ? " in standalone mode" : ""); if (ipc_already_running()) { attrd_exit_status = CRM_EX_OK; g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, "Aborting start-up because an attribute manager " "instance is already active"); crm_crit("%s", error->message); goto done; } initialized = true; attributes = pcmk__strkey_table(NULL, attrd_free_attribute); /* Connect to the CIB before connecting to the cluster or listening for IPC. * This allows us to assume the CIB is connected whenever we process a * cluster or IPC message (which also avoids start-up race conditions). */ if (!stand_alone) { if (attrd_cib_connect(30) != pcmk_ok) { attrd_exit_status = CRM_EX_FATAL; g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, "Could not connect to the CIB"); goto done; } crm_info("CIB connection active"); } if (attrd_cluster_connect() != pcmk_ok) { attrd_exit_status = CRM_EX_FATAL; g_set_error(&error, PCMK__EXITC_ERROR, attrd_exit_status, "Could not connect to the cluster"); goto done; } crm_info("Cluster connection active"); // Initialization that requires the cluster to be connected attrd_election_init(); if (!stand_alone) { attrd_cib_init(); } /* Set a private attribute for ourselves with the protocol version we * support. This lets all nodes determine the minimum supported version * across all nodes. It also ensures that the writer learns our node name, * so it can send our attributes to the CIB. */ attrd_broadcast_protocol(); attrd_init_ipc(); crm_notice("Pacemaker node attribute manager successfully started and accepting connections"); attrd_run_mainloop(); done: if (initialized) { crm_info("Shutting down attribute manager"); - attrd_election_fini(); attrd_ipc_fini(); attrd_lrmd_disconnect(); if (!stand_alone) { attrd_cib_disconnect(); } attrd_free_waitlist(); pcmk_cluster_disconnect(attrd_cluster); pcmk_cluster_free(attrd_cluster); g_hash_table_destroy(attributes); } g_strfreev(processed_args); pcmk__free_arg_context(context); g_strfreev(log_files); pcmk__output_and_clear_error(&error, out); if (out != NULL) { out->finish(out, attrd_exit_status, true, NULL); pcmk__output_free(out); } pcmk__unregister_formats(); crm_exit(attrd_exit_status); } diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h index b23a6e88c8..13646b8e51 100644 --- a/daemons/attrd/pacemaker-attrd.h +++ b/daemons/attrd/pacemaker-attrd.h @@ -1,248 +1,247 @@ /* * Copyright 2013-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #ifndef PACEMAKER_ATTRD__H # define PACEMAKER_ATTRD__H #include #include #include #include #include #include #include /* * Legacy attrd (all pre-1.1.11 Pacemaker versions, plus all versions when used * with the no-longer-supported CMAN or corosync-plugin stacks) is unversioned. * * With atomic attrd, each attrd will send ATTRD_PROTOCOL_VERSION with every * peer request and reply. As of Pacemaker 2.0.0, at start-up each attrd will * also set a private attribute for itself with its version, so any attrd can * determine the minimum version supported by all peers. * * Protocol Pacemaker Significant changes * -------- --------- ------------------- * 1 1.1.11 PCMK__ATTRD_CMD_UPDATE (PCMK__XA_ATTR_NAME only), * PCMK__ATTRD_CMD_PEER_REMOVE, PCMK__ATTRD_CMD_REFRESH, * "flush", PCMK__ATTRD_CMD_SYNC_RESPONSE * 1 1.1.13 PCMK__ATTRD_CMD_UPDATE (with PCMK__XA_ATTR_REGEX), * PCMK__ATTRD_CMD_QUERY * 1 1.1.15 PCMK__ATTRD_CMD_UPDATE_BOTH, * PCMK__ATTRD_CMD_UPDATE_DELAY * 2 1.1.17 PCMK__ATTRD_CMD_CLEAR_FAILURE * 3 2.1.1 PCMK__ATTRD_CMD_SYNC_RESPONSE indicates remote nodes * 4 2.1.5 Multiple attributes can be updated in a single IPC * message * 5 2.1.5 Peers can request confirmation of a sent message * 6 2.1.7 PCMK__ATTRD_CMD_PEER_REMOVE supports PCMK__XA_REAP * 7 3.0.0 "flush" support dropped */ #define ATTRD_PROTOCOL_VERSION "7" #define ATTRD_SUPPORTS_MULTI_MESSAGE(x) ((x) >= 4) #define ATTRD_SUPPORTS_CONFIRMATION(x) ((x) >= 5) #define attrd_send_ack(client, id, flags) \ pcmk__ipc_send_ack((client), (id), (flags), PCMK__XE_ACK, \ ATTRD_PROTOCOL_VERSION, CRM_EX_INDETERMINATE) void attrd_init_mainloop(void); void attrd_run_mainloop(void); void attrd_set_requesting_shutdown(void); void attrd_clear_requesting_shutdown(void); void attrd_free_waitlist(void); bool attrd_shutting_down(bool if_requested); void attrd_shutdown(int nsig); void attrd_init_ipc(void); void attrd_ipc_fini(void); int attrd_cib_connect(int max_retry); void attrd_cib_disconnect(void); void attrd_cib_init(void); void attrd_cib_erase_transient_attrs(const char *node); bool attrd_value_needs_expansion(const char *value); int attrd_expand_value(const char *value, const char *old_value); /* regular expression to clear failures of all resources */ #define ATTRD_RE_CLEAR_ALL \ "^(" PCMK__FAIL_COUNT_PREFIX "|" PCMK__LAST_FAILURE_PREFIX ")-" /* regular expression to clear failure of all operations for one resource * (format takes resource name) */ #define ATTRD_RE_CLEAR_ONE ATTRD_RE_CLEAR_ALL "%s#.+_[0-9]+$" /* regular expression to clear failure of one operation for one resource * (format takes resource name, operation name, and interval) */ #define ATTRD_RE_CLEAR_OP ATTRD_RE_CLEAR_ALL "%s#%s_%u$" int attrd_failure_regex(regex_t *regex, const char *rsc, const char *op, guint interval_ms); extern cib_t *the_cib; extern crm_exit_t attrd_exit_status; /* Alerts */ extern lrmd_t *the_lrmd; extern crm_trigger_t *attrd_config_read; void attrd_lrmd_disconnect(void); gboolean attrd_read_options(gpointer user_data); int attrd_send_attribute_alert(const char *node, int nodeid, const char *attr, const char *value); // Elections void attrd_election_init(void); -void attrd_election_fini(void); void attrd_start_election_if_needed(void); bool attrd_election_won(void); void attrd_handle_election_op(const pcmk__node_status_t *peer, xmlNode *xml); bool attrd_check_for_new_writer(const pcmk__node_status_t *peer, const xmlNode *xml); void attrd_declare_winner(void); void attrd_remove_voter(const pcmk__node_status_t *peer); void attrd_xml_add_writer(xmlNode *xml); enum attrd_attr_flags { attrd_attr_none = 0U, attrd_attr_changed = (1U << 0), // Attribute value has changed since last write attrd_attr_uuid_missing = (1U << 1), // Whether we know we're missing a peer UUID attrd_attr_is_private = (1U << 2), // Whether to keep this attribute out of the CIB attrd_attr_force_write = (1U << 3), // Update attribute by ignoring delay }; typedef struct attribute_s { char *id; // Attribute name char *set_type; // PCMK_XE_INSTANCE_ATTRIBUTES or PCMK_XE_UTILIZATION char *set_id; // Set's XML ID to use when writing char *user; // ACL user to use for CIB writes int update; // Call ID of pending write int timeout_ms; // How long to wait for more changes before writing uint32_t flags; // Group of enum attrd_attr_flags GHashTable *values; // Key: node name, value: attribute_value_t mainloop_timer_t *timer; // Timer to use for timeout_ms } attribute_t; #define attrd_set_attr_flags(attr, flags_to_set) do { \ (attr)->flags = pcmk__set_flags_as(__func__, __LINE__, \ LOG_TRACE, "Value for attribute", (attr)->id, \ (attr)->flags, (flags_to_set), #flags_to_set); \ } while (0) #define attrd_clear_attr_flags(attr, flags_to_clear) do { \ (attr)->flags = pcmk__clear_flags_as(__func__, __LINE__, \ LOG_TRACE, "Value for attribute", (attr)->id, \ (attr)->flags, (flags_to_clear), #flags_to_clear); \ } while (0) enum attrd_value_flags { attrd_value_none = 0U, attrd_value_remote = (1U << 0), // Value is for Pacemaker Remote node attrd_value_from_peer = (1U << 1), // Value is from peer sync response }; typedef struct attribute_value_s { char *nodename; // Node that this value is for char *current; // Attribute value char *requested; // Value specified in pending CIB write, if any uint32_t nodeid; // Cluster node ID of node that this value is for uint32_t flags; // Group of attrd_value_flags } attribute_value_t; #define attrd_set_value_flags(attr_value, flags_to_set) do { \ (attr_value)->flags = pcmk__set_flags_as(__func__, __LINE__, \ LOG_TRACE, "Value for node", (attr_value)->nodename, \ (attr_value)->flags, (flags_to_set), #flags_to_set); \ } while (0) #define attrd_clear_value_flags(attr_value, flags_to_clear) do { \ (attr_value)->flags = pcmk__clear_flags_as(__func__, __LINE__, \ LOG_TRACE, "Value for node", (attr_value)->nodename, \ (attr_value)->flags, (flags_to_clear), #flags_to_clear); \ } while (0) extern pcmk_cluster_t *attrd_cluster; extern GHashTable *attributes; extern GHashTable *peer_protocol_vers; #define CIB_OP_TIMEOUT_S 120 int attrd_cluster_connect(void); void attrd_broadcast_value(const attribute_t *a, const attribute_value_t *v); void attrd_peer_update(const pcmk__node_status_t *peer, xmlNode *xml, const char *host, bool filter); void attrd_peer_sync(pcmk__node_status_t *peer); void attrd_peer_remove(const char *host, bool uncache, const char *source); void attrd_peer_clear_failure(pcmk__request_t *request); void attrd_peer_sync_response(const pcmk__node_status_t *peer, bool peer_won, xmlNode *xml); void attrd_broadcast_protocol(void); xmlNode *attrd_client_peer_remove(pcmk__request_t *request); xmlNode *attrd_client_clear_failure(pcmk__request_t *request); xmlNode *attrd_client_update(pcmk__request_t *request); xmlNode *attrd_client_refresh(pcmk__request_t *request); xmlNode *attrd_client_query(pcmk__request_t *request); gboolean attrd_send_message(pcmk__node_status_t *node, xmlNode *data, bool confirm); xmlNode *attrd_add_value_xml(xmlNode *parent, const attribute_t *a, const attribute_value_t *v, bool force_write); void attrd_clear_value_seen(void); void attrd_free_attribute(gpointer data); void attrd_free_attribute_value(gpointer data); attribute_t *attrd_populate_attribute(xmlNode *xml, const char *attr); char *attrd_set_id(const attribute_t *attr, const char *node_state_id); char *attrd_nvpair_id(const attribute_t *attr, const char *node_state_id); enum attrd_write_options { attrd_write_changed = 0, attrd_write_all = (1 << 0), attrd_write_no_delay = (1 << 1), }; void attrd_write_attributes(uint32_t options); void attrd_write_or_elect_attribute(attribute_t *a); extern int minimum_protocol_version; void attrd_remove_peer_protocol_ver(const char *host); void attrd_update_minimum_protocol_ver(const char *host, const char *value); mainloop_timer_t *attrd_add_timer(const char *id, int timeout_ms, attribute_t *attr); void attrd_unregister_handlers(void); void attrd_handle_request(pcmk__request_t *request); enum attrd_sync_point { attrd_sync_point_local, attrd_sync_point_cluster, }; typedef int (*attrd_confirmation_action_fn)(xmlNode *); void attrd_add_client_to_waitlist(pcmk__request_t *request); void attrd_ack_waitlist_clients(enum attrd_sync_point sync_point, const xmlNode *xml); int attrd_cluster_sync_point_update(xmlNode *xml); void attrd_do_not_expect_from_peer(const char *host); void attrd_do_not_wait_for_client(pcmk__client_t *client); void attrd_expect_confirmations(pcmk__request_t *request, attrd_confirmation_action_fn fn); void attrd_free_confirmations(void); void attrd_handle_confirmation(int callid, const char *host); void attrd_remove_client_from_waitlist(pcmk__client_t *client); const char *attrd_request_sync_point(xmlNode *xml); bool attrd_request_has_sync_point(xmlNode *xml); extern gboolean stand_alone; #endif /* PACEMAKER_ATTRD__H */ diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c index ff9fb68444..323a8bc724 100644 --- a/daemons/controld/controld_control.c +++ b/daemons/controld/controld_control.c @@ -1,698 +1,702 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include static qb_ipcs_service_t *ipcs = NULL; static crm_trigger_t *config_read_trigger = NULL; #if SUPPORT_COROSYNC extern gboolean crm_connect_corosync(pcmk_cluster_t *cluster); #endif static void crm_shutdown(int nsig); static gboolean crm_read_options(gpointer user_data); /* A_HA_CONNECT */ void do_ha_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { gboolean registered = FALSE; - static pcmk_cluster_t *cluster = NULL; - if (cluster == NULL) { - cluster = pcmk_cluster_new(); + if (controld_globals.cluster == NULL) { + controld_globals.cluster = pcmk_cluster_new(); } if (action & A_HA_DISCONNECT) { - pcmk_cluster_disconnect(cluster); + pcmk_cluster_disconnect(controld_globals.cluster); crm_info("Disconnected from the cluster"); controld_set_fsa_input_flags(R_HA_DISCONNECTED); } if (action & A_HA_CONNECT) { pcmk__cluster_set_status_callback(&peer_update_callback); pcmk__cluster_set_autoreap(false); #if SUPPORT_COROSYNC if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) { - registered = crm_connect_corosync(cluster); + registered = crm_connect_corosync(controld_globals.cluster); } #endif // SUPPORT_COROSYNC if (registered) { pcmk__node_status_t *node = - pcmk__get_node(cluster->priv->node_id, cluster->priv->node_name, - NULL, pcmk__node_search_cluster_member); + pcmk__get_node(controld_globals.cluster->priv->node_id, + controld_globals.cluster->priv->node_name, NULL, + pcmk__node_search_cluster_member); - controld_election_init(cluster->priv->node_name); - controld_globals.our_nodename = cluster->priv->node_name; + controld_election_init(); + controld_globals.our_nodename = + controld_globals.cluster->priv->node_name; free(controld_globals.our_uuid); controld_globals.our_uuid = pcmk__str_copy(pcmk__cluster_node_uuid(node)); if (controld_globals.our_uuid == NULL) { crm_err("Could not obtain local uuid"); registered = FALSE; } } if (!registered) { controld_set_fsa_input_flags(R_HA_DISCONNECTED); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } populate_cib_nodes(node_update_none, __func__); controld_clear_fsa_input_flags(R_HA_DISCONNECTED); crm_info("Connected to the cluster"); } if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __func__); } } /* A_SHUTDOWN */ void do_shutdown(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { /* just in case */ controld_set_fsa_input_flags(R_SHUTDOWN); controld_disconnect_fencer(FALSE); } /* A_SHUTDOWN_REQ */ void do_shutdown_req(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { xmlNode *msg = NULL; controld_set_fsa_input_flags(R_SHUTDOWN); //controld_set_fsa_input_flags(R_STAYDOWN); crm_info("Sending shutdown request to all peers (DC is %s)", pcmk__s(controld_globals.dc_name, "not set")); msg = pcmk__new_request(pcmk_ipc_controld, CRM_SYSTEM_CRMD, NULL, CRM_SYSTEM_CRMD, CRM_OP_SHUTDOWN_REQ, NULL); if (!pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg)) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } pcmk__xml_free(msg); } void crmd_fast_exit(crm_exit_t exit_code) { if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) { crm_warn("Inhibiting respawn " QB_XS " remapping exit code %d to %d", exit_code, CRM_EX_FATAL); exit_code = CRM_EX_FATAL; } else if ((exit_code == CRM_EX_OK) && pcmk_is_set(controld_globals.fsa_input_register, R_IN_RECOVERY)) { crm_err("Could not recover from internal error"); exit_code = CRM_EX_ERROR; } if (controld_globals.logger_out != NULL) { controld_globals.logger_out->finish(controld_globals.logger_out, exit_code, true, NULL); pcmk__output_free(controld_globals.logger_out); controld_globals.logger_out = NULL; } crm_exit(exit_code); } crm_exit_t crmd_exit(crm_exit_t exit_code) { GMainLoop *mloop = controld_globals.mainloop; static bool in_progress = FALSE; if (in_progress && (exit_code == CRM_EX_OK)) { crm_debug("Exit is already in progress"); return exit_code; } else if(in_progress) { crm_notice("Error during shutdown process, exiting now with status %d (%s)", exit_code, crm_exit_str(exit_code)); crm_write_blackbox(SIGTRAP, NULL); crmd_fast_exit(exit_code); } in_progress = TRUE; crm_trace("Preparing to exit with status %d (%s)", exit_code, crm_exit_str(exit_code)); /* Suppress secondary errors resulting from us disconnecting everything */ controld_set_fsa_input_flags(R_HA_DISCONNECTED); /* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */ if(ipcs) { crm_trace("Closing IPC server"); mainloop_del_ipc_server(ipcs); ipcs = NULL; } controld_close_attrd_ipc(); controld_shutdown_schedulerd_ipc(); controld_disconnect_fencer(TRUE); if ((exit_code == CRM_EX_OK) && (controld_globals.mainloop == NULL)) { crm_debug("No mainloop detected"); exit_code = CRM_EX_ERROR; } /* On an error, just get out. * * Otherwise, make the effort to have mainloop exit gracefully so * that it (mostly) cleans up after itself and valgrind has less * to report on - allowing real errors stand out */ if (exit_code != CRM_EX_OK) { crm_notice("Forcing immediate exit with status %d (%s)", exit_code, crm_exit_str(exit_code)); crm_write_blackbox(SIGTRAP, NULL); crmd_fast_exit(exit_code); } /* Clean up as much memory as possible for valgrind */ for (GList *iter = controld_globals.fsa_message_queue; iter != NULL; iter = iter->next) { fsa_data_t *fsa_data = (fsa_data_t *) iter->data; crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(controld_globals.fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); delete_fsa_input(fsa_data); } controld_clear_fsa_input_flags(R_MEMBERSHIP); g_list_free(controld_globals.fsa_message_queue); controld_globals.fsa_message_queue = NULL; controld_free_node_pending_timers(); - controld_election_fini(); + election_reset(controld_globals.cluster); // Stop any election timer /* Tear down the CIB manager connection, but don't free it yet -- it could * be used when we drain the mainloop later. */ controld_disconnect_cib_manager(); verify_stopped(controld_globals.fsa_state, LOG_WARNING); controld_clear_fsa_input_flags(R_LRM_CONNECTED); lrm_state_destroy_all(); mainloop_destroy_trigger(config_read_trigger); config_read_trigger = NULL; controld_destroy_fsa_trigger(); controld_destroy_transition_trigger(); pcmk__client_cleanup(); pcmk__cluster_destroy_node_caches(); controld_free_fsa_timers(); te_cleanup_stonith_history_sync(NULL, TRUE); controld_free_sched_timer(); free(controld_globals.our_nodename); controld_globals.our_nodename = NULL; free(controld_globals.our_uuid); controld_globals.our_uuid = NULL; free(controld_globals.dc_name); controld_globals.dc_name = NULL; free(controld_globals.dc_version); controld_globals.dc_version = NULL; free(controld_globals.cluster_name); controld_globals.cluster_name = NULL; free(controld_globals.te_uuid); controld_globals.te_uuid = NULL; free_max_generation(); controld_destroy_failed_sync_table(); controld_destroy_outside_events_table(); mainloop_destroy_signal(SIGPIPE); mainloop_destroy_signal(SIGUSR1); mainloop_destroy_signal(SIGTERM); mainloop_destroy_signal(SIGTRAP); /* leave SIGCHLD engaged as we might still want to drain some service-actions */ if (mloop) { GMainContext *ctx = g_main_loop_get_context(controld_globals.mainloop); /* Don't re-enter this block */ controld_globals.mainloop = NULL; /* no signals on final draining anymore */ mainloop_destroy_signal(SIGCHLD); crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx)); { int lpc = 0; while((g_main_context_pending(ctx) && lpc < 10)) { lpc++; crm_trace("Iteration %d", lpc); g_main_context_dispatch(ctx); } } crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx)); g_main_loop_quit(mloop); /* Won't do anything yet, since we're inside it now */ g_main_loop_unref(mloop); } else { mainloop_destroy_signal(SIGCHLD); } cib_delete(controld_globals.cib_conn); controld_globals.cib_conn = NULL; throttle_fini(); + pcmk_cluster_free(controld_globals.cluster); + controld_globals.cluster = NULL; + /* Graceful */ crm_trace("Done preparing for exit with status %d (%s)", exit_code, crm_exit_str(exit_code)); return exit_code; } /* A_EXIT_0, A_EXIT_1 */ void do_exit(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { crm_exit_t exit_code = CRM_EX_OK; if (pcmk_is_set(action, A_EXIT_1)) { exit_code = CRM_EX_ERROR; crm_err("Exiting now due to errors"); } verify_stopped(cur_state, LOG_ERR); crmd_exit(exit_code); } static void sigpipe_ignore(int nsig) { return; } /* A_STARTUP */ void do_startup(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { crm_debug("Registering Signal Handlers"); mainloop_add_signal(SIGTERM, crm_shutdown); mainloop_add_signal(SIGPIPE, sigpipe_ignore); config_read_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL); controld_init_fsa_trigger(); controld_init_transition_trigger(); crm_debug("Creating CIB manager and executor objects"); controld_globals.cib_conn = cib_new(); lrm_state_init_local(); if (controld_init_fsa_timers() == FALSE) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } // \return libqb error code (0 on success, -errno on error) static int32_t accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid) { crm_trace("Accepting new IPC client connection"); if (pcmk__new_client(c, uid, gid) == NULL) { return -ENOMEM; } return 0; } // \return libqb error code (0 on success, -errno on error) static int32_t dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size) { uint32_t id = 0; uint32_t flags = 0; pcmk__client_t *client = pcmk__find_client(c); xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags); if (msg == NULL) { pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL, CRM_EX_PROTOCOL); return 0; } pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL, CRM_EX_INDETERMINATE); CRM_ASSERT(client->user != NULL); pcmk__update_acl_user(msg, PCMK__XA_CRM_USER, client->user); crm_xml_add(msg, PCMK__XA_CRM_SYS_FROM, client->id); if (controld_authorize_ipc_message(msg, client, NULL)) { crm_trace("Processing IPC message from client %s", pcmk__client_name(client)); route_message(C_IPC_MESSAGE, msg); } controld_trigger_fsa(); pcmk__xml_free(msg); return 0; } static int32_t ipc_client_disconnected(qb_ipcs_connection_t *c) { pcmk__client_t *client = pcmk__find_client(c); if (client) { crm_trace("Disconnecting %sregistered client %s (%p/%p)", (client->userdata? "" : "un"), pcmk__client_name(client), c, client); free(client->userdata); pcmk__free_client(client); controld_trigger_fsa(); } return 0; } static void ipc_connection_destroyed(qb_ipcs_connection_t *c) { crm_trace("Connection %p", c); ipc_client_disconnected(c); } /* A_STOP */ void do_stop(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { crm_trace("Closing IPC server"); mainloop_del_ipc_server(ipcs); ipcs = NULL; register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } /* A_STARTED */ void do_started(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { static struct qb_ipcs_service_handlers crmd_callbacks = { .connection_accept = accept_controller_client, .connection_created = NULL, .msg_process = dispatch_controller_ipc, .connection_closed = ipc_client_disconnected, .connection_destroyed = ipc_connection_destroyed }; if (cur_state != S_STARTING) { crm_err("Start cancelled... %s", fsa_state2string(cur_state)); return; } else if (!pcmk_is_set(controld_globals.fsa_input_register, R_MEMBERSHIP)) { crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP); crmd_fsa_stall(TRUE); return; } else if (!pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) { crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED); crmd_fsa_stall(TRUE); return; } else if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) { crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED); crmd_fsa_stall(TRUE); return; } else if (!pcmk_is_set(controld_globals.fsa_input_register, R_READ_CONFIG)) { crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG); crmd_fsa_stall(TRUE); return; } else if (!pcmk_is_set(controld_globals.fsa_input_register, R_PEER_DATA)) { crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA); crmd_fsa_stall(TRUE); return; } crm_debug("Init server comms"); ipcs = pcmk__serve_controld_ipc(&crmd_callbacks); if (ipcs == NULL) { crm_err("Failed to create IPC server: shutting down and inhibiting respawn"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } else { crm_notice("Pacemaker controller successfully started and accepting connections"); } controld_set_fsa_input_flags(R_ST_REQUIRED); controld_timer_fencer_connect(GINT_TO_POINTER(TRUE)); controld_clear_fsa_input_flags(R_STARTING); register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL); } /* A_RECOVER */ void do_recover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { controld_set_fsa_input_flags(R_IN_RECOVERY); crm_warn("Fast-tracking shutdown in response to errors"); register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } static void config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { const char *value = NULL; GHashTable *config_hash = NULL; crm_time_t *now = crm_time_new(NULL); xmlNode *crmconfig = NULL; xmlNode *alerts = NULL; if (rc != pcmk_ok) { fsa_data_t *msg_data = NULL; crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); if (rc == -EACCES || rc == -pcmk_err_schema_validation) { crm_err("The cluster is mis-configured - shutting down and staying down"); controld_set_fsa_input_flags(R_STAYDOWN); } goto bail; } crmconfig = output; if ((crmconfig != NULL) && !pcmk__xe_is(crmconfig, PCMK_XE_CRM_CONFIG)) { crmconfig = pcmk__xe_first_child(crmconfig, PCMK_XE_CRM_CONFIG, NULL, NULL); } if (!crmconfig) { fsa_data_t *msg_data = NULL; crm_err("Local CIB query for " PCMK_XE_CRM_CONFIG " section failed"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); goto bail; } crm_debug("Call %d : Parsing CIB options", call_id); config_hash = pcmk__strkey_table(free, free); pe_unpack_nvpairs(crmconfig, crmconfig, PCMK_XE_CLUSTER_PROPERTY_SET, NULL, config_hash, PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, FALSE, now, NULL); // Validate all options, and use defaults if not already present in hash pcmk__validate_cluster_options(config_hash); /* Validate the watchdog timeout in the context of the local node * environment. If invalid, the controller will exit with a fatal error. * * We do this via a wrapper in the controller, so that we call * pcmk__valid_stonith_watchdog_timeout() only if watchdog fencing is * enabled for the local node. Otherwise, we may exit unnecessarily. * * A validator function in libcrmcommon can't act as such a wrapper, because * it doesn't have a stonith API connection or the local node name. */ value = g_hash_table_lookup(config_hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT); controld_verify_stonith_watchdog_timeout(value); value = g_hash_table_lookup(config_hash, PCMK_OPT_NO_QUORUM_POLICY); if (pcmk__str_eq(value, PCMK_VALUE_FENCE_LEGACY, pcmk__str_casei) && (pcmk__locate_sbd() != 0)) { controld_set_global_flags(controld_no_quorum_suicide); } value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK); if (crm_is_true(value)) { controld_set_global_flags(controld_shutdown_lock_enabled); } else { controld_clear_global_flags(controld_shutdown_lock_enabled); } value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT); pcmk_parse_interval_spec(value, &controld_globals.shutdown_lock_limit); controld_globals.shutdown_lock_limit /= 1000; value = g_hash_table_lookup(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT); pcmk_parse_interval_spec(value, &controld_globals.node_pending_timeout); controld_globals.node_pending_timeout /= 1000; value = g_hash_table_lookup(config_hash, PCMK_OPT_CLUSTER_NAME); pcmk__str_update(&(controld_globals.cluster_name), value); // Let subcomponents initialize their own static variables controld_configure_election(config_hash); controld_configure_fencing(config_hash); controld_configure_fsa_timers(config_hash); controld_configure_throttle(config_hash); alerts = pcmk__xe_first_child(output, PCMK_XE_ALERTS, NULL, NULL); crmd_unpack_alerts(alerts); controld_set_fsa_input_flags(R_READ_CONFIG); controld_trigger_fsa(); g_hash_table_destroy(config_hash); bail: crm_time_free(now); } /*! * \internal * \brief Trigger read and processing of the configuration * * \param[in] fn Calling function name * \param[in] line Line number where call occurred */ void controld_trigger_config_as(const char *fn, int line) { if (config_read_trigger != NULL) { crm_trace("%s:%d - Triggered config processing", fn, line); mainloop_set_trigger(config_read_trigger); } } gboolean crm_read_options(gpointer user_data) { cib_t *cib_conn = controld_globals.cib_conn; int call_id = cib_conn->cmds->query(cib_conn, "//" PCMK_XE_CRM_CONFIG " | //" PCMK_XE_ALERTS, NULL, cib_xpath); fsa_register_cib_callback(call_id, NULL, config_query_callback); crm_trace("Querying the CIB... call %d", call_id); return TRUE; } /* A_READCONFIG */ void do_read_config(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { throttle_init(); controld_trigger_config(); } static void crm_shutdown(int nsig) { const char *value = NULL; guint default_period_ms = 0; if ((controld_globals.mainloop == NULL) || !g_main_loop_is_running(controld_globals.mainloop)) { crmd_exit(CRM_EX_OK); return; } if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { crm_err("Escalating shutdown"); register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL); return; } controld_set_fsa_input_flags(R_SHUTDOWN); register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL); /* If shutdown timer doesn't have a period set, use the default * * @TODO: Evaluate whether this is still necessary. As long as * config_query_callback() has been run at least once, it doesn't look like * anything could have changed the timer period since then. */ value = pcmk__cluster_option(NULL, PCMK_OPT_SHUTDOWN_ESCALATION); pcmk_parse_interval_spec(value, &default_period_ms); controld_shutdown_start_countdown(default_period_ms); } diff --git a/daemons/controld/controld_election.c b/daemons/controld/controld_election.c index d9bb231ad3..47a5d4419f 100644 --- a/daemons/controld/controld_election.c +++ b/daemons/controld/controld_election.c @@ -1,291 +1,281 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU General Public License version 2 * or later (GPLv2+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include -static election_t *fsa_election = NULL; - -static gboolean -election_win_cb(gpointer data) +static void +election_win_cb(pcmk_cluster_t *cluster) { register_fsa_input(C_FSA_INTERNAL, I_ELECTION_DC, NULL); - return FALSE; } void -controld_election_init(const char *uname) +controld_election_init(void) { - fsa_election = election_init(pcmk_ipc_controld, "DC", uname, 60000, - election_win_cb); + election_init(controld_globals.cluster, election_win_cb); } /*! * \internal * \brief Configure election options based on the CIB * * \param[in,out] options Name/value pairs for configured options */ void controld_configure_election(GHashTable *options) { const char *value = g_hash_table_lookup(options, PCMK_OPT_ELECTION_TIMEOUT); guint interval_ms = 0U; pcmk_parse_interval_spec(value, &interval_ms); - election_timeout_set_period(fsa_election, interval_ms); + election_timeout_set_period(controld_globals.cluster, interval_ms); } void controld_remove_voter(const char *uname) { - election_remove(fsa_election, uname); + election_remove(controld_globals.cluster, uname); if (pcmk__str_eq(uname, controld_globals.dc_name, pcmk__str_casei)) { /* Clear any election dampening in effect. Otherwise, if the lost DC had * just won, an immediate new election could fizzle out with no new DC. */ - election_clear_dampening(fsa_election); + election_clear_dampening(controld_globals.cluster); } } -void -controld_election_fini(void) -{ - election_fini(fsa_election); - fsa_election = NULL; -} - void controld_stop_current_election_timeout(void) { - election_timeout_stop(fsa_election); + election_timeout_stop(controld_globals.cluster); } /* A_ELECTION_VOTE */ void do_election_vote(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { gboolean not_voting = FALSE; /* don't vote if we're in one of these states or wanting to shut down */ switch (cur_state) { case S_STARTING: case S_RECOVERY: case S_STOPPING: case S_TERMINATE: crm_warn("Not voting in election, we're in state %s", fsa_state2string(cur_state)); not_voting = TRUE; break; case S_ELECTION: case S_INTEGRATION: case S_RELEASE_DC: break; default: crm_err("Broken? Voting in state %s", fsa_state2string(cur_state)); break; } if (not_voting == FALSE) { if (pcmk_is_set(controld_globals.fsa_input_register, R_STARTING)) { not_voting = TRUE; } } if (not_voting) { if (AM_I_DC) { register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL); } else { register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL); } return; } - election_vote(fsa_election); + election_vote(controld_globals.cluster); return; } void do_election_check(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { if (controld_globals.fsa_state == S_ELECTION) { - election_check(fsa_election); + election_check(controld_globals.cluster); } else { crm_debug("Ignoring election check because we are not in an election"); } } /* A_ELECTION_COUNT */ void do_election_count_vote(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { enum election_result rc = 0; ha_msg_input_t *vote = fsa_typed_data(fsa_dt_ha_msg); if (pcmk__peer_cache == NULL) { if (!pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { crm_err("Internal error, no peer cache"); } return; } - rc = election_count_vote(fsa_election, vote->msg, cur_state != S_STARTING); + rc = election_count_vote(controld_globals.cluster, vote->msg, + (cur_state != S_STARTING)); switch(rc) { case election_start: - election_reset(fsa_election); + election_reset(controld_globals.cluster); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); break; case election_lost: update_dc(NULL); if (pcmk_is_set(controld_globals.fsa_input_register, R_THE_DC)) { cib_t *cib_conn = controld_globals.cib_conn; register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL); cib_conn->cmds->set_secondary(cib_conn, cib_none); } else if (cur_state != S_STARTING) { register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL); } break; default: crm_trace("Election message resulted in state %d", rc); } } static void feature_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { if (rc != pcmk_ok) { fsa_data_t *msg_data = NULL; crm_notice("Feature update failed: %s " QB_XS " rc=%d", pcmk_strerror(rc), rc); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } /*! * \internal * \brief Update a node attribute in the CIB during a DC takeover * * \param[in] name Name of attribute to update * \param[in] value New attribute value */ #define dc_takeover_update_attr(name, value) do { \ cib__update_node_attr(controld_globals.logger_out, \ controld_globals.cib_conn, cib_none, \ PCMK_XE_CRM_CONFIG, NULL, NULL, NULL, NULL, \ name, value, NULL, NULL); \ } while (0) /* A_DC_TAKEOVER */ void do_dc_takeover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { xmlNode *cib = NULL; const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer(); const char *cluster_layer_s = pcmk_cluster_layer_text(cluster_layer); pid_t watchdog = pcmk__locate_sbd(); crm_info("Taking over DC status for this partition"); controld_set_fsa_input_flags(R_THE_DC); execute_stonith_cleanup(); - election_reset(fsa_election); + election_reset(controld_globals.cluster); controld_set_fsa_input_flags(R_JOIN_OK|R_INVOKE_PE); controld_globals.cib_conn->cmds->set_primary(controld_globals.cib_conn, cib_none); cib = pcmk__xe_create(NULL, PCMK_XE_CIB); crm_xml_add(cib, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET); controld_update_cib(PCMK_XE_CIB, cib, cib_none, feature_update_callback); dc_takeover_update_attr(PCMK_OPT_HAVE_WATCHDOG, pcmk__btoa(watchdog)); dc_takeover_update_attr(PCMK_OPT_DC_VERSION, PACEMAKER_VERSION "-" BUILD_VERSION); dc_takeover_update_attr(PCMK_OPT_CLUSTER_INFRASTRUCTURE, cluster_layer_s); #if SUPPORT_COROSYNC if ((controld_globals.cluster_name == NULL) && (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync)) { char *cluster_name = pcmk__corosync_cluster_name(); if (cluster_name != NULL) { dc_takeover_update_attr(PCMK_OPT_CLUSTER_NAME, cluster_name); } free(cluster_name); } #endif controld_trigger_config(); pcmk__xml_free(cib); } /* A_DC_RELEASE */ void do_dc_release(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { if (action & A_DC_RELEASE) { crm_debug("Releasing the role of DC"); controld_clear_fsa_input_flags(R_THE_DC); controld_expect_sched_reply(NULL); } else if (action & A_DC_RELEASED) { crm_info("DC role released"); if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) { xmlNode *update = NULL; pcmk__node_status_t *node = pcmk__get_node(0, controld_globals.our_nodename, NULL, pcmk__node_search_cluster_member); pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN); update = create_node_state_update(node, node_update_expected, NULL, __func__); /* Don't need a based response because controld will stop. */ fsa_cib_anon_update_discard_reply(PCMK_XE_STATUS, update); pcmk__xml_free(update); } register_fsa_input(C_FSA_INTERNAL, I_RELEASE_SUCCESS, NULL); } else { crm_err("Unknown DC action %s", fsa_action2string(action)); } crm_trace("Am I still the DC? %s", pcmk__btoa(AM_I_DC)); } diff --git a/daemons/controld/controld_globals.h b/daemons/controld/controld_globals.h index 9cf105f40a..f2b0166ff0 100644 --- a/daemons/controld/controld_globals.h +++ b/daemons/controld/controld_globals.h @@ -1,151 +1,156 @@ /* * Copyright 2022-2023 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef CONTROLD_GLOBALS__H # define CONTROLD_GLOBALS__H #include // pcmk__output_t, etc. #include // uint32_t, uint64_t #include // GList, GMainLoop #include // cib_t #include // pcmk__graph_t #include // enum crmd_fsa_state typedef struct { - // Booleans - - //! Group of \p controld_flags values + // Group of \p controld_flags values uint32_t flags; - // Controller FSA + /* Controller finite state automaton */ - //! FSA state + // FSA state enum crmd_fsa_state fsa_state; - //! FSA actions (group of \p A_* flags) + // FSA actions (group of \p A_* flags) uint64_t fsa_actions; - //! FSA input register contents (group of \p R_* flags) + // FSA input register contents (group of \p R_* flags) uint64_t fsa_input_register; - //! FSA message queue + // FSA message queue GList *fsa_message_queue; - // CIB + /* CIB */ - //! Connection to the CIB + // Connection to the CIB cib_t *cib_conn; - // Scheduler + /* Scheduler */ - //! Reference of the scheduler request being waited on + // Reference of the scheduler request being waited on char *fsa_pe_ref; - // Transitioner + /* Transitioner */ - //! Transitioner UUID + // Transitioner UUID char *te_uuid; - //! Graph of transition currently being processed + // Graph of transition currently being processed pcmk__graph_t *transition_graph; - // Logging + /* Logging */ - //! Output object for controller log messages + // Output object for controller log messages pcmk__output_t *logger_out; - // Other + /* Cluster layer */ - //! Cluster name + // Cluster name char *cluster_name; - //! Designated controller name + // Cluster connection + pcmk_cluster_t *cluster; + + /* @TODO Figure out, document, and clean up the code involving + * controld_globals.membership_id, controld_globals.peer_seq, and + * highest_seq. It's convoluted with no comments. It has something to do + * with corosync quorum notifications and the current ring ID, but it's + * unclear why we need three separate variables for it. + */ + // Last saved cluster communication layer membership ID + unsigned long long membership_id; + + unsigned long long peer_seq; + + + /* Other */ + + // Designated controller name char *dc_name; - //! Designated controller's Pacemaker version + // Designated controller's Pacemaker version char *dc_version; - //! Local node's node name + // Local node's node name + // @TODO Use controld_globals.cluster->priv->node_name instead char *our_nodename; - //! Local node's UUID + // Local node's UUID char *our_uuid; - //! Last saved cluster communication layer membership ID - unsigned long long membership_id; - - /* @TODO Figure out, document, and clean up the code involving - * controld_peer_seq, controld_globals.membership_id, and highest_seq. It's - * convoluted with no comments. It has something to do with corosync quorum - * notifications and the current ring ID, but it's unclear why we need three - * separate variables for it. - */ - unsigned long long peer_seq; - - //! Max lifetime (in seconds) of a resource's shutdown lock to a node + // Max lifetime (in seconds) of a resource's shutdown lock to a node guint shutdown_lock_limit; - //! Node pending timeout + // Node pending timeout guint node_pending_timeout; - //! Main event loop + // Main event loop GMainLoop *mainloop; } controld_globals_t; extern controld_globals_t controld_globals; /*! * \internal * \enum controld_flags * \brief Bit flags to store various controller state and configuration info */ enum controld_flags { //! The DC left in a membership change that is being processed controld_dc_left = (1 << 0), //! The FSA is stalled waiting for further input controld_fsa_is_stalled = (1 << 1), //! The local node has been in a quorate partition at some point controld_ever_had_quorum = (1 << 2), //! The local node is currently in a quorate partition controld_has_quorum = (1 << 3), //! Panic the local node if it loses quorum controld_no_quorum_suicide = (1 << 4), //! Lock resources to the local node when it shuts down cleanly controld_shutdown_lock_enabled = (1 << 5), }; # define controld_set_global_flags(flags_to_set) do { \ controld_globals.flags = pcmk__set_flags_as(__func__, __LINE__, \ LOG_TRACE, \ "Global", "controller", \ controld_globals.flags, \ (flags_to_set), \ #flags_to_set); \ } while (0) # define controld_clear_global_flags(flags_to_clear) do { \ controld_globals.flags \ = pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE, "Global", \ "controller", controld_globals.flags, \ (flags_to_clear), #flags_to_clear); \ } while (0) #endif // ifndef CONTROLD_GLOBALS__H diff --git a/daemons/controld/pacemaker-controld.h b/daemons/controld/pacemaker-controld.h index ba8dc8f5b0..e596b78007 100644 --- a/daemons/controld/pacemaker-controld.h +++ b/daemons/controld/pacemaker-controld.h @@ -1,41 +1,40 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef CRMD__H # define CRMD__H #include #include #include #include #include #include #include #include #include #include #include #include #include #include # define controld_trigger_config() \ controld_trigger_config_as(__func__, __LINE__) void controld_trigger_config_as(const char *fn, int line); -void controld_election_init(const char *uname); +void controld_election_init(void); void controld_configure_election(GHashTable *options); void controld_remove_voter(const char *uname); -void controld_election_fini(void); void controld_stop_current_election_timeout(void); void set_join_state(const char *start_state, const char *node_name, const char *node_uuid, bool remote); #endif diff --git a/include/crm/cluster/election_internal.h b/include/crm/cluster/election_internal.h index dcc6b83ab8..dd803d2f15 100644 --- a/include/crm/cluster/election_internal.h +++ b/include/crm/cluster/election_internal.h @@ -1,94 +1,91 @@ /* * Copyright 2009-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__CRM_CLUSTER_ELECTION_INTERNAL__H #define PCMK__CRM_CLUSTER_ELECTION_INTERNAL__H #include // bool #include // guint, GSourceFunc #include // xmlNode #include // enum pcmk_ipc_server +#include // pcmk_cluster_t #ifdef __cplusplus extern "C" { #endif /** * \file * \brief Functions for conducting elections * * An election is useful for a daemon that runs on all nodes but needs any one * instance to perform a special role. * * Elections are closely tied to the cluster peer cache. Peers in the cache that * are active members are eligible to vote. Elections are named for logging * purposes, but only one election may exist at any time, so typically an * election would be created at daemon start-up and freed at shutdown. * * Pacemaker's election procedure has been heavily adapted from the * Invitation Algorithm variant of the Garcia-Molina Bully Algorithm: * * https://en.wikipedia.org/wiki/Bully_algorithm * * Elections are conducted via cluster messages. There are two types of * messages: a "vote" is a declaration of the voting node's candidacy, and is * always broadcast; a "no-vote" is a concession by the responding node, and is * always a reply to the preferred node's vote. (These correspond to "invite" * and "accept" in the traditional algorithm.) * * A vote together with any no-vote replies to it is considered an election * round. Rounds are numbered with a simple counter unique to each node * (this would be the group number in the traditional algorithm). Concurrent * election rounds are possible. * * An election round is started when any node broadcasts a vote. When a node * receives another node's vote, it compares itself against the sending node * according to certain metrics, and either starts a new round (if it prefers * itself) or replies to the other node with a no-vote (if it prefers that * node). * * If a node receives no-votes from all other active nodes, it declares itself * the winner. The library API does not notify other nodes of this; callers * must implement that if desired. */ -typedef struct election_s election_t; - -/*! Possible election states */ +// Possible election results enum election_result { - election_start = 0, /*! new election needed */ - election_in_progress, /*! election started but not all peers have voted */ - election_lost, /*! local node lost most recent election */ - election_won, /*! local node won most recent election */ - election_error, /*! election message or election object invalid */ + election_start = 0, // New election needed + election_in_progress, // Election started but not all peers have voted + election_lost, // Local node lost most recent election + election_won, // Local node won most recent election + election_error, // Election message or object invalid }; -void election_fini(election_t *e); -void election_reset(election_t *e); -election_t *election_init(enum pcmk_ipc_server, const char *name, - const char *uname, guint period_ms, GSourceFunc cb); +void election_reset(pcmk_cluster_t *cluster); +void election_init(pcmk_cluster_t *cluster, void (*cb)(pcmk_cluster_t *)); -void election_timeout_set_period(election_t *e, guint period_ms); -void election_timeout_stop(election_t *e); +void election_timeout_set_period(pcmk_cluster_t *cluster, guint period_ms); +void election_timeout_stop(pcmk_cluster_t *cluster); -void election_vote(election_t *e); -bool election_check(election_t *e); -void election_remove(election_t *e, const char *uname); -enum election_result election_state(const election_t *e); -enum election_result election_count_vote(election_t *e, const xmlNode *message, - bool can_win); -void election_clear_dampening(election_t *e); +void election_vote(pcmk_cluster_t *cluster); +bool election_check(pcmk_cluster_t *cluster); +void election_remove(pcmk_cluster_t *cluster, const char *uname); +enum election_result election_state(const pcmk_cluster_t *cluster); +enum election_result election_count_vote(pcmk_cluster_t *cluster, + const xmlNode *message, bool can_win); +void election_clear_dampening(pcmk_cluster_t *cluster); #ifdef __cplusplus } #endif #endif // PCMK__CRM_CLUSTER_ELECTION_INTERNAL__H diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h index bb882f2252..44235e290a 100644 --- a/include/crm/cluster/internal.h +++ b/include/crm/cluster/internal.h @@ -1,319 +1,324 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__CRM_CLUSTER_INTERNAL__H #define PCMK__CRM_CLUSTER_INTERNAL__H #include #include // uint32_t, uint64_t #include // gboolean #include // xmlNode #include // enum crm_ipc_server #include #if SUPPORT_COROSYNC #include // cpg_name, cpg_handle_t #endif #ifdef __cplusplus extern "C" { #endif // @TODO Replace this with a pcmk__node_status_flags value enum crm_proc_flag { crm_proc_none = 0x00000001, // Cluster layers crm_proc_cpg = 0x04000000, }; /*! * \internal * \enum pcmk__node_status_flags * \brief Boolean flags for a \c pcmk__node_status_t object * * Some flags may not be related to status specifically. However, we keep these * separate from enum pcmk__node_flags because they're used with * different object types. */ enum pcmk__node_status_flags { /*! * Node is a Pacemaker Remote node and should not be considered for cluster * membership */ pcmk__node_status_remote = (UINT32_C(1) << 0), //! Node's cache entry is dirty pcmk__node_status_dirty = (UINT32_C(1) << 1), }; // Used with node cache search functions enum pcmk__node_search_flags { //! Does not affect search pcmk__node_search_none = 0, //! Search for cluster nodes from membership cache pcmk__node_search_cluster_member = (1 << 0), //! Search for remote nodes pcmk__node_search_remote = (1 << 1), //! Search for cluster member nodes and remote nodes pcmk__node_search_any = pcmk__node_search_cluster_member |pcmk__node_search_remote, //! Search for cluster nodes from CIB (as of last cache refresh) pcmk__node_search_cluster_cib = (1 << 2), }; /*! * \internal * \enum pcmk__node_update * \brief Type of update to a \c pcmk__node_status_t object */ enum pcmk__node_update { pcmk__node_update_name, //!< Node name updated pcmk__node_update_state, //!< Node connection state updated pcmk__node_update_processes, //!< Node process group membership updated }; +typedef struct pcmk__election pcmk__election_t; + //! Implementation of pcmk__cluster_private_t struct pcmk__cluster_private { - // @TODO Drop and replace with per-daemon cluster-layer ID global variables? - uint32_t node_id; //!< Local node ID at cluster layer - - // @TODO Drop and replace with per-daemon node name global variables? + enum pcmk_ipc_server server; //!< Server this connection is for (if any) char *node_name; //!< Local node name at cluster layer + pcmk__election_t *election; //!< Election state (if election is needed) + + /* @TODO Corosync uses an integer node ID, but cluster layers in the + * abstract do not necessarily need to + */ + uint32_t node_id; //!< Local node ID at cluster layer #if SUPPORT_COROSYNC /* @TODO Make these members a separate struct and use void *cluster_data * here instead, to abstract the cluster layer further. */ struct cpg_name group; //!< Corosync CPG name cpg_handle_t cpg_handle; //!< Corosync CPG handle #endif // SUPPORT_COROSYNC }; //! Node status data (may be a cluster node or a Pacemaker Remote node) typedef struct pcmk__node_status { //! Node name as known to cluster layer, or Pacemaker Remote node name char *name; /* @COMPAT This is less than ideal since the value is not a valid XML ID * (for Corosync, it's the string equivalent of the node's numeric node ID, * but XML IDs can't start with a number) and the three elements should have * different IDs. * * Ideally, we would use something like node-NODEID, node_state-NODEID, and * transient_attributes-NODEID as the element IDs. Unfortunately changing it * would be impractical due to backward compatibility; older nodes in a * rolling upgrade will always write and expect the value in the old format. */ /*! * Value of the PCMK_XA_ID XML attribute to use with the node's * PCMK_XE_NODE, PCMK_XE_NODE_STATE, and PCMK_XE_TRANSIENT_ATTRIBUTES * XML elements in the CIB */ char *xml_id; char *state; // @TODO change to enum //! Group of enum pcmk__node_status_flags uint32_t flags; /*! * Most recent cluster membership in which node was seen (0 for Pacemaker * Remote nodes) */ uint64_t membership_id; uint32_t processes; // @TODO most not needed, merge into flags /* @TODO When we can break public API compatibility, we can make the rest of * these members separate structs and use void *cluster_data and * void *user_data here instead, to abstract the cluster layer further. */ //! Arbitrary data (must be freeable by \c free()) void *user_data; char *expected; time_t peer_lost; char *conn_host; time_t when_member; // Since when node has been a cluster member time_t when_online; // Since when peer has been online in CPG /* @TODO The following are currently needed only by the Corosync stack. * Eventually consider moving them to a cluster-layer-specific data object. */ uint32_t cluster_layer_id; //!< Cluster-layer numeric node ID time_t when_lost; //!< When CPG membership was last lost } pcmk__node_status_t; /*! * \internal * \brief Return the process bit corresponding to the current cluster stack * * \return Process flag if detectable, otherwise 0 */ static inline uint32_t crm_get_cluster_proc(void) { switch (pcmk_get_cluster_layer()) { case pcmk_cluster_layer_corosync: return crm_proc_cpg; default: break; } return crm_proc_none; } /*! * \internal * \brief Get log-friendly string description of a Corosync return code * * \param[in] error Corosync return code * * \return Log-friendly string description corresponding to \p error */ static inline const char * pcmk__cs_err_str(int error) { # if SUPPORT_COROSYNC switch (error) { case CS_OK: return "OK"; case CS_ERR_LIBRARY: return "Library error"; case CS_ERR_VERSION: return "Version error"; case CS_ERR_INIT: return "Initialization error"; case CS_ERR_TIMEOUT: return "Timeout"; case CS_ERR_TRY_AGAIN: return "Try again"; case CS_ERR_INVALID_PARAM: return "Invalid parameter"; case CS_ERR_NO_MEMORY: return "No memory"; case CS_ERR_BAD_HANDLE: return "Bad handle"; case CS_ERR_BUSY: return "Busy"; case CS_ERR_ACCESS: return "Access error"; case CS_ERR_NOT_EXIST: return "Doesn't exist"; case CS_ERR_NAME_TOO_LONG: return "Name too long"; case CS_ERR_EXIST: return "Exists"; case CS_ERR_NO_SPACE: return "No space"; case CS_ERR_INTERRUPT: return "Interrupt"; case CS_ERR_NAME_NOT_FOUND: return "Name not found"; case CS_ERR_NO_RESOURCES: return "No resources"; case CS_ERR_NOT_SUPPORTED: return "Not supported"; case CS_ERR_BAD_OPERATION: return "Bad operation"; case CS_ERR_FAILED_OPERATION: return "Failed operation"; case CS_ERR_MESSAGE_ERROR: return "Message error"; case CS_ERR_QUEUE_FULL: return "Queue full"; case CS_ERR_QUEUE_NOT_AVAILABLE: return "Queue not available"; case CS_ERR_BAD_FLAGS: return "Bad flags"; case CS_ERR_TOO_BIG: return "Too big"; case CS_ERR_NO_SECTIONS: return "No sections"; } # endif return "Corosync error"; } # if SUPPORT_COROSYNC #if 0 /* This is the new way to do it, but we still support all Corosync 2 versions, * and this isn't always available. A better alternative here would be to check * for support in the configure script and enable this conditionally. */ #define pcmk__init_cmap(handle) cmap_initialize_map((handle), CMAP_MAP_ICMAP) #else #define pcmk__init_cmap(handle) cmap_initialize(handle) #endif char *pcmk__corosync_cluster_name(void); bool pcmk__corosync_add_nodes(xmlNode *xml_parent); void pcmk__cpg_confchg_cb(cpg_handle_t handle, const struct cpg_name *group_name, const struct cpg_address *member_list, size_t member_list_entries, const struct cpg_address *left_list, size_t left_list_entries, const struct cpg_address *joined_list, size_t joined_list_entries); char *pcmk__cpg_message_data(cpg_handle_t handle, uint32_t sender_id, uint32_t pid, void *content, const char **from); # endif const char *pcmk__cluster_node_uuid(pcmk__node_status_t *node); char *pcmk__cluster_node_name(uint32_t nodeid); const char *pcmk__cluster_local_node_name(void); const char *pcmk__node_name_from_uuid(const char *uuid); pcmk__node_status_t *crm_update_peer_proc(const char *source, pcmk__node_status_t *peer, uint32_t flag, const char *status); pcmk__node_status_t *pcmk__update_peer_state(const char *source, pcmk__node_status_t *node, const char *state, uint64_t membership); void pcmk__update_peer_expected(const char *source, pcmk__node_status_t *node, const char *expected); void pcmk__reap_unseen_nodes(uint64_t ring_id); void pcmk__corosync_quorum_connect(gboolean (*dispatch)(unsigned long long, gboolean), void (*destroy) (gpointer)); bool pcmk__cluster_send_message(const pcmk__node_status_t *node, enum pcmk_ipc_server service, const xmlNode *data); // Membership extern GHashTable *pcmk__peer_cache; extern GHashTable *pcmk__remote_peer_cache; bool pcmk__cluster_has_quorum(void); void pcmk__cluster_init_node_caches(void); void pcmk__cluster_destroy_node_caches(void); void pcmk__cluster_set_autoreap(bool enable); void pcmk__cluster_set_status_callback(void (*dispatch)(enum pcmk__node_update, pcmk__node_status_t *, const void *)); bool pcmk__cluster_is_node_active(const pcmk__node_status_t *node); unsigned int pcmk__cluster_num_active_nodes(void); unsigned int pcmk__cluster_num_remote_nodes(void); pcmk__node_status_t *pcmk__cluster_lookup_remote_node(const char *node_name); void pcmk__cluster_forget_cluster_node(uint32_t id, const char *node_name); void pcmk__cluster_forget_remote_node(const char *node_name); pcmk__node_status_t *pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags); void pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id); void pcmk__refresh_node_caches_from_cib(xmlNode *cib); pcmk__node_status_t *pcmk__get_node(unsigned int id, const char *uname, const char *uuid, uint32_t flags); #ifdef __cplusplus } #endif #endif // PCMK__CRM_CLUSTER_INTERNAL__H diff --git a/lib/cluster/cluster.c b/lib/cluster/cluster.c index 56ee7a1690..3427a409f3 100644 --- a/lib/cluster/cluster.c +++ b/lib/cluster/cluster.c @@ -1,469 +1,471 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include // PRIu32 #include #include #include #include #include #include #include #include #include // uname() #include // gboolean #include #include #include #include #include "crmcluster_private.h" CRM_TRACE_INIT_DATA(cluster); /*! * \internal * \brief Get a node's cluster-layer UUID, setting it if not already set * * \param[in,out] node Node to check * * \return Cluster-layer node UUID of \p node, or \c NULL if unknown */ const char * pcmk__cluster_node_uuid(pcmk__node_status_t *node) { const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer(); if (node == NULL) { return NULL; } if (node->xml_id != NULL) { return node->xml_id; } switch (cluster_layer) { #if SUPPORT_COROSYNC case pcmk_cluster_layer_corosync: node->xml_id = pcmk__corosync_uuid(node); return node->xml_id; #endif // SUPPORT_COROSYNC default: crm_err("Unsupported cluster layer %s", pcmk_cluster_layer_text(cluster_layer)); return NULL; } } /*! * \internal * \brief Connect to the cluster layer * * \param[in,out] cluster Initialized cluster object to connect * * \return Standard Pacemaker return code */ int pcmk_cluster_connect(pcmk_cluster_t *cluster) { const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer(); const char *cluster_layer_s = pcmk_cluster_layer_text(cluster_layer); if (cluster == NULL) { return EINVAL; } // cts-lab looks for this message crm_notice("Connecting to %s cluster layer", cluster_layer_s); switch (cluster_layer) { #if SUPPORT_COROSYNC case pcmk_cluster_layer_corosync: return pcmk__corosync_connect(cluster); #endif // SUPPORT_COROSYNC default: break; } crm_err("Failed to connect to unsupported cluster layer %s", cluster_layer_s); return EPROTONOSUPPORT; } /*! * \brief Disconnect from the cluster layer * * \param[in,out] cluster Cluster object to disconnect * * \return Standard Pacemaker return code */ int pcmk_cluster_disconnect(pcmk_cluster_t *cluster) { const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer(); const char *cluster_layer_s = pcmk_cluster_layer_text(cluster_layer); crm_info("Disconnecting from %s cluster layer", cluster_layer_s); switch (cluster_layer) { #if SUPPORT_COROSYNC case pcmk_cluster_layer_corosync: pcmk__corosync_disconnect(cluster); pcmk__cluster_destroy_node_caches(); return pcmk_rc_ok; #endif // SUPPORT_COROSYNC default: break; } crm_err("Failed to disconnect from unsupported cluster layer %s", cluster_layer_s); return EPROTONOSUPPORT; } /*! * \brief Allocate a new \p pcmk_cluster_t object * * \return A newly allocated \p pcmk_cluster_t object (guaranteed not \c NULL) * \note The caller is responsible for freeing the return value using * \p pcmk_cluster_free(). */ pcmk_cluster_t * pcmk_cluster_new(void) { pcmk_cluster_t *cluster = pcmk__assert_alloc(1, sizeof(pcmk_cluster_t)); cluster->priv = pcmk__assert_alloc(1, sizeof(pcmk__cluster_private_t)); + cluster->priv->server = pcmk__parse_server(crm_system_name); return cluster; } /*! * \brief Free a \p pcmk_cluster_t object and its dynamically allocated members * * \param[in,out] cluster Cluster object to free */ void pcmk_cluster_free(pcmk_cluster_t *cluster) { if (cluster == NULL) { return; } + election_fini(cluster); free(cluster->priv->node_name); free(cluster->priv); free(cluster); } /*! * \brief Set the destroy function for a cluster object * * \param[in,out] cluster Cluster object * \param[in] fn Destroy function to set * * \return Standard Pacemaker return code */ int pcmk_cluster_set_destroy_fn(pcmk_cluster_t *cluster, void (*fn)(gpointer)) { if (cluster == NULL) { return EINVAL; } cluster->destroy = fn; return pcmk_rc_ok; } /*! * \internal * \brief Send an XML message via the cluster messaging layer * * \param[in] node Cluster node to send message to * \param[in] service Message type to use in message host info * \param[in] data XML message to send * * \return \c true on success, or \c false otherwise */ bool pcmk__cluster_send_message(const pcmk__node_status_t *node, enum pcmk_ipc_server service, const xmlNode *data) { // @TODO Return standard Pacemaker return code switch (pcmk_get_cluster_layer()) { #if SUPPORT_COROSYNC case pcmk_cluster_layer_corosync: return pcmk__cpg_send_xml(data, node, service); #endif // SUPPORT_COROSYNC default: break; } return false; } /*! * \internal * \brief Get the node name corresponding to a cluster-layer node ID * * Get the node name from the cluster layer if possible. Otherwise, if for the * local node, call \c uname() and get the \c nodename member from the * struct utsname object. * * \param[in] nodeid Node ID to check (or 0 for the local node) * * \return Node name corresponding to \p nodeid * * \note This will fatally exit if \c uname() fails to get the local node name * or we run out of memory. * \note The caller is responsible for freeing the return value using \c free(). */ char * pcmk__cluster_node_name(uint32_t nodeid) { char *name = NULL; const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer(); const char *cluster_layer_s = pcmk_cluster_layer_text(cluster_layer); switch (cluster_layer) { #if SUPPORT_COROSYNC case pcmk_cluster_layer_corosync: name = pcmk__corosync_name(0, nodeid); if (name != NULL) { return name; } break; #endif // SUPPORT_COROSYNC default: crm_err("Unsupported cluster layer: %s", cluster_layer_s); break; } if (nodeid == 0) { struct utsname hostinfo; crm_notice("Could not get local node name from %s cluster layer, " "defaulting to local hostname", cluster_layer_s); if (uname(&hostinfo) < 0) { // @TODO Maybe let the caller decide what to do crm_err("Failed to get the local hostname"); crm_exit(CRM_EX_FATAL); } return pcmk__str_copy(hostinfo.nodename); } crm_notice("Could not obtain a node name for node with " PCMK_XA_ID "=" PRIu32, nodeid); return NULL; } /*! * \internal * \brief Get the local node's cluster-layer node name * * If getting the node name from the cluster layer is impossible, call * \c uname() and get the \c nodename member from the struct utsname * object. * * \return Local node's name * * \note This will fatally exit if \c uname() fails to get the local node name * or we run out of memory. */ const char * pcmk__cluster_local_node_name(void) { // @TODO Refactor to avoid trivially leaking name at exit static char *name = NULL; if (name == NULL) { name = pcmk__cluster_node_name(0); } return name; } /*! * \internal * \brief Get the node name corresonding to a node UUID * * Look for the UUID in both the remote node cache and the cluster member cache. * * \param[in] uuid UUID to search for * * \return Node name corresponding to \p uuid if found, or \c NULL otherwise */ const char * pcmk__node_name_from_uuid(const char *uuid) { /* @TODO There are too many functions in libcrmcluster that look up a node * from the node caches (possibly creating a cache entry if none exists). * There are at least the following: * * pcmk__cluster_lookup_remote_node() * * pcmk__get_node() * * pcmk__node_name_from_uuid() * * pcmk__search_node_caches() * * There's a lot of duplication among them, but they all do slightly * different things. We should try to clean them up and consolidate them to * the extent possible, likely with new helper functions. */ GHashTableIter iter; pcmk__node_status_t *node = NULL; CRM_CHECK(uuid != NULL, return NULL); // Remote nodes have the same uname and uuid if (g_hash_table_lookup(pcmk__remote_peer_cache, uuid)) { return uuid; } g_hash_table_iter_init(&iter, pcmk__peer_cache); while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) { if (pcmk__str_eq(node->xml_id, uuid, pcmk__str_casei)) { return node->name; } } return NULL; } /*! * \brief Get a log-friendly string equivalent of a cluster layer * * \param[in] layer Cluster layer * * \return Log-friendly string corresponding to \p layer */ const char * pcmk_cluster_layer_text(enum pcmk_cluster_layer layer) { switch (layer) { case pcmk_cluster_layer_corosync: return "corosync"; case pcmk_cluster_layer_unknown: return "unknown"; case pcmk_cluster_layer_invalid: return "invalid"; default: crm_err("Invalid cluster layer: %d", layer); return "invalid"; } } /*! * \brief Get and validate the local cluster layer * * If a cluster layer is not configured via the \c PCMK__ENV_CLUSTER_TYPE local * option, this will try to detect an active cluster from among the supported * cluster layers. * * \return Local cluster layer * * \note This will fatally exit if the configured cluster layer is invalid. */ enum pcmk_cluster_layer pcmk_get_cluster_layer(void) { static enum pcmk_cluster_layer cluster_layer = pcmk_cluster_layer_unknown; const char *cluster = NULL; // Cluster layer is stable once set if (cluster_layer != pcmk_cluster_layer_unknown) { return cluster_layer; } cluster = pcmk__env_option(PCMK__ENV_CLUSTER_TYPE); if (cluster != NULL) { crm_info("Verifying configured cluster layer '%s'", cluster); cluster_layer = pcmk_cluster_layer_invalid; #if SUPPORT_COROSYNC if (pcmk__str_eq(cluster, PCMK_VALUE_COROSYNC, pcmk__str_casei)) { cluster_layer = pcmk_cluster_layer_corosync; } #endif // SUPPORT_COROSYNC if (cluster_layer == pcmk_cluster_layer_invalid) { crm_notice("This installation does not support the '%s' cluster " "infrastructure: terminating", cluster); crm_exit(CRM_EX_FATAL); } crm_info("Assuming an active '%s' cluster", cluster); } else { // Nothing configured, so test supported cluster layers #if SUPPORT_COROSYNC crm_debug("Testing with Corosync"); if (pcmk__corosync_is_active()) { cluster_layer = pcmk_cluster_layer_corosync; } #endif // SUPPORT_COROSYNC if (cluster_layer == pcmk_cluster_layer_unknown) { crm_notice("Could not determine the current cluster layer"); } else { crm_info("Detected an active '%s' cluster", pcmk_cluster_layer_text(cluster_layer)); } } return cluster_layer; } // Deprecated functions kept only for backward API compatibility // LCOV_EXCL_START #include gboolean crm_cluster_connect(pcmk_cluster_t *cluster) { if (cluster == NULL) { return FALSE; } if (cluster->priv == NULL) { /* sbd (as of at least 1.5.2) doesn't call pcmk_cluster_new() to * allocate the pcmk_cluster_t */ cluster->priv = pcmk__assert_alloc(1, sizeof(pcmk__cluster_private_t)); } return pcmk_cluster_connect(cluster) == pcmk_rc_ok; } const char * name_for_cluster_type(enum cluster_type_e type) { switch (type) { case pcmk_cluster_corosync: return "corosync"; case pcmk_cluster_unknown: return "unknown"; case pcmk_cluster_invalid: return "invalid"; } crm_err("Invalid cluster type: %d", type); return "invalid"; } enum cluster_type_e get_cluster_type(void) { return (enum cluster_type_e) pcmk_get_cluster_layer(); } // LCOV_EXCL_STOP // End deprecated API diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c index e9f65abfd3..a5d219127c 100644 --- a/lib/cluster/cpg.c +++ b/lib/cluster/cpg.c @@ -1,1052 +1,1052 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include // PRIu32 #include #include #include #include // uint32_t #include #include // size_t #include #include #include #include #include #include #include #include #include #include #include #include // PCMK__SPECIAL_PID #include #include #include "crmcluster_private.h" /* @TODO Once we can update the public API to require pcmk_cluster_t* in more * functions, we can ditch this in favor of cluster->cpg_handle. */ static cpg_handle_t pcmk_cpg_handle = 0; // @TODO These could be moved to pcmk_cluster_t* at that time as well static bool cpg_evicted = false; static GList *cs_message_queue = NULL; static int cs_message_timer = 0; struct pcmk__cpg_host_s { uint32_t id; uint32_t pid; enum pcmk_ipc_server type; // For logging only uint32_t size; char uname[MAX_NAME]; } __attribute__ ((packed)); typedef struct pcmk__cpg_host_s pcmk__cpg_host_t; struct pcmk__cpg_msg_s { struct qb_ipc_response_header header __attribute__ ((aligned(8))); uint32_t id; gboolean is_compressed; pcmk__cpg_host_t host; pcmk__cpg_host_t sender; uint32_t size; uint32_t compressed_size; /* 584 bytes */ char data[0]; } __attribute__ ((packed)); typedef struct pcmk__cpg_msg_s pcmk__cpg_msg_t; static void crm_cs_flush(gpointer data); #define msg_data_len(msg) (msg->is_compressed?msg->compressed_size:msg->size) #define cs_repeat(rc, counter, max, code) do { \ rc = code; \ if ((rc == CS_ERR_TRY_AGAIN) || (rc == CS_ERR_QUEUE_FULL)) { \ counter++; \ crm_debug("Retrying operation after %ds", counter); \ sleep(counter); \ } else { \ break; \ } \ } while (counter < max) /*! * \internal * \brief Get the local Corosync node ID (via CPG) * * \param[in] handle CPG connection to use (or 0 to use new connection) * * \return Corosync ID of local node (or 0 if not known) */ uint32_t pcmk__cpg_local_nodeid(cpg_handle_t handle) { cs_error_t rc = CS_OK; int retries = 0; static uint32_t local_nodeid = 0; cpg_handle_t local_handle = handle; cpg_model_v1_data_t cpg_model_info = {CPG_MODEL_V1, NULL, NULL, NULL, 0}; int fd = -1; uid_t found_uid = 0; gid_t found_gid = 0; pid_t found_pid = 0; int rv = 0; if (local_nodeid != 0) { return local_nodeid; } if (handle == 0) { crm_trace("Creating connection"); cs_repeat(rc, retries, 5, cpg_model_initialize(&local_handle, CPG_MODEL_V1, (cpg_model_data_t *) &cpg_model_info, NULL)); if (rc != CS_OK) { crm_err("Could not connect to the CPG API: %s (%d)", cs_strerror(rc), rc); return 0; } rc = cpg_fd_get(local_handle, &fd); if (rc != CS_OK) { crm_err("Could not obtain the CPG API connection: %s (%d)", cs_strerror(rc), rc); goto bail; } // CPG provider run as root (at least in given user namespace)? rv = crm_ipc_is_authentic_process(fd, (uid_t) 0, (gid_t) 0, &found_pid, &found_uid, &found_gid); if (rv == 0) { crm_err("CPG provider is not authentic:" " process %lld (uid: %lld, gid: %lld)", (long long) PCMK__SPECIAL_PID_AS_0(found_pid), (long long) found_uid, (long long) found_gid); goto bail; } else if (rv < 0) { crm_err("Could not verify authenticity of CPG provider: %s (%d)", strerror(-rv), -rv); goto bail; } } if (rc == CS_OK) { retries = 0; crm_trace("Performing lookup"); cs_repeat(rc, retries, 5, cpg_local_get(local_handle, &local_nodeid)); } if (rc != CS_OK) { crm_err("Could not get local node id from the CPG API: %s (%d)", pcmk__cs_err_str(rc), rc); } bail: if (handle == 0) { crm_trace("Closing connection"); cpg_finalize(local_handle); } crm_debug("Local nodeid is %u", local_nodeid); return local_nodeid; } /*! * \internal * \brief Callback function for Corosync message queue timer * * \param[in] data CPG handle * * \return FALSE (to indicate to glib that timer should not be removed) */ static gboolean crm_cs_flush_cb(gpointer data) { cs_message_timer = 0; crm_cs_flush(data); return FALSE; } // Send no more than this many CPG messages in one flush #define CS_SEND_MAX 200 /*! * \internal * \brief Send messages in Corosync CPG message queue * * \param[in] data CPG handle */ static void crm_cs_flush(gpointer data) { unsigned int sent = 0; guint queue_len = 0; cs_error_t rc = 0; cpg_handle_t *handle = (cpg_handle_t *) data; if (*handle == 0) { crm_trace("Connection is dead"); return; } queue_len = g_list_length(cs_message_queue); if (((queue_len % 1000) == 0) && (queue_len > 1)) { crm_err("CPG queue has grown to %d", queue_len); } else if (queue_len == CS_SEND_MAX) { crm_warn("CPG queue has grown to %d", queue_len); } if (cs_message_timer != 0) { /* There is already a timer, wait until it goes off */ crm_trace("Timer active %d", cs_message_timer); return; } while ((cs_message_queue != NULL) && (sent < CS_SEND_MAX)) { struct iovec *iov = cs_message_queue->data; rc = cpg_mcast_joined(*handle, CPG_TYPE_AGREED, iov, 1); if (rc != CS_OK) { break; } sent++; crm_trace("CPG message sent, size=%llu", (unsigned long long) iov->iov_len); cs_message_queue = g_list_remove(cs_message_queue, iov); free(iov->iov_base); free(iov); } queue_len -= sent; do_crm_log((queue_len > 5)? LOG_INFO : LOG_TRACE, "Sent %u CPG message%s (%d still queued): %s (rc=%d)", sent, pcmk__plural_s(sent), queue_len, pcmk__cs_err_str(rc), (int) rc); if (cs_message_queue) { uint32_t delay_ms = 100; if (rc != CS_OK) { /* Proportionally more if sending failed but cap at 1s */ delay_ms = QB_MIN(1000, CS_SEND_MAX + (10 * queue_len)); } cs_message_timer = g_timeout_add(delay_ms, crm_cs_flush_cb, data); } } /*! * \internal * \brief Dispatch function for CPG handle * * \param[in,out] user_data Cluster object * * \return 0 on success, -1 on error (per mainloop_io_t interface) */ static int pcmk_cpg_dispatch(gpointer user_data) { cs_error_t rc = CS_OK; pcmk_cluster_t *cluster = (pcmk_cluster_t *) user_data; rc = cpg_dispatch(cluster->priv->cpg_handle, CS_DISPATCH_ONE); if (rc != CS_OK) { crm_err("Connection to the CPG API failed: %s (%d)", pcmk__cs_err_str(rc), rc); cpg_finalize(cluster->priv->cpg_handle); cluster->priv->cpg_handle = 0; return -1; } else if (cpg_evicted) { crm_err("Evicted from CPG membership"); return -1; } return 0; } static inline const char * ais_dest(const pcmk__cpg_host_t *host) { return (host->size > 0)? host->uname : ""; } static inline const char * msg_type2text(enum pcmk_ipc_server type) { const char *name = pcmk__server_message_type(type); return pcmk__s(name, "unknown"); } /*! * \internal * \brief Check whether a Corosync CPG message is valid * * \param[in] msg Corosync CPG message to check * * \return true if \p msg is valid, otherwise false */ static bool check_message_sanity(const pcmk__cpg_msg_t *msg) { int32_t payload_size = msg->header.size - sizeof(pcmk__cpg_msg_t); if (payload_size < 1) { crm_err("%sCPG message %d from %s invalid: " "Claimed size of %d bytes is too small " QB_XS " from %s[%u] to %s@%s", (msg->is_compressed? "Compressed " : ""), msg->id, ais_dest(&(msg->sender)), (int) msg->header.size, msg_type2text(msg->sender.type), msg->sender.pid, msg_type2text(msg->host.type), ais_dest(&(msg->host))); return false; } if (msg->header.error != CS_OK) { crm_err("%sCPG message %d from %s invalid: " "Sender indicated error %d " QB_XS " from %s[%u] to %s@%s", (msg->is_compressed? "Compressed " : ""), msg->id, ais_dest(&(msg->sender)), msg->header.error, msg_type2text(msg->sender.type), msg->sender.pid, msg_type2text(msg->host.type), ais_dest(&(msg->host))); return false; } if (msg_data_len(msg) != payload_size) { crm_err("%sCPG message %d from %s invalid: " "Total size %d inconsistent with payload size %d " QB_XS " from %s[%u] to %s@%s", (msg->is_compressed? "Compressed " : ""), msg->id, ais_dest(&(msg->sender)), (int) msg->header.size, (int) msg_data_len(msg), msg_type2text(msg->sender.type), msg->sender.pid, msg_type2text(msg->host.type), ais_dest(&(msg->host))); return false; } if (!msg->is_compressed && /* msg->size != (strlen(msg->data) + 1) would be a stronger check, * but checking the last byte or two should be quick */ (((msg->size > 1) && (msg->data[msg->size - 2] == '\0')) || (msg->data[msg->size - 1] != '\0'))) { crm_err("CPG message %d from %s invalid: " "Payload does not end at byte %llu " QB_XS " from %s[%u] to %s@%s", msg->id, ais_dest(&(msg->sender)), (unsigned long long) msg->size, msg_type2text(msg->sender.type), msg->sender.pid, msg_type2text(msg->host.type), ais_dest(&(msg->host))); return false; } crm_trace("Verified %d-byte %sCPG message %d from %s[%u]@%s to %s@%s", (int) msg->header.size, (msg->is_compressed? "compressed " : ""), msg->id, msg_type2text(msg->sender.type), msg->sender.pid, ais_dest(&(msg->sender)), msg_type2text(msg->host.type), ais_dest(&(msg->host))); return true; } /*! * \internal * \brief Extract text data from a Corosync CPG message * * \param[in] handle CPG connection (to get local node ID if not known) * \param[in] sender_id Corosync ID of node that sent message * \param[in] pid Process ID of message sender (for logging only) * \param[in,out] content CPG message * \param[out] from If not \c NULL, will be set to sender uname * (valid for the lifetime of \p content) * * \return Newly allocated string with message data * * \note The caller is responsible for freeing the return value using \c free(). */ char * pcmk__cpg_message_data(cpg_handle_t handle, uint32_t sender_id, uint32_t pid, void *content, const char **from) { char *data = NULL; pcmk__cpg_msg_t *msg = content; if (handle != 0) { // Do filtering and field massaging uint32_t local_nodeid = pcmk__cpg_local_nodeid(handle); const char *local_name = pcmk__cluster_local_node_name(); if ((msg->sender.id != 0) && (msg->sender.id != sender_id)) { crm_err("Nodeid mismatch from %" PRIu32 ".%" PRIu32 ": claimed nodeid=%" PRIu32, sender_id, pid, msg->sender.id); return NULL; } if ((msg->host.id != 0) && (local_nodeid != msg->host.id)) { crm_trace("Not for us: %" PRIu32" != %" PRIu32, msg->host.id, local_nodeid); return NULL; } if ((msg->host.size > 0) && !pcmk__str_eq(msg->host.uname, local_name, pcmk__str_casei)) { crm_trace("Not for us: %s != %s", msg->host.uname, local_name); return NULL; } msg->sender.id = sender_id; if (msg->sender.size == 0) { const pcmk__node_status_t *peer = pcmk__get_node(sender_id, NULL, NULL, pcmk__node_search_cluster_member); if (peer->name == NULL) { crm_err("No node name for peer with nodeid=%u", sender_id); } else { crm_notice("Fixing node name for peer with nodeid=%u", sender_id); msg->sender.size = strlen(peer->name); memset(msg->sender.uname, 0, MAX_NAME); memcpy(msg->sender.uname, peer->name, msg->sender.size); } } } crm_trace("Got new%s message (size=%d, %d, %d)", msg->is_compressed ? " compressed" : "", msg_data_len(msg), msg->size, msg->compressed_size); if (from != NULL) { *from = msg->sender.uname; } if (msg->is_compressed && (msg->size > 0)) { int rc = BZ_OK; char *uncompressed = NULL; unsigned int new_size = msg->size + 1; if (!check_message_sanity(msg)) { goto badmsg; } crm_trace("Decompressing message data"); uncompressed = pcmk__assert_alloc(1, new_size); rc = BZ2_bzBuffToBuffDecompress(uncompressed, &new_size, msg->data, msg->compressed_size, 1, 0); rc = pcmk__bzlib2rc(rc); if (rc != pcmk_rc_ok) { crm_err("Decompression failed: %s " QB_XS " rc=%d", pcmk_rc_str(rc), rc); free(uncompressed); goto badmsg; } CRM_ASSERT(new_size == msg->size); data = uncompressed; } else if (!check_message_sanity(msg)) { goto badmsg; } else { data = strdup(msg->data); } // Is this necessary? pcmk__get_node(msg->sender.id, msg->sender.uname, NULL, pcmk__node_search_cluster_member); crm_trace("Payload: %.200s", data); return data; badmsg: crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):" " min=%d, total=%d, size=%d, bz2_size=%d", msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, (int)sizeof(pcmk__cpg_msg_t), msg->header.size, msg->size, msg->compressed_size); free(data); return NULL; } /*! * \internal * \brief Compare cpg_address objects by node ID * * \param[in] first First cpg_address structure to compare * \param[in] second Second cpg_address structure to compare * * \return Negative number if first's node ID is lower, * positive number if first's node ID is greater, * or 0 if both node IDs are equal */ static int cmp_member_list_nodeid(const void *first, const void *second) { const struct cpg_address *const a = *((const struct cpg_address **) first), *const b = *((const struct cpg_address **) second); if (a->nodeid < b->nodeid) { return -1; } else if (a->nodeid > b->nodeid) { return 1; } /* don't bother with "reason" nor "pid" */ return 0; } /*! * \internal * \brief Get a readable string equivalent of a cpg_reason_t value * * \param[in] reason CPG reason value * * \return Readable string suitable for logging */ static const char * cpgreason2str(cpg_reason_t reason) { switch (reason) { case CPG_REASON_JOIN: return " via cpg_join"; case CPG_REASON_LEAVE: return " via cpg_leave"; case CPG_REASON_NODEDOWN: return " via cluster exit"; case CPG_REASON_NODEUP: return " via cluster join"; case CPG_REASON_PROCDOWN: return " for unknown reason"; default: break; } return ""; } /*! * \internal * \brief Get a log-friendly node name * * \param[in] peer Node to check * * \return Node's uname, or readable string if not known */ static inline const char * peer_name(const pcmk__node_status_t *peer) { return (peer != NULL)? pcmk__s(peer->name, "peer node") : "unknown node"; } /*! * \internal * \brief Process a CPG peer's leaving the cluster * * \param[in] cpg_group_name CPG group name (for logging) * \param[in] event_counter Event number (for logging) * \param[in] local_nodeid Node ID of local node * \param[in] cpg_peer CPG peer that left * \param[in] sorted_member_list List of remaining members, qsort()-ed by ID * \param[in] member_list_entries Number of entries in \p sorted_member_list */ static void node_left(const char *cpg_group_name, int event_counter, uint32_t local_nodeid, const struct cpg_address *cpg_peer, const struct cpg_address **sorted_member_list, size_t member_list_entries) { pcmk__node_status_t *peer = pcmk__search_node_caches(cpg_peer->nodeid, NULL, pcmk__node_search_cluster_member); const struct cpg_address **rival = NULL; /* Most CPG-related Pacemaker code assumes that only one process on a node * can be in the process group, but Corosync does not impose this * limitation, and more than one can be a member in practice due to a * daemon attempting to start while another instance is already running. * * Check for any such duplicate instances, because we don't want to process * their leaving as if our actual peer left. If the peer that left still has * an entry in sorted_member_list (with a different PID), we will ignore the * leaving. * * @TODO Track CPG members' PIDs so we can tell exactly who left. */ if (peer != NULL) { rival = bsearch(&cpg_peer, sorted_member_list, member_list_entries, sizeof(const struct cpg_address *), cmp_member_list_nodeid); } if (rival == NULL) { crm_info("Group %s event %d: %s (node %u pid %u) left%s", cpg_group_name, event_counter, peer_name(peer), cpg_peer->nodeid, cpg_peer->pid, cpgreason2str(cpg_peer->reason)); if (peer != NULL) { crm_update_peer_proc(__func__, peer, crm_proc_cpg, PCMK_VALUE_OFFLINE); } } else if (cpg_peer->nodeid == local_nodeid) { crm_warn("Group %s event %d: duplicate local pid %u left%s", cpg_group_name, event_counter, cpg_peer->pid, cpgreason2str(cpg_peer->reason)); } else { crm_warn("Group %s event %d: " "%s (node %u) duplicate pid %u left%s (%u remains)", cpg_group_name, event_counter, peer_name(peer), cpg_peer->nodeid, cpg_peer->pid, cpgreason2str(cpg_peer->reason), (*rival)->pid); } } /*! * \internal * \brief Handle a CPG configuration change event * * \param[in] handle CPG connection * \param[in] group_name CPG group name * \param[in] member_list List of current CPG members * \param[in] member_list_entries Number of entries in \p member_list * \param[in] left_list List of CPG members that left * \param[in] left_list_entries Number of entries in \p left_list * \param[in] joined_list List of CPG members that joined * \param[in] joined_list_entries Number of entries in \p joined_list * * \note This is of type \c cpg_confchg_fn_t, intended to be used in a * \c cpg_callbacks_t object. */ void pcmk__cpg_confchg_cb(cpg_handle_t handle, const struct cpg_name *group_name, const struct cpg_address *member_list, size_t member_list_entries, const struct cpg_address *left_list, size_t left_list_entries, const struct cpg_address *joined_list, size_t joined_list_entries) { static int counter = 0; bool found = false; uint32_t local_nodeid = pcmk__cpg_local_nodeid(handle); const struct cpg_address **sorted = NULL; sorted = pcmk__assert_alloc(member_list_entries, sizeof(const struct cpg_address *)); for (size_t iter = 0; iter < member_list_entries; iter++) { sorted[iter] = member_list + iter; } // So that the cross-matching of multiply-subscribed nodes is then cheap qsort(sorted, member_list_entries, sizeof(const struct cpg_address *), cmp_member_list_nodeid); for (int i = 0; i < left_list_entries; i++) { node_left(group_name->value, counter, local_nodeid, &left_list[i], sorted, member_list_entries); } free(sorted); sorted = NULL; for (int i = 0; i < joined_list_entries; i++) { crm_info("Group %s event %d: node %u pid %u joined%s", group_name->value, counter, joined_list[i].nodeid, joined_list[i].pid, cpgreason2str(joined_list[i].reason)); } for (int i = 0; i < member_list_entries; i++) { pcmk__node_status_t *peer = pcmk__get_node(member_list[i].nodeid, NULL, NULL, pcmk__node_search_cluster_member); if (member_list[i].nodeid == local_nodeid && member_list[i].pid != getpid()) { // See the note in node_left() crm_warn("Group %s event %d: detected duplicate local pid %u", group_name->value, counter, member_list[i].pid); continue; } crm_info("Group %s event %d: %s (node %u pid %u) is member", group_name->value, counter, peer_name(peer), member_list[i].nodeid, member_list[i].pid); /* If the caller left auto-reaping enabled, this will also update the * state to member. */ peer = crm_update_peer_proc(__func__, peer, crm_proc_cpg, PCMK_VALUE_ONLINE); if (peer && peer->state && strcmp(peer->state, PCMK_VALUE_MEMBER)) { /* The node is a CPG member, but we currently think it's not a * cluster member. This is possible only if auto-reaping was * disabled. The node may be joining, and we happened to get the CPG * notification before the quorum notification; or the node may have * just died, and we are processing its final messages; or a bug * has affected the peer cache. */ time_t now = time(NULL); if (peer->when_lost == 0) { // Track when we first got into this contradictory state peer->when_lost = now; } else if (now > (peer->when_lost + 60)) { // If it persists for more than a minute, update the state crm_warn("Node %u is member of group %s but was believed " "offline", member_list[i].nodeid, group_name->value); pcmk__update_peer_state(__func__, peer, PCMK_VALUE_MEMBER, 0); } } if (local_nodeid == member_list[i].nodeid) { found = true; } } if (!found) { crm_err("Local node was evicted from group %s", group_name->value); cpg_evicted = true; } counter++; } /*! * \brief Set the CPG deliver callback function for a cluster object * * \param[in,out] cluster Cluster object * \param[in] fn Deliver callback function to set * * \return Standard Pacemaker return code */ int pcmk_cpg_set_deliver_fn(pcmk_cluster_t *cluster, cpg_deliver_fn_t fn) { if (cluster == NULL) { return EINVAL; } cluster->cpg.cpg_deliver_fn = fn; return pcmk_rc_ok; } /*! * \brief Set the CPG config change callback function for a cluster object * * \param[in,out] cluster Cluster object * \param[in] fn Configuration change callback function to set * * \return Standard Pacemaker return code */ int pcmk_cpg_set_confchg_fn(pcmk_cluster_t *cluster, cpg_confchg_fn_t fn) { if (cluster == NULL) { return EINVAL; } cluster->cpg.cpg_confchg_fn = fn; return pcmk_rc_ok; } /*! * \brief Connect to Corosync CPG * * \param[in,out] cluster Initialized cluster object to connect * * \return Standard Pacemaker return code */ int pcmk__cpg_connect(pcmk_cluster_t *cluster) { cs_error_t rc; int fd = -1; int retries = 0; uint32_t id = 0; pcmk__node_status_t *peer = NULL; cpg_handle_t handle = 0; - enum pcmk_ipc_server server = pcmk__parse_server(crm_system_name); - const char *cpg_group_name = pcmk__server_message_type(server); + const char *cpg_group_name = NULL; uid_t found_uid = 0; gid_t found_gid = 0; pid_t found_pid = 0; int rv; struct mainloop_fd_callbacks cpg_fd_callbacks = { .dispatch = pcmk_cpg_dispatch, .destroy = cluster->destroy, }; cpg_model_v1_data_t cpg_model_info = { .model = CPG_MODEL_V1, .cpg_deliver_fn = cluster->cpg.cpg_deliver_fn, .cpg_confchg_fn = cluster->cpg.cpg_confchg_fn, .cpg_totem_confchg_fn = NULL, .flags = 0, }; cpg_evicted = false; + cpg_group_name = pcmk__server_message_type(cluster->priv->server); if (cpg_group_name == NULL) { /* The name will already be non-NULL for Pacemaker servers. If a * command-line tool or external caller connects to the cluster, * they will join this CPG group. */ cpg_group_name = pcmk__s(crm_system_name, "unknown"); } memset(cluster->priv->group.value, 0, 128); strncpy(cluster->priv->group.value, cpg_group_name, 127); cluster->priv->group.length = strlen(cluster->priv->group.value) + 1; cs_repeat(rc, retries, 30, cpg_model_initialize(&handle, CPG_MODEL_V1, (cpg_model_data_t *)&cpg_model_info, NULL)); if (rc != CS_OK) { crm_err("Could not connect to the CPG API: %s (%d)", cs_strerror(rc), rc); goto bail; } rc = cpg_fd_get(handle, &fd); if (rc != CS_OK) { crm_err("Could not obtain the CPG API connection: %s (%d)", cs_strerror(rc), rc); goto bail; } /* CPG provider run as root (in given user namespace, anyway)? */ if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid, &found_uid, &found_gid))) { crm_err("CPG provider is not authentic:" " process %lld (uid: %lld, gid: %lld)", (long long) PCMK__SPECIAL_PID_AS_0(found_pid), (long long) found_uid, (long long) found_gid); rc = CS_ERR_ACCESS; goto bail; } else if (rv < 0) { crm_err("Could not verify authenticity of CPG provider: %s (%d)", strerror(-rv), -rv); rc = CS_ERR_ACCESS; goto bail; } id = pcmk__cpg_local_nodeid(handle); if (id == 0) { crm_err("Could not get local node id from the CPG API"); goto bail; } cluster->priv->node_id = id; retries = 0; cs_repeat(rc, retries, 30, cpg_join(handle, &cluster->priv->group)); if (rc != CS_OK) { crm_err("Could not join the CPG group '%s': %d", cpg_group_name, rc); goto bail; } pcmk_cpg_handle = handle; cluster->priv->cpg_handle = handle; mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks); bail: if (rc != CS_OK) { cpg_finalize(handle); // @TODO Map rc to more specific Pacemaker return code return ENOTCONN; } peer = pcmk__get_node(id, NULL, NULL, pcmk__node_search_cluster_member); crm_update_peer_proc(__func__, peer, crm_proc_cpg, PCMK_VALUE_ONLINE); return pcmk_rc_ok; } /*! * \internal * \brief Disconnect from Corosync CPG * * \param[in,out] cluster Cluster object to disconnect */ void pcmk__cpg_disconnect(pcmk_cluster_t *cluster) { pcmk_cpg_handle = 0; if (cluster->priv->cpg_handle != 0) { crm_trace("Disconnecting CPG"); cpg_leave(cluster->priv->cpg_handle, &cluster->priv->group); cpg_finalize(cluster->priv->cpg_handle); cluster->priv->cpg_handle = 0; } else { crm_info("No CPG connection"); } } /*! * \internal * \brief Send string data via Corosync CPG * * \param[in] data Data to send * \param[in] node Cluster node to send message to * \param[in] dest Type of message to send * * \return \c true on success, or \c false otherwise */ static bool send_cpg_text(const char *data, const pcmk__node_status_t *node, enum pcmk_ipc_server dest) { static int msg_id = 0; static int local_pid = 0; static int local_name_len = 0; static const char *local_name = NULL; char *target = NULL; struct iovec *iov; pcmk__cpg_msg_t *msg = NULL; if (local_name == NULL) { local_name = pcmk__cluster_local_node_name(); } if ((local_name_len == 0) && (local_name != NULL)) { local_name_len = strlen(local_name); } if (data == NULL) { data = ""; } if (local_pid == 0) { local_pid = getpid(); } msg = pcmk__assert_alloc(1, sizeof(pcmk__cpg_msg_t)); msg_id++; msg->id = msg_id; msg->header.error = CS_OK; msg->host.type = dest; if (node != NULL) { if (node->name != NULL) { target = pcmk__str_copy(node->name); msg->host.size = strlen(node->name); memset(msg->host.uname, 0, MAX_NAME); memcpy(msg->host.uname, node->name, msg->host.size); } else { target = crm_strdup_printf("%" PRIu32, node->cluster_layer_id); } msg->host.id = node->cluster_layer_id; } else { target = pcmk__str_copy("all"); } msg->sender.id = 0; msg->sender.type = pcmk__parse_server(crm_system_name); msg->sender.pid = local_pid; msg->sender.size = local_name_len; memset(msg->sender.uname, 0, MAX_NAME); if ((local_name != NULL) && (msg->sender.size != 0)) { memcpy(msg->sender.uname, local_name, msg->sender.size); } msg->size = 1 + strlen(data); msg->header.size = sizeof(pcmk__cpg_msg_t) + msg->size; if (msg->size < CRM_BZ2_THRESHOLD) { msg = pcmk__realloc(msg, msg->header.size); memcpy(msg->data, data, msg->size); } else { char *compressed = NULL; unsigned int new_size = 0; if (pcmk__compress(data, (unsigned int) msg->size, 0, &compressed, &new_size) == pcmk_rc_ok) { msg->header.size = sizeof(pcmk__cpg_msg_t) + new_size; msg = pcmk__realloc(msg, msg->header.size); memcpy(msg->data, compressed, new_size); msg->is_compressed = TRUE; msg->compressed_size = new_size; } else { // cppcheck seems not to understand the abort logic in pcmk__realloc // cppcheck-suppress memleak msg = pcmk__realloc(msg, msg->header.size); memcpy(msg->data, data, msg->size); } free(compressed); } iov = pcmk__assert_alloc(1, sizeof(struct iovec)); iov->iov_base = msg; iov->iov_len = msg->header.size; if (msg->compressed_size > 0) { crm_trace("Queueing CPG message %u to %s " "(%llu bytes, %d bytes compressed payload): %.200s", msg->id, target, (unsigned long long) iov->iov_len, msg->compressed_size, data); } else { crm_trace("Queueing CPG message %u to %s " "(%llu bytes, %d bytes payload): %.200s", msg->id, target, (unsigned long long) iov->iov_len, msg->size, data); } free(target); cs_message_queue = g_list_append(cs_message_queue, iov); crm_cs_flush(&pcmk_cpg_handle); return true; } /*! * \internal * \brief Send an XML message via Corosync CPG * * \param[in] msg XML message to send * \param[in] node Cluster node to send message to * \param[in] dest Type of message to send * * \return TRUE on success, otherwise FALSE */ bool pcmk__cpg_send_xml(const xmlNode *msg, const pcmk__node_status_t *node, enum pcmk_ipc_server dest) { bool rc = true; GString *data = g_string_sized_new(1024); pcmk__xml_string(msg, 0, data, 0); rc = send_cpg_text(data->str, node, dest); g_string_free(data, TRUE); return rc; } diff --git a/lib/cluster/crmcluster_private.h b/lib/cluster/crmcluster_private.h index f99a21c9b9..44a9d33798 100644 --- a/lib/cluster/crmcluster_private.h +++ b/lib/cluster/crmcluster_private.h @@ -1,80 +1,83 @@ /* * Copyright 2020-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #ifndef PCMK__CLUSTER_CRMCLUSTER_PRIVATE__H #define PCMK__CLUSTER_CRMCLUSTER_PRIVATE__H /* This header is for the sole use of libcrmcluster, so that functions can be * declared with G_GNUC_INTERNAL for efficiency. */ #include // bool #include // uint32_t, uint64_t #include // G_GNUC_INTERNAL, gboolean #include // xmlNode #if SUPPORT_COROSYNC #include // cpg_handle_t #endif // SUPPORT_COROSYNC #include // pcmk_cluster_t #include // pcmk__node_status_t #ifdef __cplusplus extern "C" { #endif G_GNUC_INTERNAL void pcmk__cluster_set_quorum(bool quorate); +G_GNUC_INTERNAL +void election_fini(pcmk_cluster_t *cluster); + #if SUPPORT_COROSYNC G_GNUC_INTERNAL bool pcmk__corosync_is_active(void); G_GNUC_INTERNAL bool pcmk__corosync_has_nodelist(void); G_GNUC_INTERNAL char *pcmk__corosync_uuid(const pcmk__node_status_t *peer); G_GNUC_INTERNAL char *pcmk__corosync_name(uint64_t /*cmap_handle_t */ cmap_handle, uint32_t nodeid); G_GNUC_INTERNAL int pcmk__corosync_connect(pcmk_cluster_t *cluster); G_GNUC_INTERNAL void pcmk__corosync_disconnect(pcmk_cluster_t *cluster); G_GNUC_INTERNAL bool pcmk__corosync_is_peer_active(const pcmk__node_status_t *node); G_GNUC_INTERNAL int pcmk__cpg_connect(pcmk_cluster_t *cluster); G_GNUC_INTERNAL void pcmk__cpg_disconnect(pcmk_cluster_t *cluster); G_GNUC_INTERNAL uint32_t pcmk__cpg_local_nodeid(cpg_handle_t handle); G_GNUC_INTERNAL bool pcmk__cpg_send_xml(const xmlNode *msg, const pcmk__node_status_t *node, enum pcmk_ipc_server dest); #endif // SUPPORT_COROSYNC #ifdef __cplusplus } #endif #endif // PCMK__CLUSTER_CRMCLUSTER_PRIVATE__H diff --git a/lib/cluster/election.c b/lib/cluster/election.c index ce4d822b7b..7474d0a385 100644 --- a/lib/cluster/election.c +++ b/lib/cluster/election.c @@ -1,738 +1,738 @@ /* * Copyright 2004-2024 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include +#include +#include #include -#include #include #include -#include +#include "crmcluster_private.h" #define STORM_INTERVAL 2 /* in seconds */ -struct election_s { - enum pcmk_ipc_server server; // For message type - enum election_result state; - guint count; // How many times local node has voted - char *name; // Descriptive name for this election - char *uname; // Local node's name - GSourceFunc cb; // Function to call if election is won +struct pcmk__election { + enum election_result state; // Current state of election + guint count; // How many times local node has voted + void (*cb)(pcmk_cluster_t *); // Function to call if election is won GHashTable *voted; // Key = node name, value = how node voted mainloop_timer_t *timeout; // When to abort if all votes not received int election_wins; // Track wins, for storm detection bool wrote_blackbox; // Write a storm blackbox at most once time_t expires; // When storm detection period ends time_t last_election_loss; // When dampening period ends }; static void -election_complete(election_t *e) +election_complete(pcmk_cluster_t *cluster) { - e->state = election_won; - if (e->cb != NULL) { - e->cb(e); + CRM_ASSERT((cluster != NULL) && (cluster->priv->election != NULL)); + cluster->priv->election->state = election_won; + if (cluster->priv->election->cb != NULL) { + cluster->priv->election->cb(cluster); } - election_reset(e); + election_reset(cluster); } static gboolean election_timer_cb(gpointer user_data) { - election_t *e = user_data; + pcmk_cluster_t *cluster = user_data; - crm_info("%s timed out, declaring local node as winner", e->name); - election_complete(e); + crm_info("Declaring local node as winner after election timed out"); + election_complete(cluster); return FALSE; } /*! + * \internal * \brief Get current state of an election * - * \param[in] e Election object + * \param[in] cluster Cluster with election * * \return Current state of \e */ enum election_result -election_state(const election_t *e) +election_state(const pcmk_cluster_t *cluster) { - return (e == NULL)? election_error : e->state; + if ((cluster == NULL) || (cluster->priv->election == NULL)) { + return election_error; + } + return cluster->priv->election->state; } +/* The local node will be declared the winner if missing votes are not received + * within this time. The value is chosen to be the same as the default for the + * election-timeout cluster option. + */ +#define ELECTION_TIMEOUT_MS 120000 + /*! - * \brief Create a new election object + * \internal + * \brief Track election state in a cluster * - * Every node that wishes to participate in an election must create an election - * object. Typically, this should be done once, at start-up. A caller should - * only create a single election object. + * Every node that wishes to participate in an election must initialize the + * election once, typically at start-up. * - * \param[in] server Server to use for message type in election messages - * \param[in] name Label for election (for logging) - * \param[in] uname Local node's name - * \param[in] period_ms How long to wait for all peers to vote + * \param[in] cluster Cluster that election is for * \param[in] cb Function to call if local node wins election - * - * \return Newly allocated election object on success, NULL on error - * \note The caller is responsible for freeing the returned value using - * election_fini(). */ -election_t * -election_init(enum pcmk_ipc_server server, const char *name, const char *uname, - guint period_ms, GSourceFunc cb) +void +election_init(pcmk_cluster_t *cluster, void (*cb)(pcmk_cluster_t *)) { - election_t *e = NULL; - - static guint count = 0; - - CRM_CHECK(uname != NULL, return NULL); + const char *name = pcmk__s(crm_system_name, "election"); - e = calloc(1, sizeof(election_t)); - if (e == NULL) { - crm_perror(LOG_CRIT, "Cannot create election"); - return NULL; - } - - e->server = server; - e->uname = strdup(uname); - if (e->uname == NULL) { - crm_perror(LOG_CRIT, "Cannot create election"); - free(e); - return NULL; - } + CRM_CHECK(cluster->priv->election == NULL, return); - e->name = name? crm_strdup_printf("election-%s", name) - : crm_strdup_printf("election-%u", count++); - e->cb = cb; - e->timeout = mainloop_timer_add(e->name, period_ms, FALSE, - election_timer_cb, e); - crm_trace("Created %s", e->name); - return e; + cluster->priv->election = pcmk__assert_alloc(1, sizeof(pcmk__election_t)); + cluster->priv->election->cb = cb; + cluster->priv->election->timeout = mainloop_timer_add(name, + ELECTION_TIMEOUT_MS, + FALSE, + election_timer_cb, + cluster); } /*! + * \internal * \brief Disregard any previous vote by specified peer * * This discards any recorded vote from a specified peer. Election users should * call this whenever a voting peer becomes inactive. * - * \param[in,out] e Election object - * \param[in] uname Name of peer to disregard + * \param[in,out] cluster Cluster with election + * \param[in] uname Name of peer to disregard */ void -election_remove(election_t *e, const char *uname) +election_remove(pcmk_cluster_t *cluster, const char *uname) { - if ((e != NULL) && (uname != NULL) && (e->voted != NULL)) { - crm_trace("Discarding %s (no-)vote from lost peer %s", e->name, uname); - g_hash_table_remove(e->voted, uname); + if ((cluster != NULL) && (cluster->priv->election != NULL) + && (uname != NULL) && (cluster->priv->election->voted != NULL)) { + crm_trace("Discarding (no-)vote from lost peer %s", uname); + g_hash_table_remove(cluster->priv->election->voted, uname); } } /*! + * \internal * \brief Stop election timer and disregard all votes * - * \param[in,out] e Election object + * \param[in,out] cluster Cluster with election */ void -election_reset(election_t *e) +election_reset(pcmk_cluster_t *cluster) { - if (e != NULL) { - crm_trace("Resetting election %s", e->name); - mainloop_timer_stop(e->timeout); - if (e->voted) { - crm_trace("Destroying voted cache with %d members", g_hash_table_size(e->voted)); - g_hash_table_destroy(e->voted); - e->voted = NULL; + if ((cluster != NULL) && (cluster->priv->election != NULL)) { + crm_trace("Resetting election"); + mainloop_timer_stop(cluster->priv->election->timeout); + if (cluster->priv->election->voted != NULL) { + g_hash_table_destroy(cluster->priv->election->voted); + cluster->priv->election->voted = NULL; } } } /*! + * \internal * \brief Free an election object * * Free all memory associated with an election object, stopping its * election timer (if running). * - * \param[in,out] e Election object + * \param[in,out] cluster Cluster with election */ void -election_fini(election_t *e) +election_fini(pcmk_cluster_t *cluster) { - if (e != NULL) { - election_reset(e); - crm_trace("Destroying %s", e->name); - mainloop_timer_del(e->timeout); - free(e->uname); - free(e->name); - free(e); + if ((cluster != NULL) && (cluster->priv->election != NULL)) { + election_reset(cluster); + crm_trace("Destroying election"); + mainloop_timer_del(cluster->priv->election->timeout); + free(cluster->priv->election); + cluster->priv->election = NULL; } } static void -election_timeout_start(election_t *e) +election_timeout_start(pcmk_cluster_t *cluster) { - if (e != NULL) { - mainloop_timer_start(e->timeout); - } + mainloop_timer_start(cluster->priv->election->timeout); } /*! + * \internal * \brief Stop an election's timer, if running * - * \param[in,out] e Election object + * \param[in,out] cluster Cluster with election */ void -election_timeout_stop(election_t *e) +election_timeout_stop(pcmk_cluster_t *cluster) { - if (e != NULL) { - mainloop_timer_stop(e->timeout); + if ((cluster != NULL) && (cluster->priv->election != NULL)) { + mainloop_timer_stop(cluster->priv->election->timeout); } } /*! + * \internal * \brief Change an election's timeout (restarting timer if running) * - * \param[in,out] e Election object - * \param[in] period New timeout + * \param[in,out] cluster Cluster with election + * \param[in] period New timeout */ void -election_timeout_set_period(election_t *e, guint period) +election_timeout_set_period(pcmk_cluster_t *cluster, guint period) { - if (e != NULL) { - mainloop_timer_set_period(e->timeout, period); - } else { - crm_err("No election defined"); - } + CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL), return); + mainloop_timer_set_period(cluster->priv->election->timeout, period); } static int get_uptime(struct timeval *output) { static time_t expires = 0; static struct rusage info; time_t tm_now = time(NULL); if (expires < tm_now) { int rc = 0; info.ru_utime.tv_sec = 0; info.ru_utime.tv_usec = 0; rc = getrusage(RUSAGE_SELF, &info); output->tv_sec = 0; output->tv_usec = 0; if (rc < 0) { crm_perror(LOG_ERR, "Could not calculate the current uptime"); expires = 0; return -1; } crm_debug("Current CPU usage is: %lds, %ldus", (long)info.ru_utime.tv_sec, (long)info.ru_utime.tv_usec); } expires = tm_now + STORM_INTERVAL; /* N seconds after the last _access_ */ output->tv_sec = info.ru_utime.tv_sec; output->tv_usec = info.ru_utime.tv_usec; return 1; } static int compare_age(struct timeval your_age) { struct timeval our_age; get_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */ if (our_age.tv_sec > your_age.tv_sec) { crm_debug("Win: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec); return 1; } else if (our_age.tv_sec < your_age.tv_sec) { crm_debug("Lose: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec); return -1; } else if (our_age.tv_usec > your_age.tv_usec) { crm_debug("Win: %ld.%06ld vs %ld.%06ld (usec)", (long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec); return 1; } else if (our_age.tv_usec < your_age.tv_usec) { crm_debug("Lose: %ld.%06ld vs %ld.%06ld (usec)", (long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec); return -1; } return 0; } /*! + * \internal * \brief Start a new election by offering local node's candidacy * * Broadcast a "vote" election message containing the local node's ID, * (incremented) election counter, and uptime, and start the election timer. * - * \param[in,out] e Election object + * \param[in,out] cluster Cluster with election * * \note Any nodes agreeing to the candidacy will send a "no-vote" reply, and if * all active peers do so, or if the election times out, the local node * wins the election. (If we lose to any peer vote, we will stop the * timer, so a timeout means we did not lose -- either some peer did not * vote, or we did not call election_check() in time.) */ void -election_vote(election_t *e) +election_vote(pcmk_cluster_t *cluster) { struct timeval age; xmlNode *vote = NULL; pcmk__node_status_t *our_node = NULL; const char *message_type = NULL; - if (e == NULL) { - crm_trace("Election vote requested, but no election available"); + CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL), return); + + if (cluster->priv->node_name == NULL) { + crm_err("Cannot start an election: Local node name unknown"); return; } - our_node = pcmk__get_node(0, e->uname, NULL, + our_node = pcmk__get_node(0, cluster->priv->node_name, NULL, pcmk__node_search_cluster_member); if (!pcmk__cluster_is_node_active(our_node)) { - crm_trace("Cannot vote in %s yet: local node not connected to cluster", - e->name); + crm_trace("Cannot vote yet: local node not connected to cluster"); return; } - election_reset(e); - e->state = election_in_progress; - message_type = pcmk__server_message_type(e->server); + election_reset(cluster); + cluster->priv->election->state = election_in_progress; + message_type = pcmk__server_message_type(cluster->priv->server); /* @COMPAT We use message_type as the sender and recipient system for * backward compatibility (see T566). */ - vote = pcmk__new_request(e->server, message_type, NULL, - message_type, CRM_OP_VOTE, NULL); + vote = pcmk__new_request(cluster->priv->server, message_type, + NULL, message_type, CRM_OP_VOTE, NULL); - e->count++; + cluster->priv->election->count++; crm_xml_add(vote, PCMK__XA_ELECTION_OWNER, our_node->xml_id); - crm_xml_add_int(vote, PCMK__XA_ELECTION_ID, e->count); + crm_xml_add_int(vote, PCMK__XA_ELECTION_ID, cluster->priv->election->count); // Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is actually microseconds get_uptime(&age); crm_xml_add_timeval(vote, PCMK__XA_ELECTION_AGE_SEC, PCMK__XA_ELECTION_AGE_NANO_SEC, &age); - pcmk__cluster_send_message(NULL, e->server, vote); + pcmk__cluster_send_message(NULL, cluster->priv->server, vote); pcmk__xml_free(vote); - crm_debug("Started %s round %d", e->name, e->count); - election_timeout_start(e); + crm_debug("Started election round %u", cluster->priv->election->count); + election_timeout_start(cluster); return; } /*! + * \internal * \brief Check whether local node has won an election * * If all known peers have sent no-vote messages, stop the election timer, set * the election state to won, and call any registered win callback. * - * \param[in,out] e Election object + * \param[in,out] cluster Cluster with election * * \return TRUE if local node has won, FALSE otherwise * \note If all known peers have sent no-vote messages, but the election owner * does not call this function, the election will not be won (and the * callback will not be called) until the election times out. * \note This should be called when election_count_vote() returns * \c election_in_progress. */ bool -election_check(election_t *e) +election_check(pcmk_cluster_t *cluster) { int voted_size = 0; int num_members = 0; - if (e == NULL) { - crm_trace("Election check requested, but no election available"); - return FALSE; - } - if (e->voted == NULL) { - crm_trace("%s check requested, but no votes received yet", e->name); + CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL), + return false); + + if (cluster->priv->election->voted == NULL) { + crm_trace("Election check requested, but no votes received yet"); return FALSE; } - voted_size = g_hash_table_size(e->voted); + voted_size = g_hash_table_size(cluster->priv->election->voted); num_members = pcmk__cluster_num_active_nodes(); /* in the case of #voted > #members, it is better to * wait for the timeout and give the cluster time to * stabilize */ if (voted_size >= num_members) { /* we won and everyone has voted */ - election_timeout_stop(e); + election_timeout_stop(cluster); if (voted_size > num_members) { GHashTableIter gIter; const pcmk__node_status_t *node = NULL; char *key = NULL; - crm_warn("Received too many votes in %s", e->name); + crm_warn("Received too many votes in election"); g_hash_table_iter_init(&gIter, pcmk__peer_cache); while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) & node)) { if (pcmk__cluster_is_node_active(node)) { crm_warn("* expected vote: %s", node->name); } } - g_hash_table_iter_init(&gIter, e->voted); + g_hash_table_iter_init(&gIter, cluster->priv->election->voted); while (g_hash_table_iter_next(&gIter, (gpointer *) & key, NULL)) { crm_warn("* actual vote: %s", key); } } - crm_info("%s won by local node", e->name); - election_complete(e); + crm_info("Election won by local node"); + election_complete(cluster); return TRUE; } else { - crm_debug("%s still waiting on %d of %d votes", - e->name, num_members - voted_size, num_members); + crm_debug("Election still waiting on %d of %d vote%s", + num_members - voted_size, num_members, + pcmk__plural_s(num_members)); } return FALSE; } #define LOSS_DAMPEN 2 /* in seconds */ struct vote { const char *op; const char *from; const char *version; const char *election_owner; int election_id; struct timeval age; }; /*! + * \internal * \brief Unpack an election message * - * \param[in] e Election object (for logging only) * \param[in] message Election message XML * \param[out] vote Parsed fields from message * * \return TRUE if election message and election are valid, FALSE otherwise * \note The parsed struct's pointer members are valid only for the lifetime of * the message argument. */ static bool -parse_election_message(const election_t *e, const xmlNode *message, - struct vote *vote) +parse_election_message(const xmlNode *message, struct vote *vote) { CRM_CHECK(message && vote, return FALSE); vote->election_id = -1; vote->age.tv_sec = -1; vote->age.tv_usec = -1; vote->op = crm_element_value(message, PCMK__XA_CRM_TASK); vote->from = crm_element_value(message, PCMK__XA_SRC); vote->version = crm_element_value(message, PCMK_XA_VERSION); vote->election_owner = crm_element_value(message, PCMK__XA_ELECTION_OWNER); crm_element_value_int(message, PCMK__XA_ELECTION_ID, &(vote->election_id)); if ((vote->op == NULL) || (vote->from == NULL) || (vote->version == NULL) || (vote->election_owner == NULL) || (vote->election_id < 0)) { - crm_warn("Invalid %s message from %s in %s ", - (vote->op? vote->op : "election"), - (vote->from? vote->from : "unspecified node"), - (e? e->name : "election")); + crm_warn("Invalid %s message from %s", + pcmk__s(vote->op, "election"), + pcmk__s(vote->from, "unspecified node")); + crm_log_xml_trace(message, "bad-vote"); return FALSE; } // Op-specific validation if (pcmk__str_eq(vote->op, CRM_OP_VOTE, pcmk__str_none)) { /* Only vote ops have uptime. Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is in microseconds. */ crm_element_value_timeval(message, PCMK__XA_ELECTION_AGE_SEC, PCMK__XA_ELECTION_AGE_NANO_SEC, &(vote->age)); if ((vote->age.tv_sec < 0) || (vote->age.tv_usec < 0)) { - crm_warn("Cannot count %s %s from %s because it is missing uptime", - (e? e->name : "election"), vote->op, vote->from); + crm_warn("Cannot count election %s from %s " + "because it is missing uptime", vote->op, vote->from); return FALSE; } } else if (!pcmk__str_eq(vote->op, CRM_OP_NOVOTE, pcmk__str_none)) { - crm_info("Cannot process %s message from %s because %s is not a known election op", - (e? e->name : "election"), vote->from, vote->op); - return FALSE; - } - - // Election validation - - if (e == NULL) { - crm_info("Cannot count %s from %s because no election available", - vote->op, vote->from); + crm_info("Cannot process election message from %s " + "because %s is not a known election op", vote->from, vote->op); return FALSE; } /* If the membership cache is NULL, we REALLY shouldn't be voting -- * the question is how we managed to get here. */ if (pcmk__peer_cache == NULL) { - crm_info("Cannot count %s %s from %s because no peer information available", - e->name, vote->op, vote->from); + crm_info("Cannot count election %s from %s " + "because no peer information available", vote->op, vote->from); return FALSE; } return TRUE; } static void -record_vote(election_t *e, struct vote *vote) +record_vote(pcmk_cluster_t *cluster, struct vote *vote) { - CRM_ASSERT(e && vote && vote->from && vote->op); + CRM_ASSERT((vote->from != NULL) && (vote->op != NULL)); - if (e->voted == NULL) { - e->voted = pcmk__strkey_table(free, free); + if (cluster->priv->election->voted == NULL) { + cluster->priv->election->voted = pcmk__strkey_table(free, free); } - pcmk__insert_dup(e->voted, vote->from, vote->op); + pcmk__insert_dup(cluster->priv->election->voted, vote->from, vote->op); } static void -send_no_vote(election_t *e, pcmk__node_status_t *peer, struct vote *vote) +send_no_vote(pcmk_cluster_t *cluster, pcmk__node_status_t *peer, + struct vote *vote) { - const char *message_type = pcmk__server_message_type(e->server); - xmlNode *novote = pcmk__new_request(e->server, message_type, - vote->from, message_type, - CRM_OP_NOVOTE, NULL); + const char *message_type = NULL; + xmlNode *novote = NULL; + message_type = pcmk__server_message_type(cluster->priv->server); + novote = pcmk__new_request(cluster->priv->server, message_type, + vote->from, message_type, CRM_OP_NOVOTE, NULL); crm_xml_add(novote, PCMK__XA_ELECTION_OWNER, vote->election_owner); crm_xml_add_int(novote, PCMK__XA_ELECTION_ID, vote->election_id); - pcmk__cluster_send_message(peer, e->server, novote); + pcmk__cluster_send_message(peer, cluster->priv->server, novote); pcmk__xml_free(novote); } /*! + * \internal * \brief Process an election message (vote or no-vote) from a peer * - * \param[in,out] e Election object + * \param[in,out] cluster Cluster with election * \param[in] message Election message XML from peer * \param[in] can_win Whether local node is eligible to win * * \return Election state after new vote is considered * \note If the peer message is a vote, and we prefer the peer to win, this will * send a no-vote reply to the peer. * \note The situations "we lost to this vote" from "this is a late no-vote * after we've already lost" both return election_lost. If a caller needs * to distinguish them, it should save the current state before calling * this function, and then compare the result. */ enum election_result -election_count_vote(election_t *e, const xmlNode *message, bool can_win) +election_count_vote(pcmk_cluster_t *cluster, const xmlNode *message, + bool can_win) { int log_level = LOG_INFO; gboolean done = FALSE; gboolean we_lose = FALSE; const char *reason = "unknown"; bool we_are_owner = FALSE; pcmk__node_status_t *our_node = NULL; pcmk__node_status_t *your_node = NULL; time_t tm_now = time(NULL); struct vote vote; - CRM_CHECK(message != NULL, return election_error); - if (parse_election_message(e, message, &vote) == FALSE) { + CRM_CHECK((cluster != NULL) && (cluster->priv->election != NULL) + && (message != NULL) && (cluster->priv->node_name != NULL), + return election_error); + + if (!parse_election_message(message, &vote)) { return election_error; } your_node = pcmk__get_node(0, vote.from, NULL, pcmk__node_search_cluster_member); - our_node = pcmk__get_node(0, e->uname, NULL, + our_node = pcmk__get_node(0, cluster->priv->node_name, NULL, pcmk__node_search_cluster_member); we_are_owner = (our_node != NULL) && pcmk__str_eq(our_node->xml_id, vote.election_owner, pcmk__str_none); if (!can_win) { reason = "Not eligible"; we_lose = TRUE; } else if (!pcmk__cluster_is_node_active(our_node)) { reason = "We are not part of the cluster"; log_level = LOG_ERR; we_lose = TRUE; - } else if (we_are_owner && (vote.election_id != e->count)) { + } else if (we_are_owner + && (vote.election_id != cluster->priv->election->count)) { log_level = LOG_TRACE; reason = "Superseded"; done = TRUE; } else if (!pcmk__cluster_is_node_active(your_node)) { /* Possibly we cached the message in the FSA queue at a point that it wasn't */ reason = "Peer is not part of our cluster"; log_level = LOG_WARNING; done = TRUE; } else if (pcmk__str_eq(vote.op, CRM_OP_NOVOTE, pcmk__str_none) - || pcmk__str_eq(vote.from, e->uname, pcmk__str_none)) { + || pcmk__str_eq(vote.from, cluster->priv->node_name, + pcmk__str_casei)) { /* Receiving our own broadcast vote, or a no-vote from peer, is a vote * for us to win */ if (!we_are_owner) { - crm_warn("Cannot count %s round %d %s from %s because we are not election owner (%s)", - e->name, vote.election_id, vote.op, vote.from, + crm_warn("Cannot count election round %d %s from %s " + "because we did not start election (node ID %s did)", + vote.election_id, vote.op, vote.from, vote.election_owner); return election_error; } - if (e->state != election_in_progress) { + if (cluster->priv->election->state != election_in_progress) { // Should only happen if we already lost - crm_debug("Not counting %s round %d %s from %s because no election in progress", - e->name, vote.election_id, vote.op, vote.from); - return e->state; + crm_debug("Not counting election round %d %s from %s " + "because no election in progress", + vote.election_id, vote.op, vote.from); + return cluster->priv->election->state; } - record_vote(e, &vote); + record_vote(cluster, &vote); reason = "Recorded"; done = TRUE; } else { // A peer vote requires a comparison to determine which node is better int age_result = compare_age(vote.age); int version_result = compare_version(vote.version, CRM_FEATURE_SET); if (version_result < 0) { reason = "Version"; we_lose = TRUE; } else if (version_result > 0) { reason = "Version"; } else if (age_result < 0) { reason = "Uptime"; we_lose = TRUE; } else if (age_result > 0) { reason = "Uptime"; - } else if (strcasecmp(e->uname, vote.from) > 0) { + } else if (strcasecmp(cluster->priv->node_name, vote.from) > 0) { reason = "Host name"; we_lose = TRUE; } else { reason = "Host name"; } } - if (e->expires < tm_now) { - e->election_wins = 0; - e->expires = tm_now + STORM_INTERVAL; + if (cluster->priv->election->expires < tm_now) { + cluster->priv->election->election_wins = 0; + cluster->priv->election->expires = tm_now + STORM_INTERVAL; } else if (done == FALSE && we_lose == FALSE) { int peers = 1 + g_hash_table_size(pcmk__peer_cache); /* If every node has to vote down every other node, thats N*(N-1) total elections * Allow some leeway before _really_ complaining */ - e->election_wins++; - if (e->election_wins > (peers * peers)) { - crm_warn("%s election storm detected: %d wins in %d seconds", - e->name, e->election_wins, STORM_INTERVAL); - e->election_wins = 0; - e->expires = tm_now + STORM_INTERVAL; - if (e->wrote_blackbox == FALSE) { + cluster->priv->election->election_wins++; + if (cluster->priv->election->election_wins > (peers * peers)) { + crm_warn("Election storm detected: %d wins in %d seconds", + cluster->priv->election->election_wins, STORM_INTERVAL); + cluster->priv->election->election_wins = 0; + cluster->priv->election->expires = tm_now + STORM_INTERVAL; + if (!(cluster->priv->election->wrote_blackbox)) { /* It's questionable whether a black box (from every node in the * cluster) would be truly helpful in diagnosing an election * storm. It's also highly doubtful a production environment * would get multiple election storms from distinct causes, so * saving one blackbox per process lifetime should be * sufficient. Alternatives would be to save a timestamp of the * last blackbox write instead of a boolean, and write a new one * if some amount of time has passed; or to save a storm count, * write a blackbox on every Nth occurrence. */ crm_write_blackbox(0, NULL); - e->wrote_blackbox = TRUE; + cluster->priv->election->wrote_blackbox = true; } } } if (done) { do_crm_log(log_level + 1, - "Processed %s round %d %s (current round %d) from %s (%s)", - e->name, vote.election_id, vote.op, e->count, vote.from, - reason); - return e->state; + "Processed election round %u %s (current round %d) " + "from %s (%s)", + vote.election_id, vote.op, cluster->priv->election->count, + vote.from, reason); + return cluster->priv->election->state; } else if (we_lose == FALSE) { /* We track the time of the last election loss to implement an election * dampening period, reducing the likelihood of an election storm. If * this node has lost within the dampening period, don't start a new * election, even if we win against a peer's vote -- the peer we lost to * should win again. * * @TODO This has a problem case: if an election winner immediately * leaves the cluster, and a new election is immediately called, all * nodes could lose, with no new winner elected. The ideal solution * would be to tie the election structure with the peer caches, which * would allow us to clear the dampening when the previous winner * leaves (and would allow other improvements as well). */ - if ((e->last_election_loss == 0) - || ((tm_now - e->last_election_loss) > (time_t) LOSS_DAMPEN)) { - - do_crm_log(log_level, "%s round %d (owner node ID %s) pass: %s from %s (%s)", - e->name, vote.election_id, vote.election_owner, vote.op, + if ((cluster->priv->election->last_election_loss == 0) + || ((tm_now - cluster->priv->election->last_election_loss) + > (time_t) LOSS_DAMPEN)) { + + do_crm_log(log_level, + "Election round %d (started by node ID %s) pass: " + "%s from %s (%s)", + vote.election_id, vote.election_owner, vote.op, vote.from, reason); - e->last_election_loss = 0; - election_timeout_stop(e); + cluster->priv->election->last_election_loss = 0; + election_timeout_stop(cluster); /* Start a new election by voting down this, and other, peers */ - e->state = election_start; - return e->state; + cluster->priv->election->state = election_start; + return cluster->priv->election->state; } else { - char *loss_time = ctime(&e->last_election_loss); + char *loss_time = NULL; + loss_time = ctime(&(cluster->priv->election->last_election_loss)); if (loss_time) { // Show only HH:MM:SS loss_time += 11; loss_time[8] = '\0'; } - crm_info("Ignoring %s round %d (owner node ID %s) pass vs %s because we lost less than %ds ago at %s", - e->name, vote.election_id, vote.election_owner, vote.from, + crm_info("Ignoring election round %d (started by node ID %s) pass " + "vs %s because we lost less than %ds ago at %s", + vote.election_id, vote.election_owner, vote.from, LOSS_DAMPEN, (loss_time? loss_time : "unknown")); } } - e->last_election_loss = tm_now; + cluster->priv->election->last_election_loss = tm_now; - do_crm_log(log_level, "%s round %d (owner node ID %s) lost: %s from %s (%s)", - e->name, vote.election_id, vote.election_owner, vote.op, + do_crm_log(log_level, + "Election round %d (started by node ID %s) lost: " + "%s from %s (%s)", + vote.election_id, vote.election_owner, vote.op, vote.from, reason); - election_reset(e); - send_no_vote(e, your_node, &vote); - e->state = election_lost; - return e->state; + election_reset(cluster); + send_no_vote(cluster, your_node, &vote); + cluster->priv->election->state = election_lost; + return cluster->priv->election->state; } /*! + * \internal * \brief Reset any election dampening currently in effect * - * \param[in,out] e Election object to clear + * \param[in,out] cluster Cluster with election */ void -election_clear_dampening(election_t *e) +election_clear_dampening(pcmk_cluster_t *cluster) { - e->last_election_loss = 0; + if ((cluster != NULL) && (cluster->priv->election != NULL)) { + cluster->priv->election->last_election_loss = 0; + } }