diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c
index 530e786374..ac33096542 100644
--- a/daemons/controld/controld_callbacks.c
+++ b/daemons/controld/controld_callbacks.c
@@ -1,396 +1,397 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <inttypes.h> // PRIu32
#include <stdbool.h> // bool
#include <stdio.h> // NULL
#include <sys/param.h>
#include <string.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/cluster.h>
#include <crm/cib.h>
#include <pacemaker-controld.h>
/* From join_dc... */
extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
void
crmd_ha_msg_filter(xmlNode * msg)
{
if (AM_I_DC) {
const char *sys_from = crm_element_value(msg, PCMK__XA_CRM_SYS_FROM);
if (pcmk__str_eq(sys_from, CRM_SYSTEM_DC, pcmk__str_casei)) {
const char *from = crm_element_value(msg, PCMK__XA_SRC);
if (!pcmk__str_eq(from, controld_globals.our_nodename,
pcmk__str_casei)) {
int level = LOG_INFO;
const char *op = crm_element_value(msg, PCMK__XA_CRM_TASK);
/* make sure the election happens NOW */
if (controld_globals.fsa_state != S_ELECTION) {
ha_msg_input_t new_input;
level = LOG_WARNING;
new_input.msg = msg;
register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
__func__);
}
do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
goto done;
}
}
} else {
const char *sys_to = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
if (pcmk__str_eq(sys_to, CRM_SYSTEM_DC, pcmk__str_casei)) {
return;
}
}
/* crm_log_xml_trace(msg, "HA[inbound]"); */
route_message(C_HA_MESSAGE, msg);
done:
controld_trigger_fsa();
}
/*!
* \internal
* \brief Check whether a node is online
*
* \param[in] node Node to check
*
* \retval -1 if completely dead
* \retval 0 if partially alive
* \retval 1 if completely alive
*/
static int
node_alive(const pcmk__node_status_t *node)
{
if (pcmk_is_set(node->flags, pcmk__node_status_remote)) {
// Pacemaker Remote nodes can't be partially alive
if (pcmk__str_eq(node->state, PCMK_VALUE_MEMBER, pcmk__str_none)) {
return 1;
}
return -1;
} else if (pcmk__cluster_is_node_active(node)) {
// Completely up cluster node: both cluster member and peer
return 1;
} else if (!pcmk_is_set(node->processes, crm_get_cluster_proc())
&& !pcmk__str_eq(node->state, PCMK_VALUE_MEMBER,
pcmk__str_none)) {
// Completely down cluster node: neither cluster member nor peer
return -1;
}
// Partially up cluster node: only cluster member or only peer
return 0;
}
#define state_text(state) ((state)? (const char *)(state) : "in unknown state")
void
peer_update_callback(enum pcmk__node_update type, pcmk__node_status_t *node,
const void *data)
{
uint32_t old = 0;
bool appeared = FALSE;
bool is_remote = pcmk_is_set(node->flags, pcmk__node_status_remote);
controld_node_pending_timer(node);
/* The controller waits to receive some information from the membership
* layer before declaring itself operational. If this is being called for a
* cluster node, indicate that we have it.
*/
if (!is_remote) {
controld_set_fsa_input_flags(R_PEER_DATA);
}
if ((type == pcmk__node_update_processes)
&& pcmk_is_set(node->processes, crm_get_cluster_proc())
&& !AM_I_DC
&& !is_remote) {
/*
* This is a hack until we can send to a nodeid and/or we fix node name lookups
* These messages are ignored in crmd_ha_msg_filter()
*/
- xmlNode *query = create_request(CRM_OP_HELLO, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+ xmlNode *query = create_request(CRM_OP_HELLO, NULL, NULL,
+ CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD);
crm_debug("Sending hello to node %" PRIu32 " so that it learns our "
"node name",
node->cluster_layer_id);
pcmk__cluster_send_message(node, pcmk_ipc_controld, query);
pcmk__xml_free(query);
}
if (node->name == NULL) {
return;
}
switch (type) {
case pcmk__node_update_name:
/* If we've never seen the node, then it also won't be in the status section */
crm_info("%s node %s is now %s",
(is_remote? "Remote" : "Cluster"),
node->name, state_text(node->state));
return;
case pcmk__node_update_state:
/* This callback should not be called unless the state actually
* changed, but here's a failsafe just in case.
*/
CRM_CHECK(!pcmk__str_eq(data, node->state, pcmk__str_casei),
return);
crm_info("%s node %s is now %s (was %s)",
(is_remote? "Remote" : "Cluster"),
node->name, state_text(node->state), state_text(data));
if (pcmk__str_eq(PCMK_VALUE_MEMBER, node->state, pcmk__str_none)) {
appeared = TRUE;
if (!is_remote) {
remove_stonith_cleanup(node->name);
}
} else {
controld_remove_failed_sync_node(node->name);
controld_remove_voter(node->name);
}
crmd_alert_node_event(node);
break;
case pcmk__node_update_processes:
CRM_CHECK(data != NULL, return);
old = *(const uint32_t *)data;
appeared = pcmk_is_set(node->processes, crm_get_cluster_proc());
{
const char *dc_s = controld_globals.dc_name;
if ((dc_s == NULL) && AM_I_DC) {
dc_s = PCMK_VALUE_TRUE;
}
crm_info("Node %s is %s a peer " QB_XS
" DC=%s old=%#07x new=%#07x",
node->name, (appeared? "now" : "no longer"),
pcmk__s(dc_s, "<none>"), old, node->processes);
}
if (!pcmk_is_set((node->processes ^ old), crm_get_cluster_proc())) {
/* Peer status did not change. This should not be possible,
* since we don't track process flags other than peer status.
*/
crm_trace("Process flag %#7x did not change from %#7x to %#7x",
crm_get_cluster_proc(), old, node->processes);
return;
}
if (!appeared) {
node->peer_lost = time(NULL);
controld_remove_failed_sync_node(node->name);
controld_remove_voter(node->name);
}
if (!pcmk_is_set(controld_globals.fsa_input_register,
R_CIB_CONNECTED)) {
crm_trace("Ignoring peer status change because not connected to CIB");
return;
} else if (controld_globals.fsa_state == S_STOPPING) {
crm_trace("Ignoring peer status change because stopping");
return;
}
if (!appeared
&& pcmk__str_eq(node->name, controld_globals.our_nodename,
pcmk__str_casei)) {
/* Did we get evicted? */
crm_notice("Our peer connection failed");
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
} else if (pcmk__str_eq(node->name, controld_globals.dc_name,
pcmk__str_casei)
&& !pcmk__cluster_is_node_active(node)) {
/* The DC has left, so delete its transient attributes and
* trigger a new election.
*
* A DC sends its shutdown request to all peers, who update the
* DC's expected state to down. This avoids fencing upon
* deletion of its transient attributes.
*/
crm_notice("Our peer on the DC (%s) is dead",
controld_globals.dc_name);
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
controld_delete_node_state(node->name, controld_section_attrs,
cib_none);
} else if (AM_I_DC
|| pcmk_is_set(controld_globals.flags, controld_dc_left)
|| (controld_globals.dc_name == NULL)) {
/* This only needs to be done once, so normally the DC should do
* it. However if there is no DC, every node must do it, since
* there is no other way to ensure some one node does it.
*/
if (appeared) {
te_trigger_stonith_history_sync(FALSE);
} else {
controld_delete_node_state(node->name,
controld_section_attrs,
cib_none);
}
}
break;
}
if (AM_I_DC) {
xmlNode *update = NULL;
int flags = node_update_peer;
int alive = node_alive(node);
pcmk__graph_action_t *down = match_down_event(node->xml_id);
crm_trace("Alive=%d, appeared=%d, down=%d",
alive, appeared, (down? down->id : -1));
if (appeared && (alive > 0) && !is_remote) {
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
if (down) {
const char *task = crm_element_value(down->xml, PCMK_XA_OPERATION);
if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) {
const bool confirmed =
pcmk_is_set(down->flags, pcmk__graph_action_confirmed);
/* tengine_stonith_callback() confirms fence actions */
crm_trace("Updating CIB %s fencer reported fencing of %s complete",
(confirmed? "after" : "before"), node->name);
} else if (!appeared && pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN,
pcmk__str_casei)) {
// Shutdown actions are immediately confirmed (i.e. no_wait)
if (!is_remote) {
flags |= node_update_join | node_update_expected;
crmd_peer_down(node, FALSE);
check_join_state(controld_globals.fsa_state, __func__);
}
if (alive >= 0) {
crm_info("%s of peer %s is in progress " QB_XS " action=%d",
task, node->name, down->id);
} else {
crm_notice("%s of peer %s is complete " QB_XS " action=%d",
task, node->name, down->id);
pcmk__update_graph(controld_globals.transition_graph, down);
trigger_graph();
}
} else {
const char *liveness = "alive";
if (alive == 0) {
liveness = "partially alive";
} else if (alive < 0) {
liveness = "dead";
}
crm_trace("Node %s is %s, was expected to %s (op %d)",
node->name, liveness, task, down->id);
}
} else if (appeared == FALSE) {
if ((controld_globals.transition_graph == NULL)
|| (controld_globals.transition_graph->id == -1)) {
crm_info("Stonith/shutdown of node %s is unknown to the "
"current DC", node->name);
} else {
crm_warn("Stonith/shutdown of node %s was not expected",
node->name);
}
if (!is_remote) {
crm_update_peer_join(__func__, node, controld_join_none);
check_join_state(controld_globals.fsa_state, __func__);
}
abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
"Node failure", NULL);
fail_incompletable_actions(controld_globals.transition_graph,
node->xml_id);
} else {
crm_trace("Node %s came up, was not expected to be down",
node->name);
}
if (is_remote) {
/* A pacemaker_remote node won't have its cluster status updated
* in the CIB by membership-layer callbacks, so do it here.
*/
flags |= node_update_cluster;
/* Trigger resource placement on newly integrated nodes */
if (appeared) {
abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
"Pacemaker Remote node integrated", NULL);
}
}
if (!appeared && (type == pcmk__node_update_processes)
&& (node->when_member > 1)) {
/* The node left CPG but is still a cluster member. Set its
* membership time to 1 to record it in the cluster state as a
* boolean, so we don't fence it due to
* PCMK_OPT_NODE_PENDING_TIMEOUT.
*/
node->when_member = 1;
flags |= node_update_cluster;
controld_node_pending_timer(node);
}
/* Update the CIB node state */
update = create_node_state_update(node, flags, NULL, __func__);
if (update == NULL) {
crm_debug("Node state update not yet possible for %s", node->name);
} else {
fsa_cib_anon_update(PCMK_XE_STATUS, update);
}
pcmk__xml_free(update);
}
controld_trigger_fsa();
}
gboolean
crm_fsa_trigger(gpointer user_data)
{
crm_trace("Invoked (queue len: %d)",
g_list_length(controld_globals.fsa_message_queue));
s_crmd_fsa(C_FSA_INTERNAL);
crm_trace("Exited (queue len: %d)",
g_list_length(controld_globals.fsa_message_queue));
return TRUE;
}
diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
index 43ab797abf..631bf56387 100644
--- a/daemons/controld/controld_control.c
+++ b/daemons/controld/controld_control.c
@@ -1,697 +1,698 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/pengine/rules.h>
#include <crm/cluster/internal.h>
#include <crm/cluster/election_internal.h>
#include <crm/common/ipc_internal.h>
#include <pacemaker-controld.h>
static qb_ipcs_service_t *ipcs = NULL;
static crm_trigger_t *config_read_trigger = NULL;
#if SUPPORT_COROSYNC
extern gboolean crm_connect_corosync(pcmk_cluster_t *cluster);
#endif
static void crm_shutdown(int nsig);
static gboolean crm_read_options(gpointer user_data);
/* A_HA_CONNECT */
void
do_ha_control(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
gboolean registered = FALSE;
static pcmk_cluster_t *cluster = NULL;
if (cluster == NULL) {
cluster = pcmk_cluster_new();
}
if (action & A_HA_DISCONNECT) {
pcmk_cluster_disconnect(cluster);
crm_info("Disconnected from the cluster");
controld_set_fsa_input_flags(R_HA_DISCONNECTED);
}
if (action & A_HA_CONNECT) {
pcmk__cluster_set_status_callback(&peer_update_callback);
pcmk__cluster_set_autoreap(false);
#if SUPPORT_COROSYNC
if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) {
registered = crm_connect_corosync(cluster);
}
#endif // SUPPORT_COROSYNC
if (registered) {
pcmk__node_status_t *node =
pcmk__get_node(cluster->priv->node_id, cluster->priv->node_name,
NULL, pcmk__node_search_cluster_member);
controld_election_init(cluster->priv->node_name);
controld_globals.our_nodename = cluster->priv->node_name;
free(controld_globals.our_uuid);
controld_globals.our_uuid =
pcmk__str_copy(pcmk__cluster_node_uuid(node));
if (controld_globals.our_uuid == NULL) {
crm_err("Could not obtain local uuid");
registered = FALSE;
}
}
if (!registered) {
controld_set_fsa_input_flags(R_HA_DISCONNECTED);
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
return;
}
populate_cib_nodes(node_update_none, __func__);
controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
crm_info("Connected to the cluster");
}
if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) {
crm_err("Unexpected action %s in %s", fsa_action2string(action),
__func__);
}
}
/* A_SHUTDOWN */
void
do_shutdown(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
/* just in case */
controld_set_fsa_input_flags(R_SHUTDOWN);
controld_disconnect_fencer(FALSE);
}
/* A_SHUTDOWN_REQ */
void
do_shutdown_req(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *msg = NULL;
controld_set_fsa_input_flags(R_SHUTDOWN);
//controld_set_fsa_input_flags(R_STAYDOWN);
crm_info("Sending shutdown request to all peers (DC is %s)",
pcmk__s(controld_globals.dc_name, "not set"));
- msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+ msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD,
+ CRM_SYSTEM_CRMD);
if (!pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg)) {
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
pcmk__xml_free(msg);
}
void
crmd_fast_exit(crm_exit_t exit_code)
{
if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) {
crm_warn("Inhibiting respawn " QB_XS " remapping exit code %d to %d",
exit_code, CRM_EX_FATAL);
exit_code = CRM_EX_FATAL;
} else if ((exit_code == CRM_EX_OK)
&& pcmk_is_set(controld_globals.fsa_input_register,
R_IN_RECOVERY)) {
crm_err("Could not recover from internal error");
exit_code = CRM_EX_ERROR;
}
if (controld_globals.logger_out != NULL) {
controld_globals.logger_out->finish(controld_globals.logger_out,
exit_code, true, NULL);
pcmk__output_free(controld_globals.logger_out);
controld_globals.logger_out = NULL;
}
crm_exit(exit_code);
}
crm_exit_t
crmd_exit(crm_exit_t exit_code)
{
GMainLoop *mloop = controld_globals.mainloop;
static bool in_progress = FALSE;
if (in_progress && (exit_code == CRM_EX_OK)) {
crm_debug("Exit is already in progress");
return exit_code;
} else if(in_progress) {
crm_notice("Error during shutdown process, exiting now with status %d (%s)",
exit_code, crm_exit_str(exit_code));
crm_write_blackbox(SIGTRAP, NULL);
crmd_fast_exit(exit_code);
}
in_progress = TRUE;
crm_trace("Preparing to exit with status %d (%s)",
exit_code, crm_exit_str(exit_code));
/* Suppress secondary errors resulting from us disconnecting everything */
controld_set_fsa_input_flags(R_HA_DISCONNECTED);
/* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */
if(ipcs) {
crm_trace("Closing IPC server");
mainloop_del_ipc_server(ipcs);
ipcs = NULL;
}
controld_close_attrd_ipc();
controld_shutdown_schedulerd_ipc();
controld_disconnect_fencer(TRUE);
if ((exit_code == CRM_EX_OK) && (controld_globals.mainloop == NULL)) {
crm_debug("No mainloop detected");
exit_code = CRM_EX_ERROR;
}
/* On an error, just get out.
*
* Otherwise, make the effort to have mainloop exit gracefully so
* that it (mostly) cleans up after itself and valgrind has less
* to report on - allowing real errors stand out
*/
if (exit_code != CRM_EX_OK) {
crm_notice("Forcing immediate exit with status %d (%s)",
exit_code, crm_exit_str(exit_code));
crm_write_blackbox(SIGTRAP, NULL);
crmd_fast_exit(exit_code);
}
/* Clean up as much memory as possible for valgrind */
for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
iter = iter->next) {
fsa_data_t *fsa_data = (fsa_data_t *) iter->data;
crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
fsa_input2string(fsa_data->fsa_input),
fsa_state2string(controld_globals.fsa_state),
fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
delete_fsa_input(fsa_data);
}
controld_clear_fsa_input_flags(R_MEMBERSHIP);
g_list_free(controld_globals.fsa_message_queue);
controld_globals.fsa_message_queue = NULL;
controld_free_node_pending_timers();
controld_election_fini();
/* Tear down the CIB manager connection, but don't free it yet -- it could
* be used when we drain the mainloop later.
*/
controld_disconnect_cib_manager();
verify_stopped(controld_globals.fsa_state, LOG_WARNING);
controld_clear_fsa_input_flags(R_LRM_CONNECTED);
lrm_state_destroy_all();
mainloop_destroy_trigger(config_read_trigger);
config_read_trigger = NULL;
controld_destroy_fsa_trigger();
controld_destroy_transition_trigger();
pcmk__client_cleanup();
pcmk__cluster_destroy_node_caches();
controld_free_fsa_timers();
te_cleanup_stonith_history_sync(NULL, TRUE);
controld_free_sched_timer();
free(controld_globals.our_nodename);
controld_globals.our_nodename = NULL;
free(controld_globals.our_uuid);
controld_globals.our_uuid = NULL;
free(controld_globals.dc_name);
controld_globals.dc_name = NULL;
free(controld_globals.dc_version);
controld_globals.dc_version = NULL;
free(controld_globals.cluster_name);
controld_globals.cluster_name = NULL;
free(controld_globals.te_uuid);
controld_globals.te_uuid = NULL;
free_max_generation();
controld_destroy_failed_sync_table();
controld_destroy_outside_events_table();
mainloop_destroy_signal(SIGPIPE);
mainloop_destroy_signal(SIGUSR1);
mainloop_destroy_signal(SIGTERM);
mainloop_destroy_signal(SIGTRAP);
/* leave SIGCHLD engaged as we might still want to drain some service-actions */
if (mloop) {
GMainContext *ctx = g_main_loop_get_context(controld_globals.mainloop);
/* Don't re-enter this block */
controld_globals.mainloop = NULL;
/* no signals on final draining anymore */
mainloop_destroy_signal(SIGCHLD);
crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
{
int lpc = 0;
while((g_main_context_pending(ctx) && lpc < 10)) {
lpc++;
crm_trace("Iteration %d", lpc);
g_main_context_dispatch(ctx);
}
}
crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
g_main_loop_quit(mloop);
/* Won't do anything yet, since we're inside it now */
g_main_loop_unref(mloop);
} else {
mainloop_destroy_signal(SIGCHLD);
}
cib_delete(controld_globals.cib_conn);
controld_globals.cib_conn = NULL;
throttle_fini();
/* Graceful */
crm_trace("Done preparing for exit with status %d (%s)",
exit_code, crm_exit_str(exit_code));
return exit_code;
}
/* A_EXIT_0, A_EXIT_1 */
void
do_exit(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_exit_t exit_code = CRM_EX_OK;
if (pcmk_is_set(action, A_EXIT_1)) {
exit_code = CRM_EX_ERROR;
crm_err("Exiting now due to errors");
}
verify_stopped(cur_state, LOG_ERR);
crmd_exit(exit_code);
}
static void sigpipe_ignore(int nsig) { return; }
/* A_STARTUP */
void
do_startup(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_debug("Registering Signal Handlers");
mainloop_add_signal(SIGTERM, crm_shutdown);
mainloop_add_signal(SIGPIPE, sigpipe_ignore);
config_read_trigger = mainloop_add_trigger(G_PRIORITY_HIGH,
crm_read_options, NULL);
controld_init_fsa_trigger();
controld_init_transition_trigger();
crm_debug("Creating CIB manager and executor objects");
controld_globals.cib_conn = cib_new();
lrm_state_init_local();
if (controld_init_fsa_timers() == FALSE) {
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
// \return libqb error code (0 on success, -errno on error)
static int32_t
accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
{
crm_trace("Accepting new IPC client connection");
if (pcmk__new_client(c, uid, gid) == NULL) {
return -ENOMEM;
}
return 0;
}
// \return libqb error code (0 on success, -errno on error)
static int32_t
dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
{
uint32_t id = 0;
uint32_t flags = 0;
pcmk__client_t *client = pcmk__find_client(c);
xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);
if (msg == NULL) {
pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL,
CRM_EX_PROTOCOL);
return 0;
}
pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL,
CRM_EX_INDETERMINATE);
CRM_ASSERT(client->user != NULL);
pcmk__update_acl_user(msg, PCMK__XA_CRM_USER, client->user);
crm_xml_add(msg, PCMK__XA_CRM_SYS_FROM, client->id);
if (controld_authorize_ipc_message(msg, client, NULL)) {
crm_trace("Processing IPC message from client %s",
pcmk__client_name(client));
route_message(C_IPC_MESSAGE, msg);
}
controld_trigger_fsa();
pcmk__xml_free(msg);
return 0;
}
static int32_t
ipc_client_disconnected(qb_ipcs_connection_t *c)
{
pcmk__client_t *client = pcmk__find_client(c);
if (client) {
crm_trace("Disconnecting %sregistered client %s (%p/%p)",
(client->userdata? "" : "un"), pcmk__client_name(client),
c, client);
free(client->userdata);
pcmk__free_client(client);
controld_trigger_fsa();
}
return 0;
}
static void
ipc_connection_destroyed(qb_ipcs_connection_t *c)
{
crm_trace("Connection %p", c);
ipc_client_disconnected(c);
}
/* A_STOP */
void
do_stop(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_trace("Closing IPC server");
mainloop_del_ipc_server(ipcs); ipcs = NULL;
register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
}
/* A_STARTED */
void
do_started(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
static struct qb_ipcs_service_handlers crmd_callbacks = {
.connection_accept = accept_controller_client,
.connection_created = NULL,
.msg_process = dispatch_controller_ipc,
.connection_closed = ipc_client_disconnected,
.connection_destroyed = ipc_connection_destroyed
};
if (cur_state != S_STARTING) {
crm_err("Start cancelled... %s", fsa_state2string(cur_state));
return;
} else if (!pcmk_is_set(controld_globals.fsa_input_register,
R_MEMBERSHIP)) {
crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);
crmd_fsa_stall(TRUE);
return;
} else if (!pcmk_is_set(controld_globals.fsa_input_register,
R_LRM_CONNECTED)) {
crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);
crmd_fsa_stall(TRUE);
return;
} else if (!pcmk_is_set(controld_globals.fsa_input_register,
R_CIB_CONNECTED)) {
crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);
crmd_fsa_stall(TRUE);
return;
} else if (!pcmk_is_set(controld_globals.fsa_input_register,
R_READ_CONFIG)) {
crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);
crmd_fsa_stall(TRUE);
return;
} else if (!pcmk_is_set(controld_globals.fsa_input_register, R_PEER_DATA)) {
crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
crmd_fsa_stall(TRUE);
return;
}
crm_debug("Init server comms");
ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
if (ipcs == NULL) {
crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
} else {
crm_notice("Pacemaker controller successfully started and accepting connections");
}
controld_set_fsa_input_flags(R_ST_REQUIRED);
controld_timer_fencer_connect(GINT_TO_POINTER(TRUE));
controld_clear_fsa_input_flags(R_STARTING);
register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
}
/* A_RECOVER */
void
do_recover(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
controld_set_fsa_input_flags(R_IN_RECOVERY);
crm_warn("Fast-tracking shutdown in response to errors");
register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
}
static void
config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
const char *value = NULL;
GHashTable *config_hash = NULL;
crm_time_t *now = crm_time_new(NULL);
xmlNode *crmconfig = NULL;
xmlNode *alerts = NULL;
if (rc != pcmk_ok) {
fsa_data_t *msg_data = NULL;
crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
crm_err("The cluster is mis-configured - shutting down and staying down");
controld_set_fsa_input_flags(R_STAYDOWN);
}
goto bail;
}
crmconfig = output;
if ((crmconfig != NULL) && !pcmk__xe_is(crmconfig, PCMK_XE_CRM_CONFIG)) {
crmconfig = pcmk__xe_first_child(crmconfig, PCMK_XE_CRM_CONFIG, NULL,
NULL);
}
if (!crmconfig) {
fsa_data_t *msg_data = NULL;
crm_err("Local CIB query for " PCMK_XE_CRM_CONFIG " section failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
goto bail;
}
crm_debug("Call %d : Parsing CIB options", call_id);
config_hash = pcmk__strkey_table(free, free);
pe_unpack_nvpairs(crmconfig, crmconfig, PCMK_XE_CLUSTER_PROPERTY_SET, NULL,
config_hash, PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, FALSE, now,
NULL);
// Validate all options, and use defaults if not already present in hash
pcmk__validate_cluster_options(config_hash);
/* Validate the watchdog timeout in the context of the local node
* environment. If invalid, the controller will exit with a fatal error.
*
* We do this via a wrapper in the controller, so that we call
* pcmk__valid_stonith_watchdog_timeout() only if watchdog fencing is
* enabled for the local node. Otherwise, we may exit unnecessarily.
*
* A validator function in libcrmcommon can't act as such a wrapper, because
* it doesn't have a stonith API connection or the local node name.
*/
value = g_hash_table_lookup(config_hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
controld_verify_stonith_watchdog_timeout(value);
value = g_hash_table_lookup(config_hash, PCMK_OPT_NO_QUORUM_POLICY);
if (pcmk__str_eq(value, PCMK_VALUE_FENCE_LEGACY, pcmk__str_casei)
&& (pcmk__locate_sbd() != 0)) {
controld_set_global_flags(controld_no_quorum_suicide);
}
value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK);
if (crm_is_true(value)) {
controld_set_global_flags(controld_shutdown_lock_enabled);
} else {
controld_clear_global_flags(controld_shutdown_lock_enabled);
}
value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
pcmk_parse_interval_spec(value, &controld_globals.shutdown_lock_limit);
controld_globals.shutdown_lock_limit /= 1000;
value = g_hash_table_lookup(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
pcmk_parse_interval_spec(value, &controld_globals.node_pending_timeout);
controld_globals.node_pending_timeout /= 1000;
value = g_hash_table_lookup(config_hash, PCMK_OPT_CLUSTER_NAME);
pcmk__str_update(&(controld_globals.cluster_name), value);
// Let subcomponents initialize their own static variables
controld_configure_election(config_hash);
controld_configure_fencing(config_hash);
controld_configure_fsa_timers(config_hash);
controld_configure_throttle(config_hash);
alerts = pcmk__xe_first_child(output, PCMK_XE_ALERTS, NULL, NULL);
crmd_unpack_alerts(alerts);
controld_set_fsa_input_flags(R_READ_CONFIG);
controld_trigger_fsa();
g_hash_table_destroy(config_hash);
bail:
crm_time_free(now);
}
/*!
* \internal
* \brief Trigger read and processing of the configuration
*
* \param[in] fn Calling function name
* \param[in] line Line number where call occurred
*/
void
controld_trigger_config_as(const char *fn, int line)
{
if (config_read_trigger != NULL) {
crm_trace("%s:%d - Triggered config processing", fn, line);
mainloop_set_trigger(config_read_trigger);
}
}
gboolean
crm_read_options(gpointer user_data)
{
cib_t *cib_conn = controld_globals.cib_conn;
int call_id = cib_conn->cmds->query(cib_conn,
"//" PCMK_XE_CRM_CONFIG
" | //" PCMK_XE_ALERTS,
NULL, cib_xpath);
fsa_register_cib_callback(call_id, NULL, config_query_callback);
crm_trace("Querying the CIB... call %d", call_id);
return TRUE;
}
/* A_READCONFIG */
void
do_read_config(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
throttle_init();
controld_trigger_config();
}
static void
crm_shutdown(int nsig)
{
const char *value = NULL;
guint default_period_ms = 0;
if ((controld_globals.mainloop == NULL)
|| !g_main_loop_is_running(controld_globals.mainloop)) {
crmd_exit(CRM_EX_OK);
return;
}
if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
crm_err("Escalating shutdown");
register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
return;
}
controld_set_fsa_input_flags(R_SHUTDOWN);
register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
/* If shutdown timer doesn't have a period set, use the default
*
* @TODO: Evaluate whether this is still necessary. As long as
* config_query_callback() has been run at least once, it doesn't look like
* anything could have changed the timer period since then.
*/
value = pcmk__cluster_option(NULL, PCMK_OPT_SHUTDOWN_ESCALATION);
pcmk_parse_interval_spec(value, &default_period_ms);
controld_shutdown_start_countdown(default_period_ms);
}
diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c
index c1da3a8d3b..6be4f11f05 100644
--- a/daemons/controld/controld_join_client.c
+++ b/daemons/controld/controld_join_client.c
@@ -1,370 +1,370 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/common/xml.h>
#include <pacemaker-controld.h>
void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
/*!
* \internal
* \brief Remember if DC is shutting down as we join
*
* If we're joining while the current DC is shutting down, update its expected
* state, so we don't fence it if we become the new DC. (We weren't a peer
* when it broadcast its shutdown request.)
*
* \param[in] msg A join message from the DC
*/
static void
update_dc_expected(const xmlNode *msg)
{
if ((controld_globals.dc_name != NULL)
&& pcmk__xe_attr_is_true(msg, PCMK__XA_DC_LEAVING)) {
pcmk__node_status_t *dc_node =
pcmk__get_node(0, controld_globals.dc_name, NULL,
pcmk__node_search_cluster_member);
pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN);
}
}
/* A_CL_JOIN_QUERY */
/* is there a DC out there? */
void
do_cl_join_query(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
- CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
+ CRM_SYSTEM_DC, CRM_SYSTEM_CRMD);
sleep(1); // Give the cluster layer time to propagate to the DC
update_dc(NULL); /* Unset any existing value so that the result is not discarded */
crm_debug("Querying for a DC");
pcmk__cluster_send_message(NULL, pcmk_ipc_controld, req);
pcmk__xml_free(req);
}
/* A_CL_JOIN_ANNOUNCE */
/* this is kind of a workaround for the fact that we may not be around or
* are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
*/
void
do_cl_join_announce(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
/* don't announce if we're in one of these states */
if (cur_state != S_PENDING) {
crm_warn("Not announcing cluster join because in state %s",
fsa_state2string(cur_state));
return;
}
if (!pcmk_is_set(controld_globals.fsa_input_register, R_STARTING)) {
/* send as a broadcast */
xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
- CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
+ CRM_SYSTEM_DC, CRM_SYSTEM_CRMD);
crm_debug("Announcing availability");
update_dc(NULL);
pcmk__cluster_send_message(NULL, pcmk_ipc_controld, req);
pcmk__xml_free(req);
} else {
/* Delay announce until we have finished local startup */
crm_warn("Delaying announce of cluster join until local startup is complete");
return;
}
}
static int query_call_id = 0;
/* A_CL_JOIN_REQUEST */
/* aka. accept the welcome offer */
void
do_cl_join_offer_respond(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
cib_t *cib_conn = controld_globals.cib_conn;
ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
const char *welcome_from;
const char *join_id;
CRM_CHECK(input != NULL, return);
welcome_from = crm_element_value(input->msg, PCMK__XA_SRC);
join_id = crm_element_value(input->msg, PCMK__XA_JOIN_ID);
crm_trace("Accepting cluster join offer from node %s " QB_XS " join-%s",
welcome_from, crm_element_value(input->msg, PCMK__XA_JOIN_ID));
/* we only ever want the last one */
if (query_call_id > 0) {
crm_trace("Cancelling previous join query: %d", query_call_id);
remove_cib_op_callback(query_call_id, FALSE);
query_call_id = 0;
}
if (update_dc(input->msg) == FALSE) {
crm_warn("Discarding cluster join offer from node %s (expected %s)",
welcome_from, controld_globals.dc_name);
return;
}
update_dc_expected(input->msg);
query_call_id = cib_conn->cmds->query(cib_conn, NULL, NULL,
cib_no_children);
fsa_register_cib_callback(query_call_id, pcmk__str_copy(join_id),
join_query_callback);
crm_trace("Registered join query callback: %d", query_call_id);
controld_set_fsa_action_flags(A_DC_TIMER_STOP);
controld_trigger_fsa();
}
void
join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
char *join_id = user_data;
xmlNode *generation = pcmk__xe_create(NULL, PCMK__XE_GENERATION_TUPLE);
CRM_LOG_ASSERT(join_id != NULL);
if (query_call_id != call_id) {
crm_trace("Query %d superseded", call_id);
goto done;
}
query_call_id = 0;
if(rc != pcmk_ok || output == NULL) {
crm_err("Could not retrieve version details for join-%s: %s (%d)",
join_id, pcmk_strerror(rc), rc);
register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
} else if (controld_globals.dc_name == NULL) {
crm_debug("Membership is in flux, not continuing join-%s", join_id);
} else {
xmlNode *reply = NULL;
const pcmk__node_status_t *dc_node =
pcmk__get_node(0, controld_globals.dc_name, NULL,
pcmk__node_search_cluster_member);
crm_debug("Respond to join offer join-%s from %s",
join_id, controld_globals.dc_name);
pcmk__xe_copy_attrs(generation, output, pcmk__xaf_none);
reply = create_request(CRM_OP_JOIN_REQUEST, generation,
controld_globals.dc_name, CRM_SYSTEM_DC,
- CRM_SYSTEM_CRMD, NULL);
+ CRM_SYSTEM_CRMD);
crm_xml_add(reply, PCMK__XA_JOIN_ID, join_id);
crm_xml_add(reply, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
pcmk__cluster_send_message(dc_node, pcmk_ipc_controld, reply);
pcmk__xml_free(reply);
}
done:
pcmk__xml_free(generation);
}
void
set_join_state(const char *start_state, const char *node_name, const char *node_uuid,
bool remote)
{
if (pcmk__str_eq(start_state, PCMK_VALUE_STANDBY, pcmk__str_casei)) {
crm_notice("Forcing node %s to join in %s state per configured "
"environment", node_name, start_state);
cib__update_node_attr(controld_globals.logger_out,
controld_globals.cib_conn, cib_sync_call,
PCMK_XE_NODES, node_uuid,
NULL, NULL, NULL, PCMK_NODE_ATTR_STANDBY,
PCMK_VALUE_TRUE, NULL,
(remote? PCMK_VALUE_REMOTE : NULL));
} else if (pcmk__str_eq(start_state, PCMK_VALUE_ONLINE, pcmk__str_casei)) {
crm_notice("Forcing node %s to join in %s state per configured "
"environment", node_name, start_state);
cib__update_node_attr(controld_globals.logger_out,
controld_globals.cib_conn, cib_sync_call,
PCMK_XE_NODES, node_uuid,
NULL, NULL, NULL, PCMK_NODE_ATTR_STANDBY,
PCMK_VALUE_FALSE, NULL,
(remote? PCMK_VALUE_REMOTE : NULL));
} else if (pcmk__str_eq(start_state, PCMK_VALUE_DEFAULT, pcmk__str_casei)) {
crm_debug("Not forcing a starting state on node %s", node_name);
} else {
crm_warn("Unrecognized start state '%s', using "
"'" PCMK_VALUE_DEFAULT "' (%s)",
start_state, node_name);
}
}
static int
update_conn_host_cache(xmlNode *node, void *userdata)
{
const char *remote = crm_element_value(node, PCMK_XA_ID);
const char *conn_host = crm_element_value(node, PCMK__XA_CONNECTION_HOST);
const char *state = crm_element_value(node, PCMK__XA_NODE_STATE);
pcmk__node_status_t *remote_peer =
pcmk__cluster_lookup_remote_node(remote);
if (remote_peer == NULL) {
return pcmk_rc_ok;
}
if (conn_host != NULL) {
pcmk__str_update(&remote_peer->conn_host, conn_host);
}
if (state != NULL) {
pcmk__update_peer_state(__func__, remote_peer, state, 0);
}
return pcmk_rc_ok;
}
/* A_CL_JOIN_RESULT */
/* aka. this is notification that we have (or have not) been accepted */
void
do_cl_join_finalize_respond(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *tmp1 = NULL;
gboolean was_nack = TRUE;
static gboolean first_join = TRUE;
ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
const char *start_state = pcmk__env_option(PCMK__ENV_NODE_START_STATE);
int join_id = -1;
const char *op = crm_element_value(input->msg, PCMK__XA_CRM_TASK);
const char *welcome_from = crm_element_value(input->msg, PCMK__XA_SRC);
if (!pcmk__str_eq(op, CRM_OP_JOIN_ACKNAK, pcmk__str_casei)) {
crm_trace("Ignoring op=%s message", op);
return;
}
/* calculate if it was an ack or a nack */
if (pcmk__xe_attr_is_true(input->msg, CRM_OP_JOIN_ACKNAK)) {
was_nack = FALSE;
}
crm_element_value_int(input->msg, PCMK__XA_JOIN_ID, &join_id);
if (was_nack) {
crm_err("Shutting down because cluster join with leader %s failed "
QB_XS " join-%d NACK'd", welcome_from, join_id);
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
controld_set_fsa_input_flags(R_STAYDOWN);
return;
}
if (!AM_I_DC
&& pcmk__str_eq(welcome_from, controld_globals.our_nodename,
pcmk__str_casei)) {
crm_warn("Discarding our own welcome - we're no longer the DC");
return;
}
if (update_dc(input->msg) == FALSE) {
crm_warn("Discarding %s from node %s (expected from %s)",
op, welcome_from, controld_globals.dc_name);
return;
}
update_dc_expected(input->msg);
/* record the node's feature set as a transient attribute */
update_attrd(controld_globals.our_nodename, CRM_ATTR_FEATURE_SET,
CRM_FEATURE_SET, NULL, FALSE);
/* send our status section to the DC */
tmp1 = controld_query_executor_state();
if (tmp1 != NULL) {
xmlNode *remotes = NULL;
xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1,
controld_globals.dc_name, CRM_SYSTEM_DC,
- CRM_SYSTEM_CRMD, NULL);
+ CRM_SYSTEM_CRMD);
const pcmk__node_status_t *dc_node =
pcmk__get_node(0, controld_globals.dc_name, NULL,
pcmk__node_search_cluster_member);
crm_xml_add_int(reply, PCMK__XA_JOIN_ID, join_id);
crm_debug("Confirming join-%d: sending local operation history to %s",
join_id, controld_globals.dc_name);
/*
* If this is the node's first join since the controller started on it,
* set its initial state (standby or member) according to the user's
* preference.
*
* We do not clear the LRM history here. Even if the DC failed to do it
* when we last left, removing them here creates a race condition if the
* controller is being recovered. Instead of a list of active resources
* from the executor, we may end up with a blank status section. If we
* are _NOT_ lucky, we will probe for the "wrong" instance of anonymous
* clones and end up with multiple active instances on the machine.
*/
if (first_join
&& !pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
first_join = FALSE;
if (start_state) {
set_join_state(start_state, controld_globals.our_nodename,
controld_globals.our_uuid, false);
}
}
pcmk__cluster_send_message(dc_node, pcmk_ipc_controld, reply);
pcmk__xml_free(reply);
if (AM_I_DC == FALSE) {
register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE,
__func__);
}
pcmk__xml_free(tmp1);
/* Update the remote node cache with information about which node
* is hosting the connection.
*/
remotes = pcmk__xe_first_child(input->msg, PCMK_XE_NODES, NULL, NULL);
if (remotes != NULL) {
pcmk__xe_foreach_child(remotes, PCMK_XE_NODE,
update_conn_host_cache, NULL);
}
} else {
crm_err("Could not confirm join-%d with %s: Local operation history "
"failed", join_id, controld_globals.dc_name);
register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
}
}
diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
index 4d48ac0a60..34c9367f77 100644
--- a/daemons/controld/controld_join_dc.c
+++ b/daemons/controld/controld_join_dc.c
@@ -1,1083 +1,1083 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <inttypes.h> // PRIu32
#include <stdbool.h> // bool, true, false
#include <stdio.h> // NULL
#include <stdlib.h> // free(), etc.
#include <glib.h> // gboolean, etc.
#include <libxml/tree.h> // xmlNode
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/cluster.h>
#include <pacemaker-controld.h>
static char *max_generation_from = NULL;
static xmlNodePtr max_generation_xml = NULL;
/*!
* \internal
* \brief Nodes from which a CIB sync has failed since the peer joined
*
* This table is of the form (<tt>node_name -> join_id</tt>). \p node_name is
* the name of a client node from which a CIB \p sync_from() call has failed in
* \p do_dc_join_finalize() since the client joined the cluster as a peer.
* \p join_id is the ID of the join round in which the \p sync_from() failed,
* and is intended for use in nack log messages.
*/
static GHashTable *failed_sync_nodes = NULL;
void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
/* Numeric counter used to identify join rounds (an unsigned int would be
* appropriate, except we get and set it in XML as int)
*/
static int current_join_id = 0;
/*!
* \internal
* \brief Get log-friendly string equivalent of a controller group join phase
*
* \param[in] phase Join phase
*
* \return Log-friendly string equivalent of \p phase
*/
static const char *
join_phase_text(enum controld_join_phase phase)
{
switch (phase) {
case controld_join_nack:
return "nack";
case controld_join_none:
return "none";
case controld_join_welcomed:
return "welcomed";
case controld_join_integrated:
return "integrated";
case controld_join_finalized:
return "finalized";
case controld_join_confirmed:
return "confirmed";
default:
return "invalid";
}
}
/*!
* \internal
* \brief Destroy the hash table containing failed sync nodes
*/
void
controld_destroy_failed_sync_table(void)
{
if (failed_sync_nodes != NULL) {
g_hash_table_destroy(failed_sync_nodes);
failed_sync_nodes = NULL;
}
}
/*!
* \internal
* \brief Remove a node from the failed sync nodes table if present
*
* \param[in] node_name Node name to remove
*/
void
controld_remove_failed_sync_node(const char *node_name)
{
if (failed_sync_nodes != NULL) {
g_hash_table_remove(failed_sync_nodes, (gchar *) node_name);
}
}
/*!
* \internal
* \brief Add to a hash table a node whose CIB failed to sync
*
* \param[in] node_name Name of node whose CIB failed to sync
* \param[in] join_id Join round when the failure occurred
*/
static void
record_failed_sync_node(const char *node_name, gint join_id)
{
if (failed_sync_nodes == NULL) {
failed_sync_nodes = pcmk__strikey_table(g_free, NULL);
}
/* If the node is already in the table then we failed to nack it during the
* filter offer step
*/
CRM_LOG_ASSERT(g_hash_table_insert(failed_sync_nodes, g_strdup(node_name),
GINT_TO_POINTER(join_id)));
}
/*!
* \internal
* \brief Look up a node name in the failed sync table
*
* \param[in] node_name Name of node to look up
* \param[out] join_id Where to store the join ID of when the sync failed
*
* \return Standard Pacemaker return code. Specifically, \p pcmk_rc_ok if the
* node name was found, or \p pcmk_rc_node_unknown otherwise.
* \note \p *join_id is set to -1 if the node is not found.
*/
static int
lookup_failed_sync_node(const char *node_name, gint *join_id)
{
*join_id = -1;
if (failed_sync_nodes != NULL) {
gpointer result = g_hash_table_lookup(failed_sync_nodes,
(gchar *) node_name);
if (result != NULL) {
*join_id = GPOINTER_TO_INT(result);
return pcmk_rc_ok;
}
}
return pcmk_rc_node_unknown;
}
void
crm_update_peer_join(const char *source, pcmk__node_status_t *node,
enum controld_join_phase phase)
{
enum controld_join_phase last = controld_get_join_phase(node);
CRM_CHECK(node != NULL, return);
/* Remote nodes do not participate in joins */
if (pcmk_is_set(node->flags, pcmk__node_status_remote)) {
return;
}
if (phase == last) {
crm_trace("Node %s join-%d phase is still %s "
QB_XS " nodeid=%" PRIu32 " source=%s",
node->name, current_join_id, join_phase_text(last),
node->cluster_layer_id, source);
return;
}
if ((phase <= controld_join_none) || (phase == (last + 1))) {
struct controld_node_status_data *data =
pcmk__assert_alloc(1, sizeof(struct controld_node_status_data));
data->join_phase = phase;
node->user_data = data;
crm_trace("Node %s join-%d phase is now %s (was %s) "
QB_XS " nodeid=%" PRIu32 " source=%s",
node->name, current_join_id, join_phase_text(phase),
join_phase_text(last), node->cluster_layer_id,
source);
return;
}
crm_warn("Rejecting join-%d phase update for node %s because can't go from "
"%s to %s " QB_XS " nodeid=%" PRIu32 " source=%s",
current_join_id, node->name, join_phase_text(last),
join_phase_text(phase), node->cluster_layer_id, source);
}
static void
start_join_round(void)
{
GHashTableIter iter;
pcmk__node_status_t *peer = NULL;
crm_debug("Starting new join round join-%d", current_join_id);
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
crm_update_peer_join(__func__, peer, controld_join_none);
}
if (max_generation_from != NULL) {
free(max_generation_from);
max_generation_from = NULL;
}
if (max_generation_xml != NULL) {
pcmk__xml_free(max_generation_xml);
max_generation_xml = NULL;
}
controld_clear_fsa_input_flags(R_HAVE_CIB);
}
/*!
* \internal
* \brief Create a join message from the DC
*
* \param[in] join_op Join operation name
* \param[in] host_to Recipient of message
*/
static xmlNode *
create_dc_message(const char *join_op, const char *host_to)
{
xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
- CRM_SYSTEM_DC, NULL);
+ CRM_SYSTEM_DC);
/* Identify which election this is a part of */
crm_xml_add_int(msg, PCMK__XA_JOIN_ID, current_join_id);
/* Add a field specifying whether the DC is shutting down. This keeps the
* joining node from fencing the old DC if it becomes the new DC.
*/
pcmk__xe_set_bool_attr(msg, PCMK__XA_DC_LEAVING,
pcmk_is_set(controld_globals.fsa_input_register,
R_SHUTDOWN));
return msg;
}
static void
join_make_offer(gpointer key, gpointer value, gpointer user_data)
{
/* @TODO We don't use user_data except to distinguish one particular call
* from others. Make this clearer.
*/
xmlNode *offer = NULL;
pcmk__node_status_t *member = (pcmk__node_status_t *) value;
CRM_ASSERT(member != NULL);
if (!pcmk__cluster_is_node_active(member)) {
crm_info("Not making join-%d offer to inactive node %s",
current_join_id, pcmk__s(member->name, "with unknown name"));
if ((member->expected == NULL)
&& pcmk__str_eq(member->state, PCMK__VALUE_LOST, pcmk__str_none)) {
/* You would think this unsafe, but in fact this plus an
* active resource is what causes it to be fenced.
*
* Yes, this does mean that any node that dies at the same
* time as the old DC and is not running resource (still)
* won't be fenced.
*
* I'm not happy about this either.
*/
pcmk__update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
}
return;
}
if (member->name == NULL) {
crm_info("Not making join-%d offer to node uuid %s with unknown name",
current_join_id, member->xml_id);
return;
}
if (controld_globals.membership_id != controld_globals.peer_seq) {
controld_globals.membership_id = controld_globals.peer_seq;
crm_info("Making join-%d offers based on membership event %llu",
current_join_id, controld_globals.peer_seq);
}
if (user_data != NULL) {
enum controld_join_phase phase = controld_get_join_phase(member);
if (phase > controld_join_none) {
crm_info("Not making join-%d offer to already known node %s (%s)",
current_join_id, member->name, join_phase_text(phase));
return;
}
}
crm_update_peer_join(__func__, (pcmk__node_status_t*) member,
controld_join_none);
offer = create_dc_message(CRM_OP_JOIN_OFFER, member->name);
// Advertise our feature set so the joining node can bail if not compatible
crm_xml_add(offer, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
crm_info("Sending join-%d offer to %s", current_join_id, member->name);
pcmk__cluster_send_message(member, pcmk_ipc_controld, offer);
pcmk__xml_free(offer);
crm_update_peer_join(__func__, member, controld_join_welcomed);
}
/* A_DC_JOIN_OFFER_ALL */
void
do_dc_join_offer_all(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
int count;
/* Reset everyone's status back to down or in_ccm in the CIB.
* Any nodes that are active in the CIB but not in the cluster membership
* will be seen as offline by the scheduler anyway.
*/
current_join_id++;
start_join_round();
update_dc(NULL);
if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
crm_info("A new node joined the cluster");
}
g_hash_table_foreach(pcmk__peer_cache, join_make_offer, NULL);
count = crmd_join_phase_count(controld_join_welcomed);
crm_info("Waiting on join-%d requests from %d outstanding node%s",
current_join_id, count, pcmk__plural_s(count));
// Don't waste time by invoking the scheduler yet
}
/* A_DC_JOIN_OFFER_ONE */
void
do_dc_join_offer_one(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
pcmk__node_status_t *member = NULL;
ha_msg_input_t *welcome = NULL;
int count;
const char *join_to = NULL;
if (msg_data->data == NULL) {
crm_info("Making join-%d offers to any unconfirmed nodes "
"because an unknown node joined", current_join_id);
g_hash_table_foreach(pcmk__peer_cache, join_make_offer, &member);
check_join_state(cur_state, __func__);
return;
}
welcome = fsa_typed_data(fsa_dt_ha_msg);
if (welcome == NULL) {
// fsa_typed_data() already logged an error
return;
}
join_to = crm_element_value(welcome->msg, PCMK__XA_SRC);
if (join_to == NULL) {
crm_err("Can't make join-%d offer to unknown node", current_join_id);
return;
}
member = pcmk__get_node(0, join_to, NULL, pcmk__node_search_cluster_member);
/* It is possible that a node will have been sick or starting up when the
* original offer was made. However, it will either re-announce itself in
* due course, or we can re-store the original offer on the client.
*/
crm_update_peer_join(__func__, member, controld_join_none);
join_make_offer(NULL, member, NULL);
/* If the offer isn't to the local node, make an offer to the local node as
* well, to ensure the correct value for max_generation_from.
*/
if (strcasecmp(join_to, controld_globals.our_nodename) != 0) {
member = pcmk__get_node(0, controld_globals.our_nodename, NULL,
pcmk__node_search_cluster_member);
join_make_offer(NULL, member, NULL);
}
/* This was a genuine join request; cancel any existing transition and
* invoke the scheduler.
*/
abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Node join",
NULL);
count = crmd_join_phase_count(controld_join_welcomed);
crm_info("Waiting on join-%d requests from %d outstanding node%s",
current_join_id, count, pcmk__plural_s(count));
// Don't waste time by invoking the scheduler yet
}
static int
compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
{
const char *elem_l = crm_element_value(left, field);
const char *elem_r = crm_element_value(right, field);
long long int_elem_l;
long long int_elem_r;
pcmk__scan_ll(elem_l, &int_elem_l, -1LL);
pcmk__scan_ll(elem_r, &int_elem_r, -1LL);
if (int_elem_l < int_elem_r) {
return -1;
} else if (int_elem_l > int_elem_r) {
return 1;
}
return 0;
}
/* A_DC_JOIN_PROCESS_REQ */
void
do_dc_join_filter_offer(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
xmlNode *generation = NULL;
int cmp = 0;
int join_id = -1;
int count = 0;
gint value = 0;
gboolean ack_nack_bool = TRUE;
ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
const char *join_from = crm_element_value(join_ack->msg, PCMK__XA_SRC);
const char *ref = crm_element_value(join_ack->msg, PCMK_XA_REFERENCE);
const char *join_version = crm_element_value(join_ack->msg,
PCMK_XA_CRM_FEATURE_SET);
pcmk__node_status_t *join_node = NULL;
if (join_from == NULL) {
crm_err("Ignoring invalid join request without node name");
return;
}
join_node = pcmk__get_node(0, join_from, NULL,
pcmk__node_search_cluster_member);
crm_element_value_int(join_ack->msg, PCMK__XA_JOIN_ID, &join_id);
if (join_id != current_join_id) {
crm_debug("Ignoring join-%d request from %s because we are on join-%d",
join_id, join_from, current_join_id);
check_join_state(cur_state, __func__);
return;
}
generation = join_ack->xml;
if (max_generation_xml != NULL && generation != NULL) {
int lpc = 0;
const char *attributes[] = {
PCMK_XA_ADMIN_EPOCH,
PCMK_XA_EPOCH,
PCMK_XA_NUM_UPDATES,
};
/* It's not obvious that join_ack->xml is the PCMK__XE_GENERATION_TUPLE
* element from the join client. The "if" guard is for clarity.
*/
if (pcmk__xe_is(generation, PCMK__XE_GENERATION_TUPLE)) {
for (lpc = 0; cmp == 0 && lpc < PCMK__NELEM(attributes); lpc++) {
cmp = compare_int_fields(max_generation_xml, generation,
attributes[lpc]);
}
} else { // Should always be PCMK__XE_GENERATION_TUPLE
CRM_LOG_ASSERT(false);
}
}
if (ref == NULL) {
ref = "none"; // for logging only
}
if (lookup_failed_sync_node(join_from, &value) == pcmk_rc_ok) {
crm_err("Rejecting join-%d request from node %s because we failed to "
"sync its CIB in join-%d " QB_XS " ref=%s",
join_id, join_from, value, ref);
ack_nack_bool = FALSE;
} else if (!pcmk__cluster_is_node_active(join_node)) {
if (match_down_event(join_from) != NULL) {
/* The join request was received after the node was fenced or
* otherwise shutdown in a way that we're aware of. No need to log
* an error in this rare occurrence; we know the client was recently
* shut down, and receiving a lingering in-flight request is not
* cause for alarm.
*/
crm_debug("Rejecting join-%d request from inactive node %s "
QB_XS " ref=%s", join_id, join_from, ref);
} else {
crm_err("Rejecting join-%d request from inactive node %s "
QB_XS " ref=%s", join_id, join_from, ref);
}
ack_nack_bool = FALSE;
} else if (generation == NULL) {
crm_err("Rejecting invalid join-%d request from node %s "
"missing CIB generation " QB_XS " ref=%s",
join_id, join_from, ref);
ack_nack_bool = FALSE;
} else if ((join_version == NULL)
|| !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
crm_err("Rejecting join-%d request from node %s because feature set %s"
" is incompatible with ours (%s) " QB_XS " ref=%s",
join_id, join_from, (join_version? join_version : "pre-3.1.0"),
CRM_FEATURE_SET, ref);
ack_nack_bool = FALSE;
} else if (max_generation_xml == NULL) {
const char *validation = crm_element_value(generation,
PCMK_XA_VALIDATE_WITH);
if (pcmk__get_schema(validation) == NULL) {
crm_err("Rejecting join-%d request from %s (with first CIB "
"generation) due to unknown schema version %s "
QB_XS " ref=%s",
join_id, join_from, pcmk__s(validation, "(missing)"), ref);
ack_nack_bool = FALSE;
} else {
crm_debug("Accepting join-%d request from %s (with first CIB "
"generation) " QB_XS " ref=%s",
join_id, join_from, ref);
max_generation_xml = pcmk__xml_copy(NULL, generation);
pcmk__str_update(&max_generation_from, join_from);
}
} else if ((cmp < 0)
|| ((cmp == 0)
&& pcmk__str_eq(join_from, controld_globals.our_nodename,
pcmk__str_casei))) {
const char *validation = crm_element_value(generation,
PCMK_XA_VALIDATE_WITH);
if (pcmk__get_schema(validation) == NULL) {
crm_err("Rejecting join-%d request from %s (with better CIB "
"generation than current best from %s) due to unknown "
"schema version %s " QB_XS " ref=%s",
join_id, join_from, max_generation_from,
pcmk__s(validation, "(missing)"), ref);
ack_nack_bool = FALSE;
} else {
crm_debug("Accepting join-%d request from %s (with better CIB "
"generation than current best from %s) " QB_XS " ref=%s",
join_id, join_from, max_generation_from, ref);
crm_log_xml_debug(max_generation_xml, "Old max generation");
crm_log_xml_debug(generation, "New max generation");
pcmk__xml_free(max_generation_xml);
max_generation_xml = pcmk__xml_copy(NULL, join_ack->xml);
pcmk__str_update(&max_generation_from, join_from);
}
} else {
crm_debug("Accepting join-%d request from %s " QB_XS " ref=%s",
join_id, join_from, ref);
}
if (!ack_nack_bool) {
crm_update_peer_join(__func__, join_node, controld_join_nack);
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);
} else {
crm_update_peer_join(__func__, join_node, controld_join_integrated);
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
}
count = crmd_join_phase_count(controld_join_integrated);
crm_debug("%d node%s currently integrated in join-%d",
count, pcmk__plural_s(count), join_id);
if (check_join_state(cur_state, __func__) == FALSE) {
// Don't waste time by invoking the scheduler yet
count = crmd_join_phase_count(controld_join_welcomed);
crm_debug("Waiting on join-%d requests from %d outstanding node%s",
join_id, count, pcmk__plural_s(count));
}
}
/* A_DC_JOIN_FINALIZE */
void
do_dc_join_finalize(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
char *sync_from = NULL;
int rc = pcmk_ok;
int count_welcomed = crmd_join_phase_count(controld_join_welcomed);
int count_finalizable = crmd_join_phase_count(controld_join_integrated)
+ crmd_join_phase_count(controld_join_nack);
/* This we can do straight away and avoid clients timing us out
* while we compute the latest CIB
*/
if (count_welcomed != 0) {
crm_debug("Waiting on join-%d requests from %d outstanding node%s "
"before finalizing join", current_join_id, count_welcomed,
pcmk__plural_s(count_welcomed));
crmd_join_phase_log(LOG_DEBUG);
/* crmd_fsa_stall(FALSE); Needed? */
return;
} else if (count_finalizable == 0) {
crm_debug("Finalization not needed for join-%d at the current time",
current_join_id);
crmd_join_phase_log(LOG_DEBUG);
check_join_state(controld_globals.fsa_state, __func__);
return;
}
controld_clear_fsa_input_flags(R_HAVE_CIB);
if (pcmk__str_eq(max_generation_from, controld_globals.our_nodename,
pcmk__str_null_matches|pcmk__str_casei)) {
controld_set_fsa_input_flags(R_HAVE_CIB);
}
if (!controld_globals.transition_graph->complete) {
crm_warn("Delaying join-%d finalization while transition in progress",
current_join_id);
crmd_join_phase_log(LOG_DEBUG);
crmd_fsa_stall(FALSE);
return;
}
if (pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
// Send our CIB out to everyone
sync_from = pcmk__str_copy(controld_globals.our_nodename);
crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
current_join_id, count_finalizable,
pcmk__plural_s(count_finalizable));
crm_log_xml_debug(max_generation_xml, "Requested CIB version");
} else {
// Ask for the agreed best CIB
sync_from = pcmk__str_copy(max_generation_from);
crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
current_join_id, count_finalizable,
pcmk__plural_s(count_finalizable), sync_from);
crm_log_xml_notice(max_generation_xml, "Requested CIB version");
}
crmd_join_phase_log(LOG_DEBUG);
rc = controld_globals.cib_conn->cmds->sync_from(controld_globals.cib_conn,
sync_from, NULL, cib_none);
fsa_register_cib_callback(rc, sync_from, finalize_sync_callback);
}
void
free_max_generation(void)
{
free(max_generation_from);
max_generation_from = NULL;
pcmk__xml_free(max_generation_xml);
max_generation_xml = NULL;
}
void
finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
CRM_LOG_ASSERT(-EPERM != rc);
if (rc != pcmk_ok) {
const char *sync_from = (const char *) user_data;
do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
"Could not sync CIB from %s in join-%d: %s",
sync_from, current_join_id, pcmk_strerror(rc));
if (rc != -pcmk_err_old_data) {
record_failed_sync_node(sync_from, current_join_id);
}
/* restart the whole join process */
register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL,
__func__);
} else if (!AM_I_DC) {
crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);
} else if (controld_globals.fsa_state != S_FINALIZE_JOIN) {
crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN "
"(%s)", current_join_id,
fsa_state2string(controld_globals.fsa_state));
} else {
controld_set_fsa_input_flags(R_HAVE_CIB);
/* make sure dc_uuid is re-set to us */
if (!check_join_state(controld_globals.fsa_state, __func__)) {
int count_finalizable = 0;
count_finalizable = crmd_join_phase_count(controld_join_integrated)
+ crmd_join_phase_count(controld_join_nack);
crm_debug("Notifying %d node%s of join-%d results",
count_finalizable, pcmk__plural_s(count_finalizable),
current_join_id);
g_hash_table_foreach(pcmk__peer_cache, finalize_join_for, NULL);
}
}
}
static void
join_node_state_commit_callback(xmlNode *msg, int call_id, int rc,
xmlNode *output, void *user_data)
{
const char *node = user_data;
if (rc != pcmk_ok) {
fsa_data_t *msg_data = NULL; // for register_fsa_error() macro
crm_crit("join-%d node history update (via CIB call %d) for node %s "
"failed: %s",
current_join_id, call_id, node, pcmk_strerror(rc));
crm_log_xml_debug(msg, "failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
crm_debug("join-%d node history update (via CIB call %d) for node %s "
"complete",
current_join_id, call_id, node);
check_join_state(controld_globals.fsa_state, __func__);
}
/* A_DC_JOIN_PROCESS_ACK */
void
do_dc_join_ack(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
int join_id = -1;
ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
const char *op = crm_element_value(join_ack->msg, PCMK__XA_CRM_TASK);
char *join_from = crm_element_value_copy(join_ack->msg, PCMK__XA_SRC);
pcmk__node_status_t *peer = NULL;
enum controld_join_phase phase = controld_join_none;
enum controld_section_e section = controld_section_lrm;
char *xpath = NULL;
xmlNode *state = join_ack->xml;
xmlNode *execd_state = NULL;
cib_t *cib = controld_globals.cib_conn;
int rc = pcmk_ok;
// Sanity checks
if (join_from == NULL) {
crm_warn("Ignoring message received without node identification");
goto done;
}
if (op == NULL) {
crm_warn("Ignoring message received from %s without task", join_from);
goto done;
}
if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
op, join_from, CRM_OP_JOIN_CONFIRM);
goto done;
}
if (crm_element_value_int(join_ack->msg, PCMK__XA_JOIN_ID, &join_id) != 0) {
crm_warn("Ignoring join confirmation from %s without valid join ID",
join_from);
goto done;
}
peer = pcmk__get_node(0, join_from, NULL, pcmk__node_search_cluster_member);
phase = controld_get_join_phase(peer);
if (phase != controld_join_finalized) {
crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
"(currently %s not %s)",
join_id, join_from, join_phase_text(phase),
join_phase_text(controld_join_finalized));
goto done;
}
if (join_id != current_join_id) {
crm_err("Rejecting join-%d confirmation from %s "
"because currently on join-%d",
join_id, join_from, current_join_id);
crm_update_peer_join(__func__, peer, controld_join_nack);
goto done;
}
crm_update_peer_join(__func__, peer, controld_join_confirmed);
/* Update CIB with node's current executor state. A new transition will be
* triggered later, when the CIB manager notifies us of the change.
*
* The delete and modify requests are part of an atomic transaction.
*/
rc = cib->cmds->init_transaction(cib);
if (rc != pcmk_ok) {
goto done;
}
// Delete relevant parts of node's current executor state from CIB
if (pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
section = controld_section_lrm_unlocked;
}
controld_node_state_deletion_strings(join_from, section, &xpath, NULL);
rc = cib->cmds->remove(cib, xpath, NULL,
cib_xpath|cib_multiple|cib_transaction);
if (rc != pcmk_ok) {
goto done;
}
// Update CIB with node's latest known executor state
if (pcmk__str_eq(join_from, controld_globals.our_nodename,
pcmk__str_casei)) {
// Use the latest possible state if processing our own join ack
execd_state = controld_query_executor_state();
if (execd_state != NULL) {
crm_debug("Updating local node history for join-%d from query "
"result",
current_join_id);
state = execd_state;
} else {
crm_warn("Updating local node history from join-%d confirmation "
"because query failed",
current_join_id);
}
} else {
crm_debug("Updating node history for %s from join-%d confirmation",
join_from, current_join_id);
}
rc = cib->cmds->modify(cib, PCMK_XE_STATUS, state,
cib_can_create|cib_transaction);
pcmk__xml_free(execd_state);
if (rc != pcmk_ok) {
goto done;
}
// Commit the transaction
rc = cib->cmds->end_transaction(cib, true, cib_none);
fsa_register_cib_callback(rc, join_from, join_node_state_commit_callback);
if (rc > 0) {
// join_from will be freed after callback
join_from = NULL;
rc = pcmk_ok;
}
done:
if (rc != pcmk_ok) {
crm_crit("join-%d node history update for node %s failed: %s",
current_join_id, join_from, pcmk_strerror(rc));
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
free(join_from);
free(xpath);
}
void
finalize_join_for(gpointer key, gpointer value, gpointer user_data)
{
xmlNode *acknak = NULL;
xmlNode *tmp1 = NULL;
pcmk__node_status_t *join_node = value;
const char *join_to = join_node->name;
enum controld_join_phase phase = controld_get_join_phase(join_node);
bool integrated = false;
switch (phase) {
case controld_join_integrated:
integrated = true;
break;
case controld_join_nack:
break;
default:
crm_trace("Not updating non-integrated and non-nacked node %s (%s) "
"for join-%d",
join_to, join_phase_text(phase), current_join_id);
return;
}
/* Update the <node> element with the node's name and UUID, in case they
* weren't known before
*/
crm_trace("Updating node name and UUID in CIB for %s", join_to);
tmp1 = pcmk__xe_create(NULL, PCMK_XE_NODE);
crm_xml_add(tmp1, PCMK_XA_ID, pcmk__cluster_node_uuid(join_node));
crm_xml_add(tmp1, PCMK_XA_UNAME, join_to);
fsa_cib_anon_update(PCMK_XE_NODES, tmp1);
pcmk__xml_free(tmp1);
join_node = pcmk__get_node(0, join_to, NULL,
pcmk__node_search_cluster_member);
if (!pcmk__cluster_is_node_active(join_node)) {
/*
* NACK'ing nodes that the membership layer doesn't know about yet
* simply creates more churn
*
* Better to leave them waiting and let the join restart when
* the new membership event comes in
*
* All other NACKs (due to versions etc) should still be processed
*/
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
return;
}
// Acknowledge or nack node's join request
crm_debug("%sing join-%d request from %s",
integrated? "Acknowledg" : "Nack", current_join_id, join_to);
acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
pcmk__xe_set_bool_attr(acknak, CRM_OP_JOIN_ACKNAK, integrated);
if (integrated) {
// No change needed for a nacked node
crm_update_peer_join(__func__, join_node, controld_join_finalized);
pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
/* Iterate through the remote peer cache and add information on which
* node hosts each to the ACK message. This keeps new controllers in
* sync with what has already happened.
*/
if (pcmk__cluster_num_remote_nodes() > 0) {
GHashTableIter iter;
pcmk__node_status_t *node = NULL;
xmlNode *remotes = pcmk__xe_create(acknak, PCMK_XE_NODES);
g_hash_table_iter_init(&iter, pcmk__remote_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
xmlNode *remote = NULL;
if (!node->conn_host) {
continue;
}
remote = pcmk__xe_create(remotes, PCMK_XE_NODE);
pcmk__xe_set_props(remote,
PCMK_XA_ID, node->name,
PCMK__XA_NODE_STATE, node->state,
PCMK__XA_CONNECTION_HOST, node->conn_host,
NULL);
}
}
}
pcmk__cluster_send_message(join_node, pcmk_ipc_controld, acknak);
pcmk__xml_free(acknak);
return;
}
gboolean
check_join_state(enum crmd_fsa_state cur_state, const char *source)
{
static unsigned long long highest_seq = 0;
if (controld_globals.membership_id != controld_globals.peer_seq) {
crm_debug("join-%d: Membership changed from %llu to %llu "
QB_XS " highest=%llu state=%s for=%s",
current_join_id, controld_globals.membership_id,
controld_globals.peer_seq, highest_seq,
fsa_state2string(cur_state), source);
if (highest_seq < controld_globals.peer_seq) {
/* Don't spam the FSA with duplicates */
highest_seq = controld_globals.peer_seq;
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
} else if (cur_state == S_INTEGRATION) {
if (crmd_join_phase_count(controld_join_welcomed) == 0) {
int count = crmd_join_phase_count(controld_join_integrated);
crm_debug("join-%d: Integration of %d peer%s complete "
QB_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
return TRUE;
}
} else if (cur_state == S_FINALIZE_JOIN) {
if (!pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
crm_debug("join-%d: Delaying finalization until we have CIB "
QB_XS " state=%s for=%s",
current_join_id, fsa_state2string(cur_state), source);
return TRUE;
} else if (crmd_join_phase_count(controld_join_welcomed) != 0) {
int count = crmd_join_phase_count(controld_join_welcomed);
crm_debug("join-%d: Still waiting on %d welcomed node%s "
QB_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
crmd_join_phase_log(LOG_DEBUG);
} else if (crmd_join_phase_count(controld_join_integrated) != 0) {
int count = crmd_join_phase_count(controld_join_integrated);
crm_debug("join-%d: Still waiting on %d integrated node%s "
QB_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
crmd_join_phase_log(LOG_DEBUG);
} else if (crmd_join_phase_count(controld_join_finalized) != 0) {
int count = crmd_join_phase_count(controld_join_finalized);
crm_debug("join-%d: Still waiting on %d finalized node%s "
QB_XS " state=%s for=%s",
current_join_id, count, pcmk__plural_s(count),
fsa_state2string(cur_state), source);
crmd_join_phase_log(LOG_DEBUG);
} else {
crm_debug("join-%d: Complete " QB_XS " state=%s for=%s",
current_join_id, fsa_state2string(cur_state), source);
register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
return TRUE;
}
}
return FALSE;
}
void
do_dc_join_final(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
crm_update_quorum(pcmk__cluster_has_quorum(), TRUE);
}
int crmd_join_phase_count(enum controld_join_phase phase)
{
int count = 0;
pcmk__node_status_t *peer;
GHashTableIter iter;
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
if (controld_get_join_phase(peer) == phase) {
count++;
}
}
return count;
}
void crmd_join_phase_log(int level)
{
pcmk__node_status_t *peer;
GHashTableIter iter;
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->name,
join_phase_text(controld_get_join_phase(peer)));
}
}
diff --git a/daemons/controld/controld_membership.c b/daemons/controld/controld_membership.c
index 26729c09f9..b52d64ce0d 100644
--- a/daemons/controld/controld_membership.c
+++ b/daemons/controld/controld_membership.c
@@ -1,467 +1,467 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
/* put these first so that uuid_t is defined without conflicts */
#include <crm_internal.h>
#include <string.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/common/xml_internal.h>
#include <crm/cluster/internal.h>
#include <pacemaker-controld.h>
void post_cache_update(int instance);
extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
static void
reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
{
pcmk__node_status_t *node = value;
if (pcmk__cluster_is_node_active(node)) {
return;
}
crm_update_peer_join(__func__, node, controld_join_none);
if ((node != NULL) && (node->name != NULL)) {
if (pcmk__str_eq(controld_globals.our_nodename, node->name,
pcmk__str_casei)) {
crm_err("We're not part of the cluster anymore");
register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
} else if (!AM_I_DC
&& pcmk__str_eq(node->name, controld_globals.dc_name,
pcmk__str_casei)) {
crm_warn("Our DC node (%s) left the cluster", node->name);
register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
}
}
if ((controld_globals.fsa_state == S_INTEGRATION)
|| (controld_globals.fsa_state == S_FINALIZE_JOIN)) {
check_join_state(controld_globals.fsa_state, __func__);
}
if ((node != NULL) && (node->xml_id != NULL)) {
fail_incompletable_actions(controld_globals.transition_graph,
node->xml_id);
}
}
void
post_cache_update(int instance)
{
xmlNode *no_op = NULL;
controld_globals.peer_seq = instance;
crm_debug("Updated cache after membership event %d.", instance);
g_hash_table_foreach(pcmk__peer_cache, reap_dead_nodes, NULL);
controld_set_fsa_input_flags(R_MEMBERSHIP);
if (AM_I_DC) {
populate_cib_nodes(node_update_quick | node_update_cluster | node_update_peer |
node_update_expected, __func__);
}
/*
* If we lost nodes, we should re-check the election status
* Safe to call outside of an election
*/
controld_set_fsa_action_flags(A_ELECTION_CHECK);
controld_trigger_fsa();
/* Membership changed, remind everyone we're here.
* This will aid detection of duplicate DCs
*/
no_op = create_request(CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD,
- AM_I_DC ? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD, NULL);
+ (AM_I_DC? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD));
pcmk__cluster_send_message(NULL, pcmk_ipc_controld, no_op);
pcmk__xml_free(no_op);
}
static void
crmd_node_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
fsa_data_t *msg_data = NULL;
if (rc == pcmk_ok) {
crm_trace("Node update %d complete", call_id);
} else if(call_id < pcmk_ok) {
crm_err("Node update failed: %s (%d)", pcmk_strerror(call_id), call_id);
crm_log_xml_debug(msg, "failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
} else {
crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
crm_log_xml_debug(msg, "failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
/*!
* \internal
* \brief Create an XML node state tag with updates
*
* \param[in,out] node Node whose state will be used for update
* \param[in] flags Bitmask of node_update_flags indicating what to update
* \param[in,out] parent XML node to contain update (or NULL)
* \param[in] source Who requested the update (only used for logging)
*
* \return Pointer to created node state tag
*/
xmlNode *
create_node_state_update(pcmk__node_status_t *node, int flags,
xmlNode *parent, const char *source)
{
const char *value = NULL;
xmlNode *node_state;
if (!node->state) {
crm_info("Node update for %s cancelled: no state, not seen yet",
node->name);
return NULL;
}
node_state = pcmk__xe_create(parent, PCMK__XE_NODE_STATE);
if (pcmk_is_set(node->flags, pcmk__node_status_remote)) {
pcmk__xe_set_bool_attr(node_state, PCMK_XA_REMOTE_NODE, true);
}
if (crm_xml_add(node_state, PCMK_XA_ID,
pcmk__cluster_node_uuid(node)) == NULL) {
crm_info("Node update for %s cancelled: no ID", node->name);
pcmk__xml_free(node_state);
return NULL;
}
crm_xml_add(node_state, PCMK_XA_UNAME, node->name);
if ((flags & node_update_cluster) && node->state) {
if (compare_version(controld_globals.dc_version, "3.18.0") >= 0) {
// A value 0 means the node is not a cluster member.
crm_xml_add_ll(node_state, PCMK__XA_IN_CCM, node->when_member);
} else {
pcmk__xe_set_bool_attr(node_state, PCMK__XA_IN_CCM,
pcmk__str_eq(node->state, PCMK_VALUE_MEMBER,
pcmk__str_none));
}
}
if (!pcmk_is_set(node->flags, pcmk__node_status_remote)) {
if (flags & node_update_peer) {
if (compare_version(controld_globals.dc_version, "3.18.0") >= 0) {
// A value 0 means the peer is offline in CPG.
crm_xml_add_ll(node_state, PCMK_XA_CRMD, node->when_online);
} else {
// @COMPAT DCs < 2.1.7 use online/offline rather than timestamp
value = PCMK_VALUE_OFFLINE;
if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
value = PCMK_VALUE_ONLINE;
}
crm_xml_add(node_state, PCMK_XA_CRMD, value);
}
}
if (flags & node_update_join) {
if (controld_get_join_phase(node) <= controld_join_none) {
value = CRMD_JOINSTATE_DOWN;
} else {
value = CRMD_JOINSTATE_MEMBER;
}
crm_xml_add(node_state, PCMK__XA_JOIN, value);
}
if (flags & node_update_expected) {
crm_xml_add(node_state, PCMK_XA_EXPECTED, node->expected);
}
}
crm_xml_add(node_state, PCMK_XA_CRM_DEBUG_ORIGIN, source);
return node_state;
}
static void
remove_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
xmlNode * output, void *user_data)
{
char *node_uuid = user_data;
do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
"Deletion of the unknown conflicting node \"%s\": %s (rc=%d)",
node_uuid, pcmk_strerror(rc), rc);
}
static void
search_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
xmlNode * output, void *user_data)
{
char *new_node_uuid = user_data;
xmlNode *node_xml = NULL;
if (rc != pcmk_ok) {
if (rc != -ENXIO) {
crm_notice("Searching conflicting nodes for %s failed: %s (%d)",
new_node_uuid, pcmk_strerror(rc), rc);
}
return;
} else if (output == NULL) {
return;
}
if (pcmk__xe_is(output, PCMK_XE_NODE)) {
node_xml = output;
} else {
node_xml = pcmk__xe_first_child(output, PCMK_XE_NODE, NULL, NULL);
}
for (; node_xml != NULL; node_xml = pcmk__xe_next_same(node_xml)) {
const char *node_uuid = NULL;
const char *node_uname = NULL;
GHashTableIter iter;
pcmk__node_status_t *node = NULL;
gboolean known = FALSE;
node_uuid = crm_element_value(node_xml, PCMK_XA_ID);
node_uname = crm_element_value(node_xml, PCMK_XA_UNAME);
if (node_uuid == NULL || node_uname == NULL) {
continue;
}
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
if ((node != NULL)
&& pcmk__str_eq(node->xml_id, node_uuid, pcmk__str_casei)
&& pcmk__str_eq(node->name, node_uname, pcmk__str_casei)) {
known = TRUE;
break;
}
}
if (known == FALSE) {
cib_t *cib_conn = controld_globals.cib_conn;
int delete_call_id = 0;
xmlNode *node_state_xml = NULL;
crm_notice("Deleting unknown node %s/%s which has conflicting uname with %s",
node_uuid, node_uname, new_node_uuid);
delete_call_id = cib_conn->cmds->remove(cib_conn, PCMK_XE_NODES,
node_xml, cib_none);
fsa_register_cib_callback(delete_call_id, pcmk__str_copy(node_uuid),
remove_conflicting_node_callback);
node_state_xml = pcmk__xe_create(NULL, PCMK__XE_NODE_STATE);
crm_xml_add(node_state_xml, PCMK_XA_ID, node_uuid);
crm_xml_add(node_state_xml, PCMK_XA_UNAME, node_uname);
delete_call_id = cib_conn->cmds->remove(cib_conn, PCMK_XE_STATUS,
node_state_xml, cib_none);
fsa_register_cib_callback(delete_call_id, pcmk__str_copy(node_uuid),
remove_conflicting_node_callback);
pcmk__xml_free(node_state_xml);
}
}
}
static void
node_list_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
fsa_data_t *msg_data = NULL;
if(call_id < pcmk_ok) {
crm_err("Node list update failed: %s (%d)", pcmk_strerror(call_id), call_id);
crm_log_xml_debug(msg, "update:failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
} else if(rc < pcmk_ok) {
crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
crm_log_xml_debug(msg, "update:failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
void
populate_cib_nodes(enum node_update_flags flags, const char *source)
{
cib_t *cib_conn = controld_globals.cib_conn;
int call_id = 0;
gboolean from_hashtable = TRUE;
xmlNode *node_list = pcmk__xe_create(NULL, PCMK_XE_NODES);
#if SUPPORT_COROSYNC
if (!pcmk_is_set(flags, node_update_quick)
&& (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync)) {
from_hashtable = pcmk__corosync_add_nodes(node_list);
}
#endif
if (from_hashtable) {
GHashTableIter iter;
pcmk__node_status_t *node = NULL;
GString *xpath = NULL;
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
xmlNode *new_node = NULL;
if ((node->xml_id != NULL) && (node->name != NULL)) {
crm_trace("Creating node entry for %s/%s",
node->name, node->xml_id);
if (xpath == NULL) {
xpath = g_string_sized_new(512);
} else {
g_string_truncate(xpath, 0);
}
/* We need both to be valid */
new_node = pcmk__xe_create(node_list, PCMK_XE_NODE);
crm_xml_add(new_node, PCMK_XA_ID, node->xml_id);
crm_xml_add(new_node, PCMK_XA_UNAME, node->name);
/* Search and remove unknown nodes with the conflicting uname from CIB */
pcmk__g_strcat(xpath,
"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION
"/" PCMK_XE_NODES "/" PCMK_XE_NODE
"[@" PCMK_XA_UNAME "='", node->name, "']"
"[@" PCMK_XA_ID "!='", node->xml_id, "']", NULL);
call_id = cib_conn->cmds->query(cib_conn,
(const char *) xpath->str, NULL,
cib_xpath);
fsa_register_cib_callback(call_id, pcmk__str_copy(node->xml_id),
search_conflicting_node_callback);
}
}
if (xpath != NULL) {
g_string_free(xpath, TRUE);
}
}
crm_trace("Populating <nodes> section from %s", from_hashtable ? "hashtable" : "cluster");
if ((controld_update_cib(PCMK_XE_NODES, node_list, cib_none,
node_list_update_callback) == pcmk_rc_ok)
&& (pcmk__peer_cache != NULL) && AM_I_DC) {
/*
* There is no need to update the local CIB with our values if
* we've not seen valid membership data
*/
GHashTableIter iter;
pcmk__node_status_t *node = NULL;
pcmk__xml_free(node_list);
node_list = pcmk__xe_create(NULL, PCMK_XE_STATUS);
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
create_node_state_update(node, flags, node_list, source);
}
if (pcmk__remote_peer_cache != NULL) {
g_hash_table_iter_init(&iter, pcmk__remote_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
create_node_state_update(node, flags, node_list, source);
}
}
controld_update_cib(PCMK_XE_STATUS, node_list, cib_none,
crmd_node_update_complete);
}
pcmk__xml_free(node_list);
}
static void
cib_quorum_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
fsa_data_t *msg_data = NULL;
if (rc == pcmk_ok) {
crm_trace("Quorum update %d complete", call_id);
} else {
crm_err("Quorum update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
crm_log_xml_debug(msg, "failed");
register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
}
}
void
crm_update_quorum(gboolean quorum, gboolean force_update)
{
bool has_quorum = pcmk_is_set(controld_globals.flags, controld_has_quorum);
if (quorum) {
controld_set_global_flags(controld_ever_had_quorum);
} else if (pcmk_all_flags_set(controld_globals.flags,
controld_ever_had_quorum
|controld_no_quorum_suicide)) {
pcmk__panic(__func__);
}
if (AM_I_DC
&& ((has_quorum && !quorum) || (!has_quorum && quorum)
|| force_update)) {
xmlNode *update = NULL;
update = pcmk__xe_create(NULL, PCMK_XE_CIB);
crm_xml_add_int(update, PCMK_XA_HAVE_QUORUM, quorum);
crm_xml_add(update, PCMK_XA_DC_UUID, controld_globals.our_uuid);
crm_debug("Updating quorum status to %s", pcmk__btoa(quorum));
controld_update_cib(PCMK_XE_CIB, update, cib_none,
cib_quorum_update_complete);
pcmk__xml_free(update);
/* Quorum changes usually cause a new transition via other activity:
* quorum gained via a node joining will abort via the node join,
* and quorum lost via a node leaving will usually abort via resource
* activity and/or fencing.
*
* However, it is possible that nothing else causes a transition (e.g.
* someone forces quorum via corosync-cmaptcl, or quorum is lost due to
* a node in standby shutting down cleanly), so here ensure a new
* transition is triggered.
*/
if (quorum) {
/* If quorum was gained, abort after a short delay, in case multiple
* nodes are joining around the same time, so the one that brings us
* to quorum doesn't cause all the remaining ones to be fenced.
*/
abort_after_delay(PCMK_SCORE_INFINITY, pcmk__graph_restart,
"Quorum gained", 5000);
} else {
abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
"Quorum lost", NULL);
}
}
if (quorum) {
controld_set_global_flags(controld_has_quorum);
} else {
controld_clear_global_flags(controld_has_quorum);
}
}
diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
index d9fcd7423d..5e0770a122 100644
--- a/daemons/controld/controld_messages.c
+++ b/daemons/controld/controld_messages.c
@@ -1,1364 +1,1365 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <string.h>
#include <time.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/cluster/internal.h>
#include <crm/cib.h>
#include <crm/common/ipc_internal.h>
#include <pacemaker-controld.h>
static enum crmd_fsa_input handle_message(xmlNode *msg,
enum crmd_fsa_cause cause);
static void handle_response(xmlNode *stored_msg);
static enum crmd_fsa_input handle_request(xmlNode *stored_msg,
enum crmd_fsa_cause cause);
static enum crmd_fsa_input handle_shutdown_request(xmlNode *stored_msg);
static void send_msg_via_ipc(xmlNode * msg, const char *sys);
/* debug only, can wrap all it likes */
static int last_data_id = 0;
void
register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
fsa_data_t * cur_data, void *new_data, const char *raised_from)
{
/* save the current actions if any */
if (controld_globals.fsa_actions != A_NOTHING) {
register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
I_NULL, cur_data ? cur_data->data : NULL,
controld_globals.fsa_actions, TRUE, __func__);
}
/* reset the action list */
crm_info("Resetting the current action list");
fsa_dump_actions(controld_globals.fsa_actions, "Drop");
controld_globals.fsa_actions = A_NOTHING;
/* register the error */
register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
}
void
register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
void *data, uint64_t with_actions,
gboolean prepend, const char *raised_from)
{
unsigned old_len = g_list_length(controld_globals.fsa_message_queue);
fsa_data_t *fsa_data = NULL;
if (raised_from == NULL) {
raised_from = "<unknown>";
}
if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
/* no point doing anything */
crm_err("Cannot add entry to queue: no input and no action");
return;
}
if (input == I_WAIT_FOR_EVENT) {
controld_set_global_flags(controld_fsa_is_stalled);
crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
raised_from, fsa_cause2string(cause), data, old_len);
if (old_len > 0) {
fsa_dump_queue(LOG_TRACE);
prepend = FALSE;
}
if (data == NULL) {
controld_set_fsa_action_flags(with_actions);
fsa_dump_actions(with_actions, "Restored");
return;
}
/* Store everything in the new event and reset
* controld_globals.fsa_actions
*/
with_actions |= controld_globals.fsa_actions;
controld_globals.fsa_actions = A_NOTHING;
}
last_data_id++;
crm_trace("%s %s FSA input %d (%s) due to %s, %s data",
raised_from, (prepend? "prepended" : "appended"), last_data_id,
fsa_input2string(input), fsa_cause2string(cause),
(data? "with" : "without"));
fsa_data = pcmk__assert_alloc(1, sizeof(fsa_data_t));
fsa_data->id = last_data_id;
fsa_data->fsa_input = input;
fsa_data->fsa_cause = cause;
fsa_data->origin = raised_from;
fsa_data->data = NULL;
fsa_data->data_type = fsa_dt_none;
fsa_data->actions = with_actions;
if (with_actions != A_NOTHING) {
crm_trace("Adding actions %.16llx to input",
(unsigned long long) with_actions);
}
if (data != NULL) {
switch (cause) {
case C_FSA_INTERNAL:
case C_CRMD_STATUS_CALLBACK:
case C_IPC_MESSAGE:
case C_HA_MESSAGE:
CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
crm_err("Bogus data from %s", raised_from));
crm_trace("Copying %s data from %s as cluster message data",
fsa_cause2string(cause), raised_from);
fsa_data->data = copy_ha_msg_input(data);
fsa_data->data_type = fsa_dt_ha_msg;
break;
case C_LRM_OP_CALLBACK:
crm_trace("Copying %s data from %s as lrmd_event_data_t",
fsa_cause2string(cause), raised_from);
fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
fsa_data->data_type = fsa_dt_lrm;
break;
case C_TIMER_POPPED:
case C_SHUTDOWN:
case C_UNKNOWN:
case C_STARTUP:
crm_crit("Copying %s data (from %s) is not yet implemented",
fsa_cause2string(cause), raised_from);
crmd_exit(CRM_EX_SOFTWARE);
break;
}
}
/* make sure to free it properly later */
if (prepend) {
controld_globals.fsa_message_queue
= g_list_prepend(controld_globals.fsa_message_queue, fsa_data);
} else {
controld_globals.fsa_message_queue
= g_list_append(controld_globals.fsa_message_queue, fsa_data);
}
crm_trace("FSA message queue length is %d",
g_list_length(controld_globals.fsa_message_queue));
/* fsa_dump_queue(LOG_TRACE); */
if (old_len == g_list_length(controld_globals.fsa_message_queue)) {
crm_err("Couldn't add message to the queue");
}
if (input != I_WAIT_FOR_EVENT) {
controld_trigger_fsa();
}
}
void
fsa_dump_queue(int log_level)
{
int offset = 0;
for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
iter = iter->next) {
fsa_data_t *data = (fsa_data_t *) iter->data;
do_crm_log_unlikely(log_level,
"queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
offset++, data->id, fsa_input2string(data->fsa_input),
data->origin, data->data, data->data_type,
fsa_cause2string(data->fsa_cause));
}
}
ha_msg_input_t *
copy_ha_msg_input(ha_msg_input_t * orig)
{
xmlNode *wrapper = NULL;
ha_msg_input_t *copy = pcmk__assert_alloc(1, sizeof(ha_msg_input_t));
copy->msg = (orig != NULL)? pcmk__xml_copy(NULL, orig->msg) : NULL;
wrapper = pcmk__xe_first_child(copy->msg, PCMK__XE_CRM_XML, NULL, NULL);
copy->xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
return copy;
}
void
delete_fsa_input(fsa_data_t * fsa_data)
{
lrmd_event_data_t *op = NULL;
xmlNode *foo = NULL;
if (fsa_data == NULL) {
return;
}
crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));
if (fsa_data->data != NULL) {
switch (fsa_data->data_type) {
case fsa_dt_ha_msg:
delete_ha_msg_input(fsa_data->data);
break;
case fsa_dt_xml:
foo = fsa_data->data;
pcmk__xml_free(foo);
break;
case fsa_dt_lrm:
op = (lrmd_event_data_t *) fsa_data->data;
lrmd_free_event(op);
break;
case fsa_dt_none:
if (fsa_data->data != NULL) {
crm_err("Don't know how to free %s data from %s",
fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
crmd_exit(CRM_EX_SOFTWARE);
}
break;
}
crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
}
free(fsa_data);
}
/* returns the next message */
fsa_data_t *
get_message(void)
{
fsa_data_t *message
= (fsa_data_t *) controld_globals.fsa_message_queue->data;
controld_globals.fsa_message_queue
= g_list_remove(controld_globals.fsa_message_queue, message);
crm_trace("Processing input %d", message->id);
return message;
}
void *
fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
{
void *ret_val = NULL;
if (fsa_data == NULL) {
crm_err("%s: No FSA data available", caller);
} else if (fsa_data->data == NULL) {
crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);
} else if (fsa_data->data_type != a_type) {
crm_crit("%s: Message data was the wrong type! %d vs. requested=%d. Origin: %s",
caller, fsa_data->data_type, a_type, fsa_data->origin);
CRM_ASSERT(fsa_data->data_type == a_type);
} else {
ret_val = fsa_data->data;
}
return ret_val;
}
/* A_MSG_ROUTE */
void
do_msg_route(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state,
enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
route_message(msg_data->fsa_cause, input->msg);
}
void
route_message(enum crmd_fsa_cause cause, xmlNode * input)
{
ha_msg_input_t fsa_input;
enum crmd_fsa_input result = I_NULL;
fsa_input.msg = input;
CRM_CHECK(cause == C_IPC_MESSAGE || cause == C_HA_MESSAGE, return);
/* try passing the buck first */
if (relay_message(input, cause == C_IPC_MESSAGE)) {
return;
}
/* handle locally */
result = handle_message(input, cause);
/* done or process later? */
switch (result) {
case I_NULL:
case I_CIB_OP:
case I_ROUTER:
case I_NODE_JOIN:
case I_JOIN_REQUEST:
case I_JOIN_RESULT:
break;
default:
/* Defering local processing of message */
register_fsa_input_later(cause, result, &fsa_input);
return;
}
if (result != I_NULL) {
/* add to the front of the queue */
register_fsa_input(cause, result, &fsa_input);
}
}
gboolean
relay_message(xmlNode * msg, gboolean originated_locally)
{
enum pcmk_ipc_server dest = pcmk_ipc_unknown;
bool is_for_dc = false;
bool is_for_dcib = false;
bool is_for_te = false;
bool is_for_crm = false;
bool is_for_cib = false;
bool is_local = false;
bool broadcast = false;
const char *host_to = NULL;
const char *sys_to = NULL;
const char *sys_from = NULL;
const char *type = NULL;
const char *task = NULL;
const char *ref = NULL;
pcmk__node_status_t *node_to = NULL;
CRM_CHECK(msg != NULL, return TRUE);
host_to = crm_element_value(msg, PCMK__XA_CRM_HOST_TO);
sys_to = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
sys_from = crm_element_value(msg, PCMK__XA_CRM_SYS_FROM);
type = crm_element_value(msg, PCMK__XA_T);
task = crm_element_value(msg, PCMK__XA_CRM_TASK);
ref = crm_element_value(msg, PCMK_XA_REFERENCE);
broadcast = pcmk__str_empty(host_to);
if (ref == NULL) {
ref = "without reference ID";
}
if (pcmk__str_eq(task, CRM_OP_HELLO, pcmk__str_casei)) {
crm_trace("Received hello %s from %s (no processing needed)",
ref, pcmk__s(sys_from, "unidentified source"));
crm_log_xml_trace(msg, "hello");
return TRUE;
}
// Require message type (set by create_request())
if (!pcmk__str_eq(type, PCMK__VALUE_CRMD, pcmk__str_none)) {
crm_warn("Ignoring invalid message %s with type '%s' "
"(not '" PCMK__VALUE_CRMD "')",
ref, pcmk__s(type, ""));
crm_log_xml_trace(msg, "ignored");
return TRUE;
}
// Require a destination subsystem (also set by create_request())
if (sys_to == NULL) {
crm_warn("Ignoring invalid message %s with no " PCMK__XA_CRM_SYS_TO,
ref);
crm_log_xml_trace(msg, "ignored");
return TRUE;
}
// Get the message type appropriate to the destination subsystem
if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) {
dest = pcmk__parse_server(sys_to);
if (dest == pcmk_ipc_unknown) {
/* Unrecognized value, use a sane default
*
* @TODO Maybe we should bail instead
*/
dest = pcmk_ipc_controld;
}
}
is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);
// Check whether message should be processed locally
is_local = false;
if (broadcast) {
if (is_for_dc || is_for_te) {
is_local = false;
} else if (is_for_crm) {
if (pcmk__strcase_any_of(task, CRM_OP_NODE_INFO,
PCMK__CONTROLD_CMD_NODES, NULL)) {
/* Node info requests do not specify a host, which is normally
* treated as "all hosts", because the whole point is that the
* client may not know the local node name. Always handle these
* requests locally.
*/
is_local = true;
} else {
is_local = !originated_locally;
}
} else {
is_local = true;
}
} else if (pcmk__str_eq(controld_globals.our_nodename, host_to,
pcmk__str_casei)) {
is_local = true;
} else if (is_for_crm && pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL,
NULL);
xmlNode *msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
const char *mode = crm_element_value(msg_data, PCMK__XA_MODE);
if (pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) {
// Local delete of an offline node's resource history
is_local = true;
}
}
// Check whether message should be relayed
if (is_for_dc || is_for_dcib || is_for_te) {
if (AM_I_DC) {
if (is_for_te) {
crm_trace("Route message %s locally as transition request",
ref);
crm_log_xml_trace(msg, sys_to);
send_msg_via_ipc(msg, sys_to);
return TRUE; // No further processing of message is needed
}
crm_trace("Route message %s locally as DC request", ref);
return FALSE; // More to be done by caller
}
if (originated_locally
&& !pcmk__strcase_any_of(sys_from, CRM_SYSTEM_PENGINE,
CRM_SYSTEM_TENGINE, NULL)) {
crm_trace("Relay message %s to DC (via %s)",
ref, pcmk__s(host_to, "broadcast"));
crm_log_xml_trace(msg, "relayed");
if (!broadcast) {
node_to = pcmk__get_node(0, host_to, NULL,
pcmk__node_search_cluster_member);
}
pcmk__cluster_send_message(node_to, dest, msg);
return TRUE;
}
/* Transition engine and scheduler messages are sent only to the DC on
* the same node. If we are no longer the DC, discard this message.
*/
crm_trace("Ignoring message %s because we are no longer DC", ref);
crm_log_xml_trace(msg, "ignored");
return TRUE; // No further processing of message is needed
}
if (is_local) {
if (is_for_crm || is_for_cib) {
crm_trace("Route message %s locally as controller request", ref);
return FALSE; // More to be done by caller
}
crm_trace("Relay message %s locally to %s", ref, sys_to);
crm_log_xml_trace(msg, "IPC-relay");
send_msg_via_ipc(msg, sys_to);
return TRUE;
}
if (!broadcast) {
node_to = pcmk__search_node_caches(0, host_to,
pcmk__node_search_cluster_member);
if (node_to == NULL) {
crm_warn("Ignoring message %s because node %s is unknown",
ref, host_to);
crm_log_xml_trace(msg, "ignored");
return TRUE;
}
}
crm_trace("Relay message %s to %s",
ref, pcmk__s(host_to, "all peers"));
crm_log_xml_trace(msg, "relayed");
pcmk__cluster_send_message(node_to, dest, msg);
return TRUE;
}
// Return true if field contains a positive integer
static bool
authorize_version(xmlNode *message_data, const char *field,
const char *client_name, const char *ref, const char *uuid)
{
const char *version = crm_element_value(message_data, field);
long long version_num;
if ((pcmk__scan_ll(version, &version_num, -1LL) != pcmk_rc_ok)
|| (version_num < 0LL)) {
crm_warn("Rejected IPC hello from %s: '%s' is not a valid protocol %s "
QB_XS " ref=%s uuid=%s",
client_name, ((version == NULL)? "" : version),
field, (ref? ref : "none"), uuid);
return false;
}
return true;
}
/*!
* \internal
* \brief Check whether a client IPC message is acceptable
*
* If a given client IPC message is a hello, "authorize" it by ensuring it has
* valid information such as a protocol version, and return false indicating
* that nothing further needs to be done with the message. If the message is not
* a hello, just return true to indicate it needs further processing.
*
* \param[in] client_msg XML of IPC message
* \param[in,out] curr_client If IPC is not proxied, client that sent message
* \param[in] proxy_session If IPC is proxied, the session ID
*
* \return true if message needs further processing, false if it doesn't
*/
bool
controld_authorize_ipc_message(const xmlNode *client_msg, pcmk__client_t *curr_client,
const char *proxy_session)
{
xmlNode *wrapper = NULL;
xmlNode *message_data = NULL;
const char *client_name = NULL;
const char *op = crm_element_value(client_msg, PCMK__XA_CRM_TASK);
const char *ref = crm_element_value(client_msg, PCMK_XA_REFERENCE);
const char *uuid = (curr_client? curr_client->id : proxy_session);
if (uuid == NULL) {
crm_warn("IPC message from client rejected: No client identifier "
QB_XS " ref=%s", (ref? ref : "none"));
goto rejected;
}
if (!pcmk__str_eq(CRM_OP_HELLO, op, pcmk__str_casei)) {
// Only hello messages need to be authorized
return true;
}
wrapper = pcmk__xe_first_child(client_msg, PCMK__XE_CRM_XML, NULL, NULL);
message_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
client_name = crm_element_value(message_data, PCMK__XA_CLIENT_NAME);
if (pcmk__str_empty(client_name)) {
crm_warn("IPC hello from client rejected: No client name",
QB_XS " ref=%s uuid=%s", (ref? ref : "none"), uuid);
goto rejected;
}
if (!authorize_version(message_data, PCMK__XA_MAJOR_VERSION, client_name,
ref, uuid)) {
goto rejected;
}
if (!authorize_version(message_data, PCMK__XA_MINOR_VERSION, client_name,
ref, uuid)) {
goto rejected;
}
crm_trace("Validated IPC hello from client %s", client_name);
crm_log_xml_trace(client_msg, "hello");
if (curr_client) {
curr_client->userdata = pcmk__str_copy(client_name);
}
controld_trigger_fsa();
return false;
rejected:
crm_log_xml_trace(client_msg, "rejected");
if (curr_client) {
qb_ipcs_disconnect(curr_client->ipcs);
}
return false;
}
static enum crmd_fsa_input
handle_message(xmlNode *msg, enum crmd_fsa_cause cause)
{
const char *type = NULL;
CRM_CHECK(msg != NULL, return I_NULL);
type = crm_element_value(msg, PCMK__XA_SUBT);
if (pcmk__str_eq(type, PCMK__VALUE_REQUEST, pcmk__str_none)) {
return handle_request(msg, cause);
}
if (pcmk__str_eq(type, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
handle_response(msg);
return I_NULL;
}
crm_warn("Ignoring message with unknown " PCMK__XA_SUBT" '%s'",
pcmk__s(type, ""));
crm_log_xml_trace(msg, "bad");
return I_NULL;
}
static enum crmd_fsa_input
handle_failcount_op(xmlNode * stored_msg)
{
const char *rsc = NULL;
const char *uname = NULL;
const char *op = NULL;
char *interval_spec = NULL;
guint interval_ms = 0;
gboolean is_remote_node = FALSE;
xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML, NULL,
NULL);
xmlNode *xml_op = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
if (xml_op) {
xmlNode *xml_rsc = pcmk__xe_first_child(xml_op, PCMK_XE_PRIMITIVE, NULL,
NULL);
xmlNode *xml_attrs = pcmk__xe_first_child(xml_op, PCMK__XE_ATTRIBUTES,
NULL, NULL);
if (xml_rsc) {
rsc = pcmk__xe_id(xml_rsc);
}
if (xml_attrs) {
op = crm_element_value(xml_attrs,
CRM_META "_" PCMK__META_CLEAR_FAILURE_OP);
crm_element_value_ms(xml_attrs,
CRM_META "_" PCMK__META_CLEAR_FAILURE_INTERVAL,
&interval_ms);
}
}
uname = crm_element_value(xml_op, PCMK__META_ON_NODE);
if ((rsc == NULL) || (uname == NULL)) {
crm_log_xml_warn(stored_msg, "invalid failcount op");
return I_NULL;
}
if (crm_element_value(xml_op, PCMK__XA_ROUTER_NODE)) {
is_remote_node = TRUE;
}
crm_debug("Clearing failures for %s-interval %s on %s "
"from attribute manager, CIB, and executor state",
pcmk__readable_interval(interval_ms), rsc, uname);
if (interval_ms) {
interval_spec = crm_strdup_printf("%ums", interval_ms);
}
update_attrd_clear_failures(uname, rsc, op, interval_spec, is_remote_node);
free(interval_spec);
controld_cib_delete_last_failure(rsc, uname, op, interval_ms);
lrm_clear_last_failure(rsc, uname, op, interval_ms);
return I_NULL;
}
static enum crmd_fsa_input
handle_lrm_delete(xmlNode *stored_msg)
{
const char *mode = NULL;
xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML, NULL,
NULL);
xmlNode *msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
CRM_CHECK(msg_data != NULL, return I_NULL);
/* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to
* relay the operation to the affected node, which will unregister the
* resource from the local executor, clear the resource's history from the
* CIB, and do some bookkeeping in the controller.
*
* However, if the affected node is offline, the client will specify
* mode=PCMK__VALUE_CIB which means the controller receiving the operation
* should clear the resource's history from the CIB and nothing else. This
* is used to clear shutdown locks.
*/
mode = crm_element_value(msg_data, PCMK__XA_MODE);
if (!pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) {
// Relay to affected node
crm_xml_add(stored_msg, PCMK__XA_CRM_SYS_TO, CRM_SYSTEM_LRMD);
return I_ROUTER;
} else {
// Delete CIB history locally (compare with do_lrm_delete())
const char *from_sys = NULL;
const char *user_name = NULL;
const char *rsc_id = NULL;
const char *node = NULL;
xmlNode *rsc_xml = NULL;
int rc = pcmk_rc_ok;
rsc_xml = pcmk__xe_first_child(msg_data, PCMK_XE_PRIMITIVE, NULL, NULL);
CRM_CHECK(rsc_xml != NULL, return I_NULL);
rsc_id = pcmk__xe_id(rsc_xml);
from_sys = crm_element_value(stored_msg, PCMK__XA_CRM_SYS_FROM);
node = crm_element_value(msg_data, PCMK__META_ON_NODE);
user_name = pcmk__update_acl_user(stored_msg, PCMK__XA_CRM_USER, NULL);
crm_debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s "
"(clearing CIB resource history only)", rsc_id, node,
(user_name? " for user " : ""), (user_name? user_name : ""));
rc = controld_delete_resource_history(rsc_id, node, user_name,
cib_dryrun|cib_sync_call);
if (rc == pcmk_rc_ok) {
rc = controld_delete_resource_history(rsc_id, node, user_name,
crmd_cib_smart_opt());
}
/* Notify client. Also notify tengine if mode=PCMK__VALUE_CIB and
* op=CRM_OP_LRM_DELETE.
*/
if (from_sys) {
lrmd_event_data_t *op = NULL;
const char *from_host = crm_element_value(stored_msg, PCMK__XA_SRC);
const char *transition;
if (strcmp(from_sys, CRM_SYSTEM_TENGINE)) {
transition = crm_element_value(msg_data,
PCMK__XA_TRANSITION_KEY);
} else {
transition = crm_element_value(stored_msg,
PCMK__XA_TRANSITION_KEY);
}
crm_info("Notifying %s on %s that %s was%s deleted",
from_sys, (from_host? from_host : "local node"), rsc_id,
((rc == pcmk_rc_ok)? "" : " not"));
op = lrmd_new_event(rsc_id, PCMK_ACTION_DELETE, 0);
op->type = lrmd_event_exec_complete;
op->user_data = pcmk__str_copy(pcmk__s(transition, FAKE_TE_ID));
op->params = pcmk__strkey_table(free, free);
pcmk__insert_dup(op->params, PCMK_XA_CRM_FEATURE_SET,
CRM_FEATURE_SET);
controld_rc2event(op, rc);
controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
lrmd_free_event(op);
controld_trigger_delete_refresh(from_sys, rsc_id);
}
return I_NULL;
}
}
/*!
* \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache
*
* \param[in] msg Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_remote_state(const xmlNode *msg)
{
const char *conn_host = NULL;
const char *remote_uname = pcmk__xe_id(msg);
pcmk__node_status_t *remote_peer;
bool remote_is_up = false;
int rc = pcmk_rc_ok;
rc = pcmk__xe_get_bool_attr(msg, PCMK__XA_IN_CCM, &remote_is_up);
CRM_CHECK(remote_uname && rc == pcmk_rc_ok, return I_NULL);
remote_peer = pcmk__cluster_lookup_remote_node(remote_uname);
CRM_CHECK(remote_peer, return I_NULL);
pcmk__update_peer_state(__func__, remote_peer,
remote_is_up ? PCMK_VALUE_MEMBER : PCMK__VALUE_LOST,
0);
conn_host = crm_element_value(msg, PCMK__XA_CONNECTION_HOST);
if (conn_host) {
pcmk__str_update(&remote_peer->conn_host, conn_host);
} else if (remote_peer->conn_host) {
free(remote_peer->conn_host);
remote_peer->conn_host = NULL;
}
return I_NULL;
}
/*!
* \brief Handle a CRM_OP_PING message
*
* \param[in] msg Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_ping(const xmlNode *msg)
{
const char *value = NULL;
xmlNode *ping = NULL;
xmlNode *reply = NULL;
// Build reply
ping = pcmk__xe_create(NULL, PCMK__XE_PING_RESPONSE);
value = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
crm_xml_add(ping, PCMK__XA_CRM_SUBSYSTEM, value);
// Add controller state
value = fsa_state2string(controld_globals.fsa_state);
crm_xml_add(ping, PCMK__XA_CRMD_STATE, value);
crm_notice("Current ping state: %s", value); // CTS needs this
// Add controller health
// @TODO maybe do some checks to determine meaningful status
crm_xml_add(ping, PCMK_XA_RESULT, "ok");
// Send reply
reply = pcmk__new_reply(msg, ping);
pcmk__xml_free(ping);
if (reply != NULL) {
(void) relay_message(reply, TRUE);
pcmk__xml_free(reply);
}
// Nothing further to do
return I_NULL;
}
/*!
* \brief Handle a PCMK__CONTROLD_CMD_NODES message
*
* \param[in] request Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_node_list(const xmlNode *request)
{
GHashTableIter iter;
pcmk__node_status_t *node = NULL;
xmlNode *reply = NULL;
xmlNode *reply_data = NULL;
// Create message data for reply
reply_data = pcmk__xe_create(NULL, PCMK_XE_NODES);
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
xmlNode *xml = pcmk__xe_create(reply_data, PCMK_XE_NODE);
crm_xml_add_ll(xml, PCMK_XA_ID,
(long long) node->cluster_layer_id); // uint32_t
crm_xml_add(xml, PCMK_XA_UNAME, node->name);
crm_xml_add(xml, PCMK__XA_IN_CCM, node->state);
}
// Create and send reply
reply = pcmk__new_reply(request, reply_data);
pcmk__xml_free(reply_data);
if (reply) {
(void) relay_message(reply, TRUE);
pcmk__xml_free(reply);
}
// Nothing further to do
return I_NULL;
}
/*!
* \brief Handle a CRM_OP_NODE_INFO request
*
* \param[in] msg Message XML
*
* \return Next FSA input
*/
static enum crmd_fsa_input
handle_node_info_request(const xmlNode *msg)
{
const char *value = NULL;
pcmk__node_status_t *node = NULL;
int node_id = 0;
xmlNode *reply = NULL;
xmlNode *reply_data = NULL;
// Build reply
reply_data = pcmk__xe_create(NULL, PCMK_XE_NODE);
crm_xml_add(reply_data, PCMK__XA_CRM_SUBSYSTEM, CRM_SYSTEM_CRMD);
// Add whether current partition has quorum
pcmk__xe_set_bool_attr(reply_data, PCMK_XA_HAVE_QUORUM,
pcmk_is_set(controld_globals.flags,
controld_has_quorum));
/* Check whether client requested node info by ID and/or name
*
* @TODO A Corosync-layer node ID is of type uint32_t. We should be able to
* handle legitimate node IDs greater than INT_MAX, but currently we do not.
*/
crm_element_value_int(msg, PCMK_XA_ID, &node_id);
if (node_id < 0) {
node_id = 0;
}
value = crm_element_value(msg, PCMK_XA_UNAME);
// Default to local node if none given
if ((node_id == 0) && (value == NULL)) {
value = controld_globals.our_nodename;
}
node = pcmk__search_node_caches(node_id, value, pcmk__node_search_any);
if (node) {
crm_xml_add(reply_data, PCMK_XA_ID, node->xml_id);
crm_xml_add(reply_data, PCMK_XA_UNAME, node->name);
crm_xml_add(reply_data, PCMK_XA_CRMD, node->state);
pcmk__xe_set_bool_attr(reply_data, PCMK_XA_REMOTE_NODE,
pcmk_is_set(node->flags,
pcmk__node_status_remote));
}
// Send reply
reply = pcmk__new_reply(msg, reply_data);
pcmk__xml_free(reply_data);
if (reply != NULL) {
(void) relay_message(reply, TRUE);
pcmk__xml_free(reply);
}
// Nothing further to do
return I_NULL;
}
static void
verify_feature_set(xmlNode *msg)
{
const char *dc_version = crm_element_value(msg, PCMK_XA_CRM_FEATURE_SET);
if (dc_version == NULL) {
/* All we really know is that the DC feature set is older than 3.1.0,
* but that's also all that really matters.
*/
dc_version = "3.0.14";
}
if (feature_set_compatible(dc_version, CRM_FEATURE_SET)) {
crm_trace("Local feature set (%s) is compatible with DC's (%s)",
CRM_FEATURE_SET, dc_version);
} else {
crm_err("Local feature set (%s) is incompatible with DC's (%s)",
CRM_FEATURE_SET, dc_version);
// Nothing is likely to improve without administrator involvement
controld_set_fsa_input_flags(R_STAYDOWN);
crmd_exit(CRM_EX_FATAL);
}
}
// DC gets own shutdown all-clear
static enum crmd_fsa_input
handle_shutdown_self_ack(xmlNode *stored_msg)
{
const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);
if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
// The expected case -- we initiated own shutdown sequence
crm_info("Shutting down controller");
return I_STOP;
}
if (pcmk__str_eq(host_from, controld_globals.dc_name, pcmk__str_casei)) {
// Must be logic error -- DC confirming its own unrequested shutdown
crm_err("Shutting down controller immediately due to "
"unexpected shutdown confirmation");
return I_TERMINATE;
}
if (controld_globals.fsa_state != S_STOPPING) {
// Shouldn't happen -- non-DC confirming unrequested shutdown
crm_err("Starting new DC election because %s is "
"confirming shutdown we did not request",
(host_from? host_from : "another node"));
return I_ELECTION;
}
// Shouldn't happen, but we are already stopping anyway
crm_debug("Ignoring unexpected shutdown confirmation from %s",
(host_from? host_from : "another node"));
return I_NULL;
}
// Non-DC gets shutdown all-clear from DC
static enum crmd_fsa_input
handle_shutdown_ack(xmlNode *stored_msg)
{
const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);
if (host_from == NULL) {
crm_warn("Ignoring shutdown request without origin specified");
return I_NULL;
}
if (pcmk__str_eq(host_from, controld_globals.dc_name,
pcmk__str_null_matches|pcmk__str_casei)) {
if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
crm_info("Shutting down controller after confirmation from %s",
host_from);
} else {
crm_err("Shutting down controller after unexpected "
"shutdown request from %s", host_from);
controld_set_fsa_input_flags(R_STAYDOWN);
}
return I_STOP;
}
crm_warn("Ignoring shutdown request from %s because DC is %s",
host_from, controld_globals.dc_name);
return I_NULL;
}
static enum crmd_fsa_input
handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause)
{
xmlNode *msg = NULL;
const char *op = crm_element_value(stored_msg, PCMK__XA_CRM_TASK);
/* Optimize this for the DC - it has the most to do */
crm_log_xml_trace(stored_msg, "request");
if (op == NULL) {
crm_warn("Ignoring request without " PCMK__XA_CRM_TASK);
return I_NULL;
}
if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
const char *from = crm_element_value(stored_msg, PCMK__XA_SRC);
pcmk__node_status_t *node =
pcmk__search_node_caches(0, from, pcmk__node_search_cluster_member);
pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
if(AM_I_DC == FALSE) {
return I_NULL; /* Done */
}
}
/*========== DC-Only Actions ==========*/
if (AM_I_DC) {
if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
return I_NODE_JOIN;
} else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
return I_JOIN_REQUEST;
} else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
return I_JOIN_RESULT;
} else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
return handle_shutdown_self_ack(stored_msg);
} else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
// Another controller wants to shut down its node
return handle_shutdown_request(stored_msg);
}
}
/*========== common actions ==========*/
if (strcmp(op, CRM_OP_NOVOTE) == 0) {
ha_msg_input_t fsa_input;
fsa_input.msg = stored_msg;
register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE,
__func__);
} else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) {
/* a remote connection host is letting us know the node state */
return handle_remote_state(stored_msg);
} else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
throttle_update(stored_msg);
if (AM_I_DC && (controld_globals.transition_graph != NULL)
&& !controld_globals.transition_graph->complete) {
crm_debug("The throttle changed. Trigger a graph.");
trigger_graph();
}
return I_NULL;
} else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
return handle_failcount_op(stored_msg);
} else if (strcmp(op, CRM_OP_VOTE) == 0) {
/* count the vote and decide what to do after that */
ha_msg_input_t fsa_input;
fsa_input.msg = stored_msg;
register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE,
__func__);
/* Sometimes we _must_ go into S_ELECTION */
if (controld_globals.fsa_state == S_HALT) {
crm_debug("Forcing an election from S_HALT");
return I_ELECTION;
}
} else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
verify_feature_set(stored_msg);
crm_debug("Raising I_JOIN_OFFER: join-%s",
crm_element_value(stored_msg, PCMK__XA_JOIN_ID));
return I_JOIN_OFFER;
} else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
crm_debug("Raising I_JOIN_RESULT: join-%s",
crm_element_value(stored_msg, PCMK__XA_JOIN_ID));
return I_JOIN_RESULT;
} else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) {
return handle_lrm_delete(stored_msg);
} else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0)
|| (strcmp(op, CRM_OP_LRM_REFRESH) == 0) // @COMPAT
|| (strcmp(op, CRM_OP_REPROBE) == 0)) {
crm_xml_add(stored_msg, PCMK__XA_CRM_SYS_TO, CRM_SYSTEM_LRMD);
return I_ROUTER;
} else if (strcmp(op, CRM_OP_NOOP) == 0) {
return I_NULL;
} else if (strcmp(op, CRM_OP_PING) == 0) {
return handle_ping(stored_msg);
} else if (strcmp(op, CRM_OP_NODE_INFO) == 0) {
return handle_node_info_request(stored_msg);
} else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
int id = 0;
const char *name = NULL;
crm_element_value_int(stored_msg, PCMK_XA_ID, &id);
name = crm_element_value(stored_msg, PCMK_XA_UNAME);
if(cause == C_IPC_MESSAGE) {
- msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+ msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL,
+ CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD);
if (!pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg)) {
crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
} else {
crm_notice("Instructing peers to remove references to node %s/%u", name, id);
}
pcmk__xml_free(msg);
} else {
pcmk__cluster_forget_cluster_node(id, name);
/* If we're forgetting this node, also forget any failures to fence
* it, so we don't carry that over to any node added later with the
* same name.
*/
st_fail_count_reset(name);
}
} else if (strcmp(op, CRM_OP_MAINTENANCE_NODES) == 0) {
xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML,
NULL, NULL);
xmlNode *xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
remote_ra_process_maintenance_nodes(xml);
} else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) {
return handle_node_list(stored_msg);
/*========== (NOT_DC)-Only Actions ==========*/
} else if (!AM_I_DC) {
if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
return handle_shutdown_ack(stored_msg);
}
} else {
crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
crm_log_xml_err(stored_msg, "Unexpected");
}
return I_NULL;
}
static void
handle_response(xmlNode *stored_msg)
{
const char *op = crm_element_value(stored_msg, PCMK__XA_CRM_TASK);
crm_log_xml_trace(stored_msg, "reply");
if (op == NULL) {
crm_warn("Ignoring reply without " PCMK__XA_CRM_TASK);
} else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
// Check whether scheduler answer been superseded by subsequent request
const char *msg_ref = crm_element_value(stored_msg, PCMK_XA_REFERENCE);
if (msg_ref == NULL) {
crm_err("%s - Ignoring calculation with no reference", op);
} else if (pcmk__str_eq(msg_ref, controld_globals.fsa_pe_ref,
pcmk__str_none)) {
ha_msg_input_t fsa_input;
controld_stop_sched_timer();
fsa_input.msg = stored_msg;
register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);
} else {
crm_info("%s calculation %s is obsolete", op, msg_ref);
}
} else if (strcmp(op, CRM_OP_VOTE) == 0
|| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 || strcmp(op, CRM_OP_SHUTDOWN) == 0) {
} else {
const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);
crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
op, host_from, AM_I_DC ? "DC" : "controller");
}
}
static enum crmd_fsa_input
handle_shutdown_request(xmlNode * stored_msg)
{
/* handle here to avoid potential version issues
* where the shutdown message/procedure may have
* been changed in later versions.
*
* This way the DC is always in control of the shutdown
*/
char *now_s = NULL;
const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);
if (host_from == NULL) {
/* we're shutting down and the DC */
host_from = controld_globals.our_nodename;
}
crm_info("Creating shutdown request for %s (state=%s)", host_from,
fsa_state2string(controld_globals.fsa_state));
crm_log_xml_trace(stored_msg, "message");
now_s = pcmk__ttoa(time(NULL));
update_attrd(host_from, PCMK__NODE_ATTR_SHUTDOWN, now_s, NULL, FALSE);
free(now_s);
/* will be picked up by the TE as long as its running */
return I_NULL;
}
static void
send_msg_via_ipc(xmlNode * msg, const char *sys)
{
pcmk__client_t *client_channel = NULL;
CRM_CHECK(sys != NULL, return);
client_channel = pcmk__find_client_by_id(sys);
if (crm_element_value(msg, PCMK__XA_SRC) == NULL) {
crm_xml_add(msg, PCMK__XA_SRC, controld_globals.our_nodename);
}
if (client_channel != NULL) {
/* Transient clients such as crmadmin */
pcmk__ipc_send_xml(client_channel, 0, msg, crm_ipc_server_event);
} else if (pcmk__str_eq(sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL,
NULL);
xmlNode *data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
process_te_message(msg, data);
} else if (pcmk__str_eq(sys, CRM_SYSTEM_LRMD, pcmk__str_none)) {
fsa_data_t fsa_data;
ha_msg_input_t fsa_input;
xmlNode *wrapper = NULL;
fsa_input.msg = msg;
wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL, NULL);
fsa_input.xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
fsa_data.id = 0;
fsa_data.actions = 0;
fsa_data.data = &fsa_input;
fsa_data.fsa_input = I_MESSAGE;
fsa_data.fsa_cause = C_IPC_MESSAGE;
fsa_data.origin = __func__;
fsa_data.data_type = fsa_dt_ha_msg;
do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, controld_globals.fsa_state,
I_MESSAGE, &fsa_data);
} else if (crmd_is_proxy_session(sys)) {
crmd_proxy_send(sys, msg);
} else {
crm_info("Received invalid request: unknown subsystem '%s'", sys);
}
}
void
delete_ha_msg_input(ha_msg_input_t * orig)
{
if (orig == NULL) {
return;
}
pcmk__xml_free(orig->msg);
free(orig);
}
/*!
* \internal
* \brief Notify the cluster of a remote node state change
*
* \param[in] node_name Node's name
* \param[in] node_up true if node is up, false if down
*/
void
broadcast_remote_state_message(const char *node_name, bool node_up)
{
xmlNode *msg = create_request(CRM_OP_REMOTE_STATE, NULL, NULL,
- CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+ CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD);
crm_info("Notifying cluster of Pacemaker Remote node %s %s",
node_name, node_up? "coming up" : "going down");
crm_xml_add(msg, PCMK_XA_ID, node_name);
pcmk__xe_set_bool_attr(msg, PCMK__XA_IN_CCM, node_up);
if (node_up) {
crm_xml_add(msg, PCMK__XA_CONNECTION_HOST,
controld_globals.our_nodename);
}
pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg);
pcmk__xml_free(msg);
}
diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c
index 0cc19250e0..779552a267 100644
--- a/daemons/controld/controld_te_actions.c
+++ b/daemons/controld/controld_te_actions.c
@@ -1,765 +1,766 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/param.h>
#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/lrmd.h> // lrmd_event_data_t, lrmd_free_event()
#include <crm/common/xml.h>
#include <crm/cluster.h>
#include <pacemaker-internal.h>
#include <pacemaker-controld.h>
static GHashTable *te_targets = NULL;
void send_rsc_command(pcmk__graph_action_t *action);
static void te_update_job_count(pcmk__graph_action_t *action, int offset);
static void
te_start_action_timer(const pcmk__graph_t *graph, pcmk__graph_action_t *action)
{
action->timer = g_timeout_add(action->timeout + graph->network_delay,
action_timer_callback, (void *) action);
CRM_ASSERT(action->timer != 0);
}
/*!
* \internal
* \brief Execute a graph pseudo-action
*
* \param[in,out] graph Transition graph being executed
* \param[in,out] pseudo Pseudo-action to execute
*
* \return Standard Pacemaker return code
*/
static int
execute_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *pseudo)
{
const char *task = crm_element_value(pseudo->xml, PCMK_XA_OPERATION);
/* send to peers as well? */
if (pcmk__str_eq(task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_casei)) {
GHashTableIter iter;
pcmk__node_status_t *node = NULL;
g_hash_table_iter_init(&iter, pcmk__peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
xmlNode *cmd = NULL;
if (pcmk__str_eq(controld_globals.our_nodename, node->name,
pcmk__str_casei)) {
continue;
}
cmd = create_request(task, pseudo->xml, node->name,
- CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
+ CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE);
pcmk__cluster_send_message(node, pcmk_ipc_controld, cmd);
pcmk__xml_free(cmd);
}
remote_ra_process_maintenance_nodes(pseudo->xml);
} else {
/* Check action for Pacemaker Remote node side effects */
remote_ra_process_pseudo(pseudo->xml);
}
crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
crm_element_value(pseudo->xml, PCMK__XA_OPERATION_KEY));
te_action_confirmed(pseudo, graph);
return pcmk_rc_ok;
}
static int
get_target_rc(pcmk__graph_action_t *action)
{
int exit_status;
pcmk__scan_min_int(crm_meta_value(action->params, PCMK__META_OP_TARGET_RC),
&exit_status, 0);
return exit_status;
}
/*!
* \internal
* \brief Execute a cluster action from a transition graph
*
* \param[in,out] graph Transition graph being executed
* \param[in,out] action Cluster action to execute
*
* \return Standard Pacemaker return code
*/
static int
execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
{
char *counter = NULL;
xmlNode *cmd = NULL;
gboolean is_local = FALSE;
const char *id = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
const char *router_node = NULL;
gboolean rc = TRUE;
gboolean no_wait = FALSE;
const pcmk__node_status_t *node = NULL;
id = pcmk__xe_id(action->xml);
CRM_CHECK(!pcmk__str_empty(id), return EPROTO);
task = crm_element_value(action->xml, PCMK_XA_OPERATION);
CRM_CHECK(!pcmk__str_empty(task), return EPROTO);
on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown);
router_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
if (router_node == NULL) {
router_node = on_node;
if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);
if (pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) {
router_node = controld_globals.our_nodename;
}
}
}
if (pcmk__str_eq(router_node, controld_globals.our_nodename,
pcmk__str_casei)) {
is_local = TRUE;
}
value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT);
if (crm_is_true(value)) {
no_wait = TRUE;
}
crm_info("Handling controller request '%s' (%s on %s)%s%s",
id, task, on_node, (is_local? " locally" : ""),
(no_wait? " without waiting" : ""));
if (is_local
&& pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
/* defer until everything else completes */
crm_info("Controller request '%s' is a local shutdown", id);
graph->completion_action = pcmk__graph_shutdown;
graph->abort_reason = "local shutdown";
te_action_confirmed(action, graph);
return pcmk_rc_ok;
} else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
pcmk__node_status_t *peer =
pcmk__get_node(0, router_node, NULL,
pcmk__node_search_cluster_member);
pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
}
- cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
+ cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD,
+ CRM_SYSTEM_TENGINE);
counter = pcmk__transition_key(controld_globals.transition_graph->id,
action->id, get_target_rc(action),
controld_globals.te_uuid);
crm_xml_add(cmd, PCMK__XA_TRANSITION_KEY, counter);
node = pcmk__get_node(0, router_node, NULL,
pcmk__node_search_cluster_member);
rc = pcmk__cluster_send_message(node, pcmk_ipc_controld, cmd);
free(counter);
pcmk__xml_free(cmd);
if (rc == FALSE) {
crm_err("Action %d failed: send", action->id);
return ECOMM;
} else if (no_wait) {
te_action_confirmed(action, graph);
} else {
if (action->timeout <= 0) {
crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %ums instead",
action->id, task, on_node, action->timeout, graph->network_delay);
action->timeout = (int) graph->network_delay;
}
te_start_action_timer(graph, action);
}
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Synthesize an executor event for a resource action timeout
*
* \param[in] action Resource action that timed out
* \param[in] target_rc Expected result of action that timed out
*
* Synthesize an executor event for a resource action timeout. (If the executor
* gets a timeout while waiting for a resource action to complete, that will be
* reported via the usual callback. This timeout means we didn't hear from the
* executor itself or the controller that relayed the action to the executor.)
*
* \return Newly created executor event for result of \p action
* \note The caller is responsible for freeing the return value using
* lrmd_free_event().
*/
static lrmd_event_data_t *
synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc)
{
lrmd_event_data_t *op = NULL;
const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
const char *reason = NULL;
char *dynamic_reason = NULL;
if (pcmk__str_eq(target, pcmk__cluster_local_node_name(),
pcmk__str_casei)) {
reason = "Local executor did not return result in time";
} else {
const char *router_node = NULL;
router_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
if (router_node == NULL) {
router_node = target;
}
dynamic_reason = crm_strdup_printf("Controller on %s did not return "
"result in time", router_node);
reason = dynamic_reason;
}
op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT,
PCMK_OCF_UNKNOWN_ERROR, reason);
op->call_id = -1;
op->user_data = pcmk__transition_key(controld_globals.transition_graph->id,
action->id, target_rc,
controld_globals.te_uuid);
free(dynamic_reason);
return op;
}
static void
controld_record_action_event(pcmk__graph_action_t *action,
lrmd_event_data_t *op)
{
cib_t *cib_conn = controld_globals.cib_conn;
xmlNode *state = NULL;
xmlNode *rsc = NULL;
xmlNode *action_rsc = NULL;
int rc = pcmk_ok;
const char *rsc_id = NULL;
const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
const char *task_uuid = crm_element_value(action->xml,
PCMK__XA_OPERATION_KEY);
const char *target_uuid = crm_element_value(action->xml,
PCMK__META_ON_NODE_UUID);
int target_rc = get_target_rc(action);
action_rsc = pcmk__xe_first_child(action->xml, PCMK_XE_PRIMITIVE, NULL,
NULL);
if (action_rsc == NULL) {
return;
}
rsc_id = pcmk__xe_id(action_rsc);
CRM_CHECK(rsc_id != NULL,
crm_log_xml_err(action->xml, "Bad:action"); return);
/*
update the CIB
<node_state id="hadev">
<lrm>
<lrm_resources>
<lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
*/
state = pcmk__xe_create(NULL, PCMK__XE_NODE_STATE);
crm_xml_add(state, PCMK_XA_ID, target_uuid);
crm_xml_add(state, PCMK_XA_UNAME, target);
rsc = pcmk__xe_create(state, PCMK__XE_LRM);
crm_xml_add(rsc, PCMK_XA_ID, target_uuid);
rsc = pcmk__xe_create(rsc, PCMK__XE_LRM_RESOURCES);
rsc = pcmk__xe_create(rsc, PCMK__XE_LRM_RESOURCE);
crm_xml_add(rsc, PCMK_XA_ID, rsc_id);
crm_copy_xml_element(action_rsc, rsc, PCMK_XA_TYPE);
crm_copy_xml_element(action_rsc, rsc, PCMK_XA_CLASS);
crm_copy_xml_element(action_rsc, rsc, PCMK_XA_PROVIDER);
pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target,
__func__);
rc = cib_conn->cmds->modify(cib_conn, PCMK_XE_STATUS, state, cib_none);
fsa_register_cib_callback(rc, NULL, cib_action_updated);
pcmk__xml_free(state);
crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)",
rc, action->id, task_uuid, target);
pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update);
}
void
controld_record_action_timeout(pcmk__graph_action_t *action)
{
lrmd_event_data_t *op = NULL;
const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
const char *task_uuid = crm_element_value(action->xml,
PCMK__XA_OPERATION_KEY);
int target_rc = get_target_rc(action);
crm_warn("%s %d: %s on %s timed out",
action->xml->name, action->id, task_uuid, target);
op = synthesize_timeout_event(action, target_rc);
controld_record_action_event(action, op);
lrmd_free_event(op);
}
/*!
* \internal
* \brief Execute a resource action from a transition graph
*
* \param[in,out] graph Transition graph being executed
* \param[in,out] action Resource action to execute
*
* \return Standard Pacemaker return code
*/
static int
execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
{
/* never overwrite stop actions in the CIB with
* anything other than completed results
*
* Writing pending stops makes it look like the
* resource is running again
*/
xmlNode *cmd = NULL;
xmlNode *rsc_op = NULL;
gboolean rc = TRUE;
gboolean no_wait = FALSE;
gboolean is_local = FALSE;
char *counter = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
const char *router_node = NULL;
const char *task_uuid = NULL;
CRM_ASSERT(action != NULL);
CRM_ASSERT(action->xml != NULL);
pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed);
on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
CRM_CHECK(!pcmk__str_empty(on_node),
crm_err("Corrupted command(id=%s) %s: no node",
pcmk__xe_id(action->xml), pcmk__s(task, "without task"));
return pcmk_rc_node_unknown);
rsc_op = action->xml;
task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY);
router_node = crm_element_value(rsc_op, PCMK__XA_ROUTER_NODE);
if (!router_node) {
router_node = on_node;
}
counter = pcmk__transition_key(controld_globals.transition_graph->id,
action->id, get_target_rc(action),
controld_globals.te_uuid);
crm_xml_add(rsc_op, PCMK__XA_TRANSITION_KEY, counter);
if (pcmk__str_eq(router_node, controld_globals.our_nodename,
pcmk__str_casei)) {
is_local = TRUE;
}
value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT);
if (crm_is_true(value)) {
no_wait = TRUE;
}
crm_notice("Initiating %s operation %s%s on %s%s " QB_XS " action %d",
task, task_uuid, (is_local? " locally" : ""), on_node,
(no_wait? " without waiting" : ""), action->id);
cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
- CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
+ CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE);
if (is_local) {
/* shortcut local resource commands */
ha_msg_input_t data = {
.msg = cmd,
.xml = rsc_op,
};
fsa_data_t msg = {
.id = 0,
.data = &data,
.data_type = fsa_dt_ha_msg,
.fsa_input = I_NULL,
.fsa_cause = C_FSA_INTERNAL,
.actions = A_LRM_INVOKE,
.origin = __func__,
};
do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state,
I_NULL, &msg);
} else {
const pcmk__node_status_t *node =
pcmk__get_node(0, router_node, NULL,
pcmk__node_search_cluster_member);
rc = pcmk__cluster_send_message(node, pcmk_ipc_execd, cmd);
}
free(counter);
pcmk__xml_free(cmd);
pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);
if (rc == FALSE) {
crm_err("Action %d failed: send", action->id);
return ECOMM;
} else if (no_wait) {
/* Just mark confirmed. Don't bump the job count only to immediately
* decrement it.
*/
crm_info("Action %d confirmed - no wait", action->id);
pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
pcmk__update_graph(controld_globals.transition_graph, action);
trigger_graph();
} else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
action->id, task, task_uuid, on_node, action->timeout);
} else {
if (action->timeout <= 0) {
crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %ums instead",
action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
action->timeout = (int) graph->network_delay;
}
te_update_job_count(action, 1);
te_start_action_timer(graph, action);
}
return pcmk_rc_ok;
}
struct te_peer_s
{
char *name;
int jobs;
int migrate_jobs;
};
static void te_peer_free(gpointer p)
{
struct te_peer_s *peer = p;
free(peer->name);
free(peer);
}
void te_reset_job_counts(void)
{
GHashTableIter iter;
struct te_peer_s *peer = NULL;
if(te_targets == NULL) {
te_targets = pcmk__strkey_table(NULL, te_peer_free);
}
g_hash_table_iter_init(&iter, te_targets);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
peer->jobs = 0;
peer->migrate_jobs = 0;
}
}
static void
te_update_job_count_on(const char *target, int offset, bool migrate)
{
struct te_peer_s *r = NULL;
if(target == NULL || te_targets == NULL) {
return;
}
r = g_hash_table_lookup(te_targets, target);
if(r == NULL) {
r = pcmk__assert_alloc(1, sizeof(struct te_peer_s));
r->name = pcmk__str_copy(target);
g_hash_table_insert(te_targets, r->name, r);
}
r->jobs += offset;
if(migrate) {
r->migrate_jobs += offset;
}
crm_trace("jobs[%s] = %d", target, r->jobs);
}
static void
te_update_job_count(pcmk__graph_action_t *action, int offset)
{
const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
if ((action->type != pcmk__rsc_graph_action) || (target == NULL)) {
/* No limit on these */
return;
}
/* if we have a router node, this means the action is performing
* on a remote node. For now, we count all actions occurring on a
* remote node against the job list on the cluster node hosting
* the connection resources */
target = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
if ((target == NULL)
&& pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
PCMK_ACTION_MIGRATE_FROM, NULL)) {
const char *t1 = crm_meta_value(action->params,
PCMK__META_MIGRATE_SOURCE);
const char *t2 = crm_meta_value(action->params,
PCMK__META_MIGRATE_TARGET);
te_update_job_count_on(t1, offset, TRUE);
te_update_job_count_on(t2, offset, TRUE);
return;
} else if (target == NULL) {
target = crm_element_value(action->xml, PCMK__META_ON_NODE);
}
te_update_job_count_on(target, offset, FALSE);
}
/*!
* \internal
* \brief Check whether a graph action is allowed to be executed on a node
*
* \param[in] graph Transition graph being executed
* \param[in] action Graph action being executed
* \param[in] target Name of node where action should be executed
*
* \return true if action is allowed, otherwise false
*/
static bool
allowed_on_node(const pcmk__graph_t *graph, const pcmk__graph_action_t *action,
const char *target)
{
int limit = 0;
struct te_peer_s *r = NULL;
const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
const char *id = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY);
if(target == NULL) {
/* No limit on these */
return true;
} else if(te_targets == NULL) {
return false;
}
r = g_hash_table_lookup(te_targets, target);
limit = throttle_get_job_limit(target);
if(r == NULL) {
r = pcmk__assert_alloc(1, sizeof(struct te_peer_s));
r->name = pcmk__str_copy(target);
g_hash_table_insert(te_targets, r->name, r);
}
if(limit <= r->jobs) {
crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
target, limit, r->jobs, id);
return false;
} else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
if (pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
PCMK_ACTION_MIGRATE_FROM, NULL)) {
crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
target, graph->migration_limit, r->migrate_jobs, id);
return false;
}
}
crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);
return true;
}
/*!
* \internal
* \brief Check whether a graph action is allowed to be executed
*
* \param[in] graph Transition graph being executed
* \param[in] action Graph action being executed
*
* \return true if action is allowed, otherwise false
*/
static bool
graph_action_allowed(pcmk__graph_t *graph, pcmk__graph_action_t *action)
{
const char *target = NULL;
const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
if (action->type != pcmk__rsc_graph_action) {
/* No limit on these */
return true;
}
/* if we have a router node, this means the action is performing
* on a remote node. For now, we count all actions occurring on a
* remote node against the job list on the cluster node hosting
* the connection resources */
target = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
if ((target == NULL)
&& pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
PCMK_ACTION_MIGRATE_FROM, NULL)) {
target = crm_meta_value(action->params, PCMK__META_MIGRATE_SOURCE);
if (!allowed_on_node(graph, action, target)) {
return false;
}
target = crm_meta_value(action->params, PCMK__META_MIGRATE_TARGET);
} else if (target == NULL) {
target = crm_element_value(action->xml, PCMK__META_ON_NODE);
}
return allowed_on_node(graph, action, target);
}
/*!
* \brief Confirm a graph action (and optionally update graph)
*
* \param[in,out] action Action to confirm
* \param[in,out] graph Update and trigger this graph (if non-NULL)
*/
void
te_action_confirmed(pcmk__graph_action_t *action, pcmk__graph_t *graph)
{
if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
if ((action->type == pcmk__rsc_graph_action)
&& (crm_element_value(action->xml, PCMK__META_ON_NODE) != NULL)) {
te_update_job_count(action, -1);
}
pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
}
if (graph) {
pcmk__update_graph(graph, action);
trigger_graph();
}
}
static pcmk__graph_functions_t te_graph_fns = {
execute_pseudo_action,
execute_rsc_action,
execute_cluster_action,
controld_execute_fence_action,
graph_action_allowed,
};
/*
* \internal
* \brief Register the transitioner's graph functions with \p libpacemaker
*/
void
controld_register_graph_functions(void)
{
pcmk__set_graph_functions(&te_graph_fns);
}
void
notify_crmd(pcmk__graph_t *graph)
{
const char *type = "unknown";
enum crmd_fsa_input event = I_NULL;
crm_debug("Processing transition completion in state %s",
fsa_state2string(controld_globals.fsa_state));
CRM_CHECK(graph->complete, graph->complete = true);
switch (graph->completion_action) {
case pcmk__graph_wait:
type = "stop";
if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
event = I_TE_SUCCESS;
}
break;
case pcmk__graph_done:
type = "done";
if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
event = I_TE_SUCCESS;
}
break;
case pcmk__graph_restart:
type = "restart";
if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
if (controld_get_period_transition_timer() > 0) {
controld_stop_transition_timer();
controld_start_transition_timer();
} else {
event = I_PE_CALC;
}
} else if (controld_globals.fsa_state == S_POLICY_ENGINE) {
controld_set_fsa_action_flags(A_PE_INVOKE);
controld_trigger_fsa();
}
break;
case pcmk__graph_shutdown:
type = "shutdown";
if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
event = I_STOP;
} else {
crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
event = I_TERMINATE;
}
}
crm_debug("Transition %d status: %s - %s", graph->id, type,
pcmk__s(graph->abort_reason, "unspecified reason"));
graph->abort_reason = NULL;
graph->completion_action = pcmk__graph_done;
if (event != I_NULL) {
register_fsa_input(C_FSA_INTERNAL, event, NULL);
} else {
controld_trigger_fsa();
}
}
diff --git a/daemons/controld/controld_throttle.c b/daemons/controld/controld_throttle.c
index 3e45b1c0db..37a82d0b01 100644
--- a/daemons/controld/controld_throttle.c
+++ b/daemons/controld/controld_throttle.c
@@ -1,574 +1,575 @@
/*
* Copyright 2013-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <ctype.h>
#include <dirent.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/cluster.h>
#include <pacemaker-controld.h>
/* These values don't need to be bits, but these particular values must be kept
* for backward compatibility during rolling upgrades.
*/
enum throttle_state_e {
throttle_none = 0x0000,
throttle_low = 0x0001,
throttle_med = 0x0010,
throttle_high = 0x0100,
throttle_extreme = 0x1000,
};
struct throttle_record_s {
int max;
enum throttle_state_e mode;
char *node;
};
static int throttle_job_max = 0;
static float throttle_load_target = 0.0;
#define THROTTLE_FACTOR_LOW 1.2
#define THROTTLE_FACTOR_MEDIUM 1.6
#define THROTTLE_FACTOR_HIGH 2.0
static GHashTable *throttle_records = NULL;
static mainloop_timer_t *throttle_timer = NULL;
static const char *
load2str(enum throttle_state_e mode)
{
switch (mode) {
case throttle_extreme: return "extreme";
case throttle_high: return "high";
case throttle_med: return "medium";
case throttle_low: return "low";
case throttle_none: return "negligible";
default: return "undetermined";
}
}
#if HAVE_LINUX_PROCFS
/*!
* \internal
* \brief Return name of /proc file containing the CIB daemon's load statistics
*
* \return Newly allocated memory with file name on success, NULL otherwise
*
* \note It is the caller's responsibility to free the return value.
* This will return NULL if the daemon is being run via valgrind.
* This should be called only on Linux systems.
*/
static char *
find_cib_loadfile(void)
{
pid_t pid = pcmk__procfs_pid_of("pacemaker-based");
return pid? crm_strdup_printf("/proc/%lld/stat", (long long) pid) : NULL;
}
static bool
throttle_cib_load(float *load)
{
/*
/proc/[pid]/stat
Status information about the process. This is used by ps(1). It is defined in /usr/src/linux/fs/proc/array.c.
The fields, in order, with their proper scanf(3) format specifiers, are:
pid %d (1) The process ID.
comm %s (2) The filename of the executable, in parentheses. This is visible whether or not the executable is swapped out.
state %c (3) One character from the string "RSDZTW" where R is running, S is sleeping in an interruptible wait, D is waiting in uninterruptible disk sleep, Z is zombie, T is traced or stopped (on a signal), and W is paging.
ppid %d (4) The PID of the parent.
pgrp %d (5) The process group ID of the process.
session %d (6) The session ID of the process.
tty_nr %d (7) The controlling terminal of the process. (The minor device number is contained in the combination of bits 31 to 20 and 7 to 0; the major device number is in bits 15 to 8.)
tpgid %d (8) The ID of the foreground process group of the controlling terminal of the process.
flags %u (%lu before Linux 2.6.22)
(9) The kernel flags word of the process. For bit meanings, see the PF_* defines in the Linux kernel source file include/linux/sched.h. Details depend on the kernel version.
minflt %lu (10) The number of minor faults the process has made which have not required loading a memory page from disk.
cminflt %lu (11) The number of minor faults that the process's waited-for children have made.
majflt %lu (12) The number of major faults the process has made which have required loading a memory page from disk.
cmajflt %lu (13) The number of major faults that the process's waited-for children have made.
utime %lu (14) Amount of time that this process has been scheduled in user mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)). This includes guest time, guest_time (time spent running a virtual CPU, see below), so that applications that are not aware of the guest time field do not lose that time from their calculations.
stime %lu (15) Amount of time that this process has been scheduled in kernel mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)).
*/
static char *loadfile = NULL;
static time_t last_call = 0;
static long ticks_per_s = 0;
static unsigned long last_utime, last_stime;
char buffer[64*1024];
FILE *stream = NULL;
time_t now = time(NULL);
if(load == NULL) {
return FALSE;
} else {
*load = 0.0;
}
if(loadfile == NULL) {
last_call = 0;
last_utime = 0;
last_stime = 0;
loadfile = find_cib_loadfile();
if (loadfile == NULL) {
crm_warn("Couldn't find CIB load file");
return FALSE;
}
ticks_per_s = sysconf(_SC_CLK_TCK);
crm_trace("Found %s", loadfile);
}
stream = fopen(loadfile, "r");
if(stream == NULL) {
int rc = errno;
crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_rc_str(rc), rc);
free(loadfile); loadfile = NULL;
return FALSE;
}
if(fgets(buffer, sizeof(buffer), stream)) {
char *comm = pcmk__assert_alloc(1, 256);
char state = 0;
int rc = 0, pid = 0, ppid = 0, pgrp = 0, session = 0, tty_nr = 0, tpgid = 0;
unsigned long flags = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0, utime = 0, stime = 0;
rc = sscanf(buffer, "%d %[^ ] %c %d %d %d %d %d %lu %lu %lu %lu %lu %lu %lu",
&pid, comm, &state,
&ppid, &pgrp, &session, &tty_nr, &tpgid,
&flags, &minflt, &cminflt, &majflt, &cmajflt, &utime, &stime);
free(comm);
if(rc != 15) {
crm_err("Only %d of 15 fields found in %s", rc, loadfile);
fclose(stream);
return FALSE;
} else if(last_call > 0
&& last_call < now
&& last_utime <= utime
&& last_stime <= stime) {
time_t elapsed = now - last_call;
unsigned long delta_utime = utime - last_utime;
unsigned long delta_stime = stime - last_stime;
*load = (delta_utime + delta_stime); /* Cast to a float before division */
*load /= ticks_per_s;
*load /= elapsed;
crm_debug("cib load: %f (%lu ticks in %lds)", *load, delta_utime + delta_stime, (long)elapsed);
} else {
crm_debug("Init %lu + %lu ticks at %ld (%lu tps)", utime, stime, (long)now, ticks_per_s);
}
last_call = now;
last_utime = utime;
last_stime = stime;
fclose(stream);
return TRUE;
}
fclose(stream);
return FALSE;
}
static bool
throttle_load_avg(float *load)
{
char buffer[256];
FILE *stream = NULL;
const char *loadfile = "/proc/loadavg";
if(load == NULL) {
return FALSE;
}
stream = fopen(loadfile, "r");
if(stream == NULL) {
int rc = errno;
crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_rc_str(rc), rc);
return FALSE;
}
if(fgets(buffer, sizeof(buffer), stream)) {
char *nl = strstr(buffer, "\n");
/* Grab the 1-minute average, ignore the rest */
*load = strtof(buffer, NULL);
if(nl) { nl[0] = 0; }
fclose(stream);
return TRUE;
}
fclose(stream);
return FALSE;
}
/*!
* \internal
* \brief Check a load value against throttling thresholds
*
* \param[in] load Load value to check
* \param[in] desc Description of metric (for logging)
* \param[in] thresholds Low/medium/high/extreme thresholds
*
* \return Throttle mode corresponding to load value
*/
static enum throttle_state_e
throttle_check_thresholds(float load, const char *desc,
const float thresholds[4])
{
if (load > thresholds[3]) {
crm_notice("Extreme %s detected: %f", desc, load);
return throttle_extreme;
} else if (load > thresholds[2]) {
crm_notice("High %s detected: %f", desc, load);
return throttle_high;
} else if (load > thresholds[1]) {
crm_info("Moderate %s detected: %f", desc, load);
return throttle_med;
} else if (load > thresholds[0]) {
crm_debug("Noticeable %s detected: %f", desc, load);
return throttle_low;
}
crm_trace("Negligible %s detected: %f", desc, load);
return throttle_none;
}
static enum throttle_state_e
throttle_handle_load(float load, const char *desc, int cores)
{
float normalize;
float thresholds[4];
if (cores == 1) {
/* On a single core machine, a load of 1.0 is already too high */
normalize = 0.6;
} else {
/* Normalize the load to be per-core */
normalize = cores;
}
thresholds[0] = throttle_load_target * normalize * THROTTLE_FACTOR_LOW;
thresholds[1] = throttle_load_target * normalize * THROTTLE_FACTOR_MEDIUM;
thresholds[2] = throttle_load_target * normalize * THROTTLE_FACTOR_HIGH;
thresholds[3] = load + 1.0; /* never extreme */
return throttle_check_thresholds(load, desc, thresholds);
}
#endif // HAVE_LINUX_PROCFS
static enum throttle_state_e
throttle_mode(void)
{
enum throttle_state_e mode = throttle_none;
#if HAVE_LINUX_PROCFS
unsigned int cores;
float load;
float thresholds[4];
cores = pcmk__procfs_num_cores();
if(throttle_cib_load(&load)) {
float cib_max_cpu = 0.95;
/* The CIB is a single-threaded task and thus cannot consume
* more than 100% of a CPU (and 1/cores of the overall system
* load).
*
* On a many-cored system, the CIB might therefore be maxed out
* (causing operations to fail or appear to fail) even though
* the overall system load is still reasonable.
*
* Therefore, the 'normal' thresholds can not apply here, and we
* need a special case.
*/
if(cores == 1) {
cib_max_cpu = 0.4;
}
if(throttle_load_target > 0.0 && throttle_load_target < cib_max_cpu) {
cib_max_cpu = throttle_load_target;
}
thresholds[0] = cib_max_cpu * 0.8;
thresholds[1] = cib_max_cpu * 0.9;
thresholds[2] = cib_max_cpu;
/* Can only happen on machines with a low number of cores */
thresholds[3] = cib_max_cpu * 1.5;
mode = throttle_check_thresholds(load, "CIB load", thresholds);
}
if(throttle_load_target <= 0) {
/* If we ever make this a valid value, the cluster will at least behave as expected */
return mode;
}
if(throttle_load_avg(&load)) {
enum throttle_state_e cpu_load;
cpu_load = throttle_handle_load(load, "CPU load", cores);
if (cpu_load > mode) {
mode = cpu_load;
}
crm_debug("Current load is %f across %u core(s)", load, cores);
}
#endif // HAVE_LINUX_PROCFS
return mode;
}
static void
throttle_send_command(enum throttle_state_e mode)
{
xmlNode *xml = NULL;
static enum throttle_state_e last = -1;
if(mode != last) {
crm_info("New throttle mode: %s load (was %s)",
load2str(mode), load2str(last));
last = mode;
- xml = create_request(CRM_OP_THROTTLE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+ xml = create_request(CRM_OP_THROTTLE, NULL, NULL, CRM_SYSTEM_CRMD,
+ CRM_SYSTEM_CRMD);
crm_xml_add_int(xml, PCMK__XA_CRM_LIMIT_MODE, mode);
crm_xml_add_int(xml, PCMK__XA_CRM_LIMIT_MAX, throttle_job_max);
pcmk__cluster_send_message(NULL, pcmk_ipc_controld, xml);
pcmk__xml_free(xml);
}
}
static gboolean
throttle_timer_cb(gpointer data)
{
throttle_send_command(throttle_mode());
return TRUE;
}
static void
throttle_record_free(gpointer p)
{
struct throttle_record_s *r = p;
free(r->node);
free(r);
}
static void
throttle_set_load_target(float target)
{
throttle_load_target = target;
}
/*!
* \internal
* \brief Update the maximum number of simultaneous jobs
*
* \param[in] preference Cluster-wide \c PCMK_OPT_NODE_ACTION_LIMIT from the
* CIB
*/
static void
throttle_update_job_max(const char *preference)
{
long long max = 0LL;
const char *env_limit = pcmk__env_option(PCMK__ENV_NODE_ACTION_LIMIT);
if (env_limit != NULL) {
preference = env_limit; // Per-node override
}
if (preference != NULL) {
pcmk__scan_ll(preference, &max, 0LL);
}
if (max > 0) {
throttle_job_max = (max >= INT_MAX)? INT_MAX : (int) max;
} else {
// Default is based on the number of cores detected
throttle_job_max = 2 * pcmk__procfs_num_cores();
}
}
void
throttle_init(void)
{
if(throttle_records == NULL) {
throttle_records = pcmk__strkey_table(NULL, throttle_record_free);
throttle_timer = mainloop_timer_add("throttle", 30 * 1000, TRUE, throttle_timer_cb, NULL);
}
throttle_update_job_max(NULL);
mainloop_timer_start(throttle_timer);
}
/*!
* \internal
* \brief Configure throttle options based on the CIB
*
* \param[in,out] options Name/value pairs for configured options
*/
void
controld_configure_throttle(GHashTable *options)
{
const char *value = g_hash_table_lookup(options, PCMK_OPT_LOAD_THRESHOLD);
if (value != NULL) {
throttle_set_load_target(strtof(value, NULL) / 100.0);
}
value = g_hash_table_lookup(options, PCMK_OPT_NODE_ACTION_LIMIT);
throttle_update_job_max(value);
}
void
throttle_fini(void)
{
if (throttle_timer != NULL) {
mainloop_timer_del(throttle_timer);
throttle_timer = NULL;
}
if (throttle_records != NULL) {
g_hash_table_destroy(throttle_records);
throttle_records = NULL;
}
}
int
throttle_get_total_job_limit(int l)
{
/* Cluster-wide limit */
GHashTableIter iter;
int limit = l;
int peers = pcmk__cluster_num_active_nodes();
struct throttle_record_s *r = NULL;
g_hash_table_iter_init(&iter, throttle_records);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &r)) {
switch(r->mode) {
case throttle_extreme:
if(limit == 0 || limit > peers/4) {
limit = QB_MAX(1, peers/4);
}
break;
case throttle_high:
if(limit == 0 || limit > peers/2) {
limit = QB_MAX(1, peers/2);
}
break;
default:
break;
}
}
if(limit == l) {
} else if(l == 0) {
crm_trace("Using " PCMK_OPT_BATCH_LIMIT "=%d", limit);
} else {
crm_trace("Using " PCMK_OPT_BATCH_LIMIT "=%d instead of %d", limit, l);
}
return limit;
}
int
throttle_get_job_limit(const char *node)
{
int jobs = 1;
struct throttle_record_s *r = NULL;
r = g_hash_table_lookup(throttle_records, node);
if(r == NULL) {
r = pcmk__assert_alloc(1, sizeof(struct throttle_record_s));
r->node = pcmk__str_copy(node);
r->mode = throttle_low;
r->max = throttle_job_max;
crm_trace("Defaulting to local values for unknown node %s", node);
g_hash_table_insert(throttle_records, r->node, r);
}
switch(r->mode) {
case throttle_extreme:
case throttle_high:
jobs = 1; /* At least one job must always be allowed */
break;
case throttle_med:
jobs = QB_MAX(1, r->max / 4);
break;
case throttle_low:
jobs = QB_MAX(1, r->max / 2);
break;
case throttle_none:
jobs = QB_MAX(1, r->max);
break;
default:
crm_err("Unknown throttle mode %.4x on %s", r->mode, node);
break;
}
return jobs;
}
void
throttle_update(xmlNode *xml)
{
int max = 0;
int mode = 0;
struct throttle_record_s *r = NULL;
const char *from = crm_element_value(xml, PCMK__XA_SRC);
crm_element_value_int(xml, PCMK__XA_CRM_LIMIT_MODE, &mode);
crm_element_value_int(xml, PCMK__XA_CRM_LIMIT_MAX, &max);
r = g_hash_table_lookup(throttle_records, from);
if(r == NULL) {
r = pcmk__assert_alloc(1, sizeof(struct throttle_record_s));
r->node = pcmk__str_copy(from);
g_hash_table_insert(throttle_records, r->node, r);
}
r->max = max;
r->mode = (enum throttle_state_e) mode;
crm_debug("Node %s has %s load and supports at most %d jobs; new job limit %d",
from, load2str((enum throttle_state_e) mode), max,
throttle_get_job_limit(from));
}
diff --git a/include/crm/common/ipc.h b/include/crm/common/ipc.h
index 5d91a2580f..9a0d5f1dcf 100644
--- a/include/crm/common/ipc.h
+++ b/include/crm/common/ipc.h
@@ -1,240 +1,238 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#ifndef PCMK__CRM_COMMON_IPC__H
#define PCMK__CRM_COMMON_IPC__H
#include <sys/uio.h>
#include <qb/qbipcc.h>
#include <crm/common/xml.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* \file
* \brief IPC interface to Pacemaker daemons
*
* \ingroup core
*/
/*
* Message creation utilities
*
* These are used for both IPC messages and cluster layer messages. However,
* since this is public API, they stay in this header for backward
* compatibility.
*/
// @COMPAT Make internal when we can break API backward compatibility
//! \deprecated Do not use
-#define create_request(task, xml_data, host_to, sys_to, sys_from, uuid_from) \
- create_request_adv(task, xml_data, host_to, sys_to, sys_from, uuid_from, \
- __func__)
+#define create_request(task, xml_data, host_to, sys_to, sys_from) \
+ create_request_adv(task, xml_data, host_to, sys_to, sys_from, __func__)
// @COMPAT Make internal when we can break API backward compatibility
//! \deprecated Do not use
xmlNode *create_request_adv(const char *task, xmlNode *xml_data,
const char *host_to, const char *sys_to,
- const char *sys_from, const char *uuid_from,
- const char *origin);
+ const char *sender_system, const char *origin);
/*
* The library supports two methods of creating IPC connections. The older code
* allows connecting to any arbitrary IPC name. The newer code only allows
* connecting to one of the Pacemaker daemons.
*
* As daemons are converted to use the new model, the old functions should be
* considered deprecated for use with those daemons. Once all daemons are
* converted, the old functions should be officially deprecated as public API
* and eventually made internal API.
*/
/*
* Pacemaker daemon IPC
*/
/* @COMPAT This is also used internally for cluster message types, but it's not
* worth the hassle of redefining this public API just to change the name.
*/
//! Available IPC interfaces
enum pcmk_ipc_server {
pcmk_ipc_unknown, //!< Unknown or invalid
pcmk_ipc_attrd, //!< Attribute manager
pcmk_ipc_based, //!< CIB manager
pcmk_ipc_controld, //!< Controller
pcmk_ipc_execd, //!< Executor
pcmk_ipc_fenced, //!< Fencer
pcmk_ipc_pacemakerd, //!< Launcher
pcmk_ipc_schedulerd, //!< Scheduler
};
// NOTE: sbd (as of at least 1.5.2) uses this enum
//! Possible event types that an IPC event callback can be called for
enum pcmk_ipc_event {
pcmk_ipc_event_connect, //!< Result of asynchronous connection attempt
// NOTE: sbd (as of at least 1.5.2) uses this value
pcmk_ipc_event_disconnect, //!< Termination of IPC connection
// NOTE: sbd (as of at least 1.5.2) uses this value
pcmk_ipc_event_reply, //!< Daemon's reply to client IPC request
pcmk_ipc_event_notify, //!< Notification from daemon
};
//! How IPC replies should be dispatched
enum pcmk_ipc_dispatch {
pcmk_ipc_dispatch_main, //!< Attach IPC to GMainLoop for dispatch
pcmk_ipc_dispatch_poll, //!< Caller will poll and dispatch IPC
pcmk_ipc_dispatch_sync, //!< Sending a command will wait for any reply
};
// NOTE: sbd (as of at least 1.5.2) uses this
//! Client connection to Pacemaker IPC
typedef struct pcmk_ipc_api_s pcmk_ipc_api_t;
/*!
* \brief Callback function type for Pacemaker daemon IPC APIs
*
* \param[in,out] api IPC API connection
* \param[in] event_type The type of event that occurred
* \param[in] status Event status
* \param[in,out] event_data Event-specific data
* \param[in,out] user_data Caller data provided when callback was registered
*
* \note For connection and disconnection events, event_data may be NULL (for
* local IPC) or the name of the connected node (for remote IPC, for
* daemons that support that). For reply and notify events, event_data is
* defined by the specific daemon API.
*/
typedef void (*pcmk_ipc_callback_t)(pcmk_ipc_api_t *api,
enum pcmk_ipc_event event_type,
crm_exit_t status,
void *event_data, void *user_data);
// NOTE: sbd (as of at least 1.5.2) uses this
int pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server);
// NOTE: sbd (as of at least 1.5.2) uses this
void pcmk_free_ipc_api(pcmk_ipc_api_t *api);
// NOTE: sbd (as of at least 1.5.2) uses this
int pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type);
void pcmk_disconnect_ipc(pcmk_ipc_api_t *api);
int pcmk_poll_ipc(const pcmk_ipc_api_t *api, int timeout_ms);
void pcmk_dispatch_ipc(pcmk_ipc_api_t *api);
// NOTE: sbd (as of at least 1.5.2) uses this
void pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
void *user_data);
const char *pcmk_ipc_name(const pcmk_ipc_api_t *api, bool for_log);
bool pcmk_ipc_is_connected(pcmk_ipc_api_t *api);
int pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name,
uint32_t nodeid);
/*
* Generic IPC API (to eventually be deprecated as public API and made internal)
*/
/* *INDENT-OFF* */
enum crm_ipc_flags
{
crm_ipc_flags_none = 0x00000000,
crm_ipc_compressed = 0x00000001, /* Message has been compressed */
crm_ipc_proxied = 0x00000100, /* _ALL_ replies to proxied connections need to be sent as events */
crm_ipc_client_response = 0x00000200, /* A Response is expected in reply */
// These are options for Pacemaker's internal use only (pcmk__ipc_send_*())
crm_ipc_server_event = 0x00010000, /* Send an Event instead of a Response */
crm_ipc_server_free = 0x00020000, /* Free the iovec after sending */
crm_ipc_proxied_relay_response = 0x00040000, /* all replies to proxied connections are sent as events, this flag preserves whether the event should be treated as an actual event, or a response.*/
};
/* *INDENT-ON* */
typedef struct crm_ipc_s crm_ipc_t;
crm_ipc_t *crm_ipc_new(const char *name, size_t max_size);
bool crm_ipc_connect(crm_ipc_t * client);
void crm_ipc_close(crm_ipc_t * client);
void crm_ipc_destroy(crm_ipc_t * client);
void pcmk_free_ipc_event(struct iovec *event);
int crm_ipc_send(crm_ipc_t *client, const xmlNode *message,
enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply);
int crm_ipc_get_fd(crm_ipc_t * client);
bool crm_ipc_connected(crm_ipc_t * client);
int crm_ipc_ready(crm_ipc_t * client);
long crm_ipc_read(crm_ipc_t * client);
const char *crm_ipc_buffer(crm_ipc_t * client);
uint32_t crm_ipc_buffer_flags(crm_ipc_t * client);
const char *crm_ipc_name(crm_ipc_t * client);
unsigned int crm_ipc_default_buffer_size(void);
/*!
* \brief Check the authenticity of the IPC socket peer process (legacy)
*
* If everything goes well, peer's authenticity is verified by the means
* of comparing against provided referential UID and GID (either satisfies),
* and the result of this check can be deduced from the return value.
* As an exception, detected UID of 0 ("root") satisfies arbitrary
* provided referential daemon's credentials.
*
* \param[in] sock IPC related, connected Unix socket to check peer of
* \param[in] refuid referential UID to check against
* \param[in] refgid referential GID to check against
* \param[out] gotpid to optionally store obtained PID of the peer
* (not available on FreeBSD, special value of 1
* used instead, and the caller is required to
* special case this value respectively)
* \param[out] gotuid to optionally store obtained UID of the peer
* \param[out] gotgid to optionally store obtained GID of the peer
*
* \return 0 if IPC related socket's peer is not authentic given the
* referential credentials (see above), 1 if it is,
* negative value on error (generally expressing -errno unless
* it was zero even on nonhappy path, -pcmk_err_generic is
* returned then; no message is directly emitted)
*
* \note While this function is tolerant on what constitutes authorized
* IPC daemon process (its effective user matches UID=0 or \p refuid,
* or at least its group matches \p refgid), either or both (in case
* of UID=0) mismatches on the expected credentials of such peer
* process \e shall be investigated at the caller when value of 1
* gets returned there, since higher-than-expected privileges in
* respect to the expected/intended credentials possibly violate
* the least privilege principle and may pose an additional risk
* (i.e. such accidental inconsistency shall be eventually fixed).
*/
int crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid,
pid_t *gotpid, uid_t *gotuid, gid_t *gotgid);
// @COMPAT Make internal when we can break API backward compatibility
//! \deprecated Do not use
xmlNode *create_hello_message(const char *uuid, const char *client_name,
const char *major_version, const char *minor_version);
#ifdef __cplusplus
}
#endif
#endif
diff --git a/lib/cluster/election.c b/lib/cluster/election.c
index c419609c3b..5d7dc0999c 100644
--- a/lib/cluster/election.c
+++ b/lib/cluster/election.c
@@ -1,727 +1,728 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <crm/common/xml.h>
#include <crm/common/mainloop.h>
#include <crm/cluster/internal.h>
#include <crm/cluster/election_internal.h>
#include <crm/crm.h>
#define STORM_INTERVAL 2 /* in seconds */
struct election_s {
enum election_result state;
guint count; // How many times local node has voted
char *name; // Descriptive name for this election
char *uname; // Local node's name
GSourceFunc cb; // Function to call if election is won
GHashTable *voted; // Key = node name, value = how node voted
mainloop_timer_t *timeout; // When to abort if all votes not received
int election_wins; // Track wins, for storm detection
bool wrote_blackbox; // Write a storm blackbox at most once
time_t expires; // When storm detection period ends
time_t last_election_loss; // When dampening period ends
};
static void
election_complete(election_t *e)
{
e->state = election_won;
if (e->cb != NULL) {
e->cb(e);
}
election_reset(e);
}
static gboolean
election_timer_cb(gpointer user_data)
{
election_t *e = user_data;
crm_info("%s timed out, declaring local node as winner", e->name);
election_complete(e);
return FALSE;
}
/*!
* \brief Get current state of an election
*
* \param[in] e Election object
*
* \return Current state of \e
*/
enum election_result
election_state(const election_t *e)
{
return (e == NULL)? election_error : e->state;
}
/*!
* \brief Create a new election object
*
* Every node that wishes to participate in an election must create an election
* object. Typically, this should be done once, at start-up. A caller should
* only create a single election object.
*
* \param[in] name Label for election (for logging)
* \param[in] uname Local node's name
* \param[in] period_ms How long to wait for all peers to vote
* \param[in] cb Function to call if local node wins election
*
* \return Newly allocated election object on success, NULL on error
* \note The caller is responsible for freeing the returned value using
* election_fini().
*/
election_t *
election_init(const char *name, const char *uname, guint period_ms, GSourceFunc cb)
{
election_t *e = NULL;
static guint count = 0;
CRM_CHECK(uname != NULL, return NULL);
e = calloc(1, sizeof(election_t));
if (e == NULL) {
crm_perror(LOG_CRIT, "Cannot create election");
return NULL;
}
e->uname = strdup(uname);
if (e->uname == NULL) {
crm_perror(LOG_CRIT, "Cannot create election");
free(e);
return NULL;
}
e->name = name? crm_strdup_printf("election-%s", name)
: crm_strdup_printf("election-%u", count++);
e->cb = cb;
e->timeout = mainloop_timer_add(e->name, period_ms, FALSE,
election_timer_cb, e);
crm_trace("Created %s", e->name);
return e;
}
/*!
* \brief Disregard any previous vote by specified peer
*
* This discards any recorded vote from a specified peer. Election users should
* call this whenever a voting peer becomes inactive.
*
* \param[in,out] e Election object
* \param[in] uname Name of peer to disregard
*/
void
election_remove(election_t *e, const char *uname)
{
if ((e != NULL) && (uname != NULL) && (e->voted != NULL)) {
crm_trace("Discarding %s (no-)vote from lost peer %s", e->name, uname);
g_hash_table_remove(e->voted, uname);
}
}
/*!
* \brief Stop election timer and disregard all votes
*
* \param[in,out] e Election object
*/
void
election_reset(election_t *e)
{
if (e != NULL) {
crm_trace("Resetting election %s", e->name);
mainloop_timer_stop(e->timeout);
if (e->voted) {
crm_trace("Destroying voted cache with %d members", g_hash_table_size(e->voted));
g_hash_table_destroy(e->voted);
e->voted = NULL;
}
}
}
/*!
* \brief Free an election object
*
* Free all memory associated with an election object, stopping its
* election timer (if running).
*
* \param[in,out] e Election object
*/
void
election_fini(election_t *e)
{
if (e != NULL) {
election_reset(e);
crm_trace("Destroying %s", e->name);
mainloop_timer_del(e->timeout);
free(e->uname);
free(e->name);
free(e);
}
}
static void
election_timeout_start(election_t *e)
{
if (e != NULL) {
mainloop_timer_start(e->timeout);
}
}
/*!
* \brief Stop an election's timer, if running
*
* \param[in,out] e Election object
*/
void
election_timeout_stop(election_t *e)
{
if (e != NULL) {
mainloop_timer_stop(e->timeout);
}
}
/*!
* \brief Change an election's timeout (restarting timer if running)
*
* \param[in,out] e Election object
* \param[in] period New timeout
*/
void
election_timeout_set_period(election_t *e, guint period)
{
if (e != NULL) {
mainloop_timer_set_period(e->timeout, period);
} else {
crm_err("No election defined");
}
}
static int
get_uptime(struct timeval *output)
{
static time_t expires = 0;
static struct rusage info;
time_t tm_now = time(NULL);
if (expires < tm_now) {
int rc = 0;
info.ru_utime.tv_sec = 0;
info.ru_utime.tv_usec = 0;
rc = getrusage(RUSAGE_SELF, &info);
output->tv_sec = 0;
output->tv_usec = 0;
if (rc < 0) {
crm_perror(LOG_ERR, "Could not calculate the current uptime");
expires = 0;
return -1;
}
crm_debug("Current CPU usage is: %lds, %ldus", (long)info.ru_utime.tv_sec,
(long)info.ru_utime.tv_usec);
}
expires = tm_now + STORM_INTERVAL; /* N seconds after the last _access_ */
output->tv_sec = info.ru_utime.tv_sec;
output->tv_usec = info.ru_utime.tv_usec;
return 1;
}
static int
compare_age(struct timeval your_age)
{
struct timeval our_age;
get_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */
if (our_age.tv_sec > your_age.tv_sec) {
crm_debug("Win: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
return 1;
} else if (our_age.tv_sec < your_age.tv_sec) {
crm_debug("Lose: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
return -1;
} else if (our_age.tv_usec > your_age.tv_usec) {
crm_debug("Win: %ld.%06ld vs %ld.%06ld (usec)",
(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
return 1;
} else if (our_age.tv_usec < your_age.tv_usec) {
crm_debug("Lose: %ld.%06ld vs %ld.%06ld (usec)",
(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
return -1;
}
return 0;
}
/*!
* \brief Start a new election by offering local node's candidacy
*
* Broadcast a "vote" election message containing the local node's ID,
* (incremented) election counter, and uptime, and start the election timer.
*
* \param[in,out] e Election object
*
* \note Any nodes agreeing to the candidacy will send a "no-vote" reply, and if
* all active peers do so, or if the election times out, the local node
* wins the election. (If we lose to any peer vote, we will stop the
* timer, so a timeout means we did not lose -- either some peer did not
* vote, or we did not call election_check() in time.)
*/
void
election_vote(election_t *e)
{
struct timeval age;
xmlNode *vote = NULL;
pcmk__node_status_t *our_node = NULL;
if (e == NULL) {
crm_trace("Election vote requested, but no election available");
return;
}
our_node = pcmk__get_node(0, e->uname, NULL,
pcmk__node_search_cluster_member);
if (!pcmk__cluster_is_node_active(our_node)) {
crm_trace("Cannot vote in %s yet: local node not connected to cluster",
e->name);
return;
}
election_reset(e);
e->state = election_in_progress;
- vote = create_request(CRM_OP_VOTE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+ vote = create_request(CRM_OP_VOTE, NULL, NULL, CRM_SYSTEM_CRMD,
+ CRM_SYSTEM_CRMD);
e->count++;
crm_xml_add(vote, PCMK__XA_ELECTION_OWNER, our_node->xml_id);
crm_xml_add_int(vote, PCMK__XA_ELECTION_ID, e->count);
// Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is actually microseconds
get_uptime(&age);
crm_xml_add_timeval(vote, PCMK__XA_ELECTION_AGE_SEC,
PCMK__XA_ELECTION_AGE_NANO_SEC, &age);
pcmk__cluster_send_message(NULL, pcmk_ipc_controld, vote);
pcmk__xml_free(vote);
crm_debug("Started %s round %d", e->name, e->count);
election_timeout_start(e);
return;
}
/*!
* \brief Check whether local node has won an election
*
* If all known peers have sent no-vote messages, stop the election timer, set
* the election state to won, and call any registered win callback.
*
* \param[in,out] e Election object
*
* \return TRUE if local node has won, FALSE otherwise
* \note If all known peers have sent no-vote messages, but the election owner
* does not call this function, the election will not be won (and the
* callback will not be called) until the election times out.
* \note This should be called when election_count_vote() returns
* \c election_in_progress.
*/
bool
election_check(election_t *e)
{
int voted_size = 0;
int num_members = 0;
if (e == NULL) {
crm_trace("Election check requested, but no election available");
return FALSE;
}
if (e->voted == NULL) {
crm_trace("%s check requested, but no votes received yet", e->name);
return FALSE;
}
voted_size = g_hash_table_size(e->voted);
num_members = pcmk__cluster_num_active_nodes();
/* in the case of #voted > #members, it is better to
* wait for the timeout and give the cluster time to
* stabilize
*/
if (voted_size >= num_members) {
/* we won and everyone has voted */
election_timeout_stop(e);
if (voted_size > num_members) {
GHashTableIter gIter;
const pcmk__node_status_t *node = NULL;
char *key = NULL;
crm_warn("Received too many votes in %s", e->name);
g_hash_table_iter_init(&gIter, pcmk__peer_cache);
while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) & node)) {
if (pcmk__cluster_is_node_active(node)) {
crm_warn("* expected vote: %s", node->name);
}
}
g_hash_table_iter_init(&gIter, e->voted);
while (g_hash_table_iter_next(&gIter, (gpointer *) & key, NULL)) {
crm_warn("* actual vote: %s", key);
}
}
crm_info("%s won by local node", e->name);
election_complete(e);
return TRUE;
} else {
crm_debug("%s still waiting on %d of %d votes",
e->name, num_members - voted_size, num_members);
}
return FALSE;
}
#define LOSS_DAMPEN 2 /* in seconds */
struct vote {
const char *op;
const char *from;
const char *version;
const char *election_owner;
int election_id;
struct timeval age;
};
/*!
* \brief Unpack an election message
*
* \param[in] e Election object (for logging only)
* \param[in] message Election message XML
* \param[out] vote Parsed fields from message
*
* \return TRUE if election message and election are valid, FALSE otherwise
* \note The parsed struct's pointer members are valid only for the lifetime of
* the message argument.
*/
static bool
parse_election_message(const election_t *e, const xmlNode *message,
struct vote *vote)
{
CRM_CHECK(message && vote, return FALSE);
vote->election_id = -1;
vote->age.tv_sec = -1;
vote->age.tv_usec = -1;
vote->op = crm_element_value(message, PCMK__XA_CRM_TASK);
vote->from = crm_element_value(message, PCMK__XA_SRC);
vote->version = crm_element_value(message, PCMK_XA_VERSION);
vote->election_owner = crm_element_value(message, PCMK__XA_ELECTION_OWNER);
crm_element_value_int(message, PCMK__XA_ELECTION_ID, &(vote->election_id));
if ((vote->op == NULL) || (vote->from == NULL) || (vote->version == NULL)
|| (vote->election_owner == NULL) || (vote->election_id < 0)) {
crm_warn("Invalid %s message from %s in %s ",
(vote->op? vote->op : "election"),
(vote->from? vote->from : "unspecified node"),
(e? e->name : "election"));
return FALSE;
}
// Op-specific validation
if (pcmk__str_eq(vote->op, CRM_OP_VOTE, pcmk__str_none)) {
/* Only vote ops have uptime.
Warning: PCMK__XA_ELECTION_AGE_NANO_SEC value is in microseconds.
*/
crm_element_value_timeval(message, PCMK__XA_ELECTION_AGE_SEC,
PCMK__XA_ELECTION_AGE_NANO_SEC, &(vote->age));
if ((vote->age.tv_sec < 0) || (vote->age.tv_usec < 0)) {
crm_warn("Cannot count %s %s from %s because it is missing uptime",
(e? e->name : "election"), vote->op, vote->from);
return FALSE;
}
} else if (!pcmk__str_eq(vote->op, CRM_OP_NOVOTE, pcmk__str_none)) {
crm_info("Cannot process %s message from %s because %s is not a known election op",
(e? e->name : "election"), vote->from, vote->op);
return FALSE;
}
// Election validation
if (e == NULL) {
crm_info("Cannot count %s from %s because no election available",
vote->op, vote->from);
return FALSE;
}
/* If the membership cache is NULL, we REALLY shouldn't be voting --
* the question is how we managed to get here.
*/
if (pcmk__peer_cache == NULL) {
crm_info("Cannot count %s %s from %s because no peer information available",
e->name, vote->op, vote->from);
return FALSE;
}
return TRUE;
}
static void
record_vote(election_t *e, struct vote *vote)
{
CRM_ASSERT(e && vote && vote->from && vote->op);
if (e->voted == NULL) {
e->voted = pcmk__strkey_table(free, free);
}
pcmk__insert_dup(e->voted, vote->from, vote->op);
}
static void
send_no_vote(pcmk__node_status_t *peer, struct vote *vote)
{
// @TODO probably shouldn't hardcode CRM_SYSTEM_CRMD and pcmk_ipc_controld
xmlNode *novote = create_request(CRM_OP_NOVOTE, NULL, vote->from,
- CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
+ CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD);
crm_xml_add(novote, PCMK__XA_ELECTION_OWNER, vote->election_owner);
crm_xml_add_int(novote, PCMK__XA_ELECTION_ID, vote->election_id);
pcmk__cluster_send_message(peer, pcmk_ipc_controld, novote);
pcmk__xml_free(novote);
}
/*!
* \brief Process an election message (vote or no-vote) from a peer
*
* \param[in,out] e Election object
* \param[in] message Election message XML from peer
* \param[in] can_win Whether local node is eligible to win
*
* \return Election state after new vote is considered
* \note If the peer message is a vote, and we prefer the peer to win, this will
* send a no-vote reply to the peer.
* \note The situations "we lost to this vote" from "this is a late no-vote
* after we've already lost" both return election_lost. If a caller needs
* to distinguish them, it should save the current state before calling
* this function, and then compare the result.
*/
enum election_result
election_count_vote(election_t *e, const xmlNode *message, bool can_win)
{
int log_level = LOG_INFO;
gboolean done = FALSE;
gboolean we_lose = FALSE;
const char *reason = "unknown";
bool we_are_owner = FALSE;
pcmk__node_status_t *our_node = NULL;
pcmk__node_status_t *your_node = NULL;
time_t tm_now = time(NULL);
struct vote vote;
CRM_CHECK(message != NULL, return election_error);
if (parse_election_message(e, message, &vote) == FALSE) {
return election_error;
}
your_node = pcmk__get_node(0, vote.from, NULL,
pcmk__node_search_cluster_member);
our_node = pcmk__get_node(0, e->uname, NULL,
pcmk__node_search_cluster_member);
we_are_owner = (our_node != NULL)
&& pcmk__str_eq(our_node->xml_id, vote.election_owner,
pcmk__str_none);
if (!can_win) {
reason = "Not eligible";
we_lose = TRUE;
} else if (!pcmk__cluster_is_node_active(our_node)) {
reason = "We are not part of the cluster";
log_level = LOG_ERR;
we_lose = TRUE;
} else if (we_are_owner && (vote.election_id != e->count)) {
log_level = LOG_TRACE;
reason = "Superseded";
done = TRUE;
} else if (!pcmk__cluster_is_node_active(your_node)) {
/* Possibly we cached the message in the FSA queue at a point that it wasn't */
reason = "Peer is not part of our cluster";
log_level = LOG_WARNING;
done = TRUE;
} else if (pcmk__str_eq(vote.op, CRM_OP_NOVOTE, pcmk__str_none)
|| pcmk__str_eq(vote.from, e->uname, pcmk__str_none)) {
/* Receiving our own broadcast vote, or a no-vote from peer, is a vote
* for us to win
*/
if (!we_are_owner) {
crm_warn("Cannot count %s round %d %s from %s because we are not election owner (%s)",
e->name, vote.election_id, vote.op, vote.from,
vote.election_owner);
return election_error;
}
if (e->state != election_in_progress) {
// Should only happen if we already lost
crm_debug("Not counting %s round %d %s from %s because no election in progress",
e->name, vote.election_id, vote.op, vote.from);
return e->state;
}
record_vote(e, &vote);
reason = "Recorded";
done = TRUE;
} else {
// A peer vote requires a comparison to determine which node is better
int age_result = compare_age(vote.age);
int version_result = compare_version(vote.version, CRM_FEATURE_SET);
if (version_result < 0) {
reason = "Version";
we_lose = TRUE;
} else if (version_result > 0) {
reason = "Version";
} else if (age_result < 0) {
reason = "Uptime";
we_lose = TRUE;
} else if (age_result > 0) {
reason = "Uptime";
} else if (strcasecmp(e->uname, vote.from) > 0) {
reason = "Host name";
we_lose = TRUE;
} else {
reason = "Host name";
}
}
if (e->expires < tm_now) {
e->election_wins = 0;
e->expires = tm_now + STORM_INTERVAL;
} else if (done == FALSE && we_lose == FALSE) {
int peers = 1 + g_hash_table_size(pcmk__peer_cache);
/* If every node has to vote down every other node, thats N*(N-1) total elections
* Allow some leeway before _really_ complaining
*/
e->election_wins++;
if (e->election_wins > (peers * peers)) {
crm_warn("%s election storm detected: %d wins in %d seconds",
e->name, e->election_wins, STORM_INTERVAL);
e->election_wins = 0;
e->expires = tm_now + STORM_INTERVAL;
if (e->wrote_blackbox == FALSE) {
/* It's questionable whether a black box (from every node in the
* cluster) would be truly helpful in diagnosing an election
* storm. It's also highly doubtful a production environment
* would get multiple election storms from distinct causes, so
* saving one blackbox per process lifetime should be
* sufficient. Alternatives would be to save a timestamp of the
* last blackbox write instead of a boolean, and write a new one
* if some amount of time has passed; or to save a storm count,
* write a blackbox on every Nth occurrence.
*/
crm_write_blackbox(0, NULL);
e->wrote_blackbox = TRUE;
}
}
}
if (done) {
do_crm_log(log_level + 1,
"Processed %s round %d %s (current round %d) from %s (%s)",
e->name, vote.election_id, vote.op, e->count, vote.from,
reason);
return e->state;
} else if (we_lose == FALSE) {
/* We track the time of the last election loss to implement an election
* dampening period, reducing the likelihood of an election storm. If
* this node has lost within the dampening period, don't start a new
* election, even if we win against a peer's vote -- the peer we lost to
* should win again.
*
* @TODO This has a problem case: if an election winner immediately
* leaves the cluster, and a new election is immediately called, all
* nodes could lose, with no new winner elected. The ideal solution
* would be to tie the election structure with the peer caches, which
* would allow us to clear the dampening when the previous winner
* leaves (and would allow other improvements as well).
*/
if ((e->last_election_loss == 0)
|| ((tm_now - e->last_election_loss) > (time_t) LOSS_DAMPEN)) {
do_crm_log(log_level, "%s round %d (owner node ID %s) pass: %s from %s (%s)",
e->name, vote.election_id, vote.election_owner, vote.op,
vote.from, reason);
e->last_election_loss = 0;
election_timeout_stop(e);
/* Start a new election by voting down this, and other, peers */
e->state = election_start;
return e->state;
} else {
char *loss_time = ctime(&e->last_election_loss);
if (loss_time) {
// Show only HH:MM:SS
loss_time += 11;
loss_time[8] = '\0';
}
crm_info("Ignoring %s round %d (owner node ID %s) pass vs %s because we lost less than %ds ago at %s",
e->name, vote.election_id, vote.election_owner, vote.from,
LOSS_DAMPEN, (loss_time? loss_time : "unknown"));
}
}
e->last_election_loss = tm_now;
do_crm_log(log_level, "%s round %d (owner node ID %s) lost: %s from %s (%s)",
e->name, vote.election_id, vote.election_owner, vote.op,
vote.from, reason);
election_reset(e);
send_no_vote(your_node, &vote);
e->state = election_lost;
return e->state;
}
/*!
* \brief Reset any election dampening currently in effect
*
* \param[in,out] e Election object to clear
*/
void
election_clear_dampening(election_t *e)
{
e->last_election_loss = 0;
}
diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
index 3d17f9f4e6..e396a92fa2 100644
--- a/lib/common/ipc_client.c
+++ b/lib/common/ipc_client.c
@@ -1,1687 +1,1687 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#if defined(HAVE_UCRED) || defined(HAVE_SOCKPEERCRED)
#include <sys/socket.h>
#elif defined(HAVE_GETPEERUCRED)
#include <ucred.h>
#endif
#include <stdio.h>
#include <sys/types.h>
#include <errno.h>
#include <bzlib.h>
#include <crm/crm.h> /* indirectly: pcmk_err_generic */
#include <crm/common/xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include "crmcommon_private.h"
static int is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
uid_t refuid, gid_t refgid, pid_t *gotpid,
uid_t *gotuid, gid_t *gotgid);
/*!
* \brief Create a new object for using Pacemaker daemon IPC
*
* \param[out] api Where to store new IPC object
* \param[in] server Which Pacemaker daemon the object is for
*
* \return Standard Pacemaker result code
*
* \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
* \note This is intended to supersede crm_ipc_new() but currently only
* supports the controller, pacemakerd, and schedulerd IPC API.
*/
int
pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
{
if (api == NULL) {
return EINVAL;
}
*api = calloc(1, sizeof(pcmk_ipc_api_t));
if (*api == NULL) {
return errno;
}
(*api)->server = server;
if (pcmk_ipc_name(*api, false) == NULL) {
pcmk_free_ipc_api(*api);
*api = NULL;
return EOPNOTSUPP;
}
(*api)->ipc_size_max = 0;
// Set server methods and max_size (if not default)
switch (server) {
case pcmk_ipc_attrd:
(*api)->cmds = pcmk__attrd_api_methods();
break;
case pcmk_ipc_based:
(*api)->ipc_size_max = 512 * 1024; // 512KB
break;
case pcmk_ipc_controld:
(*api)->cmds = pcmk__controld_api_methods();
break;
case pcmk_ipc_execd:
break;
case pcmk_ipc_fenced:
break;
case pcmk_ipc_pacemakerd:
(*api)->cmds = pcmk__pacemakerd_api_methods();
break;
case pcmk_ipc_schedulerd:
(*api)->cmds = pcmk__schedulerd_api_methods();
// @TODO max_size could vary by client, maybe take as argument?
(*api)->ipc_size_max = 5 * 1024 * 1024; // 5MB
break;
default: // pcmk_ipc_unknown
pcmk_free_ipc_api(*api);
*api = NULL;
return EINVAL;
}
if ((*api)->cmds == NULL) {
pcmk_free_ipc_api(*api);
*api = NULL;
return ENOMEM;
}
(*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false),
(*api)->ipc_size_max);
if ((*api)->ipc == NULL) {
pcmk_free_ipc_api(*api);
*api = NULL;
return ENOMEM;
}
// If daemon API has its own data to track, allocate it
if ((*api)->cmds->new_data != NULL) {
if ((*api)->cmds->new_data(*api) != pcmk_rc_ok) {
pcmk_free_ipc_api(*api);
*api = NULL;
return ENOMEM;
}
}
crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true));
return pcmk_rc_ok;
}
static void
free_daemon_specific_data(pcmk_ipc_api_t *api)
{
if ((api != NULL) && (api->cmds != NULL)) {
if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) {
api->cmds->free_data(api->api_data);
api->api_data = NULL;
}
free(api->cmds);
api->cmds = NULL;
}
}
/*!
* \internal
* \brief Call an IPC API event callback, if one is registed
*
* \param[in,out] api IPC API connection
* \param[in] event_type The type of event that occurred
* \param[in] status Event status
* \param[in,out] event_data Event-specific data
*/
void
pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
crm_exit_t status, void *event_data)
{
if ((api != NULL) && (api->cb != NULL)) {
api->cb(api, event_type, status, event_data, api->user_data);
}
}
/*!
* \internal
* \brief Clean up after an IPC disconnect
*
* \param[in,out] user_data IPC API connection that disconnected
*
* \note This function can be used as a main loop IPC destroy callback.
*/
static void
ipc_post_disconnect(gpointer user_data)
{
pcmk_ipc_api_t *api = user_data;
crm_info("Disconnected from %s", pcmk_ipc_name(api, true));
// Perform any daemon-specific handling needed
if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) {
api->cmds->post_disconnect(api);
}
// Call client's registered event callback
pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT,
NULL);
/* If this is being called from a running main loop, mainloop_gio_destroy()
* will free ipc and mainloop_io immediately after calling this function.
* If this is called from a stopped main loop, these will leak, so the best
* practice is to close the connection before stopping the main loop.
*/
api->ipc = NULL;
api->mainloop_io = NULL;
if (api->free_on_disconnect) {
/* pcmk_free_ipc_api() has already been called, but did not free api
* or api->cmds because this function needed them. Do that now.
*/
free_daemon_specific_data(api);
crm_trace("Freeing IPC API object after disconnect");
free(api);
}
}
/*!
* \brief Free the contents of an IPC API object
*
* \param[in,out] api IPC API object to free
*/
void
pcmk_free_ipc_api(pcmk_ipc_api_t *api)
{
bool free_on_disconnect = false;
if (api == NULL) {
return;
}
crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true));
if (api->ipc != NULL) {
if (api->mainloop_io != NULL) {
/* We need to keep the api pointer itself around, because it is the
* user data for the IPC client destroy callback. That will be
* triggered by the pcmk_disconnect_ipc() call below, but it might
* happen later in the main loop (if still running).
*
* This flag tells the destroy callback to free the object. It can't
* do that unconditionally, because the application might call this
* function after a disconnect that happened by other means.
*/
free_on_disconnect = api->free_on_disconnect = true;
}
pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true
}
if (!free_on_disconnect) {
free_daemon_specific_data(api);
crm_trace("Freeing IPC API object");
free(api);
}
}
/*!
* \brief Get the IPC name used with an IPC API connection
*
* \param[in] api IPC API connection
* \param[in] for_log If true, return human-friendly name instead of IPC name
*
* \return IPC API's human-friendly or connection name, or if none is available,
* "Pacemaker" if for_log is true and NULL if for_log is false
*/
const char *
pcmk_ipc_name(const pcmk_ipc_api_t *api, bool for_log)
{
if (api == NULL) {
return for_log? "Pacemaker" : NULL;
}
if (for_log) {
const char *name = pcmk__server_log_name(api->server);
return pcmk__s(name, "Pacemaker");
}
switch (api->server) {
// These servers do not have pcmk_ipc_api_t implementations yet
case pcmk_ipc_based:
case pcmk_ipc_execd:
case pcmk_ipc_fenced:
return NULL;
default:
return pcmk__server_ipc_name(api->server);
}
}
/*!
* \brief Check whether an IPC API connection is active
*
* \param[in,out] api IPC API connection
*
* \return true if IPC is connected, false otherwise
*/
bool
pcmk_ipc_is_connected(pcmk_ipc_api_t *api)
{
return (api != NULL) && crm_ipc_connected(api->ipc);
}
/*!
* \internal
* \brief Call the daemon-specific API's dispatch function
*
* Perform daemon-specific handling of IPC reply dispatch. It is the daemon
* method's responsibility to call the client's registered event callback, as
* well as allocate and free any event data.
*
* \param[in,out] api IPC API connection
* \param[in,out] message IPC reply XML to dispatch
*/
static bool
call_api_dispatch(pcmk_ipc_api_t *api, xmlNode *message)
{
crm_log_xml_trace(message, "ipc-received");
if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) {
return api->cmds->dispatch(api, message);
}
return false;
}
/*!
* \internal
* \brief Dispatch previously read IPC data
*
* \param[in] buffer Data read from IPC
* \param[in,out] api IPC object
*
* \return Standard Pacemaker return code. In particular:
*
* pcmk_rc_ok: There are no more messages expected from the server. Quit
* reading.
* EINPROGRESS: There are more messages expected from the server. Keep reading.
*
* All other values indicate an error.
*/
static int
dispatch_ipc_data(const char *buffer, pcmk_ipc_api_t *api)
{
bool more = false;
xmlNode *msg;
if (buffer == NULL) {
crm_warn("Empty message received from %s IPC",
pcmk_ipc_name(api, true));
return ENOMSG;
}
msg = pcmk__xml_parse(buffer);
if (msg == NULL) {
crm_warn("Malformed message received from %s IPC",
pcmk_ipc_name(api, true));
return EPROTO;
}
more = call_api_dispatch(api, msg);
pcmk__xml_free(msg);
if (more) {
return EINPROGRESS;
} else {
return pcmk_rc_ok;
}
}
/*!
* \internal
* \brief Dispatch data read from IPC source
*
* \param[in] buffer Data read from IPC
* \param[in] length Number of bytes of data in buffer (ignored)
* \param[in,out] user_data IPC object
*
* \return Always 0 (meaning connection is still required)
*
* \note This function can be used as a main loop IPC dispatch callback.
*/
static int
dispatch_ipc_source_data(const char *buffer, ssize_t length, gpointer user_data)
{
pcmk_ipc_api_t *api = user_data;
CRM_CHECK(api != NULL, return 0);
dispatch_ipc_data(buffer, api);
return 0;
}
/*!
* \brief Check whether an IPC connection has data available (without main loop)
*
* \param[in] api IPC API connection
* \param[in] timeout_ms If less than 0, poll indefinitely; if 0, poll once
* and return immediately; otherwise, poll for up to
* this many milliseconds
*
* \return Standard Pacemaker return code
*
* \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
* this function to check whether IPC data is available. Return values of
* interest include pcmk_rc_ok meaning data is available, and EAGAIN
* meaning no data is available; all other values indicate errors.
* \todo This does not allow the caller to poll multiple file descriptors at
* once. If there is demand for that, we could add a wrapper for
* pcmk__ipc_fd(api->ipc), so the caller can call poll() themselves.
*/
int
pcmk_poll_ipc(const pcmk_ipc_api_t *api, int timeout_ms)
{
int rc;
struct pollfd pollfd = { 0, };
if ((api == NULL) || (api->dispatch_type != pcmk_ipc_dispatch_poll)) {
return EINVAL;
}
rc = pcmk__ipc_fd(api->ipc, &(pollfd.fd));
if (rc != pcmk_rc_ok) {
crm_debug("Could not obtain file descriptor for %s IPC: %s",
pcmk_ipc_name(api, true), pcmk_rc_str(rc));
return rc;
}
pollfd.events = POLLIN;
rc = poll(&pollfd, 1, timeout_ms);
if (rc < 0) {
/* Some UNIX systems return negative and set EAGAIN for failure to
* allocate memory; standardize the return code in that case
*/
return (errno == EAGAIN)? ENOMEM : errno;
} else if (rc == 0) {
return EAGAIN;
}
return pcmk_rc_ok;
}
/*!
* \brief Dispatch available messages on an IPC connection (without main loop)
*
* \param[in,out] api IPC API connection
*
* \return Standard Pacemaker return code
*
* \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
* this function when IPC data is available.
*/
void
pcmk_dispatch_ipc(pcmk_ipc_api_t *api)
{
if (api == NULL) {
return;
}
while (crm_ipc_ready(api->ipc) > 0) {
if (crm_ipc_read(api->ipc) > 0) {
dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);
}
}
}
// \return Standard Pacemaker return code
static int
connect_with_main_loop(pcmk_ipc_api_t *api)
{
int rc;
struct ipc_client_callbacks callbacks = {
.dispatch = dispatch_ipc_source_data,
.destroy = ipc_post_disconnect,
};
rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api,
&callbacks, &(api->mainloop_io));
if (rc != pcmk_rc_ok) {
return rc;
}
crm_debug("Connected to %s IPC (attached to main loop)",
pcmk_ipc_name(api, true));
/* After this point, api->mainloop_io owns api->ipc, so api->ipc
* should not be explicitly freed.
*/
return pcmk_rc_ok;
}
// \return Standard Pacemaker return code
static int
connect_without_main_loop(pcmk_ipc_api_t *api)
{
int rc = pcmk__connect_generic_ipc(api->ipc);
if (rc != pcmk_rc_ok) {
crm_ipc_close(api->ipc);
} else {
crm_debug("Connected to %s IPC (without main loop)",
pcmk_ipc_name(api, true));
}
return rc;
}
/*!
* \internal
* \brief Connect to a Pacemaker daemon via IPC (retrying after soft errors)
*
* \param[in,out] api IPC API instance
* \param[in] dispatch_type How IPC replies should be dispatched
* \param[in] attempts How many times to try (in case of soft error)
*
* \return Standard Pacemaker return code
*/
int
pcmk__connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type,
int attempts)
{
int rc = pcmk_rc_ok;
if ((api == NULL) || (attempts < 1)) {
return EINVAL;
}
if (api->ipc == NULL) {
api->ipc = crm_ipc_new(pcmk_ipc_name(api, false), api->ipc_size_max);
if (api->ipc == NULL) {
return ENOMEM;
}
}
if (crm_ipc_connected(api->ipc)) {
crm_trace("Already connected to %s", pcmk_ipc_name(api, true));
return pcmk_rc_ok;
}
api->dispatch_type = dispatch_type;
crm_debug("Attempting connection to %s (up to %d time%s)",
pcmk_ipc_name(api, true), attempts, pcmk__plural_s(attempts));
for (int remaining = attempts - 1; remaining >= 0; --remaining) {
switch (dispatch_type) {
case pcmk_ipc_dispatch_main:
rc = connect_with_main_loop(api);
break;
case pcmk_ipc_dispatch_sync:
case pcmk_ipc_dispatch_poll:
rc = connect_without_main_loop(api);
break;
}
if ((remaining == 0) || ((rc != EAGAIN) && (rc != EALREADY))) {
break; // Result is final
}
// Retry after soft error (interrupted by signal, etc.)
pcmk__sleep_ms((attempts - remaining) * 500);
crm_debug("Re-attempting connection to %s (%d attempt%s remaining)",
pcmk_ipc_name(api, true), remaining,
pcmk__plural_s(remaining));
}
if (rc != pcmk_rc_ok) {
return rc;
}
if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) {
rc = api->cmds->post_connect(api);
if (rc != pcmk_rc_ok) {
crm_ipc_close(api->ipc);
}
}
return rc;
}
/*!
* \brief Connect to a Pacemaker daemon via IPC
*
* \param[in,out] api IPC API instance
* \param[in] dispatch_type How IPC replies should be dispatched
*
* \return Standard Pacemaker return code
*/
int
pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type)
{
int rc = pcmk__connect_ipc(api, dispatch_type, 2);
if (rc != pcmk_rc_ok) {
crm_err("Connection to %s failed: %s",
pcmk_ipc_name(api, true), pcmk_rc_str(rc));
}
return rc;
}
/*!
* \brief Disconnect an IPC API instance
*
* \param[in,out] api IPC API connection
*
* \return Standard Pacemaker return code
*
* \note If the connection is attached to a main loop, this function should be
* called before quitting the main loop, to ensure that all memory is
* freed.
*/
void
pcmk_disconnect_ipc(pcmk_ipc_api_t *api)
{
if ((api == NULL) || (api->ipc == NULL)) {
return;
}
switch (api->dispatch_type) {
case pcmk_ipc_dispatch_main:
{
mainloop_io_t *mainloop_io = api->mainloop_io;
// Make sure no code with access to api can use these again
api->mainloop_io = NULL;
api->ipc = NULL;
mainloop_del_ipc_client(mainloop_io);
// After this point api might have already been freed
}
break;
case pcmk_ipc_dispatch_poll:
case pcmk_ipc_dispatch_sync:
{
crm_ipc_t *ipc = api->ipc;
// Make sure no code with access to api can use ipc again
api->ipc = NULL;
// This should always be the case already, but to be safe
api->free_on_disconnect = false;
crm_ipc_close(ipc);
crm_ipc_destroy(ipc);
ipc_post_disconnect(api);
}
break;
}
}
/*!
* \brief Register a callback for IPC API events
*
* \param[in,out] api IPC API connection
* \param[in] callback Callback to register
* \param[in] userdata Caller data to pass to callback
*
* \note This function may be called multiple times to update the callback
* and/or user data. The caller remains responsible for freeing
* userdata in any case (after the IPC is disconnected, if the
* user data is still registered with the IPC).
*/
void
pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
void *user_data)
{
if (api == NULL) {
return;
}
api->cb = cb;
api->user_data = user_data;
}
/*!
* \internal
* \brief Send an XML request across an IPC API connection
*
* \param[in,out] api IPC API connection
* \param[in] request XML request to send
*
* \return Standard Pacemaker return code
*
* \note Daemon-specific IPC API functions should call this function to send
* requests, because it handles different dispatch types appropriately.
*/
int
pcmk__send_ipc_request(pcmk_ipc_api_t *api, const xmlNode *request)
{
int rc;
xmlNode *reply = NULL;
enum crm_ipc_flags flags = crm_ipc_flags_none;
if ((api == NULL) || (api->ipc == NULL) || (request == NULL)) {
return EINVAL;
}
crm_log_xml_trace(request, "ipc-sent");
// Synchronous dispatch requires waiting for a reply
if ((api->dispatch_type == pcmk_ipc_dispatch_sync)
&& (api->cmds != NULL)
&& (api->cmds->reply_expected != NULL)
&& (api->cmds->reply_expected(api, request))) {
flags = crm_ipc_client_response;
}
// The 0 here means a default timeout of 5 seconds
rc = crm_ipc_send(api->ipc, request, flags, 0, &reply);
if (rc < 0) {
return pcmk_legacy2rc(rc);
} else if (rc == 0) {
return ENODATA;
}
// With synchronous dispatch, we dispatch any reply now
if (reply != NULL) {
bool more = call_api_dispatch(api, reply);
pcmk__xml_free(reply);
while (more) {
rc = crm_ipc_read(api->ipc);
if (rc == -EAGAIN) {
continue;
} else if (rc == -ENOMSG || rc == pcmk_ok) {
return pcmk_rc_ok;
} else if (rc < 0) {
return -rc;
}
rc = dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);
if (rc == pcmk_rc_ok) {
more = false;
} else if (rc == EINPROGRESS) {
more = true;
} else {
continue;
}
}
}
return pcmk_rc_ok;
}
/*!
* \internal
* \brief Create the XML for an IPC request to purge a node from the peer cache
*
* \param[in] api IPC API connection
* \param[in] node_name If not NULL, name of node to purge
* \param[in] nodeid If not 0, node ID of node to purge
*
* \return Newly allocated IPC request XML
*
* \note The controller, fencer, and pacemakerd use the same request syntax, but
* the attribute manager uses a different one. The CIB manager doesn't
* have any syntax for it. The executor and scheduler don't connect to the
* cluster layer and thus don't have or need any syntax for it.
*
* \todo Modify the attribute manager to accept the common syntax (as well
* as its current one, for compatibility with older clients). Modify
* the CIB manager to accept and honor the common syntax. Modify the
* executor and scheduler to accept the syntax (immediately returning
* success), just for consistency. Modify this function to use the
* common syntax with all daemons if their version supports it.
*/
static xmlNode *
create_purge_node_request(const pcmk_ipc_api_t *api, const char *node_name,
uint32_t nodeid)
{
xmlNode *request = NULL;
const char *client = crm_system_name? crm_system_name : "client";
switch (api->server) {
case pcmk_ipc_attrd:
request = pcmk__xe_create(NULL, __func__);
crm_xml_add(request, PCMK__XA_T, PCMK__VALUE_ATTRD);
crm_xml_add(request, PCMK__XA_SRC, crm_system_name);
crm_xml_add(request, PCMK_XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE);
pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, true);
pcmk__xe_add_node(request, node_name, nodeid);
break;
case pcmk_ipc_controld:
case pcmk_ipc_fenced:
case pcmk_ipc_pacemakerd:
request = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL,
- pcmk_ipc_name(api, false), client, NULL);
+ pcmk_ipc_name(api, false), client);
if (nodeid > 0) {
crm_xml_add_ll(request, PCMK_XA_ID, (long long) nodeid);
}
crm_xml_add(request, PCMK_XA_UNAME, node_name);
break;
case pcmk_ipc_based:
case pcmk_ipc_execd:
case pcmk_ipc_schedulerd:
break;
default: // pcmk_ipc_unknown (shouldn't be possible)
return NULL;
}
return request;
}
/*!
* \brief Ask a Pacemaker daemon to purge a node from its peer cache
*
* \param[in,out] api IPC API connection
* \param[in] node_name If not NULL, name of node to purge
* \param[in] nodeid If not 0, node ID of node to purge
*
* \return Standard Pacemaker return code
*
* \note At least one of node_name or nodeid must be specified.
*/
int
pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid)
{
int rc = 0;
xmlNode *request = NULL;
if (api == NULL) {
return EINVAL;
}
if ((node_name == NULL) && (nodeid == 0)) {
return EINVAL;
}
request = create_purge_node_request(api, node_name, nodeid);
if (request == NULL) {
return EOPNOTSUPP;
}
rc = pcmk__send_ipc_request(api, request);
pcmk__xml_free(request);
crm_debug("%s peer cache purge of node %s[%lu]: rc=%d",
pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc);
return rc;
}
/*
* Generic IPC API (to eventually be deprecated as public API and made internal)
*/
struct crm_ipc_s {
struct pollfd pfd;
unsigned int max_buf_size; // maximum bytes we can send or receive over IPC
unsigned int buf_size; // size of allocated buffer
int msg_size;
int need_reply;
char *buffer;
char *server_name; // server IPC name being connected to
qb_ipcc_connection_t *ipc;
};
/*!
* \brief Create a new (legacy) object for using Pacemaker daemon IPC
*
* \param[in] name IPC system name to connect to
* \param[in] max_size Use a maximum IPC buffer size of at least this size
*
* \return Newly allocated IPC object on success, NULL otherwise
*
* \note The caller is responsible for freeing the result using
* crm_ipc_destroy().
* \note This should be considered deprecated for use with daemons supported by
* pcmk_new_ipc_api().
*/
crm_ipc_t *
crm_ipc_new(const char *name, size_t max_size)
{
crm_ipc_t *client = NULL;
client = calloc(1, sizeof(crm_ipc_t));
if (client == NULL) {
crm_err("Could not create IPC connection: %s", strerror(errno));
return NULL;
}
client->server_name = strdup(name);
if (client->server_name == NULL) {
crm_err("Could not create %s IPC connection: %s",
name, strerror(errno));
free(client);
return NULL;
}
client->buf_size = pcmk__ipc_buffer_size(max_size);
client->buffer = malloc(client->buf_size);
if (client->buffer == NULL) {
crm_err("Could not create %s IPC connection: %s",
name, strerror(errno));
free(client->server_name);
free(client);
return NULL;
}
/* Clients initiating connection pick the max buf size */
client->max_buf_size = client->buf_size;
client->pfd.fd = -1;
client->pfd.events = POLLIN;
client->pfd.revents = 0;
return client;
}
/*!
* \internal
* \brief Connect a generic (not daemon-specific) IPC object
*
* \param[in,out] ipc Generic IPC object to connect
*
* \return Standard Pacemaker return code
*/
int
pcmk__connect_generic_ipc(crm_ipc_t *ipc)
{
uid_t cl_uid = 0;
gid_t cl_gid = 0;
pid_t found_pid = 0;
uid_t found_uid = 0;
gid_t found_gid = 0;
int rc = pcmk_rc_ok;
if (ipc == NULL) {
return EINVAL;
}
ipc->need_reply = FALSE;
ipc->ipc = qb_ipcc_connect(ipc->server_name, ipc->buf_size);
if (ipc->ipc == NULL) {
return errno;
}
rc = qb_ipcc_fd_get(ipc->ipc, &ipc->pfd.fd);
if (rc < 0) { // -errno
crm_ipc_close(ipc);
return -rc;
}
rc = pcmk_daemon_user(&cl_uid, &cl_gid);
rc = pcmk_legacy2rc(rc);
if (rc != pcmk_rc_ok) {
crm_ipc_close(ipc);
return rc;
}
rc = is_ipc_provider_expected(ipc->ipc, ipc->pfd.fd, cl_uid, cl_gid,
&found_pid, &found_uid, &found_gid);
if (rc != pcmk_rc_ok) {
if (rc == pcmk_rc_ipc_unauthorized) {
crm_info("%s IPC provider authentication failed: process %lld has "
"uid %lld (expected %lld) and gid %lld (expected %lld)",
ipc->server_name,
(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
(long long) found_uid, (long long) cl_uid,
(long long) found_gid, (long long) cl_gid);
}
crm_ipc_close(ipc);
return rc;
}
ipc->max_buf_size = qb_ipcc_get_buffer_size(ipc->ipc);
if (ipc->max_buf_size > ipc->buf_size) {
free(ipc->buffer);
ipc->buffer = calloc(ipc->max_buf_size, sizeof(char));
if (ipc->buffer == NULL) {
rc = errno;
crm_ipc_close(ipc);
return rc;
}
ipc->buf_size = ipc->max_buf_size;
}
return pcmk_rc_ok;
}
/*!
* \brief Establish an IPC connection to a Pacemaker component
*
* \param[in,out] client Connection instance obtained from crm_ipc_new()
*
* \return true on success, false otherwise (in which case errno will be set;
* specifically, in case of discovering the remote side is not
* authentic, its value is set to ECONNABORTED).
*/
bool
crm_ipc_connect(crm_ipc_t *client)
{
int rc = pcmk__connect_generic_ipc(client);
if (rc == pcmk_rc_ok) {
return true;
}
if ((client != NULL) && (client->ipc == NULL)) {
errno = (rc > 0)? rc : ENOTCONN;
crm_debug("Could not establish %s IPC connection: %s (%d)",
client->server_name, pcmk_rc_str(errno), errno);
} else if (rc == pcmk_rc_ipc_unauthorized) {
crm_err("%s IPC provider authentication failed",
(client == NULL)? "Pacemaker" : client->server_name);
errno = ECONNABORTED;
} else {
crm_perror(LOG_ERR,
"Could not verify authenticity of %s IPC provider",
(client == NULL)? "Pacemaker" : client->server_name);
errno = ENOTCONN;
}
return false;
}
void
crm_ipc_close(crm_ipc_t * client)
{
if (client) {
if (client->ipc) {
qb_ipcc_connection_t *ipc = client->ipc;
client->ipc = NULL;
qb_ipcc_disconnect(ipc);
}
}
}
void
crm_ipc_destroy(crm_ipc_t * client)
{
if (client) {
if (client->ipc && qb_ipcc_is_connected(client->ipc)) {
crm_notice("Destroying active %s IPC connection",
client->server_name);
/* The next line is basically unsafe
*
* If this connection was attached to mainloop and mainloop is active,
* the 'disconnected' callback will end up back here and we'll end
* up free'ing the memory twice - something that can still happen
* even without this if we destroy a connection and it closes before
* we call exit
*/
/* crm_ipc_close(client); */
} else {
crm_trace("Destroying inactive %s IPC connection",
client->server_name);
}
free(client->buffer);
free(client->server_name);
free(client);
}
}
/*!
* \internal
* \brief Get the file descriptor for a generic IPC object
*
* \param[in,out] ipc Generic IPC object to get file descriptor for
* \param[out] fd Where to store file descriptor
*
* \return Standard Pacemaker return code
*/
int
pcmk__ipc_fd(crm_ipc_t *ipc, int *fd)
{
if ((ipc == NULL) || (fd == NULL)) {
return EINVAL;
}
if ((ipc->ipc == NULL) || (ipc->pfd.fd < 0)) {
return ENOTCONN;
}
*fd = ipc->pfd.fd;
return pcmk_rc_ok;
}
int
crm_ipc_get_fd(crm_ipc_t * client)
{
int fd = -1;
if (pcmk__ipc_fd(client, &fd) != pcmk_rc_ok) {
crm_err("Could not obtain file descriptor for %s IPC",
((client == NULL)? "unspecified" : client->server_name));
errno = EINVAL;
return -EINVAL;
}
return fd;
}
bool
crm_ipc_connected(crm_ipc_t * client)
{
bool rc = FALSE;
if (client == NULL) {
crm_trace("No client");
return FALSE;
} else if (client->ipc == NULL) {
crm_trace("No connection");
return FALSE;
} else if (client->pfd.fd < 0) {
crm_trace("Bad descriptor");
return FALSE;
}
rc = qb_ipcc_is_connected(client->ipc);
if (rc == FALSE) {
client->pfd.fd = -EINVAL;
}
return rc;
}
/*!
* \brief Check whether an IPC connection is ready to be read
*
* \param[in,out] client Connection to check
*
* \return Positive value if ready to be read, 0 if not ready, -errno on error
*/
int
crm_ipc_ready(crm_ipc_t *client)
{
int rc;
CRM_ASSERT(client != NULL);
if (!crm_ipc_connected(client)) {
return -ENOTCONN;
}
client->pfd.revents = 0;
rc = poll(&(client->pfd), 1, 0);
return (rc < 0)? -errno : rc;
}
// \return Standard Pacemaker return code
static int
crm_ipc_decompress(crm_ipc_t * client)
{
pcmk__ipc_header_t *header = (pcmk__ipc_header_t *)(void*)client->buffer;
if (header->size_compressed) {
int rc = 0;
unsigned int size_u = 1 + header->size_uncompressed;
/* never let buf size fall below our max size required for ipc reads. */
unsigned int new_buf_size = QB_MAX((sizeof(pcmk__ipc_header_t) + size_u), client->max_buf_size);
char *uncompressed = pcmk__assert_alloc(1, new_buf_size);
crm_trace("Decompressing message data %u bytes into %u bytes",
header->size_compressed, size_u);
rc = BZ2_bzBuffToBuffDecompress(uncompressed + sizeof(pcmk__ipc_header_t), &size_u,
client->buffer + sizeof(pcmk__ipc_header_t), header->size_compressed, 1, 0);
rc = pcmk__bzlib2rc(rc);
if (rc != pcmk_rc_ok) {
crm_err("Decompression failed: %s " QB_XS " rc=%d",
pcmk_rc_str(rc), rc);
free(uncompressed);
return rc;
}
/*
* This assert no longer holds true. For an identical msg, some clients may
* require compression, and others may not. If that same msg (event) is sent
* to multiple clients, it could result in some clients receiving a compressed
* msg even though compression was not explicitly required for them.
*
* CRM_ASSERT((header->size_uncompressed + sizeof(pcmk__ipc_header_t)) >= ipc_buffer_max);
*/
CRM_ASSERT(size_u == header->size_uncompressed);
memcpy(uncompressed, client->buffer, sizeof(pcmk__ipc_header_t)); /* Preserve the header */
header = (pcmk__ipc_header_t *)(void*)uncompressed;
free(client->buffer);
client->buf_size = new_buf_size;
client->buffer = uncompressed;
}
CRM_ASSERT(client->buffer[sizeof(pcmk__ipc_header_t) + header->size_uncompressed - 1] == 0);
return pcmk_rc_ok;
}
long
crm_ipc_read(crm_ipc_t * client)
{
pcmk__ipc_header_t *header = NULL;
CRM_ASSERT(client != NULL);
CRM_ASSERT(client->ipc != NULL);
CRM_ASSERT(client->buffer != NULL);
client->buffer[0] = 0;
client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer,
client->buf_size, 0);
if (client->msg_size >= 0) {
int rc = crm_ipc_decompress(client);
if (rc != pcmk_rc_ok) {
return pcmk_rc2legacy(rc);
}
header = (pcmk__ipc_header_t *)(void*)client->buffer;
if (!pcmk__valid_ipc_header(header)) {
return -EBADMSG;
}
crm_trace("Received %s IPC event %d size=%u rc=%d text='%.100s'",
client->server_name, header->qb.id, header->qb.size,
client->msg_size,
client->buffer + sizeof(pcmk__ipc_header_t));
} else {
crm_trace("No message received from %s IPC: %s",
client->server_name, pcmk_strerror(client->msg_size));
if (client->msg_size == -EAGAIN) {
return -EAGAIN;
}
}
if (!crm_ipc_connected(client) || client->msg_size == -ENOTCONN) {
crm_err("Connection to %s IPC failed", client->server_name);
}
if (header) {
/* Data excluding the header */
return header->size_uncompressed;
}
return -ENOMSG;
}
const char *
crm_ipc_buffer(crm_ipc_t * client)
{
CRM_ASSERT(client != NULL);
return client->buffer + sizeof(pcmk__ipc_header_t);
}
uint32_t
crm_ipc_buffer_flags(crm_ipc_t * client)
{
pcmk__ipc_header_t *header = NULL;
CRM_ASSERT(client != NULL);
if (client->buffer == NULL) {
return 0;
}
header = (pcmk__ipc_header_t *)(void*)client->buffer;
return header->flags;
}
const char *
crm_ipc_name(crm_ipc_t * client)
{
CRM_ASSERT(client != NULL);
return client->server_name;
}
// \return Standard Pacemaker return code
static int
internal_ipc_get_reply(crm_ipc_t *client, int request_id, int ms_timeout,
ssize_t *bytes)
{
time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
int rc = pcmk_rc_ok;
/* get the reply */
crm_trace("Waiting on reply to %s IPC message %d",
client->server_name, request_id);
do {
*bytes = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000);
if (*bytes > 0) {
pcmk__ipc_header_t *hdr = NULL;
rc = crm_ipc_decompress(client);
if (rc != pcmk_rc_ok) {
return rc;
}
hdr = (pcmk__ipc_header_t *)(void*)client->buffer;
if (hdr->qb.id == request_id) {
/* Got it */
break;
} else if (hdr->qb.id < request_id) {
xmlNode *bad = pcmk__xml_parse(crm_ipc_buffer(client));
crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id);
crm_log_xml_notice(bad, "OldIpcReply");
} else {
xmlNode *bad = pcmk__xml_parse(crm_ipc_buffer(client));
crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id);
crm_log_xml_notice(bad, "ImpossibleReply");
CRM_ASSERT(hdr->qb.id <= request_id);
}
} else if (!crm_ipc_connected(client)) {
crm_err("%s IPC provider disconnected while waiting for message %d",
client->server_name, request_id);
break;
}
} while (time(NULL) < timeout);
if (*bytes < 0) {
rc = (int) -*bytes; // System errno
}
return rc;
}
/*!
* \brief Send an IPC XML message
*
* \param[in,out] client Connection to IPC server
* \param[in] message XML message to send
* \param[in] flags Bitmask of crm_ipc_flags
* \param[in] ms_timeout Give up if not sent within this much time
* (5 seconds if 0, or no timeout if negative)
* \param[out] reply Reply from server (or NULL if none)
*
* \return Negative errno on error, otherwise size of reply received in bytes
* if reply was needed, otherwise number of bytes sent
*/
int
crm_ipc_send(crm_ipc_t *client, const xmlNode *message,
enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply)
{
int rc = 0;
ssize_t qb_rc = 0;
ssize_t bytes = 0;
struct iovec *iov;
static uint32_t id = 0;
static int factor = 8;
pcmk__ipc_header_t *header;
if (client == NULL) {
crm_notice("Can't send IPC request without connection (bug?): %.100s",
message);
return -ENOTCONN;
} else if (!crm_ipc_connected(client)) {
/* Don't even bother */
crm_notice("Can't send %s IPC requests: Connection closed",
client->server_name);
return -ENOTCONN;
}
if (ms_timeout == 0) {
ms_timeout = 5000;
}
if (client->need_reply) {
qb_rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout);
if (qb_rc < 0) {
crm_warn("Sending %s IPC disabled until pending reply received",
client->server_name);
return -EALREADY;
} else {
crm_notice("Sending %s IPC re-enabled after pending reply received",
client->server_name);
client->need_reply = FALSE;
}
}
id++;
CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */
rc = pcmk__ipc_prepare_iov(id, message, client->max_buf_size, &iov, &bytes);
if (rc != pcmk_rc_ok) {
crm_warn("Couldn't prepare %s IPC request: %s " QB_XS " rc=%d",
client->server_name, pcmk_rc_str(rc), rc);
return pcmk_rc2legacy(rc);
}
header = iov[0].iov_base;
pcmk__set_ipc_flags(header->flags, client->server_name, flags);
if (pcmk_is_set(flags, crm_ipc_proxied)) {
/* Don't look for a synchronous response */
pcmk__clear_ipc_flags(flags, "client", crm_ipc_client_response);
}
if(header->size_compressed) {
if(factor < 10 && (client->max_buf_size / 10) < (bytes / factor)) {
crm_notice("Compressed message exceeds %d0%% of configured IPC "
"limit (%u bytes); consider setting PCMK_ipc_buffer to "
"%u or higher",
factor, client->max_buf_size, 2 * client->max_buf_size);
factor++;
}
}
crm_trace("Sending %s IPC request %d of %u bytes using %dms timeout",
client->server_name, header->qb.id, header->qb.size, ms_timeout);
if ((ms_timeout > 0) || !pcmk_is_set(flags, crm_ipc_client_response)) {
time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
do {
/* @TODO Is this check really needed? Won't qb_ipcc_sendv() return
* an error if it's not connected?
*/
if (!crm_ipc_connected(client)) {
goto send_cleanup;
}
qb_rc = qb_ipcc_sendv(client->ipc, iov, 2);
} while ((qb_rc == -EAGAIN) && (time(NULL) < timeout));
rc = (int) qb_rc; // Negative of system errno, or bytes sent
if (qb_rc <= 0) {
goto send_cleanup;
} else if (!pcmk_is_set(flags, crm_ipc_client_response)) {
crm_trace("Not waiting for reply to %s IPC request %d",
client->server_name, header->qb.id);
goto send_cleanup;
}
rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout, &bytes);
if (rc != pcmk_rc_ok) {
/* We didn't get the reply in time, so disable future sends for now.
* The only alternative would be to close the connection since we
* don't know how to detect and discard out-of-sequence replies.
*
* @TODO Implement out-of-sequence detection
*/
client->need_reply = TRUE;
}
rc = (int) bytes; // Negative system errno, or size of reply received
} else {
// No timeout, and client response needed
do {
qb_rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer,
client->buf_size, -1);
} while ((qb_rc == -EAGAIN) && crm_ipc_connected(client));
rc = (int) qb_rc; // Negative system errno, or size of reply received
}
if (rc > 0) {
pcmk__ipc_header_t *hdr = (pcmk__ipc_header_t *)(void*)client->buffer;
crm_trace("Received %d-byte reply %d to %s IPC %d: %.100s",
rc, hdr->qb.id, client->server_name, header->qb.id,
crm_ipc_buffer(client));
if (reply) {
*reply = pcmk__xml_parse(crm_ipc_buffer(client));
}
} else {
crm_trace("No reply to %s IPC %d: rc=%d",
client->server_name, header->qb.id, rc);
}
send_cleanup:
if (!crm_ipc_connected(client)) {
crm_notice("Couldn't send %s IPC request %d: Connection closed "
QB_XS " rc=%d", client->server_name, header->qb.id, rc);
} else if (rc == -ETIMEDOUT) {
crm_warn("%s IPC request %d failed: %s after %dms " QB_XS " rc=%d",
client->server_name, header->qb.id, pcmk_strerror(rc),
ms_timeout, rc);
crm_write_blackbox(0, NULL);
} else if (rc <= 0) {
crm_warn("%s IPC request %d failed: %s " QB_XS " rc=%d",
client->server_name, header->qb.id,
((rc == 0)? "No bytes sent" : pcmk_strerror(rc)), rc);
}
pcmk_free_ipc_event(iov);
return rc;
}
/*!
* \brief Ensure an IPC provider has expected user or group
*
* \param[in] qb_ipc libqb client connection if available
* \param[in] sock Connected Unix socket for IPC
* \param[in] refuid Expected user ID
* \param[in] refgid Expected group ID
* \param[out] gotpid If not NULL, where to store provider's actual process ID
* (or 1 on platforms where ID is not available)
* \param[out] gotuid If not NULL, where to store provider's actual user ID
* \param[out] gotgid If not NULL, where to store provider's actual group ID
*
* \return Standard Pacemaker return code
* \note An actual user ID of 0 (root) will always be considered authorized,
* regardless of the expected values provided. The caller can use the
* output arguments to be stricter than this function.
*/
static int
is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
uid_t refuid, gid_t refgid,
pid_t *gotpid, uid_t *gotuid, gid_t *gotgid)
{
int rc = EOPNOTSUPP;
pid_t found_pid = 0;
uid_t found_uid = 0;
gid_t found_gid = 0;
#ifdef HAVE_QB_IPCC_AUTH_GET
if (qb_ipc != NULL) {
rc = qb_ipcc_auth_get(qb_ipc, &found_pid, &found_uid, &found_gid);
rc = -rc; // libqb returns 0 or -errno
if (rc == pcmk_rc_ok) {
goto found;
}
}
#endif
#ifdef HAVE_UCRED
{
struct ucred ucred;
socklen_t ucred_len = sizeof(ucred);
if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &ucred_len) < 0) {
rc = errno;
} else if (ucred_len != sizeof(ucred)) {
rc = EOPNOTSUPP;
} else {
found_pid = ucred.pid;
found_uid = ucred.uid;
found_gid = ucred.gid;
goto found;
}
}
#endif
#ifdef HAVE_SOCKPEERCRED
{
struct sockpeercred sockpeercred;
socklen_t sockpeercred_len = sizeof(sockpeercred);
if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED,
&sockpeercred, &sockpeercred_len) < 0) {
rc = errno;
} else if (sockpeercred_len != sizeof(sockpeercred)) {
rc = EOPNOTSUPP;
} else {
found_pid = sockpeercred.pid;
found_uid = sockpeercred.uid;
found_gid = sockpeercred.gid;
goto found;
}
}
#endif
#ifdef HAVE_GETPEEREID // For example, FreeBSD
if (getpeereid(sock, &found_uid, &found_gid) < 0) {
rc = errno;
} else {
found_pid = PCMK__SPECIAL_PID;
goto found;
}
#endif
#ifdef HAVE_GETPEERUCRED
{
ucred_t *ucred = NULL;
if (getpeerucred(sock, &ucred) < 0) {
rc = errno;
} else {
found_pid = ucred_getpid(ucred);
found_uid = ucred_geteuid(ucred);
found_gid = ucred_getegid(ucred);
ucred_free(ucred);
goto found;
}
}
#endif
return rc; // If we get here, nothing succeeded
found:
if (gotpid != NULL) {
*gotpid = found_pid;
}
if (gotuid != NULL) {
*gotuid = found_uid;
}
if (gotgid != NULL) {
*gotgid = found_gid;
}
if ((found_uid != 0) && (found_uid != refuid) && (found_gid != refgid)) {
return pcmk_rc_ipc_unauthorized;
}
return pcmk_rc_ok;
}
int
crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid,
pid_t *gotpid, uid_t *gotuid, gid_t *gotgid)
{
int ret = is_ipc_provider_expected(NULL, sock, refuid, refgid,
gotpid, gotuid, gotgid);
/* The old function had some very odd return codes*/
if (ret == 0) {
return 1;
} else if (ret == pcmk_rc_ipc_unauthorized) {
return 0;
} else {
return pcmk_rc2legacy(ret);
}
}
int
pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid,
gid_t refgid, pid_t *gotpid)
{
static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */
int fd;
int rc = pcmk_rc_ipc_unresponsive;
int auth_rc = 0;
int32_t qb_rc;
pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0;
qb_ipcc_connection_t *c;
#ifdef HAVE_QB_IPCC_CONNECT_ASYNC
struct pollfd pollfd = { 0, };
int poll_rc;
c = qb_ipcc_connect_async(name, 0,
&(pollfd.fd));
#else
c = qb_ipcc_connect(name, 0);
#endif
if (c == NULL) {
crm_info("Could not connect to %s IPC: %s", name, strerror(errno));
rc = pcmk_rc_ipc_unresponsive;
goto bail;
}
#ifdef HAVE_QB_IPCC_CONNECT_ASYNC
pollfd.events = POLLIN;
do {
poll_rc = poll(&pollfd, 1, 2000);
} while ((poll_rc == -1) && (errno == EINTR));
/* If poll() failed, given that disconnect function is not registered yet,
* qb_ipcc_disconnect() won't clean up the socket. In any case, call
* qb_ipcc_connect_continue() here so that it may fail and do the cleanup
* for us.
*/
if (qb_ipcc_connect_continue(c) != 0) {
crm_info("Could not connect to %s IPC: %s", name,
(poll_rc == 0)?"timeout":strerror(errno));
rc = pcmk_rc_ipc_unresponsive;
c = NULL; // qb_ipcc_connect_continue cleaned up for us
goto bail;
}
#endif
qb_rc = qb_ipcc_fd_get(c, &fd);
if (qb_rc != 0) {
rc = (int) -qb_rc; // System errno
crm_err("Could not get fd from %s IPC: %s " QB_XS " rc=%d",
name, pcmk_rc_str(rc), rc);
goto bail;
}
auth_rc = is_ipc_provider_expected(c, fd, refuid, refgid,
&found_pid, &found_uid, &found_gid);
if (auth_rc == pcmk_rc_ipc_unauthorized) {
crm_err("Daemon (IPC %s) effectively blocked with unauthorized"
" process %lld (uid: %lld, gid: %lld)",
name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
(long long) found_uid, (long long) found_gid);
rc = pcmk_rc_ipc_unauthorized;
goto bail;
}
if (auth_rc != pcmk_rc_ok) {
rc = auth_rc;
crm_err("Could not get peer credentials from %s IPC: %s "
QB_XS " rc=%d", name, pcmk_rc_str(rc), rc);
goto bail;
}
if (gotpid != NULL) {
*gotpid = found_pid;
}
rc = pcmk_rc_ok;
if ((found_uid != refuid || found_gid != refgid)
&& strncmp(last_asked_name, name, sizeof(last_asked_name))) {
if ((found_uid == 0) && (refuid != 0)) {
crm_warn("Daemon (IPC %s) runs as root, whereas the expected"
" credentials are %lld:%lld, hazard of violating"
" the least privilege principle",
name, (long long) refuid, (long long) refgid);
} else {
crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the"
" expected credentials are %lld:%lld, which may"
" mean a different set of privileges than expected",
name, (long long) found_uid, (long long) found_gid,
(long long) refuid, (long long) refgid);
}
memccpy(last_asked_name, name, '\0', sizeof(last_asked_name));
}
bail:
if (c != NULL) {
qb_ipcc_disconnect(c);
}
return rc;
}
diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
index 75b79e2f42..7f53f1cbf7 100644
--- a/lib/common/ipc_controld.c
+++ b/lib/common/ipc_controld.c
@@ -1,663 +1,670 @@
/*
* Copyright 2020-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <errno.h>
#include <inttypes.h> // PRIu32
#include <stdbool.h>
#include <stdint.h> // uint32_t
#include <stdio.h>
#include <libxml/tree.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/ipc_controld.h>
#include "crmcommon_private.h"
struct controld_api_private_s {
char *client_uuid;
unsigned int replies_expected;
};
/*!
* \internal
* \brief Get a string representation of a controller API reply type
*
* \param[in] reply Controller API reply type
*
* \return String representation of a controller API reply type
*/
const char *
pcmk__controld_api_reply2str(enum pcmk_controld_api_reply reply)
{
switch (reply) {
case pcmk_controld_reply_reprobe:
return "reprobe";
case pcmk_controld_reply_info:
return "info";
case pcmk_controld_reply_resource:
return "resource";
case pcmk_controld_reply_ping:
return "ping";
case pcmk_controld_reply_nodes:
return "nodes";
default:
return "unknown";
}
}
// \return Standard Pacemaker return code
static int
new_data(pcmk_ipc_api_t *api)
{
struct controld_api_private_s *private = NULL;
api->api_data = calloc(1, sizeof(struct controld_api_private_s));
if (api->api_data == NULL) {
return errno;
}
private = api->api_data;
/* This is set to the PID because that's how it was always done, but PIDs
* are not unique because clients can be remote. The value appears to be
* unused other than as part of PCMK__XA_CRM_SYS_FROM in IPC requests, which
* is only compared against the internal system names (CRM_SYSTEM_TENGINE,
* etc.), so it shouldn't be a problem.
*/
private->client_uuid = pcmk__getpid_s();
/* @TODO Implement a call ID model similar to the CIB, executor, and fencer
* IPC APIs, so that requests and replies can be matched, and
* duplicate replies can be discarded.
*/
return pcmk_rc_ok;
}
static void
free_data(void *data)
{
free(((struct controld_api_private_s *) data)->client_uuid);
free(data);
}
// \return Standard Pacemaker return code
static int
post_connect(pcmk_ipc_api_t *api)
{
/* The controller currently requires clients to register via a hello
* request, but does not reply back.
*/
struct controld_api_private_s *private = api->api_data;
const char *client_name = crm_system_name? crm_system_name : "client";
xmlNode *hello;
int rc;
hello = create_hello_message(private->client_uuid, client_name,
PCMK__CONTROLD_API_MAJOR,
PCMK__CONTROLD_API_MINOR);
rc = pcmk__send_ipc_request(api, hello);
pcmk__xml_free(hello);
if (rc != pcmk_rc_ok) {
crm_info("Could not send IPC hello to %s: %s " QB_XS " rc=%s",
pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
} else {
crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
}
return rc;
}
static void
set_node_info_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
{
data->reply_type = pcmk_controld_reply_info;
if (msg_data == NULL) {
return;
}
data->data.node_info.have_quorum =
pcmk__xe_attr_is_true(msg_data, PCMK_XA_HAVE_QUORUM);
data->data.node_info.is_remote =
pcmk__xe_attr_is_true(msg_data, PCMK_XA_REMOTE_NODE);
/* Integer node_info.id is currently valid only for Corosync nodes.
*
* @TODO: Improve handling after pcmk__node_status_t is refactored to handle
* layer-specific data better.
*/
crm_element_value_int(msg_data, PCMK_XA_ID, &(data->data.node_info.id));
data->data.node_info.uuid = crm_element_value(msg_data, PCMK_XA_ID);
data->data.node_info.uname = crm_element_value(msg_data, PCMK_XA_UNAME);
data->data.node_info.state = crm_element_value(msg_data, PCMK_XA_CRMD);
}
static void
set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
{
data->reply_type = pcmk_controld_reply_ping;
if (msg_data == NULL) {
return;
}
data->data.ping.sys_from = crm_element_value(msg_data,
PCMK__XA_CRM_SUBSYSTEM);
data->data.ping.fsa_state = crm_element_value(msg_data,
PCMK__XA_CRMD_STATE);
data->data.ping.result = crm_element_value(msg_data, PCMK_XA_RESULT);
}
static void
set_nodes_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
{
pcmk_controld_api_node_t *node_info;
data->reply_type = pcmk_controld_reply_nodes;
for (xmlNode *node = pcmk__xe_first_child(msg_data, PCMK_XE_NODE, NULL,
NULL);
node != NULL; node = pcmk__xe_next_same(node)) {
long long id_ll = 0;
node_info = pcmk__assert_alloc(1, sizeof(pcmk_controld_api_node_t));
crm_element_value_ll(node, PCMK_XA_ID, &id_ll);
if (id_ll > 0) {
node_info->id = id_ll;
}
node_info->uname = crm_element_value(node, PCMK_XA_UNAME);
node_info->state = crm_element_value(node, PCMK__XA_IN_CCM);
data->data.nodes = g_list_prepend(data->data.nodes, node_info);
}
}
static bool
reply_expected(pcmk_ipc_api_t *api, const xmlNode *request)
{
// We only need to handle commands that API functions can send
return pcmk__str_any_of(crm_element_value(request, PCMK__XA_CRM_TASK),
PCMK__CONTROLD_CMD_NODES,
CRM_OP_LRM_DELETE,
CRM_OP_LRM_FAIL,
CRM_OP_NODE_INFO,
CRM_OP_PING,
CRM_OP_REPROBE,
CRM_OP_RM_NODE_CACHE,
NULL);
}
static bool
dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
{
struct controld_api_private_s *private = api->api_data;
crm_exit_t status = CRM_EX_OK;
xmlNode *wrapper = NULL;
xmlNode *msg_data = NULL;
const char *value = NULL;
pcmk_controld_api_reply_t reply_data = {
pcmk_controld_reply_unknown, NULL, NULL,
};
if (pcmk__xe_is(reply, PCMK__XE_ACK)) {
/* ACKs are trivial responses that do not count toward expected replies,
* and do not have all the fields that validation requires, so skip that
* processing.
*/
return private->replies_expected > 0;
}
if (private->replies_expected > 0) {
private->replies_expected--;
}
// Do some basic validation of the reply
value = crm_element_value(reply, PCMK__XA_SUBT);
if (pcmk__str_eq(value, PCMK__VALUE_REQUEST, pcmk__str_none)) {
/* @COMPAT Controllers <3.0.0 set PCMK__XA_SUBT to PCMK__VALUE_REQUEST
* for certain replies. Once we no longer support Pacemaker Remote nodes
* connecting to cluster nodes <3.0.0, or rolling upgrades from <3.0.0,
* we can drop this check.
*/
crm_trace("Received a reply that was marked as a request "
"(bug unless sent by a controller <3.0.0)");
} else if (!pcmk__str_eq(value, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
crm_info("Unrecognizable message from controller: "
"invalid message type '%s'", pcmk__s(value, ""));
status = CRM_EX_PROTOCOL;
goto done;
}
if (pcmk__str_empty(crm_element_value(reply, PCMK_XA_REFERENCE))) {
crm_info("Unrecognizable message from controller: no reference");
status = CRM_EX_PROTOCOL;
goto done;
}
value = crm_element_value(reply, PCMK__XA_CRM_TASK);
if (pcmk__str_empty(value)) {
crm_info("Unrecognizable message from controller: no command name");
status = CRM_EX_PROTOCOL;
goto done;
}
// Parse useful info from reply
reply_data.feature_set = crm_element_value(reply, PCMK_XA_VERSION);
reply_data.host_from = crm_element_value(reply, PCMK__XA_SRC);
wrapper = pcmk__xe_first_child(reply, PCMK__XE_CRM_XML, NULL, NULL);
msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
if (!strcmp(value, CRM_OP_REPROBE)) {
reply_data.reply_type = pcmk_controld_reply_reprobe;
} else if (!strcmp(value, CRM_OP_NODE_INFO)) {
set_node_info_data(&reply_data, msg_data);
} else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
reply_data.reply_type = pcmk_controld_reply_resource;
reply_data.data.resource.node_state = msg_data;
} else if (!strcmp(value, CRM_OP_PING)) {
set_ping_data(&reply_data, msg_data);
} else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
set_nodes_data(&reply_data, msg_data);
} else {
crm_info("Unrecognizable message from controller: unknown command '%s'",
value);
status = CRM_EX_PROTOCOL;
}
done:
pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
// Free any reply data that was allocated
if (pcmk__str_eq(value, PCMK__CONTROLD_CMD_NODES, pcmk__str_casei)) {
g_list_free_full(reply_data.data.nodes, free);
}
return false; // No further replies needed
}
pcmk__ipc_methods_t *
pcmk__controld_api_methods(void)
{
pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));
if (cmds != NULL) {
cmds->new_data = new_data;
cmds->free_data = free_data;
cmds->post_connect = post_connect;
cmds->reply_expected = reply_expected;
cmds->dispatch = dispatch;
}
return cmds;
}
/*!
* \internal
* \brief Create XML for a controller IPC request
*
* \param[in] api Controller connection
* \param[in] op Controller IPC command name
* \param[in] node Node name to set as destination host
* \param[in] msg_data XML to attach to request as message data
*
* \return Newly allocated XML for request
*/
static xmlNode *
create_controller_request(const pcmk_ipc_api_t *api, const char *op,
const char *node, xmlNode *msg_data)
{
struct controld_api_private_s *private = NULL;
const char *sys_to = NULL;
+ char *sender_system = NULL;
+ xmlNode *request = NULL;
if (api == NULL) {
return NULL;
}
private = api->api_data;
if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
sys_to = CRM_SYSTEM_DC;
} else {
sys_to = CRM_SYSTEM_CRMD;
}
- return create_request(op, msg_data, node, sys_to,
- (crm_system_name? crm_system_name : "client"),
- private->client_uuid);
+ sender_system = crm_strdup_printf("%s_%s", private->client_uuid,
+ pcmk__s(crm_system_name, "client"));
+ request = create_request(op, msg_data, node, sys_to, sender_system);
+ free(sender_system);
+ return request;
}
// \return Standard Pacemaker return code
static int
send_controller_request(pcmk_ipc_api_t *api, const xmlNode *request,
bool reply_is_expected)
{
if (crm_element_value(request, PCMK_XA_REFERENCE) == NULL) {
return EINVAL;
}
if (reply_is_expected) {
struct controld_api_private_s *private = api->api_data;
private->replies_expected++;
}
return pcmk__send_ipc_request(api, request);
}
static xmlNode *
create_reprobe_message_data(const char *target_node, const char *router_node)
{
xmlNode *msg_data;
msg_data = pcmk__xe_create(NULL, "data_for_" CRM_OP_REPROBE);
crm_xml_add(msg_data, PCMK__META_ON_NODE, target_node);
if ((router_node != NULL) && !pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
crm_xml_add(msg_data, PCMK__XA_ROUTER_NODE, router_node);
}
return msg_data;
}
/*!
* \brief Send a reprobe controller operation
*
* \param[in,out] api Controller connection
* \param[in] target_node Name of node to reprobe
* \param[in] router_node Router node for host
*
* \return Standard Pacemaker return code
* \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
*/
int
pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node,
const char *router_node)
{
xmlNode *request;
xmlNode *msg_data;
int rc = pcmk_rc_ok;
if (api == NULL) {
return EINVAL;
}
if (router_node == NULL) {
router_node = target_node;
}
crm_debug("Sending %s IPC request to reprobe %s via %s",
pcmk_ipc_name(api, true), pcmk__s(target_node, "local node"),
pcmk__s(router_node, "local node"));
msg_data = create_reprobe_message_data(target_node, router_node);
request = create_controller_request(api, CRM_OP_REPROBE, router_node,
msg_data);
rc = send_controller_request(api, request, true);
pcmk__xml_free(msg_data);
pcmk__xml_free(request);
return rc;
}
/*!
* \brief Send a "node info" controller operation
*
* \param[in,out] api Controller connection
* \param[in] nodeid ID of node to get info for (or 0 for local node)
*
* \return Standard Pacemaker return code
* \note Event callback will get a reply of type pcmk_controld_reply_info.
*/
int
pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
{
xmlNode *request;
int rc = pcmk_rc_ok;
request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
if (request == NULL) {
return EINVAL;
}
if (nodeid > 0) {
crm_xml_add_ll(request, PCMK_XA_ID, nodeid);
}
rc = send_controller_request(api, request, true);
pcmk__xml_free(request);
return rc;
}
/*!
* \brief Ask the controller for status
*
* \param[in,out] api Controller connection
* \param[in] node_name Name of node whose status is desired (NULL for DC)
*
* \return Standard Pacemaker return code
* \note Event callback will get a reply of type pcmk_controld_reply_ping.
*/
int
pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name)
{
xmlNode *request;
int rc = pcmk_rc_ok;
request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
if (request == NULL) {
return EINVAL;
}
rc = send_controller_request(api, request, true);
pcmk__xml_free(request);
return rc;
}
/*!
* \brief Ask the controller for cluster information
*
* \param[in,out] api Controller connection
*
* \return Standard Pacemaker return code
* \note Event callback will get a reply of type pcmk_controld_reply_nodes.
*/
int
pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api)
{
xmlNode *request;
int rc = EINVAL;
request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL,
NULL);
if (request != NULL) {
rc = send_controller_request(api, request, true);
pcmk__xml_free(request);
}
return rc;
}
// \return Standard Pacemaker return code
static int
controller_resource_op(pcmk_ipc_api_t *api, const char *op,
const char *target_node, const char *router_node,
bool cib_only, const char *rsc_id,
const char *rsc_long_id, const char *standard,
const char *provider, const char *type)
{
int rc = pcmk_rc_ok;
char *key;
xmlNode *request, *msg_data, *xml_rsc, *params;
if (api == NULL) {
return EINVAL;
}
if (router_node == NULL) {
router_node = target_node;
}
msg_data = pcmk__xe_create(NULL, PCMK__XE_RSC_OP);
/* The controller logs the transition key from resource op requests, so we
* need to have *something* for it.
* @TODO don't use "crm-resource"
*/
key = pcmk__transition_key(0, getpid(), 0,
"xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
crm_xml_add(msg_data, PCMK__XA_TRANSITION_KEY, key);
free(key);
crm_xml_add(msg_data, PCMK__META_ON_NODE, target_node);
if (!pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
crm_xml_add(msg_data, PCMK__XA_ROUTER_NODE, router_node);
}
if (cib_only) {
// Indicate that only the CIB needs to be cleaned
crm_xml_add(msg_data, PCMK__XA_MODE, PCMK__VALUE_CIB);
}
xml_rsc = pcmk__xe_create(msg_data, PCMK_XE_PRIMITIVE);
crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
crm_xml_add(xml_rsc, PCMK__XA_LONG_ID, rsc_long_id);
crm_xml_add(xml_rsc, PCMK_XA_CLASS, standard);
crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, provider);
crm_xml_add(xml_rsc, PCMK_XA_TYPE, type);
params = pcmk__xe_create(msg_data, PCMK__XE_ATTRIBUTES);
crm_xml_add(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
// The controller parses the timeout from the request
key = crm_meta_name(PCMK_META_TIMEOUT);
crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg
free(key);
request = create_controller_request(api, op, router_node, msg_data);
rc = send_controller_request(api, request, true);
pcmk__xml_free(msg_data);
pcmk__xml_free(request);
return rc;
}
/*!
* \brief Ask the controller to fail a resource
*
* \param[in,out] api Controller connection
* \param[in] target_node Name of node resource is on
* \param[in] router_node Router node for target
* \param[in] rsc_id ID of resource to fail
* \param[in] rsc_long_id Long ID of resource (if any)
* \param[in] standard Standard of resource
* \param[in] provider Provider of resource (if any)
* \param[in] type Type of resource to fail
*
* \return Standard Pacemaker return code
* \note Event callback will get a reply of type pcmk_controld_reply_resource.
*/
int
pcmk_controld_api_fail(pcmk_ipc_api_t *api,
const char *target_node, const char *router_node,
const char *rsc_id, const char *rsc_long_id,
const char *standard, const char *provider,
const char *type)
{
crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
pcmk__s(rsc_long_id, "no other names"),
pcmk__s(target_node, "unspecified node"),
pcmk__s(router_node, "unspecified node"));
return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
router_node, false, rsc_id, rsc_long_id,
standard, provider, type);
}
/*!
* \brief Ask the controller to refresh a resource
*
* \param[in,out] api Controller connection
* \param[in] target_node Name of node resource is on
* \param[in] router_node Router node for target
* \param[in] rsc_id ID of resource to refresh
* \param[in] rsc_long_id Long ID of resource (if any)
* \param[in] standard Standard of resource
* \param[in] provider Provider of resource (if any)
* \param[in] type Type of resource
* \param[in] cib_only If true, clean resource from CIB only
*
* \return Standard Pacemaker return code
* \note Event callback will get a reply of type pcmk_controld_reply_resource.
*/
int
pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
const char *router_node,
const char *rsc_id, const char *rsc_long_id,
const char *standard, const char *provider,
const char *type, bool cib_only)
{
crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
pcmk__s(rsc_long_id, "no other names"),
pcmk__s(target_node, "unspecified node"),
pcmk__s(router_node, "unspecified node"));
return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
router_node, cib_only, rsc_id, rsc_long_id,
standard, provider, type);
}
/*!
* \brief Get the number of IPC replies currently expected from the controller
*
* \param[in] api Controller IPC API connection
*
* \return Number of replies expected
*/
unsigned int
pcmk_controld_api_replies_expected(const pcmk_ipc_api_t *api)
{
struct controld_api_private_s *private = api->api_data;
return private->replies_expected;
}
/*!
* \brief Create XML for a controller IPC "hello" message
*
* \deprecated This function is deprecated as part of the public C API.
*/
// \todo make this static to this file when breaking API backward compatibility
xmlNode *
create_hello_message(const char *uuid, const char *client_name,
const char *major_version, const char *minor_version)
{
xmlNode *hello_node = NULL;
xmlNode *hello = NULL;
+ char *sender_system = NULL;
if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name)
|| pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) {
crm_err("Could not create IPC hello message from %s (UUID %s): "
"missing information",
client_name? client_name : "unknown client",
uuid? uuid : "unknown");
return NULL;
}
hello_node = pcmk__xe_create(NULL, PCMK__XE_OPTIONS);
crm_xml_add(hello_node, PCMK__XA_MAJOR_VERSION, major_version);
crm_xml_add(hello_node, PCMK__XA_MINOR_VERSION, minor_version);
crm_xml_add(hello_node, PCMK__XA_CLIENT_NAME, client_name);
// @TODO Nothing uses this. Drop, or keep for debugging?
crm_xml_add(hello_node, PCMK__XA_CLIENT_UUID, uuid);
- hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
+ sender_system = crm_strdup_printf("%s_%s", uuid, client_name);
+ hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, sender_system);
+ free(sender_system);
+ pcmk__xml_free(hello_node);
if (hello == NULL) {
crm_err("Could not create IPC hello message from %s (UUID %s): "
"Request creation failed", client_name, uuid);
return NULL;
}
- pcmk__xml_free(hello_node);
crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
return hello;
}
diff --git a/lib/common/ipc_pacemakerd.c b/lib/common/ipc_pacemakerd.c
index 7377838b29..f238b9ab27 100644
--- a/lib/common/ipc_pacemakerd.c
+++ b/lib/common/ipc_pacemakerd.c
@@ -1,321 +1,323 @@
/*
* Copyright 2020-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdlib.h>
#include <time.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/ipc_pacemakerd.h>
#include "crmcommon_private.h"
typedef struct pacemakerd_api_private_s {
enum pcmk_pacemakerd_state state;
char *client_uuid;
} pacemakerd_api_private_t;
static const char *pacemakerd_state_str[] = {
PCMK__VALUE_INIT,
PCMK__VALUE_STARTING_DAEMONS,
PCMK__VALUE_WAIT_FOR_PING,
PCMK__VALUE_RUNNING,
PCMK__VALUE_SHUTTING_DOWN,
PCMK__VALUE_SHUTDOWN_COMPLETE,
PCMK_VALUE_REMOTE,
};
enum pcmk_pacemakerd_state
pcmk_pacemakerd_api_daemon_state_text2enum(const char *state)
{
int i;
if (state == NULL) {
return pcmk_pacemakerd_state_invalid;
}
for (i=pcmk_pacemakerd_state_init; i <= pcmk_pacemakerd_state_max;
i++) {
if (pcmk__str_eq(state, pacemakerd_state_str[i], pcmk__str_none)) {
return i;
}
}
return pcmk_pacemakerd_state_invalid;
}
const char *
pcmk_pacemakerd_api_daemon_state_enum2text(
enum pcmk_pacemakerd_state state)
{
if ((state >= pcmk_pacemakerd_state_init) &&
(state <= pcmk_pacemakerd_state_max)) {
return pacemakerd_state_str[state];
}
return "invalid";
}
/*!
* \internal
* \brief Return a friendly string representation of a \p pacemakerd state
*
* \param[in] state \p pacemakerd state
*
* \return A user-friendly string representation of \p state, or
* <tt>"Invalid pacemakerd state"</tt>
*/
const char *
pcmk__pcmkd_state_enum2friendly(enum pcmk_pacemakerd_state state)
{
switch (state) {
case pcmk_pacemakerd_state_init:
return "Initializing pacemaker";
case pcmk_pacemakerd_state_starting_daemons:
return "Pacemaker daemons are starting";
case pcmk_pacemakerd_state_wait_for_ping:
return "Waiting for startup trigger from SBD";
case pcmk_pacemakerd_state_running:
return "Pacemaker is running";
case pcmk_pacemakerd_state_shutting_down:
return "Pacemaker daemons are shutting down";
case pcmk_pacemakerd_state_shutdown_complete:
/* Assuming pacemakerd won't process messages while in
* shutdown_complete state unless reporting to SBD
*/
return "Pacemaker daemons are shut down (reporting to SBD)";
case pcmk_pacemakerd_state_remote:
return "pacemaker-remoted is running (on a Pacemaker Remote node)";
default:
return "Invalid pacemakerd state";
}
}
/*!
* \internal
* \brief Get a string representation of a \p pacemakerd API reply type
*
* \param[in] reply \p pacemakerd API reply type
*
* \return String representation of a \p pacemakerd API reply type
*/
const char *
pcmk__pcmkd_api_reply2str(enum pcmk_pacemakerd_api_reply reply)
{
switch (reply) {
case pcmk_pacemakerd_reply_ping:
return "ping";
case pcmk_pacemakerd_reply_shutdown:
return "shutdown";
default:
return "unknown";
}
}
// \return Standard Pacemaker return code
static int
new_data(pcmk_ipc_api_t *api)
{
struct pacemakerd_api_private_s *private = NULL;
api->api_data = calloc(1, sizeof(struct pacemakerd_api_private_s));
if (api->api_data == NULL) {
return errno;
}
private = api->api_data;
private->state = pcmk_pacemakerd_state_invalid;
/* other as with cib, controld, ... we are addressing pacemakerd just
from the local node -> pid is unique and thus sufficient as an ID
*/
private->client_uuid = pcmk__getpid_s();
return pcmk_rc_ok;
}
static void
free_data(void *data)
{
free(((struct pacemakerd_api_private_s *) data)->client_uuid);
free(data);
}
// \return Standard Pacemaker return code
static int
post_connect(pcmk_ipc_api_t *api)
{
struct pacemakerd_api_private_s *private = NULL;
if (api->api_data == NULL) {
return EINVAL;
}
private = api->api_data;
private->state = pcmk_pacemakerd_state_invalid;
return pcmk_rc_ok;
}
static void
post_disconnect(pcmk_ipc_api_t *api)
{
struct pacemakerd_api_private_s *private = NULL;
if (api->api_data == NULL) {
return;
}
private = api->api_data;
private->state = pcmk_pacemakerd_state_invalid;
return;
}
static bool
reply_expected(pcmk_ipc_api_t *api, const xmlNode *request)
{
const char *command = crm_element_value(request, PCMK__XA_CRM_TASK);
if (command == NULL) {
return false;
}
// We only need to handle commands that functions in this file can send
return pcmk__str_any_of(command, CRM_OP_PING, CRM_OP_QUIT, NULL);
}
static bool
dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
{
crm_exit_t status = CRM_EX_OK;
xmlNode *wrapper = NULL;
xmlNode *msg_data = NULL;
pcmk_pacemakerd_api_reply_t reply_data = {
pcmk_pacemakerd_reply_unknown
};
const char *value = NULL;
long long value_ll = 0;
if (pcmk__xe_is(reply, PCMK__XE_ACK)) {
long long int ack_status = 0;
pcmk__scan_ll(crm_element_value(reply, PCMK_XA_STATUS), &ack_status,
CRM_EX_OK);
return ack_status == CRM_EX_INDETERMINATE;
}
value = crm_element_value(reply, PCMK__XA_SUBT);
if (!pcmk__str_eq(value, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
crm_info("Unrecognizable message from %s: "
"message type '%s' not '" PCMK__VALUE_RESPONSE "'",
pcmk_ipc_name(api, true), pcmk__s(value, ""));
status = CRM_EX_PROTOCOL;
goto done;
}
if (pcmk__str_empty(crm_element_value(reply, PCMK_XA_REFERENCE))) {
crm_info("Unrecognizable message from %s: no reference",
pcmk_ipc_name(api, true));
status = CRM_EX_PROTOCOL;
goto done;
}
value = crm_element_value(reply, PCMK__XA_CRM_TASK);
// Parse useful info from reply
wrapper = pcmk__xe_first_child(reply, PCMK__XE_CRM_XML, NULL, NULL);
msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
crm_element_value_ll(msg_data, PCMK_XA_CRM_TIMESTAMP, &value_ll);
if (pcmk__str_eq(value, CRM_OP_PING, pcmk__str_none)) {
reply_data.reply_type = pcmk_pacemakerd_reply_ping;
reply_data.data.ping.state =
pcmk_pacemakerd_api_daemon_state_text2enum(
crm_element_value(msg_data, PCMK__XA_PACEMAKERD_STATE));
reply_data.data.ping.status =
pcmk__str_eq(crm_element_value(msg_data, PCMK_XA_RESULT), "ok",
pcmk__str_casei)?pcmk_rc_ok:pcmk_rc_error;
reply_data.data.ping.last_good = (value_ll < 0)? 0 : (time_t) value_ll;
reply_data.data.ping.sys_from =
crm_element_value(msg_data, PCMK__XA_CRM_SUBSYSTEM);
} else if (pcmk__str_eq(value, CRM_OP_QUIT, pcmk__str_none)) {
const char *op_status = crm_element_value(msg_data, PCMK__XA_OP_STATUS);
reply_data.reply_type = pcmk_pacemakerd_reply_shutdown;
reply_data.data.shutdown.status = atoi(op_status);
} else {
crm_info("Unrecognizable message from %s: unknown command '%s'",
pcmk_ipc_name(api, true), pcmk__s(value, ""));
status = CRM_EX_PROTOCOL;
goto done;
}
done:
pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
return false;
}
pcmk__ipc_methods_t *
pcmk__pacemakerd_api_methods(void)
{
pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));
if (cmds != NULL) {
cmds->new_data = new_data;
cmds->free_data = free_data;
cmds->post_connect = post_connect;
cmds->reply_expected = reply_expected;
cmds->dispatch = dispatch;
cmds->post_disconnect = post_disconnect;
}
return cmds;
}
static int
do_pacemakerd_api_call(pcmk_ipc_api_t *api, const char *ipc_name, const char *task)
{
pacemakerd_api_private_t *private;
+ char *sender_system = NULL;
xmlNode *cmd;
int rc;
if (api == NULL) {
return EINVAL;
}
private = api->api_data;
CRM_ASSERT(private != NULL);
- cmd = create_request(task, NULL, NULL, CRM_SYSTEM_MCP,
- pcmk__ipc_sys_name(ipc_name, "client"),
- private->client_uuid);
+ sender_system = crm_strdup_printf("%s_%s", private->client_uuid,
+ pcmk__ipc_sys_name(ipc_name, "client"));
+ cmd = create_request(task, NULL, NULL, CRM_SYSTEM_MCP, sender_system);
+ free(sender_system);
if (cmd) {
rc = pcmk__send_ipc_request(api, cmd);
if (rc != pcmk_rc_ok) {
crm_debug("Couldn't send request to %s: %s rc=%d",
pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
}
pcmk__xml_free(cmd);
} else {
rc = ENOMSG;
}
return rc;
}
int
pcmk_pacemakerd_api_ping(pcmk_ipc_api_t *api, const char *ipc_name)
{
return do_pacemakerd_api_call(api, ipc_name, CRM_OP_PING);
}
int
pcmk_pacemakerd_api_shutdown(pcmk_ipc_api_t *api, const char *ipc_name)
{
return do_pacemakerd_api_call(api, ipc_name, CRM_OP_QUIT);
}
diff --git a/lib/common/ipc_schedulerd.c b/lib/common/ipc_schedulerd.c
index 61f0900646..cc2ad42a62 100644
--- a/lib/common/ipc_schedulerd.c
+++ b/lib/common/ipc_schedulerd.c
@@ -1,184 +1,186 @@
/*
* Copyright 2021-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdlib.h>
#include <time.h>
#include <crm/crm.h>
#include <crm/common/xml.h>
#include <crm/common/ipc.h>
#include <crm/common/ipc_internal.h>
#include <crm/common/ipc_schedulerd.h>
#include "crmcommon_private.h"
typedef struct schedulerd_api_private_s {
char *client_uuid;
} schedulerd_api_private_t;
// \return Standard Pacemaker return code
static int
new_data(pcmk_ipc_api_t *api)
{
struct schedulerd_api_private_s *private = NULL;
api->api_data = calloc(1, sizeof(struct schedulerd_api_private_s));
if (api->api_data == NULL) {
return errno;
}
private = api->api_data;
/* See comments in ipc_pacemakerd.c. */
private->client_uuid = pcmk__getpid_s();
return pcmk_rc_ok;
}
static void
free_data(void *data)
{
free(((struct schedulerd_api_private_s *) data)->client_uuid);
free(data);
}
// \return Standard Pacemaker return code
static int
post_connect(pcmk_ipc_api_t *api)
{
if (api->api_data == NULL) {
return EINVAL;
}
return pcmk_rc_ok;
}
static bool
reply_expected(pcmk_ipc_api_t *api, const xmlNode *request)
{
const char *command = crm_element_value(request, PCMK__XA_CRM_TASK);
if (command == NULL) {
return false;
}
// We only need to handle commands that functions in this file can send
return pcmk__str_any_of(command, CRM_OP_PECALC, NULL);
}
static bool
dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
{
crm_exit_t status = CRM_EX_OK;
xmlNode *wrapper = NULL;
xmlNode *msg_data = NULL;
pcmk_schedulerd_api_reply_t reply_data = {
pcmk_schedulerd_reply_unknown
};
const char *value = NULL;
if (pcmk__xe_is(reply, PCMK__XE_ACK)) {
return false;
}
value = crm_element_value(reply, PCMK__XA_SUBT);
if (!pcmk__str_eq(value, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
crm_info("Unrecognizable message from schedulerd: "
"message type '%s' not '" PCMK__VALUE_RESPONSE "'",
pcmk__s(value, ""));
status = CRM_EX_PROTOCOL;
goto done;
}
if (pcmk__str_empty(crm_element_value(reply, PCMK_XA_REFERENCE))) {
crm_info("Unrecognizable message from schedulerd: no reference");
status = CRM_EX_PROTOCOL;
goto done;
}
// Parse useful info from reply
wrapper = pcmk__xe_first_child(reply, PCMK__XE_CRM_XML, NULL, NULL);
msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
value = crm_element_value(reply, PCMK__XA_CRM_TASK);
if (pcmk__str_eq(value, CRM_OP_PECALC, pcmk__str_none)) {
reply_data.reply_type = pcmk_schedulerd_reply_graph;
reply_data.data.graph.reference = crm_element_value(reply,
PCMK_XA_REFERENCE);
reply_data.data.graph.input = crm_element_value(reply,
PCMK__XA_CRM_TGRAPH_IN);
reply_data.data.graph.tgraph = msg_data;
} else {
crm_info("Unrecognizable message from schedulerd: "
"unknown command '%s'", pcmk__s(value, ""));
status = CRM_EX_PROTOCOL;
goto done;
}
done:
pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
return false;
}
pcmk__ipc_methods_t *
pcmk__schedulerd_api_methods(void)
{
pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));
if (cmds != NULL) {
cmds->new_data = new_data;
cmds->free_data = free_data;
cmds->post_connect = post_connect;
cmds->reply_expected = reply_expected;
cmds->dispatch = dispatch;
}
return cmds;
}
static int
do_schedulerd_api_call(pcmk_ipc_api_t *api, const char *task, xmlNode *cib, char **ref)
{
schedulerd_api_private_t *private;
xmlNode *cmd = NULL;
int rc;
+ char *sender_system = NULL;
if (!pcmk_ipc_is_connected(api)) {
return ENOTCONN;
}
private = api->api_data;
CRM_ASSERT(private != NULL);
- cmd = create_request(task, cib, NULL, CRM_SYSTEM_PENGINE,
- crm_system_name? crm_system_name : "client",
- private->client_uuid);
+ sender_system = crm_strdup_printf("%s_%s", private->client_uuid,
+ pcmk__s(crm_system_name, "client"));
+ cmd = create_request(task, cib, NULL, CRM_SYSTEM_PENGINE, sender_system);
+ free(sender_system);
if (cmd) {
rc = pcmk__send_ipc_request(api, cmd);
if (rc != pcmk_rc_ok) {
crm_debug("Couldn't send request to schedulerd: %s rc=%d",
pcmk_rc_str(rc), rc);
}
*ref = strdup(crm_element_value(cmd, PCMK_XA_REFERENCE));
pcmk__xml_free(cmd);
} else {
rc = ENOMSG;
}
return rc;
}
int
pcmk_schedulerd_api_graph(pcmk_ipc_api_t *api, xmlNode *cib, char **ref)
{
return do_schedulerd_api_call(api, CRM_OP_PECALC, cib, ref);
}
diff --git a/lib/common/messages.c b/lib/common/messages.c
index 4111f92e1f..4a8a3ce4c1 100644
--- a/lib/common/messages.c
+++ b/lib/common/messages.c
@@ -1,292 +1,278 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU Lesser General Public License
* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <stdio.h>
#include <time.h> // time()
#include <sys/types.h>
#include <glib.h>
#include <libxml/tree.h>
#include <crm/common/xml.h>
#include <crm/common/xml_internal.h>
/*!
* \internal
* \brief Create message XML (for IPC or the cluster layer)
*
* Create standard, generic XML that can be used as a message sent via IPC or
* the cluster layer. Currently, not all IPC and cluster layer messaging uses
* this, but it should (eventually, keeping backward compatibility in mind).
*
* \param[in] origin Name of function that called this one (required)
* \param[in] server Server whose protocol defines message semantics
* \param[in] reply_to If NULL, create message as a request with a
* generated message ID, otherwise create message
* as a reply to this message ID
* \param[in] sender_system Sender's subsystem (required; this is an
* arbitrary string that may have meaning between
* the sender and recipient)
* \param[in] recipient_node If not NULL, add as message's recipient node
* (NULL typically indicates a broadcast message)
* \param[in] recipient_system If not NULL, add as message's recipient
* subsystem (this is an arbitrary string that may
* have meaning between the sender and recipient)
* \param[in] task Add as message's task (required)
* \param[in] data If not NULL, copy as message's data (callers
* should not add attributes to the returned
* message element, but instead pass any desired
* information here, though this is not always
* honored currently)
*
* \return Newly created message XML
*
* \note This function should usually not be called directly, but via the
* pcmk__new_message() wrapper.
* \note The caller is responsible for freeing the return value using
* \c pcmk__xml_free().
*/
xmlNode *
pcmk__new_message_as(const char *origin, enum pcmk_ipc_server server,
const char *reply_to, const char *sender_system,
const char *recipient_node, const char *recipient_system,
const char *task, xmlNode *data)
{
static unsigned int message_counter = 0U;
xmlNode *message = NULL;
char *message_id = NULL;
const char *subtype = PCMK__VALUE_RESPONSE;
CRM_CHECK(!pcmk__str_empty(origin)
&& !pcmk__str_empty(sender_system)
&& !pcmk__str_empty(task),
return NULL);
if (reply_to == NULL) {
subtype = PCMK__VALUE_REQUEST;
message_id = crm_strdup_printf("%s-%s-%llu-%u", task, sender_system,
(unsigned long long) time(NULL),
message_counter++);
reply_to = message_id;
}
message = pcmk__xe_create(NULL, PCMK__XE_MESSAGE);
pcmk__xe_set_props(message,
PCMK_XA_ORIGIN, origin,
PCMK__XA_T, pcmk__server_message_type(server),
PCMK__XA_SUBT, subtype,
PCMK_XA_VERSION, CRM_FEATURE_SET,
PCMK_XA_REFERENCE, reply_to,
PCMK__XA_CRM_SYS_FROM, sender_system,
PCMK__XA_CRM_HOST_TO, recipient_node,
PCMK__XA_CRM_SYS_TO, recipient_system,
PCMK__XA_CRM_TASK, task,
NULL);
if (data != NULL) {
xmlNode *wrapper = pcmk__xe_create(message, PCMK__XE_CRM_XML);
pcmk__xml_copy(wrapper, data);
}
free(message_id);
return message;
}
/*!
* \brief Create a Pacemaker request (for IPC or cluster layer)
*
* \param[in] task What to set as the request's task
* \param[in] msg_data What to add as the request's data contents
* \param[in] host_to What to set as the request's destination host
* \param[in] sys_to What to set as the request's destination system
- * \param[in] sys_from If not NULL, set as request's origin system
- * \param[in] uuid_from If not NULL, use in request's origin system
+ * \param[in] sender_system Sender's subsystem (required; this is an
+ * arbitrary string that may have meaning between
+ * the sender and recipient)
* \param[in] origin Name of function that called this one
*
* \return XML of new request
*
- * \note One of sys_from or uuid_from must be non-NULL
* \note This function should not be called directly, but via the
* create_request() wrapper.
* \note The caller is responsible for freeing the return value using
* \c pcmk__xml_free().
*/
xmlNode *
create_request_adv(const char *task, xmlNode *msg_data,
const char *host_to, const char *sys_to,
- const char *sys_from, const char *uuid_from,
- const char *origin)
+ const char *sender_system, const char *origin)
{
- char *true_from = NULL;
- xmlNode *request = NULL;
-
- if (uuid_from != NULL) {
- true_from = crm_strdup_printf("%s_%s", uuid_from,
- (sys_from? sys_from : "none"));
- } else if (sys_from != NULL) {
- true_from = strdup(sys_from);
- } else {
- crm_err("Cannot create IPC request: No originating system specified");
- }
- request = pcmk__new_message_as(origin, pcmk_ipc_controld, NULL, true_from,
- host_to, sys_to, task, msg_data);
- free(true_from);
- return request;
+ return pcmk__new_message_as(origin, pcmk_ipc_controld, NULL, sender_system,
+ host_to, sys_to, task, msg_data);
}
/*!
* \internal
* \brief Create a Pacemaker reply (for IPC or cluster layer)
*
* \param[in] origin Name of function that called this one
* \param[in] original_request XML of request being replied to
* \param[in] data If not NULL, copy as reply's data (callers
* should not add attributes to the returned
* message element, but instead pass any desired
* information here, though this is not always
* honored currently)
*
* \return Newly created reply XML
*
* \note This function should not be called directly, but via the
* pcmk__new_reply() wrapper.
* \note The caller is responsible for freeing the return value using
* \c pcmk__xml_free().
*/
xmlNode *
pcmk__new_reply_as(const char *origin, const xmlNode *original_request,
xmlNode *data)
{
const char *message_type = crm_element_value(original_request, PCMK__XA_T);
const char *host_from = crm_element_value(original_request, PCMK__XA_SRC);
const char *sys_from = crm_element_value(original_request,
PCMK__XA_CRM_SYS_FROM);
const char *sys_to = crm_element_value(original_request,
PCMK__XA_CRM_SYS_TO);
const char *type = crm_element_value(original_request, PCMK__XA_SUBT);
const char *operation = crm_element_value(original_request,
PCMK__XA_CRM_TASK);
const char *crm_msg_reference = crm_element_value(original_request,
PCMK_XA_REFERENCE);
enum pcmk_ipc_server server = pcmk__parse_server(message_type);
if (server == pcmk_ipc_unknown) {
/* @COMPAT Not all requests currently specify a message type, so use a
* default that preserves past behavior.
*
* @TODO Ensure all requests specify a message type, drop this check
* after we no longer support rolling upgrades or Pacemaker Remote
* connections involving versions before that.
*/
server = pcmk_ipc_controld;
}
if (type == NULL) {
crm_warn("Cannot reply to invalid message: No message type specified");
return NULL;
}
if (strcmp(type, PCMK__VALUE_REQUEST) != 0) {
/* Replies should only be generated for request messages, but it's possible
* we expect replies to other messages right now so this can't be enforced.
*/
crm_trace("Creating a reply for a non-request original message");
}
// Since this is a reply, we reverse the sender and recipient info
return pcmk__new_message_as(origin, server, crm_msg_reference, sys_to,
host_from, sys_from, operation, data);
}
/*!
* \internal
* \brief Register handlers for server commands
*
* \param[in] handlers Array of handler functions for supported server commands
* (the final entry must have a NULL command name, and if
* it has a handler it will be used as the default handler
* for unrecognized commands)
*
* \return Newly created hash table with commands and handlers
* \note The caller is responsible for freeing the return value with
* g_hash_table_destroy().
*/
GHashTable *
pcmk__register_handlers(const pcmk__server_command_t handlers[])
{
GHashTable *commands = g_hash_table_new(g_str_hash, g_str_equal);
if (handlers != NULL) {
int i;
for (i = 0; handlers[i].command != NULL; ++i) {
g_hash_table_insert(commands, (gpointer) handlers[i].command,
handlers[i].handler);
}
if (handlers[i].handler != NULL) {
// g_str_hash() can't handle NULL, so use empty string for default
g_hash_table_insert(commands, (gpointer) "", handlers[i].handler);
}
}
return commands;
}
/*!
* \internal
* \brief Process an incoming request
*
* \param[in,out] request Request to process
* \param[in] handlers Command table created by pcmk__register_handlers()
*
* \return XML to send as reply (or NULL if no reply is needed)
*/
xmlNode *
pcmk__process_request(pcmk__request_t *request, GHashTable *handlers)
{
xmlNode *(*handler)(pcmk__request_t *request) = NULL;
CRM_CHECK((request != NULL) && (request->op != NULL) && (handlers != NULL),
return NULL);
if (pcmk_is_set(request->flags, pcmk__request_sync)
&& (request->ipc_client != NULL)) {
CRM_CHECK(request->ipc_client->request_id == request->ipc_id,
return NULL);
}
handler = g_hash_table_lookup(handlers, request->op);
if (handler == NULL) {
handler = g_hash_table_lookup(handlers, ""); // Default handler
if (handler == NULL) {
crm_info("Ignoring %s request from %s %s with no handler",
request->op, pcmk__request_origin_type(request),
pcmk__request_origin(request));
return NULL;
}
}
return (*handler)(request);
}
/*!
* \internal
* \brief Free memory used within a request (but not the request itself)
*
* \param[in,out] request Request to reset
*/
void
pcmk__reset_request(pcmk__request_t *request)
{
free(request->op);
request->op = NULL;
pcmk__reset_result(&(request->result));
}
diff --git a/tools/crm_node.c b/tools/crm_node.c
index ba64b02720..a134e1317b 100644
--- a/tools/crm_node.c
+++ b/tools/crm_node.c
@@ -1,875 +1,875 @@
/*
* Copyright 2004-2024 the Pacemaker project contributors
*
* The version control history for this file may have further details.
*
* This source code is licensed under the GNU General Public License version 2
* or later (GPLv2+) WITHOUT ANY WARRANTY.
*/
#include <crm_internal.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <sys/types.h>
#include <crm/crm.h>
#include <crm/common/cmdline_internal.h>
#include <crm/common/output_internal.h>
#include <crm/common/mainloop.h>
#include <crm/common/xml.h>
#include <crm/cib.h>
#include <crm/cib/internal.h>
#include <crm/common/ipc_controld.h>
#include <crm/common/attrs_internal.h>
#include <pacemaker-internal.h>
#define SUMMARY "crm_node - Tool for displaying low-level node information"
struct {
gboolean corosync;
gboolean dangerous_cmd;
gboolean force_flag;
char command;
int nodeid;
char *target_uname;
} options = {
.command = '\0',
.force_flag = FALSE
};
gboolean command_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
gboolean name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
gboolean remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error);
static GError *error = NULL;
static GMainLoop *mainloop = NULL;
static crm_exit_t exit_code = CRM_EX_OK;
static pcmk__output_t *out = NULL;
#define INDENT " "
static GOptionEntry command_entries[] = {
{ "cluster-id", 'i', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
"Display this node's cluster id",
NULL },
{ "list", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
"Display all known members (past and present) of this cluster",
NULL },
{ "name", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
"Display the name used by the cluster for this node",
NULL },
{ "partition", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
"Display the members of this partition",
NULL },
{ "quorum", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
"Display a 1 if our partition has quorum, 0 if not",
NULL },
{ "name-for-id", 'N', 0, G_OPTION_ARG_CALLBACK, name_cb,
"Display the name used by the cluster for the node with the specified ID",
"ID" },
{ "remove", 'R', 0, G_OPTION_ARG_CALLBACK, remove_cb,
"(Advanced) Remove the (stopped) node with the specified name from Pacemaker's\n"
INDENT "configuration and caches (the node must already have been removed from\n"
INDENT "the underlying cluster stack configuration",
"NAME" },
{ NULL }
};
static GOptionEntry addl_entries[] = {
{ "force", 'f', 0, G_OPTION_ARG_NONE, &options.force_flag,
NULL,
NULL },
#if SUPPORT_COROSYNC
/* Unused and deprecated */
{ "corosync", 'C', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.corosync,
NULL,
NULL },
#endif
// @TODO add timeout option for when IPC replies are needed
{ NULL }
};
static pcmk__supported_format_t formats[] = {
PCMK__SUPPORTED_FORMAT_NONE,
PCMK__SUPPORTED_FORMAT_TEXT,
PCMK__SUPPORTED_FORMAT_XML,
{ NULL, NULL, NULL }
};
gboolean
command_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
if (pcmk__str_eq("-i", option_name, pcmk__str_casei) || pcmk__str_eq("--cluster-id", option_name, pcmk__str_casei)) {
options.command = 'i';
} else if (pcmk__str_eq("-l", option_name, pcmk__str_casei) || pcmk__str_eq("--list", option_name, pcmk__str_casei)) {
options.command = 'l';
} else if (pcmk__str_eq("-n", option_name, pcmk__str_casei) || pcmk__str_eq("--name", option_name, pcmk__str_casei)) {
options.command = 'n';
} else if (pcmk__str_eq("-p", option_name, pcmk__str_casei) || pcmk__str_eq("--partition", option_name, pcmk__str_casei)) {
options.command = 'p';
} else if (pcmk__str_eq("-q", option_name, pcmk__str_casei) || pcmk__str_eq("--quorum", option_name, pcmk__str_casei)) {
options.command = 'q';
} else {
g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Unknown param passed to command_cb: %s", option_name);
return FALSE;
}
return TRUE;
}
gboolean
name_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
options.command = 'N';
pcmk__scan_min_int(optarg, &(options.nodeid), 0);
return TRUE;
}
gboolean
remove_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
if (optarg == NULL) {
g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "-R option requires an argument");
return FALSE;
}
options.command = 'R';
options.dangerous_cmd = TRUE;
pcmk__str_update(&options.target_uname, optarg);
return TRUE;
}
PCMK__OUTPUT_ARGS("node-id", "uint32_t")
static int
node_id_default(pcmk__output_t *out, va_list args) {
uint32_t node_id = va_arg(args, uint32_t);
out->info(out, "%" PRIu32, node_id);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-id", "uint32_t")
static int
node_id_xml(pcmk__output_t *out, va_list args) {
uint32_t node_id = va_arg(args, uint32_t);
char *id_s = crm_strdup_printf("%" PRIu32, node_id);
pcmk__output_create_xml_node(out, PCMK_XE_NODE_INFO,
PCMK_XA_NODEID, id_s,
NULL);
free(id_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("simple-node-list", "GList *")
static int
simple_node_list_default(pcmk__output_t *out, va_list args)
{
GList *nodes = va_arg(args, GList *);
for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
pcmk_controld_api_node_t *node = node_iter->data;
out->info(out, "%" PRIu32 " %s %s", node->id, pcmk__s(node->uname, ""),
pcmk__s(node->state, ""));
}
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("simple-node-list", "GList *")
static int
simple_node_list_xml(pcmk__output_t *out, va_list args)
{
GList *nodes = va_arg(args, GList *);
out->begin_list(out, NULL, NULL, PCMK_XE_NODES);
for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
pcmk_controld_api_node_t *node = node_iter->data;
char *id_s = crm_strdup_printf("%" PRIu32, node->id);
pcmk__output_create_xml_node(out, PCMK_XE_NODE,
PCMK_XA_ID, id_s,
PCMK_XA_NAME, node->uname,
PCMK_XA_STATE, node->state,
NULL);
free(id_s);
}
out->end_list(out);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-name", "uint32_t", "const char *")
static int
node_name_default(pcmk__output_t *out, va_list args) {
uint32_t node_id G_GNUC_UNUSED = va_arg(args, uint32_t);
const char *node_name = va_arg(args, const char *);
out->info(out, "%s", node_name);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("node-name", "uint32_t", "const char *")
static int
node_name_xml(pcmk__output_t *out, va_list args) {
uint32_t node_id = va_arg(args, uint32_t);
const char *node_name = va_arg(args, const char *);
char *id_s = crm_strdup_printf("%" PRIu32, node_id);
pcmk__output_create_xml_node(out, PCMK_XE_NODE_INFO,
PCMK_XA_NODEID, id_s,
PCMK_XA_UNAME, node_name,
NULL);
free(id_s);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("partition-list", "GList *")
static int
partition_list_default(pcmk__output_t *out, va_list args)
{
GList *nodes = va_arg(args, GList *);
GString *buffer = NULL;
for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
pcmk_controld_api_node_t *node = node_iter->data;
if (pcmk__str_eq(node->state, "member", pcmk__str_none)) {
pcmk__add_separated_word(&buffer, 128, pcmk__s(node->uname, ""), " ");
}
}
if (buffer != NULL) {
out->info(out, "%s", buffer->str);
g_string_free(buffer, TRUE);
return pcmk_rc_ok;
}
return pcmk_rc_no_output;
}
PCMK__OUTPUT_ARGS("partition-list", "GList *")
static int
partition_list_xml(pcmk__output_t *out, va_list args)
{
GList *nodes = va_arg(args, GList *);
out->begin_list(out, NULL, NULL, PCMK_XE_NODES);
for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
pcmk_controld_api_node_t *node = node_iter->data;
if (pcmk__str_eq(node->state, "member", pcmk__str_none)) {
char *id_s = crm_strdup_printf("%" PRIu32, node->id);
pcmk__output_create_xml_node(out, PCMK_XE_NODE,
PCMK_XA_ID, id_s,
PCMK_XA_NAME, node->uname,
PCMK_XA_STATE, node->state,
NULL);
free(id_s);
}
}
out->end_list(out);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("quorum", "bool")
static int
quorum_default(pcmk__output_t *out, va_list args) {
bool have_quorum = va_arg(args, int);
out->info(out, "%d", have_quorum);
return pcmk_rc_ok;
}
PCMK__OUTPUT_ARGS("quorum", "bool")
static int
quorum_xml(pcmk__output_t *out, va_list args) {
bool have_quorum = va_arg(args, int);
pcmk__output_create_xml_node(out, PCMK_XE_CLUSTER_INFO,
PCMK_XA_QUORUM, pcmk__btoa(have_quorum),
NULL);
return pcmk_rc_ok;
}
static pcmk__message_entry_t fmt_functions[] = {
{ "node-id", "default", node_id_default },
{ "node-id", "xml", node_id_xml },
{ "node-name", "default", node_name_default },
{ "node-name", "xml", node_name_xml },
{ "partition-list", "default", partition_list_default },
{ "partition-list", "xml", partition_list_xml },
{ "quorum", "default", quorum_default },
{ "quorum", "xml", quorum_xml },
{ "simple-node-list", "default", simple_node_list_default },
{ "simple-node-list", "xml", simple_node_list_xml },
{ NULL, NULL, NULL }
};
static gint
sort_node(gconstpointer a, gconstpointer b)
{
const pcmk_controld_api_node_t *node_a = a;
const pcmk_controld_api_node_t *node_b = b;
return pcmk__numeric_strcasecmp((node_a->uname? node_a->uname : ""),
(node_b->uname? node_b->uname : ""));
}
static void
controller_event_cb(pcmk_ipc_api_t *controld_api,
enum pcmk_ipc_event event_type, crm_exit_t status,
void *event_data, void *user_data)
{
pcmk_controld_api_reply_t *reply = event_data;
switch (event_type) {
case pcmk_ipc_event_disconnect:
if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"Lost connection to controller");
}
goto done;
break;
case pcmk_ipc_event_reply:
break;
default:
return;
}
if (status != CRM_EX_OK) {
exit_code = status;
g_set_error(&error, PCMK__EXITC_ERROR, status,
"Bad reply from controller: %s",
crm_exit_str(status));
goto done;
}
if (reply->reply_type != pcmk_controld_reply_nodes) {
g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_INDETERMINATE,
"Unknown reply type %d from controller",
reply->reply_type);
goto done;
}
reply->data.nodes = g_list_sort(reply->data.nodes, sort_node);
if (options.command == 'p') {
out->message(out, "partition-list", reply->data.nodes);
} else if (options.command == 'l') {
out->message(out, "simple-node-list", reply->data.nodes);
}
// Success
exit_code = CRM_EX_OK;
done:
pcmk_disconnect_ipc(controld_api);
pcmk_quit_main_loop(mainloop, 10);
}
static void
run_controller_mainloop(void)
{
pcmk_ipc_api_t *controld_api = NULL;
int rc;
// Set disconnect exit code to handle unexpected disconnects
exit_code = CRM_EX_DISCONNECT;
// Create controller IPC object
rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
if (rc != pcmk_rc_ok) {
g_set_error(&error, PCMK__RC_ERROR, rc,
"Could not connect to controller: %s",
pcmk_rc_str(rc));
return;
}
pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);
// Connect to controller
rc = pcmk__connect_ipc(controld_api, pcmk_ipc_dispatch_main, 5);
if (rc != pcmk_rc_ok) {
exit_code = pcmk_rc2exitc(rc);
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"Could not connect to %s: %s",
pcmk_ipc_name(controld_api, true), pcmk_rc_str(rc));
return;
}
rc = pcmk_controld_api_list_nodes(controld_api);
if (rc != pcmk_rc_ok) {
pcmk_disconnect_ipc(controld_api);
exit_code = pcmk_rc2exitc(rc);
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"Could not ping controller: %s", pcmk_rc_str(rc));
return;
}
// Run main loop to get controller reply via controller_event_cb()
mainloop = g_main_loop_new(NULL, FALSE);
g_main_loop_run(mainloop);
g_main_loop_unref(mainloop);
mainloop = NULL;
pcmk_free_ipc_api(controld_api);
}
static void
print_node_id(void)
{
uint32_t nodeid = 0;
int rc = pcmk__query_node_info(out, &nodeid, NULL, NULL, NULL, NULL, NULL,
false, 0);
if (rc != pcmk_rc_ok) {
/* pcmk__query_node_info already sets an error message on the output object,
* so there's no need to call g_set_error here. That would just create a
* duplicate error message in the output.
*/
exit_code = pcmk_rc2exitc(rc);
return;
}
rc = out->message(out, "node-id", nodeid);
if (rc != pcmk_rc_ok) {
g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print node ID: %s",
pcmk_rc_str(rc));
}
exit_code = pcmk_rc2exitc(rc);
}
static void
print_node_name(uint32_t nodeid)
{
int rc = pcmk_rc_ok;
char *node_name = NULL;
if (nodeid == 0) {
// Check environment first (i.e. when called by resource agent)
const char *name = getenv("OCF_RESKEY_" CRM_META "_"
PCMK__META_ON_NODE);
if (name != NULL) {
rc = out->message(out, "node-name", 0UL, name);
goto done;
}
}
// Otherwise ask the controller
/* pcmk__query_node_name already sets an error message on the output object,
* so there's no need to call g_set_error here. That would just create a
* duplicate error message in the output.
*/
rc = pcmk__query_node_name(out, nodeid, &node_name, 0);
if (rc != pcmk_rc_ok) {
exit_code = pcmk_rc2exitc(rc);
return;
}
rc = out->message(out, "node-name", 0UL, node_name);
done:
if (node_name != NULL) {
free(node_name);
}
if (rc != pcmk_rc_ok) {
g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print node name: %s",
pcmk_rc_str(rc));
}
exit_code = pcmk_rc2exitc(rc);
}
static void
print_quorum(void)
{
bool quorum;
int rc = pcmk__query_node_info(out, NULL, NULL, NULL, NULL, &quorum, NULL,
false, 0);
if (rc != pcmk_rc_ok) {
/* pcmk__query_node_info already sets an error message on the output object,
* so there's no need to call g_set_error here. That would just create a
* duplicate error message in the output.
*/
exit_code = pcmk_rc2exitc(rc);
return;
}
rc = out->message(out, "quorum", quorum);
if (rc != pcmk_rc_ok) {
g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print quorum status: %s",
pcmk_rc_str(rc));
}
exit_code = pcmk_rc2exitc(rc);
}
/*!
* \internal
* \brief Extend a transaction by removing a node from a CIB section
*
* \param[in,out] cib Active CIB connection
* \param[in] element CIB element containing node name and/or ID
* \param[in] section CIB section that \p element is in
* \param[in] node_name Name of node to purge (NULL to leave unspecified)
* \param[in] node_id Node ID of node to purge (0 to leave unspecified)
*
* \note At least one of node_name and node_id must be specified.
* \return Standard Pacemaker return code
*/
static int
remove_from_section(cib_t *cib, const char *element, const char *section,
const char *node_name, long node_id)
{
int rc = pcmk_rc_ok;
xmlNode *xml = pcmk__xe_create(NULL, element);
crm_xml_add(xml, PCMK_XA_UNAME, node_name);
if (node_id > 0) {
crm_xml_add_ll(xml, PCMK_XA_ID, node_id);
}
rc = cib->cmds->remove(cib, section, xml, cib_transaction);
pcmk__xml_free(xml);
return (rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(rc);
}
/*!
* \internal
* \brief Purge a node from CIB
*
* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
*
* \note At least one of node_name and node_id must be specified.
* \return Standard Pacemaker return code
*/
static int
purge_node_from_cib(const char *node_name, long node_id)
{
int rc = pcmk_rc_ok;
int commit_rc = pcmk_rc_ok;
cib_t *cib = NULL;
// Connect to CIB and start a transaction
cib = cib_new();
if (cib == NULL) {
return ENOTCONN;
}
rc = cib__signon_attempts(cib, cib_command, 5);
if (rc == pcmk_ok) {
rc = cib->cmds->init_transaction(cib);
}
if (rc != pcmk_ok) {
rc = pcmk_legacy2rc(rc);
cib__clean_up_connection(&cib);
return rc;
}
// Remove from configuration and status
rc = remove_from_section(cib, PCMK_XE_NODE, PCMK_XE_NODES, node_name,
node_id);
if (rc == pcmk_rc_ok) {
rc = remove_from_section(cib, PCMK__XE_NODE_STATE, PCMK_XE_STATUS,
node_name, node_id);
}
// Commit the transaction
commit_rc = cib->cmds->end_transaction(cib, (rc == pcmk_rc_ok),
cib_sync_call);
cib__clean_up_connection(&cib);
if ((rc == pcmk_rc_ok) && (commit_rc == pcmk_ok)) {
crm_debug("Purged node %s (%ld) from CIB",
pcmk__s(node_name, "by ID"), node_id);
}
return rc;
}
/*!
* \internal
* \brief Purge a node from a single server's peer cache
*
* \param[in] server IPC server to send request to
* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
*
* \note At least one of node_name and node_id must be specified.
* \return Standard Pacemaker return code
*/
static int
purge_node_from(enum pcmk_ipc_server server, const char *node_name,
long node_id)
{
pcmk_ipc_api_t *api = NULL;
int rc;
rc = pcmk_new_ipc_api(&api, server);
if (rc != pcmk_rc_ok) {
goto done;
}
rc = pcmk__connect_ipc(api, pcmk_ipc_dispatch_sync, 5);
if (rc != pcmk_rc_ok) {
goto done;
}
rc = pcmk_ipc_purge_node(api, node_name, node_id);
done:
if (rc != pcmk_rc_ok) { // Debug message already logged on success
g_set_error(&error, PCMK__RC_ERROR, rc,
"Could not purge node %s from %s: %s",
pcmk__s(node_name, "by ID"), pcmk_ipc_name(api, true),
pcmk_rc_str(rc));
}
pcmk_free_ipc_api(api);
return rc;
}
/*!
* \internal
* \brief Purge a node from the fencer's peer cache
*
* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
*
* \note At least one of node_name and node_id must be specified.
* \return Standard Pacemaker return code
*/
static int
purge_node_from_fencer(const char *node_name, long node_id)
{
int rc = pcmk_rc_ok;
crm_ipc_t *conn = NULL;
xmlNode *cmd = NULL;
conn = crm_ipc_new("stonith-ng", 0);
if (conn == NULL) {
rc = ENOTCONN;
exit_code = pcmk_rc2exitc(rc);
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"Could not connect to fencer to purge node %s",
pcmk__s(node_name, "by ID"));
return rc;
}
rc = pcmk__connect_generic_ipc(conn);
if (rc != pcmk_rc_ok) {
exit_code = pcmk_rc2exitc(rc);
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"Could not connect to fencer to purge node %s: %s",
pcmk__s(node_name, "by ID"), pcmk_rc_str(rc));
crm_ipc_destroy(conn);
return rc;
}
cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, "stonith-ng",
- crm_system_name, NULL);
+ crm_system_name);
if (node_id > 0) {
crm_xml_add_ll(cmd, PCMK_XA_ID, node_id);
}
crm_xml_add(cmd, PCMK_XA_UNAME, node_name);
rc = crm_ipc_send(conn, cmd, 0, 0, NULL);
if (rc >= 0) {
rc = pcmk_rc_ok;
crm_debug("Purged node %s (%ld) from fencer",
pcmk__s(node_name, "by ID"), node_id);
} else {
rc = pcmk_legacy2rc(rc);
fprintf(stderr, "Could not purge node %s from fencer: %s\n",
pcmk__s(node_name, "by ID"), pcmk_rc_str(rc));
}
pcmk__xml_free(cmd);
crm_ipc_close(conn);
crm_ipc_destroy(conn);
return rc;
}
static void
remove_node(const char *target_uname)
{
int rc = pcmk_rc_ok;
long nodeid = 0;
const char *node_name = NULL;
char *endptr = NULL;
const enum pcmk_ipc_server servers[] = {
pcmk_ipc_controld,
pcmk_ipc_attrd,
};
// Check whether node was specified by name or numeric ID
errno = 0;
nodeid = strtol(target_uname, &endptr, 10);
if ((errno != 0) || (endptr == target_uname) || (*endptr != '\0')
|| (nodeid <= 0)) {
// It's not a positive integer, so assume it's a node name
nodeid = 0;
node_name = target_uname;
}
for (int i = 0; i < PCMK__NELEM(servers); ++i) {
rc = purge_node_from(servers[i], node_name, nodeid);
if (rc != pcmk_rc_ok) {
exit_code = pcmk_rc2exitc(rc);
return;
}
}
// The fencer hasn't been converted to pcmk_ipc_api_t yet
rc = purge_node_from_fencer(node_name, nodeid);
if (rc != pcmk_rc_ok) {
exit_code = pcmk_rc2exitc(rc);
return;
}
// Lastly, purge the node from the CIB itself
rc = purge_node_from_cib(node_name, nodeid);
exit_code = pcmk_rc2exitc(rc);
}
static GOptionContext *
build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
GOptionContext *context = NULL;
GOptionEntry extra_prog_entries[] = {
{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
"Be less descriptive in output.",
NULL },
{ NULL }
};
context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
/* Add the -q option, which cannot be part of the globally supported options
* because some tools use that flag for something else.
*/
pcmk__add_main_args(context, extra_prog_entries);
pcmk__add_arg_group(context, "commands", "Commands:",
"Show command help", command_entries);
pcmk__add_arg_group(context, "additional", "Additional Options:",
"Show additional options", addl_entries);
return context;
}
int
main(int argc, char **argv)
{
int rc = pcmk_rc_ok;
GOptionGroup *output_group = NULL;
pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
gchar **processed_args = pcmk__cmdline_preproc(argv, "NR");
GOptionContext *context = build_arg_context(args, &output_group);
pcmk__register_formats(output_group, formats);
if (!g_option_context_parse_strv(context, &processed_args, &error)) {
exit_code = CRM_EX_USAGE;
goto done;
}
pcmk__cli_init_logging("crm_node", args->verbosity);
rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
if (rc != pcmk_rc_ok) {
exit_code = pcmk_rc2exitc(rc);
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"Error creating output format %s: %s", args->output_ty,
pcmk_rc_str(rc));
goto done;
}
if (args->version) {
out->version(out, false);
goto done;
}
if (options.command == 0) {
char *help = g_option_context_get_help(context, TRUE, NULL);
out->err(out, "%s", help);
g_free(help);
exit_code = CRM_EX_USAGE;
goto done;
}
if (options.dangerous_cmd && options.force_flag == FALSE) {
exit_code = CRM_EX_USAGE;
g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
"The supplied command is considered dangerous."
" To prevent accidental destruction of the cluster,"
" the --force flag is required in order to proceed.");
goto done;
}
pcmk__register_lib_messages(out);
pcmk__register_messages(out, fmt_functions);
switch (options.command) {
case 'i':
print_node_id();
break;
case 'n':
print_node_name(0);
break;
case 'q':
print_quorum();
break;
case 'N':
print_node_name(options.nodeid);
break;
case 'R':
remove_node(options.target_uname);
break;
case 'l':
case 'p':
run_controller_mainloop();
break;
default:
break;
}
done:
g_strfreev(processed_args);
pcmk__free_arg_context(context);
pcmk__output_and_clear_error(&error, out);
if (out != NULL) {
out->finish(out, exit_code, true, NULL);
pcmk__output_free(out);
}
pcmk__unregister_formats();
return crm_exit(exit_code);
}