No OneTemporary
Actions

Size

63 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
	index 968369a83a..60c092e259 100644
	--- a/daemons/controld/controld_control.c
	+++ b/daemons/controld/controld_control.c
	@@ -1,715 +1,715 @@
	/*
	* Copyright 2004-2025 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election_internal.h>
	#include <crm/common/ipc_internal.h>

	#include <pacemaker-controld.h>

	static qb_ipcs_service_t *ipcs = NULL;

	static crm_trigger_t *config_read_trigger = NULL;

	#if SUPPORT_COROSYNC
	extern gboolean crm_connect_corosync(pcmk_cluster_t *cluster);
	#endif

	static void crm_shutdown(int nsig);
	static gboolean crm_read_options(gpointer user_data);

	/* A_HA_CONNECT */
	void
	do_ha_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean registered = FALSE;

	if (controld_globals.cluster == NULL) {
	controld_globals.cluster = pcmk_cluster_new();
	}

	if (action & A_HA_DISCONNECT) {
	pcmk_cluster_disconnect(controld_globals.cluster);
	crm_info("Disconnected from the cluster");

	controld_set_fsa_input_flags(R_HA_DISCONNECTED);
	}

	if (action & A_HA_CONNECT) {
	pcmk__cluster_set_status_callback(&peer_update_callback);
	pcmk__cluster_set_autoreap(false);

	#if SUPPORT_COROSYNC
	if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) {
	registered = crm_connect_corosync(controld_globals.cluster);
	}
	#endif // SUPPORT_COROSYNC

	if (registered) {
	pcmk__node_status_t *node = controld_get_local_node_status();

	controld_election_init();

	free(controld_globals.our_uuid);
	controld_globals.our_uuid =
	pcmk__str_copy(pcmk__cluster_get_xml_id(node));

	if (controld_globals.our_uuid == NULL) {
	crm_err("Could not obtain local uuid");
	registered = FALSE;
	}
	}

	if (!registered) {
	controld_set_fsa_input_flags(R_HA_DISCONNECTED);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	populate_cib_nodes(controld_node_update_none, __func__);
	controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
	crm_info("Connected to the cluster");
	}

	if (action & ~(A_HA_CONNECT \| A_HA_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action),
	__func__);
	}
	}

	/* A_SHUTDOWN */
	void
	do_shutdown(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* just in case */
	controld_set_fsa_input_flags(R_SHUTDOWN);
	controld_disconnect_fencer(FALSE);
	}

	/* A_SHUTDOWN_REQ */
	void
	do_shutdown_req(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *msg = NULL;

	controld_set_fsa_input_flags(R_SHUTDOWN);
	//controld_set_fsa_input_flags(R_STAYDOWN);
	crm_info("Sending shutdown request to all peers (DC is %s)",
	pcmk__s(controld_globals.dc_name, "not set"));
	msg = pcmk__new_request(pcmk_ipc_controld, CRM_SYSTEM_CRMD, NULL,
	CRM_SYSTEM_CRMD, CRM_OP_SHUTDOWN_REQ, NULL);

	if (!pcmk__cluster_send_message(NULL, pcmk_ipc_controld, msg)) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	pcmk__xml_free(msg);
	}

	void
	crmd_fast_exit(crm_exit_t exit_code)
	{
	if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) {
	crm_warn("Inhibiting respawn " QB_XS " remapping exit code %d to %d",
	exit_code, CRM_EX_FATAL);
	exit_code = CRM_EX_FATAL;

	} else if ((exit_code == CRM_EX_OK)
	&& pcmk_is_set(controld_globals.fsa_input_register,
	R_IN_RECOVERY)) {
	crm_err("Could not recover from internal error");
	exit_code = CRM_EX_ERROR;
	}

	if (controld_globals.logger_out != NULL) {
	controld_globals.logger_out->finish(controld_globals.logger_out,
	exit_code, true, NULL);
	pcmk__output_free(controld_globals.logger_out);
	controld_globals.logger_out = NULL;
	}

	crm_exit(exit_code);
	}

	crm_exit_t
	crmd_exit(crm_exit_t exit_code)
	{
	GMainLoop *mloop = controld_globals.mainloop;

	static bool in_progress = FALSE;

	if (in_progress && (exit_code == CRM_EX_OK)) {
	crm_debug("Exit is already in progress");
	return exit_code;

	} else if(in_progress) {
	crm_notice("Error during shutdown process, exiting now with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	crm_write_blackbox(SIGTRAP, NULL);
	crmd_fast_exit(exit_code);
	}

	in_progress = TRUE;
	crm_trace("Preparing to exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));

	/* Suppress secondary errors resulting from us disconnecting everything */
	controld_set_fsa_input_flags(R_HA_DISCONNECTED);

	/* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */

	if(ipcs) {
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs);
	ipcs = NULL;
	}

	controld_close_attrd_ipc();
	controld_shutdown_schedulerd_ipc();
	controld_disconnect_fencer(TRUE);

	if ((exit_code == CRM_EX_OK) && (controld_globals.mainloop == NULL)) {
	crm_debug("No mainloop detected");
	exit_code = CRM_EX_ERROR;
	}

	/* On an error, just get out.
	*
	* Otherwise, make the effort to have mainloop exit gracefully so
	* that it (mostly) cleans up after itself and valgrind has less
	* to report on - allowing real errors stand out
	*/
	if (exit_code != CRM_EX_OK) {
	crm_notice("Forcing immediate exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	crm_write_blackbox(SIGTRAP, NULL);
	crmd_fast_exit(exit_code);
	}

	/* Clean up as much memory as possible for valgrind */

	controld_clear_fsa_input_flags(R_MEMBERSHIP);

	g_queue_free_full(controld_globals.fsa_message_queue,
	(GDestroyNotify) delete_fsa_input);
	controld_globals.fsa_message_queue = NULL;

	controld_free_node_pending_timers();
	election_reset(controld_globals.cluster); // Stop any election timer

	/* Tear down the CIB manager connection, but don't free it yet -- it could
	* be used when we drain the mainloop later.
	*/

	controld_disconnect_cib_manager();

	verify_stopped(controld_globals.fsa_state, LOG_WARNING);
	controld_clear_fsa_input_flags(R_LRM_CONNECTED);
	lrm_state_destroy_all();

	mainloop_destroy_trigger(config_read_trigger);
	config_read_trigger = NULL;

	controld_destroy_fsa_trigger();
	controld_destroy_transition_trigger();

	pcmk__client_cleanup();
	pcmk__cluster_destroy_node_caches();

	controld_free_fsa_timers();
	- te_cleanup_stonith_history_sync(NULL, TRUE);
	+ controld_cleanup_fencing_history_sync(NULL, true);
	controld_free_sched_timer();

	free(controld_globals.our_uuid);
	controld_globals.our_uuid = NULL;

	free(controld_globals.dc_name);
	controld_globals.dc_name = NULL;

	free(controld_globals.dc_version);
	controld_globals.dc_version = NULL;

	free(controld_globals.cluster_name);
	controld_globals.cluster_name = NULL;

	free(controld_globals.te_uuid);
	controld_globals.te_uuid = NULL;

	free_max_generation();
	controld_destroy_failed_sync_table();
	controld_destroy_outside_events_table();

	mainloop_destroy_signal(SIGPIPE);
	mainloop_destroy_signal(SIGUSR1);
	mainloop_destroy_signal(SIGTERM);
	mainloop_destroy_signal(SIGTRAP);
	/* leave SIGCHLD engaged as we might still want to drain some service-actions */

	if (mloop) {
	GMainContext *ctx = g_main_loop_get_context(controld_globals.mainloop);

	/* Don't re-enter this block */
	controld_globals.mainloop = NULL;

	/* no signals on final draining anymore */
	mainloop_destroy_signal(SIGCHLD);

	crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));

	{
	int lpc = 0;

	while((g_main_context_pending(ctx) && lpc < 10)) {
	lpc++;
	crm_trace("Iteration %d", lpc);
	g_main_context_dispatch(ctx);
	}
	}

	crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
	g_main_loop_quit(mloop);

	/* Won't do anything yet, since we're inside it now */
	g_main_loop_unref(mloop);
	} else {
	mainloop_destroy_signal(SIGCHLD);
	}

	cib_delete(controld_globals.cib_conn);
	controld_globals.cib_conn = NULL;

	throttle_fini();

	pcmk_cluster_free(controld_globals.cluster);
	controld_globals.cluster = NULL;

	/* Graceful */
	crm_trace("Done preparing for exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	return exit_code;
	}

	/* A_EXIT_0, A_EXIT_1 */
	void
	do_exit(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_exit_t exit_code = CRM_EX_OK;

	if (pcmk_is_set(action, A_EXIT_1)) {
	exit_code = CRM_EX_ERROR;
	crm_err("Exiting now due to errors");
	}
	verify_stopped(cur_state, LOG_ERR);
	crmd_exit(exit_code);
	}

	static void sigpipe_ignore(int nsig) { return; }

	/* A_STARTUP */
	void
	do_startup(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_debug("Registering Signal Handlers");
	mainloop_add_signal(SIGTERM, crm_shutdown);
	mainloop_add_signal(SIGPIPE, sigpipe_ignore);

	config_read_trigger = mainloop_add_trigger(G_PRIORITY_HIGH,
	crm_read_options, NULL);

	controld_init_fsa_trigger();
	controld_init_transition_trigger();

	crm_debug("Creating CIB manager and executor objects");
	controld_globals.cib_conn = cib_new();

	lrm_state_init_local();
	if (controld_init_fsa_timers() == FALSE) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	// \return libqb error code (0 on success, -errno on error)
	static int32_t
	accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
	{
	crm_trace("Accepting new IPC client connection");
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -ENOMEM;
	}
	return 0;
	}

	// \return libqb error code (0 on success, -errno on error)
	static int32_t
	dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	int rc = pcmk_rc_ok;
	uint32_t id = 0;
	uint32_t flags = 0;
	pcmk__client_t *client = pcmk__find_client(c);
	xmlNode *msg = NULL;

	rc = pcmk__ipc_msg_append(&client->buffer, data);

	if (rc == pcmk_rc_ipc_more) {
	/* We haven't read the complete message yet, so just return. */
	return 0;

	} else if (rc == pcmk_rc_ok) {
	/* We've read the complete message and there's already a header on
	* the front. Pass it off for processing.
	*/
	msg = pcmk__client_data2xml(client, &id, &flags);
	g_byte_array_free(client->buffer, TRUE);
	client->buffer = NULL;

	} else {
	/* Some sort of error occurred reassembling the message. All we can
	* do is clean up, log an error and return.
	*/
	crm_err("Error when reading IPC message: %s", pcmk_rc_str(rc));

	if (client->buffer != NULL) {
	g_byte_array_free(client->buffer, TRUE);
	client->buffer = NULL;
	}

	return 0;
	}

	if (msg == NULL) {
	pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL,
	CRM_EX_PROTOCOL);
	return 0;
	}
	pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL,
	CRM_EX_INDETERMINATE);

	pcmk__assert(client->user != NULL);
	pcmk__update_acl_user(msg, PCMK__XA_CRM_USER, client->user);

	pcmk__xe_set(msg, PCMK__XA_CRM_SYS_FROM, client->id);
	if (controld_authorize_ipc_message(msg, client, NULL)) {
	crm_trace("Processing IPC message from client %s",
	pcmk__client_name(client));
	route_message(C_IPC_MESSAGE, msg);
	}

	controld_trigger_fsa();
	pcmk__xml_free(msg);
	return 0;
	}

	static int32_t
	ipc_client_disconnected(qb_ipcs_connection_t *c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client) {
	crm_trace("Disconnecting %sregistered client %s (%p/%p)",
	(client->userdata? "" : "un"), pcmk__client_name(client),
	c, client);
	free(client->userdata);
	pcmk__free_client(client);
	controld_trigger_fsa();
	}
	return 0;
	}

	static void
	ipc_connection_destroyed(qb_ipcs_connection_t *c)
	{
	crm_trace("Connection %p", c);
	ipc_client_disconnected(c);
	}

	/* A_STOP */
	void
	do_stop(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs); ipcs = NULL;
	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	/* A_STARTED */
	void
	do_started(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static struct qb_ipcs_service_handlers crmd_callbacks = {
	.connection_accept = accept_controller_client,
	.connection_created = NULL,
	.msg_process = dispatch_controller_ipc,
	.connection_closed = ipc_client_disconnected,
	.connection_destroyed = ipc_connection_destroyed
	};

	if (cur_state != S_STARTING) {
	crm_err("Start cancelled... %s", fsa_state2string(cur_state));
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_MEMBERSHIP)) {
	crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_LRM_CONNECTED)) {
	crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_CIB_CONNECTED)) {
	crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_READ_CONFIG)) {
	crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register, R_PEER_DATA)) {

	crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
	crmd_fsa_stall(TRUE);
	return;
	}

	crm_debug("Init server comms");
	ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
	if (ipcs == NULL) {
	crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	} else {
	crm_notice("Pacemaker controller successfully started and accepting connections");
	}
	controld_set_fsa_input_flags(R_ST_REQUIRED);
	controld_timer_fencer_connect(GINT_TO_POINTER(TRUE));

	controld_clear_fsa_input_flags(R_STARTING);
	register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
	}

	/* A_RECOVER */
	void
	do_recover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	controld_set_fsa_input_flags(R_IN_RECOVERY);
	crm_warn("Fast-tracking shutdown in response to errors");

	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	static void
	config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	const char *value = NULL;
	GHashTable *config_hash = NULL;
	crm_time_t *now = crm_time_new(NULL);
	xmlNode *crmconfig = NULL;
	xmlNode *alerts = NULL;
	pcmk_rule_input_t rule_input = {
	.now = now,
	};

	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL;

	crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	if (rc == -EACCES \|\| rc == -pcmk_err_schema_validation) {
	crm_err("The cluster is mis-configured - shutting down and staying down");
	controld_set_fsa_input_flags(R_STAYDOWN);
	}
	goto bail;
	}

	crmconfig = output;
	if ((crmconfig != NULL) && !pcmk__xe_is(crmconfig, PCMK_XE_CRM_CONFIG)) {
	crmconfig = pcmk__xe_first_child(crmconfig, PCMK_XE_CRM_CONFIG, NULL,
	NULL);
	}
	if (!crmconfig) {
	fsa_data_t *msg_data = NULL;

	crm_err("Local CIB query for " PCMK_XE_CRM_CONFIG " section failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	goto bail;
	}

	crm_debug("Call %d : Parsing CIB options", call_id);
	config_hash = pcmk__strkey_table(free, free);
	pcmk_unpack_nvpair_blocks(crmconfig, PCMK_XE_CLUSTER_PROPERTY_SET,
	PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, &rule_input,
	config_hash, NULL);

	// Validate all options, and use defaults if not already present in hash
	pcmk__validate_cluster_options(config_hash);

	/* Validate the watchdog timeout in the context of the local node
	* environment. If invalid, the controller will exit with a fatal error.
	*
	* We do this via a wrapper in the controller, so that we call
	* pcmk__valid_fencing_watchdog_timeout() only if watchdog fencing is
	* enabled for the local node. Otherwise, we may exit unnecessarily.
	*
	* A validator function in libcrmcommon can't act as such a wrapper, because
	* it doesn't have a stonith API connection or the local node name.
	*/
	value = g_hash_table_lookup(config_hash, PCMK_OPT_FENCING_WATCHDOG_TIMEOUT);
	controld_validate_fencing_watchdog_timeout(value);

	value = g_hash_table_lookup(config_hash, PCMK_OPT_NO_QUORUM_POLICY);
	if (pcmk__strcase_any_of(value, PCMK_VALUE_FENCE, PCMK_VALUE_FENCE_LEGACY,
	NULL)
	&& (pcmk__locate_sbd() != 0)) {
	controld_set_global_flags(controld_no_quorum_panic);
	}

	value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK);
	if (pcmk__is_true(value)) {
	controld_set_global_flags(controld_shutdown_lock_enabled);
	} else {
	controld_clear_global_flags(controld_shutdown_lock_enabled);
	}

	value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
	pcmk_parse_interval_spec(value, &controld_globals.shutdown_lock_limit);
	controld_globals.shutdown_lock_limit /= 1000;

	value = g_hash_table_lookup(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
	pcmk_parse_interval_spec(value, &controld_globals.node_pending_timeout);
	controld_globals.node_pending_timeout /= 1000;

	value = g_hash_table_lookup(config_hash, PCMK_OPT_CLUSTER_NAME);
	pcmk__str_update(&(controld_globals.cluster_name), value);

	// Let subcomponents initialize their own static variables
	controld_configure_election(config_hash);
	controld_configure_fencing(config_hash);
	controld_configure_fsa_timers(config_hash);
	controld_configure_throttle(config_hash);

	alerts = pcmk__xe_first_child(output, PCMK_XE_ALERTS, NULL, NULL);
	crmd_unpack_alerts(alerts);

	controld_set_fsa_input_flags(R_READ_CONFIG);
	controld_trigger_fsa();

	g_hash_table_destroy(config_hash);
	bail:
	crm_time_free(now);
	}

	/*!
	* \internal
	* \brief Trigger read and processing of the configuration
	*
	* \param[in] fn Calling function name
	* \param[in] line Line number where call occurred
	*/
	void
	controld_trigger_config_as(const char *fn, int line)
	{
	if (config_read_trigger != NULL) {
	crm_trace("%s:%d - Triggered config processing", fn, line);
	mainloop_set_trigger(config_read_trigger);
	}
	}

	gboolean
	crm_read_options(gpointer user_data)
	{
	cib_t *cib_conn = controld_globals.cib_conn;
	int call_id = cib_conn->cmds->query(cib_conn,
	"//" PCMK_XE_CRM_CONFIG
	" \| //" PCMK_XE_ALERTS,
	NULL, cib_xpath);

	fsa_register_cib_callback(call_id, NULL, config_query_callback);
	crm_trace("Querying the CIB... call %d", call_id);
	return TRUE;
	}

	/* A_READCONFIG */
	void
	do_read_config(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	throttle_init();
	controld_trigger_config();
	}

	static void
	crm_shutdown(int nsig)
	{
	const char *value = NULL;
	guint default_period_ms = 0;

	if ((controld_globals.mainloop == NULL)
	\|\| !g_main_loop_is_running(controld_globals.mainloop)) {
	crmd_exit(CRM_EX_OK);
	return;
	}

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	crm_err("Escalating shutdown");
	register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
	return;
	}

	controld_set_fsa_input_flags(R_SHUTDOWN);
	register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);

	/* If shutdown timer doesn't have a period set, use the default
	*
	* @TODO: Evaluate whether this is still necessary. As long as
	* config_query_callback() has been run at least once, it doesn't look like
	* anything could have changed the timer period since then.
	*/
	value = pcmk__cluster_option(NULL, PCMK_OPT_SHUTDOWN_ESCALATION);
	pcmk_parse_interval_spec(value, &default_period_ms);
	controld_shutdown_start_countdown(default_period_ms);
	}
	diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
	index 27ebebd872..1d5b185dc0 100644
	--- a/daemons/controld/controld_fencing.c
	+++ b/daemons/controld/controld_fencing.c
	@@ -1,1109 +1,1109 @@
	/*
	* Copyright 2004-2025 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>

	#include <pacemaker-controld.h>

	static void
	tengine_stonith_history_synced(stonith_t st, stonith_event_t st_event);

	/*
	* stonith failure counting
	*
	* We don't want to get stuck in a permanent fencing loop. Keep track of the
	* number of fencing failures for each target node, and the most we'll restart a
	* transition for.
	*/

	struct st_fail_rec {
	int count;
	};

	#define DEFAULT_FENCING_MAX_ATTEMPTS 10

	static bool fence_reaction_panic = false;
	static unsigned long int fencing_max_attempts = DEFAULT_FENCING_MAX_ATTEMPTS;
	static GHashTable *stonith_failures = NULL;

	/*!
	* \internal
	* \brief Update max fencing attempts before giving up
	*
	* \param[in] value New max fencing attempts
	*/
	static void
	update_fencing_max_attempts(const char *value)
	{
	int score = 0;
	int rc = pcmk_parse_score(value, &score, DEFAULT_FENCING_MAX_ATTEMPTS);

	// The option validator ensures invalid values shouldn't be possible
	CRM_CHECK((rc == pcmk_rc_ok) && (score > 0), return);

	if (fencing_max_attempts != score) {
	crm_debug("Maximum fencing attempts per transition is now %d (was %lu)",
	score, fencing_max_attempts);
	}
	fencing_max_attempts = score;
	}

	/*!
	* \internal
	* \brief Configure reaction to notification of local node being fenced
	*
	* \param[in] reaction_s Reaction type
	*/
	static void
	set_fence_reaction(const char *reaction_s)
	{
	if (pcmk__str_eq(reaction_s, "panic", pcmk__str_casei)) {
	fence_reaction_panic = true;

	} else {
	if (!pcmk__str_eq(reaction_s, PCMK_VALUE_STOP, pcmk__str_casei)) {
	crm_warn("Invalid value '%s' for %s, using 'stop'",
	reaction_s, PCMK_OPT_FENCING_REACTION);
	}
	fence_reaction_panic = false;
	}
	}

	/*!
	* \internal
	* \brief Configure fencing options based on the CIB
	*
	* \param[in,out] options Name/value pairs for configured options
	*/
	void
	controld_configure_fencing(GHashTable *options)
	{
	const char *value = NULL;

	value = g_hash_table_lookup(options, PCMK_OPT_FENCING_REACTION);
	set_fence_reaction(value);

	value = g_hash_table_lookup(options, PCMK_OPT_FENCING_MAX_ATTEMPTS);
	update_fencing_max_attempts(value);
	}

	static gboolean
	too_many_st_failures(const char *target)
	{
	GHashTableIter iter;
	const char *key = NULL;
	struct st_fail_rec *value = NULL;

	if (stonith_failures == NULL) {
	return FALSE;
	}

	if (target == NULL) {
	g_hash_table_iter_init(&iter, stonith_failures);
	while (g_hash_table_iter_next(&iter, (gpointer *) &key,
	(gpointer *) &value)) {

	if (value->count >= fencing_max_attempts) {
	target = (const char*)key;
	goto too_many;
	}
	}
	} else {
	value = g_hash_table_lookup(stonith_failures, target);
	if ((value != NULL) && (value->count >= fencing_max_attempts)) {
	goto too_many;
	}
	}
	return FALSE;

	too_many:
	crm_warn("Too many failures (%d) to fence %s, giving up",
	value->count, target);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Reset a stonith fail count
	*
	* \param[in] target Name of node to reset, or NULL for all
	*/
	void
	st_fail_count_reset(const char *target)
	{
	if (stonith_failures == NULL) {
	return;
	}

	if (target) {
	struct st_fail_rec *rec = NULL;

	rec = g_hash_table_lookup(stonith_failures, target);
	if (rec) {
	rec->count = 0;
	}
	} else {
	GHashTableIter iter;
	const char *key = NULL;
	struct st_fail_rec *rec = NULL;

	g_hash_table_iter_init(&iter, stonith_failures);
	while (g_hash_table_iter_next(&iter, (gpointer *) &key,
	(gpointer *) &rec)) {
	rec->count = 0;
	}
	}
	}

	static void
	st_fail_count_increment(const char *target)
	{
	struct st_fail_rec *rec = NULL;

	if (stonith_failures == NULL) {
	stonith_failures = pcmk__strkey_table(free, free);
	}

	rec = g_hash_table_lookup(stonith_failures, target);
	if (rec) {
	rec->count++;
	} else {
	rec = malloc(sizeof(struct st_fail_rec));
	if(rec == NULL) {
	return;
	}

	rec->count = 1;
	g_hash_table_insert(stonith_failures, pcmk__str_copy(target), rec);
	}
	}

	/* end stonith fail count functions */


	static void
	cib_fencing_updated(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	if (rc < pcmk_ok) {
	crm_err("Fencing update %d for %s: failed - %s (%d)",
	call_id, (char *)user_data, pcmk_strerror(rc), rc);
	crm_log_xml_warn(msg, "Failed update");
	abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_shutdown,
	"CIB update failed", NULL);

	} else {
	crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data);
	}
	}

	/*!
	* \internal
	* \brief Update a fencing target's node state
	*
	* \param[in] target Node that was successfully fenced
	* \param[in] target_xml_id CIB XML ID of target
	*/
	static void
	update_node_state_after_fencing(const char target, const char target_xml_id)
	{
	int rc = pcmk_ok;
	pcmk__node_status_t *peer = NULL;
	xmlNode *node_state = NULL;

	/* We (usually) rely on the membership layer to do
	* controld_node_update_cluster, and the peer status callback to do
	* controld_node_update_peer, because the node might have already rejoined
	* before we get the stonith result here.
	*/
	uint32_t flags = controld_node_update_join\|controld_node_update_expected;

	CRM_CHECK((target != NULL) && (target_xml_id != NULL), return);

	// Ensure target is cached
	peer = pcmk__get_node(0, target, target_xml_id, pcmk__node_search_any);
	CRM_CHECK(peer != NULL, return);

	if (peer->state == NULL) {
	/* Usually, we rely on the membership layer to update the cluster state
	* in the CIB. However, if the node has never been seen, do it here, so
	* the node is not considered unclean.
	*/
	flags \|= controld_node_update_cluster;
	}

	if (peer->xml_id == NULL) {
	crm_info("Recording XML ID '%s' for node '%s'", target_xml_id, target);
	peer->xml_id = pcmk__str_copy(target_xml_id);
	}

	crmd_peer_down(peer, TRUE);

	node_state = create_node_state_update(peer, flags, NULL, __func__);
	pcmk__xe_set(node_state, PCMK_XA_ID, target_xml_id);

	if (pcmk_is_set(peer->flags, pcmk__node_status_remote)) {
	char *now_s = pcmk__ttoa(time(NULL));

	pcmk__xe_set(node_state, PCMK__XA_NODE_FENCED, now_s);
	free(now_s);
	}

	rc = controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn,
	PCMK_XE_STATUS, node_state,
	cib_can_create);
	pcmk__xml_free(node_state);

	crm_debug("Updating node state for %s after fencing (call %d)", target, rc);
	fsa_register_cib_callback(rc, pcmk__str_copy(target), cib_fencing_updated);

	controld_delete_node_state(peer->name, controld_section_all, cib_none);
	}

	/*!
	* \internal
	* \brief Abort transition due to stonith failure
	*
	* \param[in] abort_action Whether to restart or stop transition
	* \param[in] target Don't restart if this (NULL for any) has too many failures
	* \param[in] reason Log this stonith action XML as abort reason (or NULL)
	*/
	static void
	abort_for_stonith_failure(enum pcmk__graph_next abort_action,
	const char target, const xmlNode reason)
	{
	/* If stonith repeatedly fails, we eventually give up on starting a new
	* transition for that reason.
	*/
	if ((abort_action != pcmk__graph_wait) && too_many_st_failures(target)) {
	abort_action = pcmk__graph_wait;
	}
	abort_transition(PCMK_SCORE_INFINITY, abort_action, "Stonith failed",
	reason);
	}


	/*
	* Fencing cleanup list
	*
	* If the DC is fenced, proper notifications might not go out. The fencing
	* cleanup list allows the cluster to (re-)send notifications once a new DC is
	* elected.
	*/

	static GList *fencing_cleanup_list = NULL;

	/*!
	* \internal
	* \brief Add a node to the fencing cleanup list
	*
	* \param[in] target Name of node to add
	*/
	static void
	add_fencing_cleanup(const char *target)
	{
	fencing_cleanup_list = g_list_append(fencing_cleanup_list,
	pcmk__str_copy(target));
	}

	/*!
	* \internal
	* \brief Remove a node from the fencing cleanup list
	*
	* \param[in] Name of node to remove
	*/
	void
	controld_remove_fencing_cleanup(const char *target)
	{
	GList *iter = fencing_cleanup_list;

	while (iter != NULL) {
	GList *tmp = iter;
	char *iter_name = tmp->data;

	iter = iter->next;
	if (pcmk__str_eq(target, iter_name, pcmk__str_casei)) {
	crm_trace("Removing %s from the cleanup list", iter_name);
	fencing_cleanup_list = g_list_delete_link(fencing_cleanup_list,
	tmp);
	free(iter_name);
	}
	}
	}

	/*!
	* \internal
	* \brief Purge all entries from the fencing cleanup list
	*/
	void
	controld_purge_fencing_cleanup(void)
	{
	for (GList *iter = fencing_cleanup_list; iter != NULL; iter = iter->next) {
	char *target = iter->data;

	crm_info("Purging %s from fencing cleanup list", target);
	free(target);
	}
	g_list_free(fencing_cleanup_list);
	fencing_cleanup_list = NULL;
	}

	/*!
	* \internal
	* \brief Send fencing updates for all entries in cleanup list, then purge it
	*/
	void
	controld_execute_fencing_cleanup(void)
	{
	for (GList *iter = fencing_cleanup_list; iter != NULL; iter = iter->next) {
	char *target = iter->data;
	pcmk__node_status_t *target_node =
	pcmk__get_node(0, target, NULL, pcmk__node_search_cluster_member);
	const char *uuid = pcmk__cluster_get_xml_id(target_node);

	crm_notice("Marking %s, target of a previous fencing action, as clean", target);
	update_node_state_after_fencing(target, uuid);
	free(target);
	}
	g_list_free(fencing_cleanup_list);
	fencing_cleanup_list = NULL;
	}

	/* end fencing cleanup list functions */


	/* stonith API client
	*
	* Functions that need to interact directly with the fencer via its API
	*/

	static stonith_t *stonith_api = NULL;
	static mainloop_timer_t *controld_fencer_connect_timer = NULL;
	static char *te_client_id = NULL;

	static gboolean
	fail_incompletable_stonith(pcmk__graph_t *graph)
	{
	GList *lpc = NULL;
	const char *task = NULL;
	xmlNode *last_action = NULL;

	if (graph == NULL) {
	return FALSE;
	}

	for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
	GList *lpc2 = NULL;
	pcmk__graph_synapse_t synapse = (pcmk__graph_synapse_t ) lpc->data;

	if (pcmk_is_set(synapse->flags, pcmk__synapse_confirmed)) {
	continue;
	}

	for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk__graph_action_t action = (pcmk__graph_action_t ) lpc2->data;

	if ((action->type != pcmk__cluster_graph_action)
	\|\| pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
	continue;
	}

	task = pcmk__xe_get(action->xml, PCMK_XA_OPERATION);
	if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) {
	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
	last_action = action->xml;
	pcmk__update_graph(graph, action);
	crm_notice("Failing action %d (%s): fencer terminated",
	action->id, pcmk__xe_id(action->xml));
	}
	}
	}

	if (last_action != NULL) {
	crm_warn("Fencer failure resulted in unrunnable actions");
	abort_for_stonith_failure(pcmk__graph_restart, NULL, last_action);
	return TRUE;
	}

	return FALSE;
	}

	static void
	tengine_stonith_connection_destroy(stonith_t st, stonith_event_t e)
	{
	- te_cleanup_stonith_history_sync(st, FALSE);
	+ controld_cleanup_fencing_history_sync(st, false);

	if (pcmk_is_set(controld_globals.fsa_input_register, R_ST_REQUIRED)) {
	crm_err("Lost fencer connection (will attempt to reconnect)");
	if (!mainloop_timer_running(controld_fencer_connect_timer)) {
	mainloop_timer_start(controld_fencer_connect_timer);
	}
	} else {
	crm_info("Disconnected from fencer");
	}

	if (stonith_api) {
	/* the client API won't properly reconnect notifications
	* if they are still in the table - so remove them
	*/
	if (stonith_api->state != stonith_disconnected) {
	stonith_api->cmds->disconnect(st);
	}
	stonith_api->cmds->remove_notification(stonith_api, NULL);
	}

	if (AM_I_DC) {
	fail_incompletable_stonith(controld_globals.transition_graph);
	trigger_graph();
	}
	}

	/*!
	* \internal
	* \brief Handle an event notification from the fencing API
	*
	* \param[in] st Fencing API connection (ignored)
	* \param[in] event Fencing API event notification
	*/
	static void
	handle_fence_notification(stonith_t st, stonith_event_t event)
	{
	bool succeeded = true;
	const char *executioner = "the cluster";
	const char *client = "a client";
	const char *reason = NULL;
	int exec_status;

	if (te_client_id == NULL) {
	te_client_id = crm_strdup_printf("%s.%lu", crm_system_name,
	(unsigned long) getpid());
	}

	if (event == NULL) {
	crm_err("Notify data not found");
	return;
	}

	if (event->executioner != NULL) {
	executioner = event->executioner;
	}
	if (event->client_origin != NULL) {
	client = event->client_origin;
	}

	exec_status = stonith__event_execution_status(event);
	if ((stonith__event_exit_status(event) != CRM_EX_OK)
	\|\| (exec_status != PCMK_EXEC_DONE)) {
	succeeded = false;
	if (exec_status == PCMK_EXEC_DONE) {
	exec_status = PCMK_EXEC_ERROR;
	}
	}
	reason = stonith__event_exit_reason(event);

	crmd_alert_fencing_op(event);

	if (pcmk__str_eq(PCMK_ACTION_ON, event->action, pcmk__str_none)) {
	// Unfencing doesn't need special handling, just a log message
	if (succeeded) {
	crm_notice("%s was unfenced by %s at the request of %s@%s",
	event->target, executioner, client, event->origin);
	} else {
	crm_err("Unfencing of %s by %s failed (%s%s%s) with exit status %d",
	event->target, executioner,
	pcmk_exec_status_str(exec_status),
	((reason == NULL)? "" : ": "),
	((reason == NULL)? "" : reason),
	stonith__event_exit_status(event));
	}
	return;
	}

	if (succeeded && controld_is_local_node(event->target)) {
	/* We were notified of our own fencing. Most likely, either fencing was
	* misconfigured, or fabric fencing that doesn't cut cluster
	* communication is in use.
	*
	* Either way, shutting down the local host is a good idea, to require
	* administrator intervention. Also, other nodes would otherwise likely
	* set our status to lost because of the fencing callback and discard
	* our subsequent election votes as "not part of our cluster".
	*/
	crm_crit("We were allegedly just fenced by %s for %s!",
	executioner, event->origin); // Dumps blackbox if enabled
	if (fence_reaction_panic) {
	pcmk__panic("Notified of own fencing");
	} else {
	crm_exit(CRM_EX_FATAL);
	}
	return; // Should never get here
	}

	/* Update the count of fencing failures for this target, in case we become
	* DC later. The current DC has already updated its fail count in
	* tengine_stonith_callback().
	*/
	if (!AM_I_DC) {
	if (succeeded) {
	st_fail_count_reset(event->target);
	} else {
	st_fail_count_increment(event->target);
	}
	}

	crm_notice("Peer %s was%s terminated (%s) by %s on behalf of %s@%s: "
	"%s%s%s%s " QB_XS " event=%s",
	event->target, (succeeded? "" : " not"),
	event->action, executioner, client, event->origin,
	(succeeded? "OK" : pcmk_exec_status_str(exec_status)),
	((reason == NULL)? "" : " ("),
	((reason == NULL)? "" : reason),
	((reason == NULL)? "" : ")"),
	event->id);

	if (succeeded) {
	const uint32_t flags = pcmk__node_search_any
	\|pcmk__node_search_cluster_cib;

	pcmk__node_status_t *peer = pcmk__search_node_caches(0, event->target,
	NULL, flags);
	const char *uuid = NULL;

	if (peer == NULL) {
	return;
	}

	uuid = pcmk__cluster_get_xml_id(peer);

	if (AM_I_DC) {
	/* The DC always sends updates */
	update_node_state_after_fencing(event->target, uuid);

	/* @TODO Ideally, at this point, we'd check whether the fenced node
	* hosted any guest nodes, and call remote_node_down() for them.
	* Unfortunately, the controller doesn't have a simple, reliable way
	* to map hosts to guests. It might be possible to track this in the
	* peer cache via refresh_remote_nodes(). For now, we rely on the
	* scheduler creating fence pseudo-events for the guests.
	*/

	if (!pcmk__str_eq(client, te_client_id, pcmk__str_casei)) {
	/* Abort the current transition if it wasn't the cluster that
	* initiated fencing.
	*/
	crm_info("External fencing operation from %s fenced %s",
	client, event->target);
	abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
	"External Fencing Operation", NULL);
	}

	} else if (pcmk__str_eq(controld_globals.dc_name, event->target,
	pcmk__str_null_matches\|pcmk__str_casei)
	&& !pcmk_is_set(peer->flags, pcmk__node_status_remote)) {
	// Assume the target was our DC if we don't currently have one

	if (controld_globals.dc_name != NULL) {
	crm_notice("Fencing target %s was our DC", event->target);
	} else {
	crm_notice("Fencing target %s may have been our DC",
	event->target);
	}

	/* Given the CIB resyncing that occurs around elections,
	* have one node update the CIB now and, if the new DC is different,
	* have them do so too after the election
	*/
	if (controld_is_local_node(event->executioner)) {
	update_node_state_after_fencing(event->target, uuid);
	}
	add_fencing_cleanup(event->target);
	}

	/* If the target is a remote node, and we host its connection,
	* immediately fail all monitors so it can be recovered quickly.
	* The connection won't necessarily drop when a remote node is fenced,
	* so the failure might not otherwise be detected until the next poke.
	*/
	if (pcmk_is_set(peer->flags, pcmk__node_status_remote)) {
	remote_ra_fail(event->target);
	}

	crmd_peer_down(peer, TRUE);
	}
	}

	/*!
	* \brief Connect to fencer
	*
	* \param[in] user_data If NULL, retry failures now, otherwise retry in mainloop timer
	*
	* \return G_SOURCE_REMOVE on success, G_SOURCE_CONTINUE to retry
	* \note If user_data is NULL, this will wait 2s between attempts, for up to
	* 30 attempts, meaning the controller could be blocked as long as 58s.
	*/
	gboolean
	controld_timer_fencer_connect(gpointer user_data)
	{
	int rc = pcmk_ok;

	if (stonith_api == NULL) {
	stonith_api = stonith__api_new();
	if (stonith_api == NULL) {
	crm_err("Could not connect to fencer: API memory allocation failed");
	return G_SOURCE_REMOVE;
	}
	}

	if (stonith_api->state != stonith_disconnected) {
	crm_trace("Already connected to fencer, no need to retry");
	return G_SOURCE_REMOVE;
	}

	if (user_data == NULL) {
	// Blocking (retry failures now until successful)
	rc = stonith__api_connect_retry(stonith_api, crm_system_name, 30);
	if (rc != pcmk_rc_ok) {
	crm_err("Could not connect to fencer in 30 attempts: %s "
	QB_XS " rc=%d", pcmk_rc_str(rc), rc);
	}
	} else {
	// Non-blocking (retry failures later in main loop)
	rc = stonith_api->cmds->connect(stonith_api, crm_system_name, NULL);

	if (controld_fencer_connect_timer == NULL) {
	controld_fencer_connect_timer =
	mainloop_timer_add("controld_fencer_connect", 1000,
	TRUE, controld_timer_fencer_connect,
	GINT_TO_POINTER(TRUE));
	}

	if (rc != pcmk_ok) {
	if (pcmk_is_set(controld_globals.fsa_input_register,
	R_ST_REQUIRED)) {
	crm_notice("Fencer connection failed (will retry): %s "
	QB_XS " rc=%d", pcmk_strerror(rc), rc);

	if (!mainloop_timer_running(controld_fencer_connect_timer)) {
	mainloop_timer_start(controld_fencer_connect_timer);
	}

	return G_SOURCE_CONTINUE;
	} else {
	crm_info("Fencer connection failed (ignoring because no longer required): %s "
	QB_XS " rc=%d", pcmk_strerror(rc), rc);
	}
	return G_SOURCE_REMOVE;
	}
	}

	if (rc == pcmk_ok) {
	stonith_api_operations_t *cmds = stonith_api->cmds;

	cmds->register_notification(stonith_api,
	PCMK__VALUE_ST_NOTIFY_DISCONNECT,
	tengine_stonith_connection_destroy);
	cmds->register_notification(stonith_api, PCMK__VALUE_ST_NOTIFY_FENCE,
	handle_fence_notification);
	cmds->register_notification(stonith_api,
	PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED,
	tengine_stonith_history_synced);
	controld_trigger_fencing_history_sync(true);
	crm_notice("Fencer successfully connected");
	}

	return G_SOURCE_REMOVE;
	}

	void
	controld_disconnect_fencer(bool destroy)
	{
	if (stonith_api) {
	// Prevent fencer connection from coming up again
	controld_clear_fsa_input_flags(R_ST_REQUIRED);

	if (stonith_api->state != stonith_disconnected) {
	stonith_api->cmds->disconnect(stonith_api);
	}
	stonith_api->cmds->remove_notification(stonith_api, NULL);
	}
	if (destroy) {
	if (stonith_api) {
	stonith_api->cmds->free(stonith_api);
	stonith_api = NULL;
	}
	if (controld_fencer_connect_timer) {
	mainloop_timer_del(controld_fencer_connect_timer);
	controld_fencer_connect_timer = NULL;
	}
	if (te_client_id) {
	free(te_client_id);
	te_client_id = NULL;
	}
	}
	}

	static gboolean
	do_stonith_history_sync(gpointer user_data)
	{
	if (stonith_api && (stonith_api->state != stonith_disconnected)) {
	stonith_history_t *history = NULL;

	- te_cleanup_stonith_history_sync(stonith_api, FALSE);
	+ controld_cleanup_fencing_history_sync(stonith_api, false);
	stonith_api->cmds->history(stonith_api,
	st_opt_sync_call \| st_opt_broadcast,
	NULL, &history, 5);
	stonith__history_free(history);
	return TRUE;
	} else {
	crm_info("Skip triggering stonith history-sync as stonith is disconnected");
	return FALSE;
	}
	}

	static void
	tengine_stonith_callback(stonith_t stonith, stonith_callback_data_t data)
	{
	char *uuid = NULL;
	int stonith_id = -1;
	int transition_id = -1;
	pcmk__graph_action_t *action = NULL;
	const char *target = NULL;

	if ((data == NULL) \|\| (data->userdata == NULL)) {
	crm_err("Ignoring fence operation %d result: "
	"No transition key given (bug?)",
	((data == NULL)? -1 : data->call_id));
	return;
	}

	if (!AM_I_DC) {
	const char *reason = stonith__exit_reason(data);

	if (reason == NULL) {
	reason = pcmk_exec_status_str(stonith__execution_status(data));
	}
	crm_notice("Result of fence operation %d: %d (%s) " QB_XS " key=%s",
	data->call_id, stonith__exit_status(data), reason,
	(const char *) data->userdata);
	return;
	}

	CRM_CHECK(decode_transition_key(data->userdata, &uuid, &transition_id,
	&stonith_id, NULL),
	goto bail);

	if (controld_globals.transition_graph->complete \|\| (stonith_id < 0)
	\|\| !pcmk__str_eq(uuid, controld_globals.te_uuid, pcmk__str_none)
	\|\| (controld_globals.transition_graph->id != transition_id)) {
	crm_info("Ignoring fence operation %d result: "
	"Not from current transition " QB_XS
	" complete=%s action=%d uuid=%s (vs %s) transition=%d (vs %d)",
	data->call_id,
	pcmk__btoa(controld_globals.transition_graph->complete),
	stonith_id, uuid, controld_globals.te_uuid, transition_id,
	controld_globals.transition_graph->id);
	goto bail;
	}

	action = controld_get_action(stonith_id);
	if (action == NULL) {
	crm_err("Ignoring fence operation %d result: "
	"Action %d not found in transition graph (bug?) "
	QB_XS " uuid=%s transition=%d",
	data->call_id, stonith_id, uuid, transition_id);
	goto bail;
	}

	target = pcmk__xe_get(action->xml, PCMK__META_ON_NODE);
	if (target == NULL) {
	crm_err("Ignoring fence operation %d result: No target given (bug?)",
	data->call_id);
	goto bail;
	}

	stop_te_timer(action);
	if (stonith__exit_status(data) == CRM_EX_OK) {
	const char *uuid = pcmk__xe_get(action->xml, PCMK__META_ON_NODE_UUID);
	const char *op = crm_meta_value(action->params,
	PCMK__META_STONITH_ACTION);

	crm_info("Fence operation %d for %s succeeded", data->call_id, target);
	if (!(pcmk_is_set(action->flags, pcmk__graph_action_confirmed))) {
	te_action_confirmed(action, NULL);
	if (pcmk__str_eq(PCMK_ACTION_ON, op, pcmk__str_casei)) {
	const char *value = NULL;
	char *now = pcmk__ttoa(time(NULL));
	gboolean is_remote_node = FALSE;

	/* This check is not 100% reliable, since this node is not
	* guaranteed to have the remote node cached. However, it
	* doesn't have to be reliable, since the attribute manager can
	* learn a node's "remoteness" by other means sooner or later.
	* This allows it to learn more quickly if this node does have
	* the information.
	*/
	if (g_hash_table_lookup(pcmk__remote_peer_cache,
	uuid) != NULL) {
	is_remote_node = TRUE;
	}

	update_attrd(target, CRM_ATTR_UNFENCED, now, NULL,
	is_remote_node);
	free(now);

	value = crm_meta_value(action->params, PCMK__META_DIGESTS_ALL);
	update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL,
	is_remote_node);

	value = crm_meta_value(action->params,
	PCMK__META_DIGESTS_SECURE);
	update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL,
	is_remote_node);

	} else if (!(pcmk_is_set(action->flags, pcmk__graph_action_sent_update))) {
	update_node_state_after_fencing(target, uuid);
	pcmk__set_graph_action_flags(action,
	pcmk__graph_action_sent_update);
	}
	}
	st_fail_count_reset(target);

	} else {
	enum pcmk__graph_next abort_action = pcmk__graph_restart;
	int status = stonith__execution_status(data);
	const char *reason = stonith__exit_reason(data);

	if (reason == NULL) {
	if (status == PCMK_EXEC_DONE) {
	reason = "Agent returned error";
	} else {
	reason = pcmk_exec_status_str(status);
	}
	}
	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);

	/* If no fence devices were available, there's no use in immediately
	* checking again, so don't start a new transition in that case.
	*/
	if (status == PCMK_EXEC_NO_FENCE_DEVICE) {
	crm_warn("Fence operation %d for %s failed: %s "
	"(aborting transition and giving up for now)",
	data->call_id, target, reason);
	abort_action = pcmk__graph_wait;
	} else {
	crm_notice("Fence operation %d for %s failed: %s "
	"(aborting transition)", data->call_id, target, reason);
	}

	/* Increment the fail count now, so abort_for_stonith_failure() can
	* check it. Non-DC nodes will increment it in
	* handle_fence_notification().
	*/
	st_fail_count_increment(target);
	abort_for_stonith_failure(abort_action, target, NULL);
	}

	pcmk__update_graph(controld_globals.transition_graph, action);
	trigger_graph();

	bail:
	free(data->userdata);
	free(uuid);
	return;
	}

	static int
	fence_with_delay(const char target, const char type, int delay)
	{
	uint32_t options = st_opt_none; // Group of enum stonith_call_options
	int timeout_sec =
	pcmk__timeout_ms2s(controld_globals.transition_graph->fencing_timeout);

	if (crmd_join_phase_count(controld_join_confirmed) == 1) {
	stonith__set_call_options(options, target, st_opt_allow_self_fencing);
	}
	return stonith_api->cmds->fence_with_delay(stonith_api, options, target,
	type, timeout_sec, 0, delay);
	}

	/*!
	* \internal
	* \brief Execute a fencing action from a transition graph
	*
	* \param[in] graph Transition graph being executed (ignored)
	* \param[in] action Fencing action to execute
	*
	* \return Standard Pacemaker return code
	*/
	int
	controld_execute_fence_action(pcmk__graph_t *graph,
	pcmk__graph_action_t *action)
	{
	int rc = 0;
	const char *id = pcmk__xe_id(action->xml);
	const char *uuid = pcmk__xe_get(action->xml, PCMK__META_ON_NODE_UUID);
	const char *target = pcmk__xe_get(action->xml, PCMK__META_ON_NODE);
	const char *type = crm_meta_value(action->params,
	PCMK__META_STONITH_ACTION);
	char *transition_key = NULL;
	const char *priority_delay = NULL;
	int delay_i = 0;
	gboolean invalid_action = FALSE;
	int timeout_sec =
	pcmk__timeout_ms2s(controld_globals.transition_graph->fencing_timeout);

	CRM_CHECK(id != NULL, invalid_action = TRUE);
	CRM_CHECK(uuid != NULL, invalid_action = TRUE);
	CRM_CHECK(type != NULL, invalid_action = TRUE);
	CRM_CHECK(target != NULL, invalid_action = TRUE);

	if (invalid_action) {
	crm_log_xml_warn(action->xml, "BadAction");
	return EPROTO;
	}

	priority_delay = crm_meta_value(action->params,
	PCMK_OPT_PRIORITY_FENCING_DELAY);

	crm_notice("Requesting fencing (%s) targeting node %s "
	QB_XS " action=%s timeout=%i%s%s",
	type, target, id, timeout_sec,
	priority_delay ? " priority_delay=" : "",
	priority_delay ? priority_delay : "");

	/* Passing NULL means block until we can connect... */
	controld_timer_fencer_connect(NULL);

	pcmk__scan_min_int(priority_delay, &delay_i, 0);
	rc = fence_with_delay(target, type, delay_i);
	transition_key = pcmk__transition_key(controld_globals.transition_graph->id,
	action->id, 0,
	controld_globals.te_uuid),
	stonith_api->cmds->register_callback(stonith_api, rc,
	(timeout_sec
	+ (delay_i > 0 ? delay_i : 0)),
	st_opt_timeout_updates, transition_key,
	"tengine_stonith_callback",
	tengine_stonith_callback);
	return pcmk_rc_ok;
	}

	void
	controld_validate_fencing_watchdog_timeout(const char *value)
	{
	const char *our_nodename = controld_globals.cluster->priv->node_name;

	// Validate only if the timeout will be used
	if ((stonith_api != NULL) && (stonith_api->state != stonith_disconnected)
	&& stonith__watchdog_fencing_enabled_for_node_api(stonith_api,
	our_nodename)) {

	pcmk__valid_fencing_watchdog_timeout(value);
	}
	}

	/* end stonith API client functions */


	/*
	* stonith history synchronization
	*
	* Each node's fencer keeps track of a cluster-wide fencing history. When a node
	* joins or leaves, we need to synchronize the history across all nodes.
	*/

	static crm_trigger_t *stonith_history_sync_trigger = NULL;
	static mainloop_timer_t *stonith_history_sync_timer_short = NULL;
	static mainloop_timer_t *stonith_history_sync_timer_long = NULL;

	void
	-te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers)
	+controld_cleanup_fencing_history_sync(stonith_t *st, bool free_timers)
	{
	if (free_timers) {
	mainloop_timer_del(stonith_history_sync_timer_short);
	stonith_history_sync_timer_short = NULL;
	mainloop_timer_del(stonith_history_sync_timer_long);
	stonith_history_sync_timer_long = NULL;
	} else {
	mainloop_timer_stop(stonith_history_sync_timer_short);
	mainloop_timer_stop(stonith_history_sync_timer_long);
	}

	if (st) {
	st->cmds->remove_notification(st, PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED);
	}
	}

	static void
	tengine_stonith_history_synced(stonith_t st, stonith_event_t st_event)
	{
	- te_cleanup_stonith_history_sync(st, FALSE);
	+ controld_cleanup_fencing_history_sync(st, false);
	crm_debug("Fence-history synced - cancel all timers");
	}

	static gboolean
	stonith_history_sync_set_trigger(gpointer user_data)
	{
	mainloop_set_trigger(stonith_history_sync_trigger);
	return FALSE;
	}

	void
	controld_trigger_fencing_history_sync(bool long_timeout)
	{
	/* trigger a sync in 5s to give more nodes the
	* chance to show up so that we don't create
	* unnecessary stonith-history-sync traffic
	*
	* the long timeout of 30s is there as a fallback
	* so that after a successful connection to fenced
	* we will wait for 30s for the DC to trigger a
	* history-sync
	* if this doesn't happen we trigger a sync locally
	* (e.g. fenced segfaults and is restarted by pacemakerd)
	*/

	/* as we are finally checking the stonith-connection
	* in do_stonith_history_sync we should be fine
	* leaving stonith_history_sync_time & stonith_history_sync_trigger
	* around
	*/
	if (stonith_history_sync_trigger == NULL) {
	stonith_history_sync_trigger =
	mainloop_add_trigger(G_PRIORITY_LOW,
	do_stonith_history_sync, NULL);
	}

	if (long_timeout) {
	if(stonith_history_sync_timer_long == NULL) {
	stonith_history_sync_timer_long =
	mainloop_timer_add("history_sync_long", 30000,
	FALSE, stonith_history_sync_set_trigger,
	NULL);
	}
	crm_info("Fence history will be synchronized cluster-wide within 30 seconds");
	mainloop_timer_start(stonith_history_sync_timer_long);
	} else {
	if(stonith_history_sync_timer_short == NULL) {
	stonith_history_sync_timer_short =
	mainloop_timer_add("history_sync_short", 5000,
	FALSE, stonith_history_sync_set_trigger,
	NULL);
	}
	crm_info("Fence history will be synchronized cluster-wide within 5 seconds");
	mainloop_timer_start(stonith_history_sync_timer_short);
	}

	}

	/* end stonith history synchronization functions */
	diff --git a/daemons/controld/controld_fencing.h b/daemons/controld/controld_fencing.h
	index 44359c22d0..b58b106d5e 100644
	--- a/daemons/controld/controld_fencing.h
	+++ b/daemons/controld/controld_fencing.h
	@@ -1,37 +1,37 @@
	/*
	* Copyright 2004-2025 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CONTROLD_FENCING__H
	# define CONTROLD_FENCING__H

	#include <stdbool.h> // bool
	#include <pacemaker-internal.h> // pcmk__graph_t, pcmk__graph_action_t

	void controld_configure_fencing(GHashTable *options);

	// stonith fail counts
	void st_fail_count_reset(const char * target);

	// stonith API client
	gboolean controld_timer_fencer_connect(gpointer user_data);
	void controld_disconnect_fencer(bool destroy);
	int controld_execute_fence_action(pcmk__graph_t *graph,
	pcmk__graph_action_t *action);
	void controld_validate_fencing_watchdog_timeout(const char *value);

	// Fencing cleanup list
	void controld_remove_fencing_cleanup(const char *target);
	void controld_purge_fencing_cleanup(void);
	void controld_execute_fencing_cleanup(void);

	// Fencing history synchronization
	void controld_trigger_fencing_history_sync(bool long_timeout);
	-void te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers);
	+void controld_cleanup_fencing_history_sync(stonith_t *st, bool free_timers);

	#endif

File Metadata

Mime Type: text/x-diff
Expires: Tue, Sep 23, 1:11 PM (23 h, 57 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2397614
Default Alt Text: (63 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions