No OneTemporary
Actions

Size

141 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
	index a727af6810..cfa821bb95 100644
	--- a/daemons/controld/controld_control.c
	+++ b/daemons/controld/controld_control.c
	@@ -1,916 +1,915 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election_internal.h>
	#include <crm/common/ipc_internal.h>

	#include <pacemaker-controld.h>

	static qb_ipcs_service_t *ipcs = NULL;

	static crm_trigger_t *config_read_trigger = NULL;

	#if SUPPORT_COROSYNC
	extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
	#endif

	static void crm_shutdown(int nsig);
	static gboolean crm_read_options(gpointer user_data);

	/* A_HA_CONNECT */
	void
	do_ha_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean registered = FALSE;
	static crm_cluster_t *cluster = NULL;

	if (cluster == NULL) {
	cluster = pcmk_cluster_new();
	}

	if (action & A_HA_DISCONNECT) {
	crm_cluster_disconnect(cluster);
	crm_info("Disconnected from the cluster");

	controld_set_fsa_input_flags(R_HA_DISCONNECTED);
	}

	if (action & A_HA_CONNECT) {
	crm_set_status_callback(&peer_update_callback);
	crm_set_autoreap(FALSE);

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	registered = crm_connect_corosync(cluster);
	}
	#endif // SUPPORT_COROSYNC

	if (registered) {
	controld_election_init(cluster->uname);
	controld_globals.our_nodename = cluster->uname;
	controld_globals.our_uuid = cluster->uuid;
	if(cluster->uuid == NULL) {
	crm_err("Could not obtain local uuid");
	registered = FALSE;
	}
	}

	if (!registered) {
	controld_set_fsa_input_flags(R_HA_DISCONNECTED);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	populate_cib_nodes(node_update_none, __func__);
	controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
	crm_info("Connected to the cluster");
	}

	if (action & ~(A_HA_CONNECT \| A_HA_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action),
	__func__);
	}
	}

	/* A_SHUTDOWN */
	void
	do_shutdown(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* just in case */
	controld_set_fsa_input_flags(R_SHUTDOWN);
	controld_disconnect_fencer(FALSE);
	}

	/* A_SHUTDOWN_REQ */
	void
	do_shutdown_req(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *msg = NULL;

	controld_set_fsa_input_flags(R_SHUTDOWN);
	//controld_set_fsa_input_flags(R_STAYDOWN);
	crm_info("Sending shutdown request to all peers (DC is %s)",
	pcmk__s(controld_globals.dc_name, "not set"));
	msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	free_xml(msg);
	}

	void
	crmd_fast_exit(crm_exit_t exit_code)
	{
	if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) {
	crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d",
	exit_code, CRM_EX_FATAL);
	exit_code = CRM_EX_FATAL;

	} else if ((exit_code == CRM_EX_OK)
	&& pcmk_is_set(controld_globals.fsa_input_register,
	R_IN_RECOVERY)) {
	crm_err("Could not recover from internal error");
	exit_code = CRM_EX_ERROR;
	}

	if (controld_globals.logger_out != NULL) {
	controld_globals.logger_out->finish(controld_globals.logger_out,
	exit_code, true, NULL);
	pcmk__output_free(controld_globals.logger_out);
	controld_globals.logger_out = NULL;
	}

	crm_exit(exit_code);
	}

	crm_exit_t
	crmd_exit(crm_exit_t exit_code)
	{
	GMainLoop *mloop = controld_globals.mainloop;

	static bool in_progress = FALSE;

	if (in_progress && (exit_code == CRM_EX_OK)) {
	crm_debug("Exit is already in progress");
	return exit_code;

	} else if(in_progress) {
	crm_notice("Error during shutdown process, exiting now with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	crm_write_blackbox(SIGTRAP, NULL);
	crmd_fast_exit(exit_code);
	}

	in_progress = TRUE;
	crm_trace("Preparing to exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));

	/* Suppress secondary errors resulting from us disconnecting everything */
	controld_set_fsa_input_flags(R_HA_DISCONNECTED);

	/* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */

	if(ipcs) {
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs);
	ipcs = NULL;
	}

	controld_close_attrd_ipc();
	controld_shutdown_schedulerd_ipc();
	controld_disconnect_fencer(TRUE);

	if ((exit_code == CRM_EX_OK) && (controld_globals.mainloop == NULL)) {
	crm_debug("No mainloop detected");
	exit_code = CRM_EX_ERROR;
	}

	/* On an error, just get out.
	*
	* Otherwise, make the effort to have mainloop exit gracefully so
	* that it (mostly) cleans up after itself and valgrind has less
	* to report on - allowing real errors stand out
	*/
	if (exit_code != CRM_EX_OK) {
	crm_notice("Forcing immediate exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	crm_write_blackbox(SIGTRAP, NULL);
	crmd_fast_exit(exit_code);
	}

	/* Clean up as much memory as possible for valgrind */

	for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
	iter = iter->next) {
	fsa_data_t fsa_data = (fsa_data_t ) iter->data;

	crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
	fsa_input2string(fsa_data->fsa_input),
	fsa_state2string(controld_globals.fsa_state),
	fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
	delete_fsa_input(fsa_data);
	}

	controld_clear_fsa_input_flags(R_MEMBERSHIP);

	g_list_free(controld_globals.fsa_message_queue);
	controld_globals.fsa_message_queue = NULL;

	controld_free_node_pending_timers();
	controld_election_fini();

	/* Tear down the CIB manager connection, but don't free it yet -- it could
	* be used when we drain the mainloop later.
	*/

	controld_disconnect_cib_manager();

	verify_stopped(controld_globals.fsa_state, LOG_WARNING);
	controld_clear_fsa_input_flags(R_LRM_CONNECTED);
	lrm_state_destroy_all();

	mainloop_destroy_trigger(config_read_trigger);
	config_read_trigger = NULL;

	controld_destroy_fsa_trigger();
	controld_destroy_transition_trigger();

	pcmk__client_cleanup();
	crm_peer_destroy();

	controld_free_fsa_timers();
	te_cleanup_stonith_history_sync(NULL, TRUE);
	controld_free_sched_timer();

	free(controld_globals.our_nodename);
	controld_globals.our_nodename = NULL;

	free(controld_globals.our_uuid);
	controld_globals.our_uuid = NULL;

	free(controld_globals.dc_name);
	controld_globals.dc_name = NULL;

	free(controld_globals.dc_version);
	controld_globals.dc_version = NULL;

	free(controld_globals.cluster_name);
	controld_globals.cluster_name = NULL;

	free(controld_globals.te_uuid);
	controld_globals.te_uuid = NULL;

	free_max_generation();
	controld_destroy_failed_sync_table();
	controld_destroy_outside_events_table();

	mainloop_destroy_signal(SIGPIPE);
	mainloop_destroy_signal(SIGUSR1);
	mainloop_destroy_signal(SIGTERM);
	mainloop_destroy_signal(SIGTRAP);
	/* leave SIGCHLD engaged as we might still want to drain some service-actions */

	if (mloop) {
	GMainContext *ctx = g_main_loop_get_context(controld_globals.mainloop);

	/* Don't re-enter this block */
	controld_globals.mainloop = NULL;

	/* no signals on final draining anymore */
	mainloop_destroy_signal(SIGCHLD);

	crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));

	{
	int lpc = 0;

	while((g_main_context_pending(ctx) && lpc < 10)) {
	lpc++;
	crm_trace("Iteration %d", lpc);
	g_main_context_dispatch(ctx);
	}
	}

	crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
	g_main_loop_quit(mloop);

	/* Won't do anything yet, since we're inside it now */
	g_main_loop_unref(mloop);
	} else {
	mainloop_destroy_signal(SIGCHLD);
	}

	cib_delete(controld_globals.cib_conn);
	controld_globals.cib_conn = NULL;

	throttle_fini();

	/* Graceful */
	crm_trace("Done preparing for exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	return exit_code;
	}

	/* A_EXIT_0, A_EXIT_1 */
	void
	do_exit(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_exit_t exit_code = CRM_EX_OK;

	if (pcmk_is_set(action, A_EXIT_1)) {
	exit_code = CRM_EX_ERROR;
	crm_err("Exiting now due to errors");
	}
	verify_stopped(cur_state, LOG_ERR);
	crmd_exit(exit_code);
	}

	static void sigpipe_ignore(int nsig) { return; }

	/* A_STARTUP */
	void
	do_startup(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_debug("Registering Signal Handlers");
	mainloop_add_signal(SIGTERM, crm_shutdown);
	mainloop_add_signal(SIGPIPE, sigpipe_ignore);

	config_read_trigger = mainloop_add_trigger(G_PRIORITY_HIGH,
	crm_read_options, NULL);

	controld_init_fsa_trigger();
	controld_init_transition_trigger();

	crm_debug("Creating CIB manager and executor objects");
	controld_globals.cib_conn = cib_new();

	lrm_state_init_local();
	if (controld_init_fsa_timers() == FALSE) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	// \return libqb error code (0 on success, -errno on error)
	static int32_t
	accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
	{
	crm_trace("Accepting new IPC client connection");
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	// \return libqb error code (0 on success, -errno on error)
	static int32_t
	dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	pcmk__client_t *client = pcmk__find_client(c);

	xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);

	if (msg == NULL) {
	pcmk__ipc_send_ack(client, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
	return 0;
	}
	pcmk__ipc_send_ack(client, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);

	CRM_ASSERT(client->user != NULL);
	pcmk__update_acl_user(msg, PCMK__XA_CRM_USER, client->user);

	crm_xml_add(msg, PCMK__XA_CRM_SYS_FROM, client->id);
	if (controld_authorize_ipc_message(msg, client, NULL)) {
	crm_trace("Processing IPC message from client %s",
	pcmk__client_name(client));
	route_message(C_IPC_MESSAGE, msg);
	}

	controld_trigger_fsa();
	free_xml(msg);
	return 0;
	}

	static int32_t
	ipc_client_disconnected(qb_ipcs_connection_t *c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client) {
	crm_trace("Disconnecting %sregistered client %s (%p/%p)",
	(client->userdata? "" : "un"), pcmk__client_name(client),
	c, client);
	free(client->userdata);
	pcmk__free_client(client);
	controld_trigger_fsa();
	}
	return 0;
	}

	static void
	ipc_connection_destroyed(qb_ipcs_connection_t *c)
	{
	crm_trace("Connection %p", c);
	ipc_client_disconnected(c);
	}

	/* A_STOP */
	void
	do_stop(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs); ipcs = NULL;
	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	/* A_STARTED */
	void
	do_started(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static struct qb_ipcs_service_handlers crmd_callbacks = {
	.connection_accept = accept_controller_client,
	.connection_created = NULL,
	.msg_process = dispatch_controller_ipc,
	.connection_closed = ipc_client_disconnected,
	.connection_destroyed = ipc_connection_destroyed
	};

	if (cur_state != S_STARTING) {
	crm_err("Start cancelled... %s", fsa_state2string(cur_state));
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_MEMBERSHIP)) {
	crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_LRM_CONNECTED)) {
	crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_CIB_CONNECTED)) {
	crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_READ_CONFIG)) {
	crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register, R_PEER_DATA)) {

	crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
	crmd_fsa_stall(TRUE);
	return;
	}

	crm_debug("Init server comms");
	ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
	if (ipcs == NULL) {
	crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	} else {
	crm_notice("Pacemaker controller successfully started and accepting connections");
	}
	controld_set_fsa_input_flags(R_ST_REQUIRED);
	controld_timer_fencer_connect(GINT_TO_POINTER(TRUE));

	controld_clear_fsa_input_flags(R_STARTING);
	register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
	}

	/* A_RECOVER */
	void
	do_recover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	controld_set_fsa_input_flags(R_IN_RECOVERY);
	crm_warn("Fast-tracking shutdown in response to errors");

	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	static pcmk__cluster_option_t controller_options[] = {
	/* name, old name, type, allowed values,
	* default value, validator,
	* context,
	* short description,
	* long description
	*/
	{
	PCMK_OPT_DC_VERSION, NULL, "string", NULL,
	PCMK__VALUE_NONE, NULL,
	pcmk__opt_context_controld,
	N_("Pacemaker version on cluster node elected Designated Controller (DC)"),
	N_("Includes a hash which identifies the exact changeset the code was "
	"built from. Used for diagnostic purposes.")
	},
	{
	PCMK_OPT_CLUSTER_INFRASTRUCTURE, NULL, "string", NULL,
	"corosync", NULL,
	pcmk__opt_context_controld,
	N_("The messaging stack on which Pacemaker is currently running"),
	N_("Used for informational and diagnostic purposes.")
	},
	{
	PCMK_OPT_CLUSTER_NAME, NULL, "string", NULL,
	NULL, NULL,
	pcmk__opt_context_controld,
	N_("An arbitrary name for the cluster"),
	N_("This optional value is mostly for users' convenience as desired "
	"in administration, but may also be used in Pacemaker "
	"configuration rules via the #cluster-name node attribute, and "
	"by higher-level tools and resource agents.")
	},
	{
	PCMK_OPT_DC_DEADTIME, NULL, "time", NULL,
	"20s", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("How long to wait for a response from other nodes during start-up"),
	N_("The optimal value will depend on the speed and load of your network "
	"and the type of switches used.")
	},
	{
	PCMK_OPT_CLUSTER_RECHECK_INTERVAL, NULL, "time",
	N_("Zero disables polling, while positive values are an interval in "
	"seconds (unless other units are specified, for example \"5min\")"),
	"15min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("Polling interval to recheck cluster state and evaluate rules "
	"with date specifications"),
	N_("Pacemaker is primarily event-driven, and looks ahead to know when to "
	"recheck cluster state for failure timeouts and most time-based "
	"rules. However, it will also recheck the cluster after this "
	"amount of inactivity, to evaluate rules with date specifications "
	"and serve as a fail-safe for certain types of scheduler bugs.")
	},
	{
	PCMK_OPT_LOAD_THRESHOLD, NULL, "percentage", NULL,
	"80%", pcmk__valid_percentage,
	pcmk__opt_context_controld,
	N_("Maximum amount of system load that should be used by cluster nodes"),
	N_("The cluster will slow down its recovery process when the amount of "
	"system resources used (currently CPU) approaches this limit"),
	},
	{
	PCMK_OPT_NODE_ACTION_LIMIT, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_controld,
	N_("Maximum number of jobs that can be scheduled per node "
	"(defaults to 2x cores)")
	},
	{
	PCMK_OPT_FENCE_REACTION, NULL, "select", "stop, panic",
	"stop", NULL,
	pcmk__opt_context_controld,
	N_("How a cluster node should react if notified of its own fencing"),
	N_("A cluster node may receive notification of its own fencing if fencing "
	"is misconfigured, or if fabric fencing is in use that doesn't cut "
	"cluster communication. Use \"stop\" to attempt to immediately "
	"stop Pacemaker and stay stopped, or \"panic\" to attempt to "
	"immediately reboot the local node, falling back to stop on "
	"failure.")
	},
	{
	PCMK_OPT_ELECTION_TIMEOUT, NULL, "time", NULL,
	"2min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	"* Advanced Use Only *",
	N_("Declare an election failed if it is not decided within this much "
	"time. If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_SHUTDOWN_ESCALATION, NULL, "time", NULL,
	"20min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	"* Advanced Use Only *",
	N_("Exit immediately if shutdown does not complete within this much "
	"time. If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_JOIN_INTEGRATION_TIMEOUT, "crmd-integration-timeout", "time",
	NULL,
	"3min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	"* Advanced Use Only *",
	N_("If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_JOIN_FINALIZATION_TIMEOUT, "crmd-finalization-timeout",
	"time", NULL,
	"30min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	"* Advanced Use Only *",
	N_("If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_TRANSITION_DELAY, "crmd-transition-delay", "time", NULL,
	"0s", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("* Advanced Use Only * Enabling this option will slow down "
	"cluster recovery under all conditions"),
	N_("Delay cluster recovery for this much time to allow for additional "
	"events to occur. Useful if your configuration is sensitive to "
	"the order in which ping updates arrive.")
	},
	{
	/* @COMPAT Currently unparsable values default to -1 (auto-calculate),
	* while missing values default to 0 (disable). All values are accepted
	* (unless the controller finds that the value conflicts with the
	* SBD_WATCHDOG_TIMEOUT).
	*
	* At a compatibility break: properly validate as a timeout, let
	* either negative values or a particular string like "auto" mean auto-
	* calculate, and use 0 as the single default for when the option either
	* is unset or fails to validate.
	*/
	PCMK_OPT_STONITH_WATCHDOG_TIMEOUT, NULL, "time", NULL,
	"0", NULL,
	pcmk__opt_context_controld,
	N_("How long before nodes can be assumed to be safely down when "
	"watchdog-based self-fencing via SBD is in use"),
	N_("If this is set to a positive value, lost nodes are assumed to "
	"self-fence using watchdog-based SBD within this much time. This "
	"does not require a fencing resource to be explicitly configured, "
	"though a fence_watchdog resource can be configured, to limit use "
	"to specific nodes. If this is set to 0 (the default), the cluster "
	"will never assume watchdog-based self-fencing. If this is set to a "
	"negative value, the cluster will use twice the local value of the "
	"`SBD_WATCHDOG_TIMEOUT` environment variable if that is positive, "
	"or otherwise treat this as 0. WARNING: When used, this timeout "
	"must be larger than `SBD_WATCHDOG_TIMEOUT` on all nodes that use "
	"watchdog-based SBD, and Pacemaker will refuse to start on any of "
	"those nodes where this is not true for the local value or SBD is "
	"not active. When this is set to a negative value, "
	"`SBD_WATCHDOG_TIMEOUT` must be set to the same value on all nodes "
	"that use SBD, otherwise data corruption or loss could occur.")
	},
	{
	PCMK_OPT_STONITH_MAX_ATTEMPTS, NULL, "integer", NULL,
	"10", pcmk__valid_positive_int,
	pcmk__opt_context_controld,
	N_("How many times fencing can fail before it will no longer be "
	"immediately re-attempted on a target")
	},

	// Already documented in libpe_status (other values must be kept identical)
	{
	PCMK_OPT_NO_QUORUM_POLICY, NULL, "select",
	"stop, freeze, ignore, demote, suicide",
	"stop", pcmk__valid_no_quorum_policy,
	pcmk__opt_context_schedulerd,
	N_("What to do when the cluster does not have quorum"), NULL
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether to lock resources to a cleanly shut down node"),
	N_("When true, resources active on a node when it is cleanly shut down "
	"are kept \"locked\" to that node (not allowed to run elsewhere) "
	"until they start again on that node after it rejoins (or for at "
	"most shutdown-lock-limit, if set). Stonith resources and "
	"Pacemaker Remote connections are never locked. Clone and bundle "
	"instances and the promoted role of promotable clones are "
	"currently never locked, though support could be added in a future "
	"release.")
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("Do not lock resources to a cleanly shut down node longer than "
	"this"),
	N_("If shutdown-lock is true and this is set to a nonzero time "
	"duration, shutdown locks will expire after this much time has "
	"passed since the shutdown was initiated, even if the node has not "
	"rejoined.")
	},
	{
	PCMK_OPT_NODE_PENDING_TIMEOUT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("How long to wait for a node that has joined the cluster to join "
	"the controller process group"),
	N_("Fence nodes that do not join the controller process group within "
	"this much time after joining the cluster, to allow the cluster "
	"to continue managing resources. A value of 0 means never fence "
	"pending nodes. Setting the value to 2h means fence nodes after "
	"2 hours.")
	},
	};

	void
	crmd_metadata(void)
	{
	const char *name = "pacemaker-controld";
	const char *desc_short = "Pacemaker controller options";
	const char *desc_long = "Cluster options used by Pacemaker's controller";

	- gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long,
	- pcmk__opt_context_controld,
	- controller_options,
	- PCMK__NELEM(controller_options));
	+ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long,
	+ pcmk__opt_context_controld);
	+
	printf("%s", s);
	g_free(s);
	}

	static void
	config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	const char *value = NULL;
	GHashTable *config_hash = NULL;
	crm_time_t *now = crm_time_new(NULL);
	xmlNode *crmconfig = NULL;
	xmlNode *alerts = NULL;

	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL;

	crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	if (rc == -EACCES \|\| rc == -pcmk_err_schema_validation) {
	crm_err("The cluster is mis-configured - shutting down and staying down");
	controld_set_fsa_input_flags(R_STAYDOWN);
	}
	goto bail;
	}

	crmconfig = output;
	if ((crmconfig != NULL) && !pcmk__xe_is(crmconfig, PCMK_XE_CRM_CONFIG)) {
	crmconfig = first_named_child(crmconfig, PCMK_XE_CRM_CONFIG);
	}
	if (!crmconfig) {
	fsa_data_t *msg_data = NULL;

	crm_err("Local CIB query for " PCMK_XE_CRM_CONFIG " section failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	goto bail;
	}

	crm_debug("Call %d : Parsing CIB options", call_id);
	config_hash = pcmk__strkey_table(free, free);
	pe_unpack_nvpairs(crmconfig, crmconfig, PCMK_XE_CLUSTER_PROPERTY_SET, NULL,
	config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);

	// Validate all options, and use defaults if not already present in hash
	pcmk__validate_cluster_options(config_hash, controller_options,
	PCMK__NELEM(controller_options));

	/* Validate the watchdog timeout in the context of the local node
	* environment. If invalid, the controller will exit with a fatal error.
	*
	* We do this via a wrapper in the controller, so that we call
	* pcmk__valid_stonith_watchdog_timeout() only if watchdog fencing is
	* enabled for the local node. Otherwise, we may exit unnecessarily.
	*
	* A validator function in libcrmcommon can't act as such a wrapper, because
	* it doesn't have a stonith API connection or the local node name.
	*/
	value = g_hash_table_lookup(config_hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
	controld_verify_stonith_watchdog_timeout(value);

	value = g_hash_table_lookup(config_hash, PCMK_OPT_NO_QUORUM_POLICY);
	if (pcmk__str_eq(value, "suicide", pcmk__str_casei) && pcmk__locate_sbd()) {
	controld_set_global_flags(controld_no_quorum_suicide);
	}

	value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK);
	if (crm_is_true(value)) {
	controld_set_global_flags(controld_shutdown_lock_enabled);
	} else {
	controld_clear_global_flags(controld_shutdown_lock_enabled);
	}

	value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
	pcmk_parse_interval_spec(value, &controld_globals.shutdown_lock_limit);
	controld_globals.shutdown_lock_limit /= 1000;

	value = g_hash_table_lookup(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
	pcmk_parse_interval_spec(value, &controld_globals.node_pending_timeout);
	controld_globals.node_pending_timeout /= 1000;

	value = g_hash_table_lookup(config_hash, PCMK_OPT_CLUSTER_NAME);
	pcmk__str_update(&(controld_globals.cluster_name), value);

	// Let subcomponents initialize their own static variables
	controld_configure_election(config_hash);
	controld_configure_fencing(config_hash);
	controld_configure_fsa_timers(config_hash);
	controld_configure_throttle(config_hash);

	alerts = first_named_child(output, PCMK_XE_ALERTS);
	crmd_unpack_alerts(alerts);

	controld_set_fsa_input_flags(R_READ_CONFIG);
	controld_trigger_fsa();

	g_hash_table_destroy(config_hash);
	bail:
	crm_time_free(now);
	}

	/*!
	* \internal
	* \brief Trigger read and processing of the configuration
	*
	* \param[in] fn Calling function name
	* \param[in] line Line number where call occurred
	*/
	void
	controld_trigger_config_as(const char *fn, int line)
	{
	if (config_read_trigger != NULL) {
	crm_trace("%s:%d - Triggered config processing", fn, line);
	mainloop_set_trigger(config_read_trigger);
	}
	}

	gboolean
	crm_read_options(gpointer user_data)
	{
	cib_t *cib_conn = controld_globals.cib_conn;
	int call_id = cib_conn->cmds->query(cib_conn,
	"//" PCMK_XE_CRM_CONFIG
	" \| //" PCMK_XE_ALERTS,
	NULL, cib_xpath\|cib_scope_local);

	fsa_register_cib_callback(call_id, NULL, config_query_callback);
	crm_trace("Querying the CIB... call %d", call_id);
	return TRUE;
	}

	/* A_READCONFIG */
	void
	do_read_config(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	throttle_init();
	controld_trigger_config();
	}

	static void
	crm_shutdown(int nsig)
	{
	const char *value = NULL;
	guint default_period_ms = 0;

	if ((controld_globals.mainloop == NULL)
	\|\| !g_main_loop_is_running(controld_globals.mainloop)) {
	crmd_exit(CRM_EX_OK);
	return;
	}

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	crm_err("Escalating shutdown");
	register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
	return;
	}

	controld_set_fsa_input_flags(R_SHUTDOWN);
	register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);

	/* If shutdown timer doesn't have a period set, use the default
	*
	* @TODO: Evaluate whether this is still necessary. As long as
	* config_query_callback() has been run at least once, it doesn't look like
	* anything could have changed the timer period since then.
	*/
	value = pcmk__cluster_option(NULL, controller_options,
	PCMK__NELEM(controller_options),
	PCMK_OPT_SHUTDOWN_ESCALATION);
	pcmk_parse_interval_spec(value, &default_period_ms);
	controld_shutdown_start_countdown(default_period_ms);
	}
	diff --git a/include/crm/common/options_internal.h b/include/crm/common/options_internal.h
	index 6db58da753..7026959f00 100644
	--- a/include/crm/common/options_internal.h
	+++ b/include/crm/common/options_internal.h
	@@ -1,204 +1,208 @@
	/*
	* Copyright 2006-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__OPTIONS_INTERNAL__H
	# define PCMK__OPTIONS_INTERNAL__H

	# ifndef PCMK__CONFIG_H
	# define PCMK__CONFIG_H
	# include <config.h> // _Noreturn
	# endif

	# include <glib.h> // GHashTable
	# include <stdbool.h> // bool

	#include <crm/common/util.h> // pcmk_parse_interval_spec()

	_Noreturn void pcmk__cli_help(char cmd);


	/*
	* Environment variable option handling
	*/

	const char pcmk__env_option(const char option);
	void pcmk__set_env_option(const char option, const char value, bool compat);
	bool pcmk__env_option_enabled(const char daemon, const char option);


	/*
	* Cluster option handling
	*/

	/*!
	* \internal
	* \enum pcmk__opt_context
	* \brief Context flags for options
	*/
	enum pcmk__opt_context {
	// @COMPAT Used only for daemon metadata
	pcmk__opt_context_none = 0, //!< No additional context
	pcmk__opt_context_based = (1 << 1), //!< CIB manager metadata
	pcmk__opt_context_controld = (1 << 2), //!< Controller metadata
	pcmk__opt_context_schedulerd = (1 << 3), //!< Scheduler metadata
	};

	typedef struct pcmk__cluster_option_s {
	const char *name;
	const char *alt_name;
	const char *type;
	const char *values;
	const char *default_value;

	bool (is_valid)(const char );

	// @COMPAT context is used only for daemon meta-data
	enum pcmk__opt_context context;

	const char *description_short;
	const char *description_long;

	} pcmk__cluster_option_t;

	const char pcmk__cluster_option(GHashTable options,
	const pcmk__cluster_option_t *option_list,
	int len, const char *name);

	gchar pcmk__format_option_metadata(const char name, const char *desc_short,
	const char *desc_long,
	enum pcmk__opt_context filter,
	pcmk__cluster_option_t *option_list,
	int len);

	+gchar pcmk__cluster_option_metadata(const char name, const char *desc_short,
	+ const char *desc_long,
	+ enum pcmk__opt_context filter);
	+
	void pcmk__validate_cluster_options(GHashTable *options,
	pcmk__cluster_option_t *option_list,
	int len);

	bool pcmk__valid_interval_spec(const char *value);
	bool pcmk__valid_boolean(const char *value);
	bool pcmk__valid_int(const char *value);
	bool pcmk__valid_positive_int(const char *value);
	bool pcmk__valid_no_quorum_policy(const char *value);
	bool pcmk__valid_percentage(const char *value);
	bool pcmk__valid_script(const char *value);
	bool pcmk__valid_placement_strategy(const char *value);

	// from watchdog.c
	long pcmk__get_sbd_watchdog_timeout(void);
	bool pcmk__get_sbd_sync_resource_startup(void);
	long pcmk__auto_stonith_watchdog_timeout(void);
	bool pcmk__valid_stonith_watchdog_timeout(const char *value);

	// Constants for environment variable names
	#define PCMK__ENV_AUTHKEY_LOCATION "authkey_location"
	#define PCMK__ENV_BLACKBOX "blackbox"
	#define PCMK__ENV_CALLGRIND_ENABLED "callgrind_enabled"
	#define PCMK__ENV_CLUSTER_TYPE "cluster_type"
	#define PCMK__ENV_DEBUG "debug"
	#define PCMK__ENV_DH_MAX_BITS "dh_max_bits"
	#define PCMK__ENV_DH_MIN_BITS "dh_min_bits"
	#define PCMK__ENV_FAIL_FAST "fail_fast"
	#define PCMK__ENV_IPC_BUFFER "ipc_buffer"
	#define PCMK__ENV_IPC_TYPE "ipc_type"
	#define PCMK__ENV_LOGFACILITY "logfacility"
	#define PCMK__ENV_LOGFILE "logfile"
	#define PCMK__ENV_LOGFILE_MODE "logfile_mode"
	#define PCMK__ENV_LOGPRIORITY "logpriority"
	#define PCMK__ENV_NODE_ACTION_LIMIT "node_action_limit"
	#define PCMK__ENV_NODE_START_STATE "node_start_state"
	#define PCMK__ENV_PANIC_ACTION "panic_action"
	#define PCMK__ENV_REMOTE_ADDRESS "remote_address"
	#define PCMK__ENV_REMOTE_SCHEMA_DIR "remote_schema_directory"
	#define PCMK__ENV_REMOTE_PID1 "remote_pid1"
	#define PCMK__ENV_REMOTE_PORT "remote_port"
	#define PCMK__ENV_RESPAWNED "respawned"
	#define PCMK__ENV_SCHEMA_DIRECTORY "schema_directory"
	#define PCMK__ENV_SERVICE "service"
	#define PCMK__ENV_STDERR "stderr"
	#define PCMK__ENV_TLS_PRIORITIES "tls_priorities"
	#define PCMK__ENV_TRACE_BLACKBOX "trace_blackbox"
	#define PCMK__ENV_TRACE_FILES "trace_files"
	#define PCMK__ENV_TRACE_FORMATS "trace_formats"
	#define PCMK__ENV_TRACE_FUNCTIONS "trace_functions"
	#define PCMK__ENV_TRACE_TAGS "trace_tags"
	#define PCMK__ENV_VALGRIND_ENABLED "valgrind_enabled"

	// @COMPAT Drop at 3.0.0; default is plenty
	#define PCMK__ENV_CIB_TIMEOUT "cib_timeout"

	// @COMPAT Drop at 3.0.0; likely last used in 1.1.24
	#define PCMK__ENV_MCP "mcp"

	// @COMPAT Drop at 3.0.0; added unused in 1.1.9
	#define PCMK__ENV_QUORUM_TYPE "quorum_type"

	/* @COMPAT Drop at 3.0.0; added to debug shutdown issues when Pacemaker is
	* managed by systemd, but no longer useful.
	*/
	#define PCMK__ENV_SHUTDOWN_DELAY "shutdown_delay"

	// @COMPAT Deprecated since 2.1.0
	#define PCMK__OPT_REMOVE_AFTER_STOP "remove-after-stop"

	// Constants for meta-attribute names
	#define PCMK__META_CLONE "clone"
	#define PCMK__META_CONTAINER "container"
	#define PCMK__META_DIGESTS_ALL "digests-all"
	#define PCMK__META_DIGESTS_SECURE "digests-secure"
	#define PCMK__META_INTERNAL_RSC "internal_rsc"
	#define PCMK__META_MIGRATE_SOURCE "migrate_source"
	#define PCMK__META_MIGRATE_TARGET "migrate_target"
	#define PCMK__META_ON_NODE "on_node"
	#define PCMK__META_ON_NODE_UUID "on_node_uuid"
	#define PCMK__META_OP_NO_WAIT "op_no_wait"
	#define PCMK__META_OP_TARGET_RC "op_target_rc"
	#define PCMK__META_PHYSICAL_HOST "physical-host"

	/* @TODO Plug these in. Currently, they're never set. These are op attrs for use
	* with https://projects.clusterlabs.org/T382.
	*/
	#define PCMK__META_CLEAR_FAILURE_OP "clear_failure_op"
	#define PCMK__META_CLEAR_FAILURE_INTERVAL "clear_failure_interval"

	// @COMPAT Deprecated meta-attribute since 2.1.0
	#define PCMK__META_CAN_FAIL "can_fail"

	// @COMPAT Deprecated alias for PCMK__META_PROMOTED_MAX since 2.0.0
	#define PCMK__META_PROMOTED_MAX_LEGACY "master-max"

	// @COMPAT Deprecated alias for PCMK__META_PROMOTED_NODE_MAX since 2.0.0
	#define PCMK__META_PROMOTED_NODE_MAX_LEGACY "master-node-max"

	// @COMPAT Deprecated meta-attribute since 2.0.0
	#define PCMK__META_RESTART_TYPE "restart-type"

	// @COMPAT Deprecated meta-attribute since 2.0.0
	#define PCMK__META_ROLE_AFTER_FAILURE "role_after_failure"

	// Constants for enumerated values for various options
	#define PCMK__VALUE_CIB "cib"
	#define PCMK__VALUE_CLUSTER "cluster"
	#define PCMK__VALUE_CUSTOM "custom"
	#define PCMK__VALUE_EN "en"
	#define PCMK__VALUE_FENCING "fencing"
	#define PCMK__VALUE_GREEN "green"
	#define PCMK__VALUE_LOCAL "local"
	#define PCMK__VALUE_MIGRATE_ON_RED "migrate-on-red"
	#define PCMK__VALUE_NONE "none"
	#define PCMK__VALUE_NOTHING "nothing"
	#define PCMK__VALUE_ONLY_GREEN "only-green"
	#define PCMK__VALUE_PROGRESSIVE "progressive"
	#define PCMK__VALUE_QUORUM "quorum"
	#define PCMK__VALUE_RED "red"
	#define PCMK__VALUE_REQUEST "request"
	#define PCMK__VALUE_RESPONSE "response"
	#define PCMK__VALUE_UNFENCING "unfencing"
	#define PCMK__VALUE_YELLOW "yellow"

	#endif // PCMK__OPTIONS_INTERNAL__H
	diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
	index d83129f3cb..30778fee53 100644
	--- a/lib/cib/cib_utils.c
	+++ b/lib/cib/cib_utils.c
	@@ -1,1118 +1,1118 @@
	/*
	* Original copyright 2004 International Business Machines
	* Later changes copyright 2008-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <sys/utsname.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/cib_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/rules.h>

	gboolean
	cib_version_details(xmlNode * cib, int admin_epoch, int epoch, int *updates)
	{
	*epoch = -1;
	*updates = -1;
	*admin_epoch = -1;

	if (cib == NULL) {
	return FALSE;

	} else {
	crm_element_value_int(cib, PCMK_XA_EPOCH, epoch);
	crm_element_value_int(cib, PCMK_XA_NUM_UPDATES, updates);
	crm_element_value_int(cib, PCMK_XA_ADMIN_EPOCH, admin_epoch);
	}
	return TRUE;
	}

	gboolean
	cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(diff, add, del);

	*admin_epoch = add[0];
	*epoch = add[1];
	*updates = add[2];

	*_admin_epoch = del[0];
	*_epoch = del[1];
	*_updates = del[2];

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get the XML patchset from a CIB diff notification
	*
	* \param[in] msg CIB diff notification
	* \param[out] patchset Where to store XML patchset
	*
	* \return Standard Pacemaker return code
	*/
	int
	cib__get_notify_patchset(const xmlNode msg, const xmlNode *patchset)
	{
	int rc = pcmk_err_generic;

	CRM_ASSERT(patchset != NULL);
	*patchset = NULL;

	if (msg == NULL) {
	crm_err("CIB diff notification received with no XML");
	return ENOMSG;
	}

	if ((crm_element_value_int(msg, F_CIB_RC, &rc) != 0) \|\| (rc != pcmk_ok)) {
	crm_warn("Ignore failed CIB update: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	return pcmk_legacy2rc(rc);
	}

	*patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	if (*patchset == NULL) {
	crm_err("CIB diff notification received with no patchset");
	return ENOMSG;
	}
	return pcmk_rc_ok;
	}

	#define XPATH_DIFF_V1 "//" F_CIB_UPDATE_RESULT "//" PCMK__XE_DIFF_ADDED

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset (v1)
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c PCMK_XE_CIB)
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	static bool
	element_in_patchset_v1(const xmlNode patchset, const char element)
	{
	char *xpath = crm_strdup_printf(XPATH_DIFF_V1 "//%s",
	pcmk__s(element, PCMK_XE_CIB));
	xmlXPathObject *xpath_obj = xpath_search(patchset, xpath);

	free(xpath);

	if (xpath_obj == NULL) {
	return false;
	}
	freeXpathObject(xpath_obj);
	return true;
	}

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset (v2)
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c PCMK_XE_CIB). Supported values include any CIB
	* element supported by \c pcmk__cib_abs_xpath_for().
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	static bool
	element_in_patchset_v2(const xmlNode patchset, const char element)
	{
	const char *element_xpath = pcmk__cib_abs_xpath_for(element);
	const char *parent_xpath = pcmk_cib_parent_name_for(element);
	char *element_regex = NULL;
	bool rc = false;

	CRM_CHECK(element_xpath != NULL, return false); // Unsupported element

	// Matches if and only if element_xpath is part of a changed path
	element_regex = crm_strdup_printf("^%s(/\|$)", element_xpath);

	for (const xmlNode *change = first_named_child(patchset, XML_DIFF_CHANGE);
	change != NULL; change = crm_next_same_xml(change)) {

	const char *op = crm_element_value(change, F_CIB_OPERATION);
	const char *diff_xpath = crm_element_value(change, PCMK_XA_PATH);

	if (pcmk__str_eq(diff_xpath, element_regex, pcmk__str_regex)) {
	// Change to an existing element
	rc = true;
	break;
	}

	if (pcmk__str_eq(op, "create", pcmk__str_none)
	&& pcmk__str_eq(diff_xpath, parent_xpath, pcmk__str_none)
	&& pcmk__xe_is(pcmk__xml_first_child(change), element)) {

	// Newly added element
	rc = true;
	break;
	}
	}

	free(element_regex);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c PCMK_XE_CIB). Supported values include any CIB
	* element supported by \c pcmk__cib_abs_xpath_for().
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	bool
	cib__element_in_patchset(const xmlNode patchset, const char element)
	{
	int format = 1;

	CRM_ASSERT(patchset != NULL);

	crm_element_value_int(patchset, PCMK_XA_FORMAT, &format);
	switch (format) {
	case 1:
	return element_in_patchset_v1(patchset, element);

	case 2:
	return element_in_patchset_v2(patchset, element);

	default:
	crm_warn("Unknown patch format: %d", format);
	return false;
	}
	}

	/*!
	* \brief Create XML for a new (empty) CIB
	*
	* \param[in] cib_epoch What to use as \c PCMK_XA_EPOCH CIB attribute
	*
	* \return Newly created XML for empty CIB
	* \note It is the caller's responsibility to free the result with free_xml().
	*/
	xmlNode *
	createEmptyCib(int cib_epoch)
	{
	xmlNode cib_root = NULL, config = NULL;

	cib_root = create_xml_node(NULL, PCMK_XE_CIB);
	crm_xml_add(cib_root, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
	crm_xml_add(cib_root, PCMK_XA_VALIDATE_WITH, xml_latest_schema());

	crm_xml_add_int(cib_root, PCMK_XA_EPOCH, cib_epoch);
	crm_xml_add_int(cib_root, PCMK_XA_NUM_UPDATES, 0);
	crm_xml_add_int(cib_root, PCMK_XA_ADMIN_EPOCH, 0);

	config = create_xml_node(cib_root, PCMK_XE_CONFIGURATION);
	create_xml_node(cib_root, PCMK_XE_STATUS);

	create_xml_node(config, PCMK_XE_CRM_CONFIG);
	create_xml_node(config, PCMK_XE_NODES);
	create_xml_node(config, PCMK_XE_RESOURCES);
	create_xml_node(config, PCMK_XE_CONSTRAINTS);

	#if PCMK__RESOURCE_STICKINESS_DEFAULT != 0
	{
	xmlNode *rsc_defaults = create_xml_node(config, PCMK_XE_RSC_DEFAULTS);
	xmlNode *meta = create_xml_node(rsc_defaults, PCMK_XE_META_ATTRIBUTES);
	xmlNode *nvpair = create_xml_node(meta, PCMK_XE_NVPAIR);

	crm_xml_add(meta, PCMK_XA_ID, "build-resource-defaults");
	crm_xml_add(nvpair, PCMK_XA_ID, "build-" PCMK_META_RESOURCE_STICKINESS);
	crm_xml_add(nvpair, PCMK_XA_NAME, PCMK_META_RESOURCE_STICKINESS);
	crm_xml_add_int(nvpair, PCMK_XA_VALUE,
	PCMK__RESOURCE_STICKINESS_DEFAULT);
	}
	#endif
	return cib_root;
	}

	static bool
	cib_acl_enabled(xmlNode xml, const char user)
	{
	bool rc = FALSE;

	if(pcmk_acl_required(user)) {
	const char *value = NULL;
	GHashTable *options = pcmk__strkey_table(free, free);

	cib_read_config(options, xml);
	value = cib_pref(options, PCMK_OPT_ENABLE_ACL);
	rc = crm_is_true(value);
	g_hash_table_destroy(options);
	}

	crm_trace("CIB ACL is %s", rc ? "enabled" : "disabled");
	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether to perform operations on a scratch copy of the CIB
	*
	* \param[in] op CIB operation
	* \param[in] section CIB section
	* \param[in] call_options CIB call options
	*
	* \return \p true if we should make a copy of the CIB, or \p false otherwise
	*/
	static bool
	should_copy_cib(const char op, const char section, int call_options)
	{
	if (pcmk_is_set(call_options, cib_dryrun)) {
	// cib_dryrun implies a scratch copy by definition; no side effects
	return true;
	}

	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_COMMIT_TRANSACT, pcmk__str_none)) {
	/* Commit-transaction must make a copy for atomicity. We must revert to
	* the original CIB if the entire transaction cannot be applied
	* successfully.
	*/
	return true;
	}

	if (pcmk_is_set(call_options, cib_transaction)) {
	/* If cib_transaction is set, then we're in the process of committing a
	* transaction. The commit-transaction request already made a scratch
	* copy, and we're accumulating changes in that copy.
	*/
	return false;
	}

	if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_none)) {
	/* Copying large CIBs accounts for a huge percentage of our CIB usage,
	* and this avoids some of it.
	*
	* @TODO: Is this safe? See discussion at
	* https://github.com/ClusterLabs/pacemaker/pull/3094#discussion_r1211400690.
	*/
	return false;
	}

	// Default behavior is to operate on a scratch copy
	return true;
	}

	int
	cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, bool is_query,
	const char section, xmlNode req, xmlNode *input,
	bool manage_counters, bool *config_changed,
	xmlNode current_cib, xmlNode result_cib, xmlNode **diff,
	xmlNode **output)
	{
	int rc = pcmk_ok;
	bool check_schema = true;
	bool make_copy = true;
	xmlNode *top = NULL;
	xmlNode *scratch = NULL;
	xmlNode *patchset_cib = NULL;
	xmlNode *local_diff = NULL;

	const char *new_version = NULL;
	const char *user = crm_element_value(req, F_CIB_USER);
	bool with_digest = false;

	crm_trace("Begin %s%s%s op",
	(pcmk_is_set(call_options, cib_dryrun)? "dry run of " : ""),
	(is_query? "read-only " : ""), op);

	CRM_CHECK(output != NULL, return -ENOMSG);
	CRM_CHECK(current_cib != NULL, return -ENOMSG);
	CRM_CHECK(result_cib != NULL, return -ENOMSG);
	CRM_CHECK(config_changed != NULL, return -ENOMSG);

	if(output) {
	*output = NULL;
	}

	*result_cib = NULL;
	*config_changed = false;

	if (fn == NULL) {
	return -EINVAL;
	}

	if (is_query) {
	xmlNode cib_ro = current_cib;
	xmlNode *cib_filtered = NULL;

	if (cib_acl_enabled(cib_ro, user)
	&& xml_acl_filtered_copy(user, current_cib, current_cib,
	&cib_filtered)) {

	if (cib_filtered == NULL) {
	crm_debug("Pre-filtered the entire cib");
	return -EACCES;
	}
	cib_ro = cib_filtered;
	crm_log_xml_trace(cib_ro, "filtered");
	}

	rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output);

	if(output == NULL \|\| *output == NULL) {
	/* nothing */

	} else if(cib_filtered == *output) {
	cib_filtered = NULL; /* Let them have this copy */

	} else if (output == current_cib) {
	/* They already know not to free it */

	} else if(cib_filtered && (*output)->doc == cib_filtered->doc) {
	/* We're about to free the document of which output is a part /
	output = copy_xml(output);

	} else if ((output)->doc == (current_cib)->doc) {
	/* Give them a copy they can free */
	output = copy_xml(output);
	}

	free_xml(cib_filtered);
	return rc;
	}

	make_copy = should_copy_cib(op, section, call_options);

	if (!make_copy) {
	/* Conditional on v2 patch style */

	scratch = *current_cib;

	// Make a copy of the top-level element to store version details
	top = create_xml_node(NULL, (const char *) scratch->name);
	copy_in_properties(top, scratch);
	patchset_cib = top;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output);

	/* If scratch points to a new object now (for example, after an erase
	* operation), then *current_cib should point to the same object.
	*/
	*current_cib = scratch;

	} else {
	scratch = copy_xml(*current_cib);
	patchset_cib = *current_cib;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (fn) (op, call_options, section, req, input, current_cib,
	&scratch, output);

	if ((scratch != NULL) && !xml_tracking_changes(scratch)) {
	crm_trace("Inferring changes after %s op", op);
	xml_track_changes(scratch, user, *current_cib,
	cib_acl_enabled(*current_cib, user));
	xml_calculate_changes(*current_cib, scratch);
	}
	CRM_CHECK(*current_cib != scratch, return -EINVAL);
	}

	xml_acl_disable(scratch); /* Allow the system to make any additional changes */

	if (rc == pcmk_ok && scratch == NULL) {
	rc = -EINVAL;
	goto done;

	} else if(rc == pcmk_ok && xml_acl_denied(scratch)) {
	crm_trace("ACL rejected part or all of the proposed changes");
	rc = -EACCES;
	goto done;

	} else if (rc != pcmk_ok) {
	goto done;
	}

	if (scratch) {
	new_version = crm_element_value(scratch, PCMK_XA_CRM_FEATURE_SET);

	if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
	crm_err("Discarding update with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	rc = -EPROTONOSUPPORT;
	goto done;
	}
	}

	if (patchset_cib != NULL) {
	int old = 0;
	int new = 0;

	crm_element_value_int(scratch, PCMK_XA_ADMIN_EPOCH, &new);
	crm_element_value_int(patchset_cib, PCMK_XA_ADMIN_EPOCH, &old);

	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: %#x)",
	PCMK_XA_ADMIN_EPOCH, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;

	} else if (old == new) {
	crm_element_value_int(scratch, PCMK_XA_EPOCH, &new);
	crm_element_value_int(patchset_cib, PCMK_XA_EPOCH, &old);
	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: %#x)",
	PCMK_XA_EPOCH, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;
	}
	}
	}

	crm_trace("Massaging CIB contents");
	pcmk__strip_xml_text(scratch);
	fix_plus_plus_recursive(scratch);

	if (!make_copy) {
	/* At this point, patchset_cib is just the PCMK_XE_CIB tag and its
	* properties.
	*
	* The v1 format would barf on this, but we know the v2 patch
	* format only needs it for the top-level version fields
	*/
	local_diff = xml_create_patchset(2, patchset_cib, scratch,
	config_changed, manage_counters);

	} else {
	static time_t expires = 0;
	time_t tm_now = time(NULL);

	if (expires < tm_now) {
	expires = tm_now + 60; /* Validate clients are correctly applying v2-style diffs at most once a minute */
	with_digest = true;
	}

	local_diff = xml_create_patchset(0, patchset_cib, scratch,
	config_changed, manage_counters);
	}

	pcmk__log_xml_changes(LOG_TRACE, scratch);
	xml_accept_changes(scratch);

	if(local_diff) {
	patchset_process_digest(local_diff, patchset_cib, scratch, with_digest);
	pcmk__log_xml_patchset(LOG_INFO, local_diff);
	crm_log_xml_trace(local_diff, "raw patch");
	}

	if (make_copy && (local_diff != NULL)) {
	// Original to compare against doesn't exist
	pcmk__if_tracing(
	{
	// Validate the calculated patch set
	int test_rc = pcmk_ok;
	int format = 1;
	xmlNode *cib_copy = copy_xml(patchset_cib);

	crm_element_value_int(local_diff, PCMK_XA_FORMAT, &format);
	test_rc = xml_apply_patchset(cib_copy, local_diff,
	manage_counters);

	if (test_rc != pcmk_ok) {
	save_xml_to_file(cib_copy, "PatchApply:calculated", NULL);
	save_xml_to_file(patchset_cib, "PatchApply:input", NULL);
	save_xml_to_file(scratch, "PatchApply:actual", NULL);
	save_xml_to_file(local_diff, "PatchApply:diff", NULL);
	crm_err("v%d patchset error, patch failed to apply: %s "
	"(%d)",
	format, pcmk_rc_str(pcmk_legacy2rc(test_rc)),
	test_rc);
	}
	free_xml(cib_copy);
	},
	{}
	);
	}

	if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) {
	/* Throttle the amount of costly validation we perform due to status updates
	* a) we don't really care whats in the status section
	* b) we don't validate any of its contents at the moment anyway
	*/
	check_schema = false;
	}

	/* === scratch must not be modified after this point ===
	* Exceptions, anything in:

	static filter_t filter[] = {
	{ 0, PCMK_XA_CRM_DEBUG_ORIGIN },
	{ 0, PCMK_XA_CIB_LAST_WRITTEN },
	{ 0, PCMK_XA_UPDATE_ORIGIN },
	{ 0, PCMK_XA_UPDATE_CLIENT },
	{ 0, PCMK_XA_UPDATE_USER },
	};
	*/

	if (*config_changed && !pcmk_is_set(call_options, cib_no_mtime)) {
	const char *schema = crm_element_value(scratch, PCMK_XA_VALIDATE_WITH);

	pcmk__xe_add_last_written(scratch);
	if (schema) {
	static int minimum_schema = 0;
	int current_schema = get_schema_version(schema);

	if (minimum_schema == 0) {
	minimum_schema = get_schema_version("pacemaker-1.2");
	}

	/* Does the CIB support the "update-" attributes... /
	if (current_schema >= minimum_schema) {
	/* Ensure values of origin, client, and user in scratch match
	* the values in req
	*/
	const char *origin = crm_element_value(req, PCMK__XA_SRC);
	const char *client = crm_element_value(req, F_CIB_CLIENTNAME);

	if (origin != NULL) {
	crm_xml_add(scratch, PCMK_XA_UPDATE_ORIGIN, origin);
	} else {
	xml_remove_prop(scratch, PCMK_XA_UPDATE_ORIGIN);
	}

	if (client != NULL) {
	crm_xml_add(scratch, PCMK_XA_UPDATE_CLIENT, user);
	} else {
	xml_remove_prop(scratch, PCMK_XA_UPDATE_CLIENT);
	}

	if (user != NULL) {
	crm_xml_add(scratch, PCMK_XA_UPDATE_USER, user);
	} else {
	xml_remove_prop(scratch, PCMK_XA_UPDATE_USER);
	}
	}
	}
	}

	crm_trace("Perform validation: %s", pcmk__btoa(check_schema));
	if ((rc == pcmk_ok) && check_schema && !validate_xml(scratch, NULL, true)) {
	const char *current_schema = crm_element_value(scratch,
	PCMK_XA_VALIDATE_WITH);

	crm_warn("Updated CIB does not validate against %s schema",
	pcmk__s(current_schema, "unspecified"));
	rc = -pcmk_err_schema_validation;
	}

	done:

	*result_cib = scratch;

	/* @TODO: This may not work correctly with !make_copy, since we don't
	* keep the original CIB.
	*/
	if ((rc != pcmk_ok) && cib_acl_enabled(patchset_cib, user)
	&& xml_acl_filtered_copy(user, patchset_cib, scratch, result_cib)) {

	if (*result_cib == NULL) {
	crm_debug("Pre-filtered the entire cib result");
	}
	free_xml(scratch);
	}

	if(diff) {
	*diff = local_diff;
	} else {
	free_xml(local_diff);
	}

	free_xml(top);
	crm_trace("Done");
	return rc;
	}

	int
	cib__create_op(cib_t cib, const char op, const char *host,
	const char section, xmlNode data, int call_options,
	const char user_name, const char client_name,
	xmlNode **op_msg)
	{
	CRM_CHECK((cib != NULL) && (op_msg != NULL), return -EPROTO);

	*op_msg = create_xml_node(NULL, T_CIB_COMMAND);
	if (*op_msg == NULL) {
	return -EPROTO;
	}

	cib->call_id++;
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	crm_xml_add(*op_msg, PCMK__XA_T, T_CIB);
	crm_xml_add(*op_msg, F_CIB_OPERATION, op);
	crm_xml_add(*op_msg, F_CIB_HOST, host);
	crm_xml_add(*op_msg, F_CIB_SECTION, section);
	crm_xml_add(*op_msg, F_CIB_USER, user_name);
	crm_xml_add(*op_msg, F_CIB_CLIENTNAME, client_name);
	crm_xml_add_int(*op_msg, F_CIB_CALLID, cib->call_id);

	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(*op_msg, F_CIB_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(*op_msg, F_CIB_CALLDATA, data);
	}

	if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
	CRM_CHECK(pcmk_is_set(call_options, cib_scope_local),
	free_xml(*op_msg); return -EPROTO);
	}
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Check whether a CIB request is supported in a transaction
	*
	* \param[in] request CIB request
	*
	* \return Standard Pacemaker return code
	*/
	static int
	validate_transaction_request(const xmlNode *request)
	{
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *host = crm_element_value(request, F_CIB_HOST);
	const cib__operation_t *operation = NULL;
	int rc = cib__get_operation(op, &operation);

	if (rc != pcmk_rc_ok) {
	// cib__get_operation() logs error
	return rc;
	}

	if (!pcmk_is_set(operation->flags, cib__op_attr_transaction)) {
	crm_err("Operation %s is not supported in CIB transactions", op);
	return EOPNOTSUPP;
	}

	if (host != NULL) {
	crm_err("Operation targeting a specific node (%s) is not supported in "
	"a CIB transaction",
	host);
	return EOPNOTSUPP;
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Append a CIB request to a CIB transaction
	*
	* \param[in,out] cib CIB client whose transaction to extend
	* \param[in,out] request Request to add to transaction
	*
	* \return Legacy Pacemaker return code
	*/
	int
	cib__extend_transaction(cib_t cib, xmlNode request)
	{
	int rc = pcmk_rc_ok;

	CRM_ASSERT((cib != NULL) && (request != NULL));

	rc = validate_transaction_request(request);

	if ((rc == pcmk_rc_ok) && (cib->transaction == NULL)) {
	rc = pcmk_rc_no_transaction;
	}

	if (rc == pcmk_rc_ok) {
	add_node_copy(cib->transaction, request);

	} else {
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *client_id = NULL;

	cib->cmds->client_id(cib, NULL, &client_id);
	crm_err("Failed to add '%s' operation to transaction for client %s: %s",
	op, pcmk__s(client_id, "(unidentified)"), pcmk_rc_str(rc));
	crm_log_xml_info(request, "failed");
	}
	return pcmk_rc2legacy(rc);
	}

	void
	cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	cib_callback_client_t *blob = NULL;

	if (msg != NULL) {
	crm_element_value_int(msg, F_CIB_RC, &rc);
	crm_element_value_int(msg, F_CIB_CALLID, &call_id);
	output = get_message_xml(msg, F_CIB_CALLDATA);
	}

	blob = cib__lookup_id(call_id);

	if (blob == NULL) {
	crm_trace("No callback found for call %d", call_id);
	}

	if (cib == NULL) {
	crm_debug("No cib object supplied");
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (blob && blob->callback && (rc == pcmk_ok \|\| blob->only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d",
	pcmk__s(blob->id, "without ID"), call_id);
	blob->callback(msg, call_id, rc, output, blob->user_data);

	} else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("CIB command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed CIB Update");
	}

	/* This may free user_data, so do it after the callback */
	if (blob) {
	remove_cib_op_callback(call_id, FALSE);
	}

	if (cib && cib->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	cib->op_callback(msg, call_id, rc, output);
	}
	crm_trace("OP callback activated for %d", call_id);
	}

	void
	cib_native_notify(gpointer data, gpointer user_data)
	{
	xmlNode *msg = user_data;
	cib_notify_client_t *entry = data;
	const char *event = NULL;

	if (msg == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(msg, PCMK__XA_SUBT);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->callback == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (!pcmk__str_eq(entry->event, event, pcmk__str_casei)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->callback(event, msg);
	crm_trace("Callback invoked...");
	}

	static pcmk__cluster_option_t cib_opts[] = {
	/* name, legacy name, type, allowed values,
	* default value, validator,
	* context,
	* short description,
	* long description
	*/
	{
	PCMK_OPT_ENABLE_ACL, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_based,
	N_("Enable Access Control Lists (ACLs) for the CIB"),
	NULL
	},
	{
	PCMK_OPT_CLUSTER_IPC_LIMIT, NULL, "integer", NULL,
	"500", pcmk__valid_positive_int,
	pcmk__opt_context_based,
	N_("Maximum IPC message backlog before disconnecting a cluster daemon"),
	N_("Raise this if log has \"Evicting client\" messages for cluster daemon"
	" PIDs (a good value is the number of resources in the cluster"
	" multiplied by the number of nodes).")
	},
	};

	void
	cib_metadata(void)
	{
	const char *name = "pacemaker-based";
	const char *desc_short = "Cluster Information Base manager options";
	const char *desc_long = "Cluster options used by Pacemaker's Cluster "
	"Information Base manager";

	- gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long,
	- pcmk__opt_context_based,
	- cib_opts, PCMK__NELEM(cib_opts));
	+ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long,
	+ pcmk__opt_context_based);
	+
	printf("%s", s);
	g_free(s);
	}

	static void
	verify_cib_options(GHashTable *options)
	{
	pcmk__validate_cluster_options(options, cib_opts, PCMK__NELEM(cib_opts));
	}

	const char *
	cib_pref(GHashTable * options, const char *name)
	{
	return pcmk__cluster_option(options, cib_opts, PCMK__NELEM(cib_opts),
	name);
	}

	gboolean
	cib_read_config(GHashTable * options, xmlNode * current_cib)
	{
	xmlNode *config = NULL;
	crm_time_t *now = NULL;

	if (options == NULL \|\| current_cib == NULL) {
	return FALSE;
	}

	now = crm_time_new(NULL);

	g_hash_table_remove_all(options);

	config = pcmk_find_cib_element(current_cib, PCMK_XE_CRM_CONFIG);
	if (config) {
	pe_unpack_nvpairs(current_cib, config, PCMK_XE_CLUSTER_PROPERTY_SET,
	NULL, options, CIB_OPTIONS_FIRST, TRUE, now, NULL);
	}

	verify_cib_options(options);

	crm_time_free(now);

	return TRUE;
	}

	int
	cib_internal_op(cib_t * cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name)
	{
	int (delegate) (cib_t cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name) =
	cib->delegate_fn;

	if(user_name == NULL) {
	user_name = getenv("CIB_user");
	}

	return delegate(cib, op, host, section, data, output_data, call_options, user_name);
	}

	/*!
	* \brief Apply a CIB update patch to a given CIB
	*
	* \param[in] event CIB update patch
	* \param[in] input CIB to patch
	* \param[out] output Resulting CIB after patch
	* \param[in] level Log the patch at this log level (unless LOG_CRIT)
	*
	* \return Legacy Pacemaker return code
	* \note sbd calls this function
	*/
	int
	cib_apply_patch_event(xmlNode event, xmlNode input, xmlNode **output,
	int level)
	{
	int rc = pcmk_err_generic;

	xmlNode *diff = NULL;

	CRM_ASSERT(event);
	CRM_ASSERT(input);
	CRM_ASSERT(output);

	crm_element_value_int(event, F_CIB_RC, &rc);
	diff = get_message_xml(event, F_CIB_UPDATE_RESULT);

	if (rc < pcmk_ok \|\| diff == NULL) {
	return rc;
	}

	if (level > LOG_CRIT) {
	pcmk__log_xml_patchset(level, diff);
	}

	if (input != NULL) {
	rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output,
	NULL);

	if (rc != pcmk_ok) {
	crm_debug("Update didn't apply: %s (%d) %p",
	pcmk_strerror(rc), rc, *output);

	if (rc == -pcmk_err_old_data) {
	crm_trace("Masking error, we already have the supplied update");
	return pcmk_ok;
	}
	free_xml(*output);
	*output = NULL;
	return rc;
	}
	}
	return rc;
	}

	#define log_signon_query_err(out, fmt, args...) do { \
	if (out != NULL) { \
	out->err(out, fmt, ##args); \
	} else { \
	crm_err(fmt, ##args); \
	} \
	} while (0)

	int
	cib__signon_query(pcmk__output_t out, cib_t cib, xmlNode *cib_object)
	{
	int rc = pcmk_rc_ok;
	cib_t *cib_conn = NULL;

	CRM_ASSERT(cib_object != NULL);

	if (cib == NULL) {
	cib_conn = cib_new();
	} else {
	if (*cib == NULL) {
	*cib = cib_new();
	}
	cib_conn = *cib;
	}

	if (cib_conn == NULL) {
	return ENOMEM;
	}

	if (cib_conn->state == cib_disconnected) {
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);
	}

	if (rc != pcmk_rc_ok) {
	log_signon_query_err(out, "Could not connect to the CIB: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	if (out != NULL) {
	out->transient(out, "Querying CIB...");
	}
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_object,
	cib_scope_local\|cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	log_signon_query_err(out, "CIB query failed: %s", pcmk_rc_str(rc));
	}

	done:
	if (cib == NULL) {
	cib__clean_up_connection(&cib_conn);
	}

	if ((rc == pcmk_rc_ok) && (*cib_object == NULL)) {
	return pcmk_rc_no_input;
	}
	return rc;
	}

	int
	cib__clean_up_connection(cib_t **cib)
	{
	int rc;

	if (*cib == NULL) {
	return pcmk_rc_ok;
	}

	rc = (cib)->cmds->signoff(cib);
	cib_delete(*cib);
	*cib = NULL;
	return pcmk_legacy2rc(rc);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/cib/util_compat.h>

	xmlNode *
	cib_get_generation(cib_t * cib)
	{
	xmlNode *the_cib = NULL;
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	cib->cmds->query(cib, NULL, &the_cib, cib_scope_local \| cib_sync_call);
	if (the_cib != NULL) {
	copy_in_properties(generation, the_cib);
	free_xml(the_cib);
	}

	return generation;
	}

	const char *
	get_object_path(const char *object_type)
	{
	return pcmk_cib_xpath_for(object_type);
	}

	const char *
	get_object_parent(const char *object_type)
	{
	return pcmk_cib_parent_name_for(object_type);
	}

	xmlNode *
	get_object_root(const char object_type, xmlNode the_root)
	{
	return pcmk_find_cib_element(the_root, object_type);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/common/options.c b/lib/common/options.c
	index fe210d6a19..d52f437328 100644
	--- a/lib/common/options.c
	+++ b/lib/common/options.c
	@@ -1,1134 +1,1158 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <stdlib.h>
	#include <sys/types.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>

	void
	pcmk__cli_help(char cmd)
	{
	if (cmd == 'v' \|\| cmd == '$') {
	printf("Pacemaker %s\n", PACEMAKER_VERSION);
	printf("Written by Andrew Beekhof and "
	"the Pacemaker project contributors\n");

	} else if (cmd == '!') {
	printf("Pacemaker %s (Build: %s): %s\n", PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
	}

	crm_exit(CRM_EX_OK);
	while(1); // above does not return
	}


	/*
	* Option metadata
	*/

	-#if 0
	static pcmk__cluster_option_t cluster_options[] = {
	/* name, old name, type, allowed values,
	* default value, validator,
	* context,
	* short description,
	* long description
	*/
	{
	PCMK_OPT_DC_VERSION, NULL, "string", NULL,
	PCMK__VALUE_NONE, NULL,
	pcmk__opt_context_controld,
	N_("Pacemaker version on cluster node elected Designated Controller "
	"(DC)"),
	N_("Includes a hash which identifies the exact changeset the code was "
	"built from. Used for diagnostic purposes."),
	},
	{
	PCMK_OPT_CLUSTER_INFRASTRUCTURE, NULL, "string", NULL,
	"corosync", NULL,
	pcmk__opt_context_controld,
	N_("The messaging stack on which Pacemaker is currently running"),
	N_("Used for informational and diagnostic purposes."),
	},
	{
	PCMK_OPT_CLUSTER_NAME, NULL, "string", NULL,
	NULL, NULL,
	pcmk__opt_context_controld,
	N_("An arbitrary name for the cluster"),
	N_("This optional value is mostly for users' convenience as desired "
	"in administration, but may also be used in Pacemaker "
	"configuration rules via the #cluster-name node attribute, and "
	"by higher-level tools and resource agents."),
	},
	{
	PCMK_OPT_DC_DEADTIME, NULL, "time", NULL,
	"20s", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("How long to wait for a response from other nodes during start-up"),
	N_("The optimal value will depend on the speed and load of your "
	"network and the type of switches used."),
	},
	{
	PCMK_OPT_CLUSTER_RECHECK_INTERVAL, NULL, "time",
	N_("Zero disables polling, while positive values are an interval in "
	"seconds (unless other units are specified, for example \"5min\")"),
	"15min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("Polling interval to recheck cluster state and evaluate rules "
	"with date specifications"),
	N_("Pacemaker is primarily event-driven, and looks ahead to know when "
	"to recheck cluster state for failure timeouts and most time-based "
	"rules. However, it will also recheck the cluster after this "
	"amount of inactivity, to evaluate rules with date specifications "
	"and serve as a fail-safe for certain types of scheduler bugs."),
	},
	{
	PCMK_OPT_FENCE_REACTION, NULL, "select", "stop, panic",
	"stop", NULL,
	pcmk__opt_context_controld,
	N_("How a cluster node should react if notified of its own fencing"),
	N_("A cluster node may receive notification of its own fencing if "
	"fencing is misconfigured, or if fabric fencing is in use that "
	"doesn't cut cluster communication. Use \"stop\" to attempt to "
	"immediately stop Pacemaker and stay stopped, or \"panic\" to "
	"attempt to immediately reboot the local node, falling back to "
	"stop on failure."),
	},
	{
	PCMK_OPT_ELECTION_TIMEOUT, NULL, "time", NULL,
	"2min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("* Advanced Use Only *"),
	N_("Declare an election failed if it is not decided within this much "
	"time. If you need to adjust this value, it probably indicates "
	"the presence of a bug."),
	},
	{
	PCMK_OPT_SHUTDOWN_ESCALATION, NULL, "time", NULL,
	"20min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("* Advanced Use Only *"),
	N_("Exit immediately if shutdown does not complete within this much "
	"time. If you need to adjust this value, it probably indicates "
	"the presence of a bug."),
	},
	{
	PCMK_OPT_JOIN_INTEGRATION_TIMEOUT, "crmd-integration-timeout", "time",
	NULL,
	"3min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("* Advanced Use Only *"),
	N_("If you need to adjust this value, it probably indicates "
	"the presence of a bug."),
	},
	{
	PCMK_OPT_JOIN_FINALIZATION_TIMEOUT, "crmd-finalization-timeout",
	"time", NULL,
	"30min", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("* Advanced Use Only *"),
	N_("If you need to adjust this value, it probably indicates "
	"the presence of a bug."),
	},
	{
	PCMK_OPT_TRANSITION_DELAY, "crmd-transition-delay", "time", NULL,
	"0s", pcmk__valid_interval_spec,
	pcmk__opt_context_controld,
	N_("* Advanced Use Only * "
	"Enabling this option will slow down cluster recovery under all "
	"conditions"),
	N_("Delay cluster recovery for this much time to allow for additional "
	"events to occur. Useful if your configuration is sensitive to "
	"the order in which ping updates arrive."),
	},
	{
	PCMK_OPT_NO_QUORUM_POLICY, NULL, "select",
	"stop, freeze, ignore, demote, suicide",
	"stop", pcmk__valid_no_quorum_policy,
	pcmk__opt_context_schedulerd,
	N_("What to do when the cluster does not have quorum"),
	NULL,
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether to lock resources to a cleanly shut down node"),
	N_("When true, resources active on a node when it is cleanly shut down "
	"are kept \"locked\" to that node (not allowed to run elsewhere) "
	"until they start again on that node after it rejoins (or for at "
	"most shutdown-lock-limit, if set). Stonith resources and "
	"Pacemaker Remote connections are never locked. Clone and bundle "
	"instances and the promoted role of promotable clones are "
	"currently never locked, though support could be added in a future "
	"release."),
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("Do not lock resources to a cleanly shut down node longer than "
	"this"),
	N_("If shutdown-lock is true and this is set to a nonzero time "
	"duration, shutdown locks will expire after this much time has "
	"passed since the shutdown was initiated, even if the node has not "
	"rejoined."),
	},
	{
	PCMK_OPT_ENABLE_ACL, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_based,
	N_("Enable Access Control Lists (ACLs) for the CIB"),
	NULL,
	},
	{
	PCMK_OPT_SYMMETRIC_CLUSTER, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether resources can run on any node by default"),
	NULL,
	},
	{
	PCMK_OPT_MAINTENANCE_MODE, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether the cluster should refrain from monitoring, starting, and "
	"stopping resources"),
	NULL,
	},
	{
	PCMK_OPT_START_FAILURE_IS_FATAL, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether a start failure should prevent a resource from being "
	"recovered on the same node"),
	N_("When true, the cluster will immediately ban a resource from a node "
	"if it fails to start there. When false, the cluster will instead "
	"check the resource's fail count against its migration-threshold.")
	},
	{
	PCMK_OPT_ENABLE_STARTUP_PROBES, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether the cluster should check for active resources during "
	"start-up"),
	NULL,
	},

	// Fencing-related options
	{
	PCMK_OPT_STONITH_ENABLED, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("* Advanced Use Only * "
	"Whether nodes may be fenced as part of recovery"),
	N_("If false, unresponsive nodes are immediately assumed to be "
	"harmless, and resources that were active on them may be recovered "
	"elsewhere. This can result in a \"split-brain\" situation, "
	"potentially leading to data loss and/or service unavailability."),
	},
	{
	PCMK_OPT_STONITH_ACTION, NULL, "select", "reboot, off, poweroff",
	PCMK_ACTION_REBOOT, pcmk__is_fencing_action,
	pcmk__opt_context_schedulerd,
	N_("Action to send to fence device when a node needs to be fenced "
	"(\"poweroff\" is a deprecated alias for \"off\")"),
	NULL,
	},
	{
	PCMK_OPT_STONITH_TIMEOUT, NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("How long to wait for on, off, and reboot fence actions to complete "
	"by default"),
	NULL,
	},
	{
	PCMK_OPT_HAVE_WATCHDOG, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether watchdog integration is enabled"),
	N_("This is set automatically by the cluster according to whether SBD "
	"is detected to be in use. User-configured values are ignored. "
	"The value `true` is meaningful if diskless SBD is used and "
	"`stonith-watchdog-timeout` is nonzero. In that case, if fencing "
	"is required, watchdog-based self-fencing will be performed via "
	"SBD without requiring a fencing resource explicitly configured."),
	},
	{
	/* @COMPAT Currently, unparsable values default to -1 (auto-calculate),
	* while missing values default to 0 (disable). All values are accepted
	* (unless the controller finds that the value conflicts with the
	* SBD_WATCHDOG_TIMEOUT).
	*
	* At a compatibility break: properly validate as a timeout, let
	* either negative values or a particular string like "auto" mean auto-
	* calculate, and use 0 as the single default for when the option either
	* is unset or fails to validate.
	*/
	PCMK_OPT_STONITH_WATCHDOG_TIMEOUT, NULL, "time", NULL,
	"0", NULL,
	pcmk__opt_context_controld,
	N_("How long before nodes can be assumed to be safely down when "
	"watchdog-based self-fencing via SBD is in use"),
	N_("If this is set to a positive value, lost nodes are assumed to "
	"self-fence using watchdog-based SBD within this much time. This "
	"does not require a fencing resource to be explicitly configured, "
	"though a fence_watchdog resource can be configured, to limit use "
	"to specific nodes. If this is set to 0 (the default), the cluster "
	"will never assume watchdog-based self-fencing. If this is set to a "
	"negative value, the cluster will use twice the local value of the "
	"`SBD_WATCHDOG_TIMEOUT` environment variable if that is positive, "
	"or otherwise treat this as 0. WARNING: When used, this timeout "
	"must be larger than `SBD_WATCHDOG_TIMEOUT` on all nodes that use "
	"watchdog-based SBD, and Pacemaker will refuse to start on any of "
	"those nodes where this is not true for the local value or SBD is "
	"not active. When this is set to a negative value, "
	"`SBD_WATCHDOG_TIMEOUT` must be set to the same value on all nodes "
	"that use SBD, otherwise data corruption or loss could occur."),
	},
	{
	PCMK_OPT_STONITH_MAX_ATTEMPTS, NULL, "integer", NULL,
	"10", pcmk__valid_positive_int,
	pcmk__opt_context_controld,
	N_("How many times fencing can fail before it will no longer be "
	"immediately re-attempted on a target"),
	NULL,
	},
	{
	PCMK_OPT_CONCURRENT_FENCING, NULL, "boolean", NULL,
	PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Allow performing fencing operations in parallel"),
	NULL,
	},
	{
	PCMK_OPT_STARTUP_FENCING, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("* Advanced Use Only * "
	"Whether to fence unseen nodes at start-up"),
	N_("Setting this to false may lead to a \"split-brain\" situation, "
	"potentially leading to data loss and/or service unavailability."),
	},
	{
	PCMK_OPT_PRIORITY_FENCING_DELAY, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("Apply fencing delay targeting the lost nodes with the highest "
	"total resource priority"),
	N_("Apply specified delay for the fencings that are targeting the lost "
	"nodes with the highest total resource priority in case we don't "
	"have the majority of the nodes in our cluster partition, so that "
	"the more significant nodes potentially win any fencing match, "
	"which is especially meaningful under split-brain of 2-node "
	"cluster. A promoted resource instance takes the base priority + 1 "
	"on calculation if the base priority is not 0. Any static/random "
	"delays that are introduced by `pcmk_delay_base/max` configured "
	"for the corresponding fencing resources will be added to this "
	"delay. This delay should be significantly greater than, safely "
	"twice, the maximum `pcmk_delay_base/max`. By default, priority "
	"fencing delay is disabled."),
	},
	{
	PCMK_OPT_NODE_PENDING_TIMEOUT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("How long to wait for a node that has joined the cluster to join "
	"the controller process group"),
	N_("Fence nodes that do not join the controller process group within "
	"this much time after joining the cluster, to allow the cluster "
	"to continue managing resources. A value of 0 means never fence "
	"pending nodes. Setting the value to 2h means fence nodes after "
	"2 hours."),
	},
	{
	PCMK_OPT_CLUSTER_DELAY, NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("Maximum time for node-to-node communication"),
	N_("The node elected Designated Controller (DC) will consider an action "
	"failed if it does not get a response from the node executing the "
	"action within this time (after considering the action's own "
	"timeout). The \"correct\" value will depend on the speed and "
	"load of your network and cluster nodes.")
	},

	// Limits
	{
	PCMK_OPT_LOAD_THRESHOLD, NULL, "percentage", NULL,
	"80%", pcmk__valid_percentage,
	pcmk__opt_context_controld,
	N_("Maximum amount of system load that should be used by cluster "
	"nodes"),
	N_("The cluster will slow down its recovery process when the amount of "
	"system resources used (currently CPU) approaches this limit"),
	},
	{
	PCMK_OPT_NODE_ACTION_LIMIT, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_controld,
	N_("Maximum number of jobs that can be scheduled per node (defaults to "
	"2x cores)"),
	NULL,
	},
	{
	PCMK_OPT_BATCH_LIMIT, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("Maximum number of jobs that the cluster may execute in parallel "
	"across all nodes"),
	N_("The \"correct\" value will depend on the speed and load of your "
	"network and cluster nodes. If set to 0, the cluster will "
	"impose a dynamically calculated limit when any node has a "
	"high load."),
	},
	{
	PCMK_OPT_MIGRATION_LIMIT, NULL, "integer", NULL,
	"-1", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of live migration actions that the cluster is allowed "
	"to execute in parallel on a node (-1 means no limit)"),
	NULL,
	},
	{
	PCMK_OPT_CLUSTER_IPC_LIMIT, NULL, "integer", NULL,
	"500", pcmk__valid_positive_int,
	pcmk__opt_context_based,
	N_("Maximum IPC message backlog before disconnecting a cluster daemon"),
	N_("Raise this if log has \"Evicting client\" messages for cluster "
	"daemon PIDs (a good value is the number of resources in the "
	"cluster multiplied by the number of nodes)."),
	},

	// Orphans and stopping
	{
	PCMK_OPT_STOP_ALL_RESOURCES, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether the cluster should stop all active resources"),
	NULL,
	},
	{
	PCMK_OPT_STOP_ORPHAN_RESOURCES, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether to stop resources that were removed from the "
	"configuration"),
	NULL,
	},
	{
	PCMK_OPT_STOP_ORPHAN_ACTIONS, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether to cancel recurring actions removed from the "
	"configuration"),
	NULL,
	},
	{
	PCMK__OPT_REMOVE_AFTER_STOP, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("* Deprecated * "
	"Whether to remove stopped resources from the executor"),
	N_("Values other than default are poorly tested and potentially "
	"dangerous. This option will be removed in a future release."),
	},

	// Storing inputs
	{
	PCMK_OPT_PE_ERROR_SERIES_MAX, NULL, "integer", NULL,
	"-1", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of scheduler inputs resulting in errors to save"),
	N_("Zero to disable, -1 to store unlimited."),
	},
	{
	PCMK_OPT_PE_WARN_SERIES_MAX, NULL, "integer", NULL,
	"5000", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of scheduler inputs resulting in warnings to save"),
	N_("Zero to disable, -1 to store unlimited."),
	},
	{
	PCMK_OPT_PE_INPUT_SERIES_MAX, NULL, "integer", NULL,
	"4000", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of scheduler inputs without errors or warnings to save"),
	N_("Zero to disable, -1 to store unlimited."),
	},

	// Node health
	{
	PCMK_OPT_NODE_HEALTH_STRATEGY, NULL, "select",
	PCMK__VALUE_NONE ", " PCMK__VALUE_MIGRATE_ON_RED ", "
	PCMK__VALUE_ONLY_GREEN ", " PCMK__VALUE_PROGRESSIVE ", "
	PCMK__VALUE_CUSTOM,
	PCMK__VALUE_NONE, pcmk__validate_health_strategy,
	pcmk__opt_context_schedulerd,
	N_("How cluster should react to node health attributes"),
	N_("Requires external entities to create node attributes (named with "
	"the prefix \"#health\") with values \"red\", \"yellow\", or "
	"\"green\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_BASE, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("Base health score assigned to a node"),
	N_("Only used when \"node-health-strategy\" is set to "
	"\"progressive\"."),
	},
	{
	PCMK_OPT_NODE_HEALTH_GREEN, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The score to use for a node health attribute whose value is "
	"\"green\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or "
	"\"progressive\"."),
	},
	{
	PCMK_OPT_NODE_HEALTH_YELLOW, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The score to use for a node health attribute whose value is "
	"\"yellow\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or "
	"\"progressive\"."),
	},
	{
	PCMK_OPT_NODE_HEALTH_RED, NULL, "integer", NULL,
	"-INFINITY", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The score to use for a node health attribute whose value is "
	"\"red\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or "
	"\"progressive\".")
	},

	// Placement strategy
	{
	PCMK_OPT_PLACEMENT_STRATEGY, NULL, "select",
	"default, utilization, minimal, balanced",
	"default", pcmk__valid_placement_strategy,
	pcmk__opt_context_schedulerd,
	N_("How the cluster should allocate resources to nodes"),
	NULL,
	},
	};
	-#endif // 0


	/*
	* Environment variable option handling
	*/

	/*!
	* \internal
	* \brief Get the value of a Pacemaker environment variable option
	*
	* If an environment variable option is set, with either a PCMK_ or (for
	* backward compatibility) HA_ prefix, log and return the value.
	*
	* \param[in] option Environment variable name (without prefix)
	*
	* \return Value of environment variable option, or NULL in case of
	* option name too long or value not found
	*/
	const char *
	pcmk__env_option(const char *option)
	{
	const char *const prefixes[] = {"PCMK_", "HA_"};
	char env_name[NAME_MAX];
	const char *value = NULL;

	CRM_CHECK(!pcmk__str_empty(option), return NULL);

	for (int i = 0; i < PCMK__NELEM(prefixes); i++) {
	int rv = snprintf(env_name, NAME_MAX, "%s%s", prefixes[i], option);

	if (rv < 0) {
	crm_err("Failed to write %s%s to buffer: %s", prefixes[i], option,
	strerror(errno));
	return NULL;
	}

	if (rv >= sizeof(env_name)) {
	crm_trace("\"%s%s\" is too long", prefixes[i], option);
	continue;
	}

	value = getenv(env_name);
	if (value != NULL) {
	crm_trace("Found %s = %s", env_name, value);
	return value;
	}
	}

	crm_trace("Nothing found for %s", option);
	return NULL;
	}

	/*!
	* \brief Set or unset a Pacemaker environment variable option
	*
	* Set an environment variable option with a \c "PCMK_" prefix and optionally
	* an \c "HA_" prefix for backward compatibility.
	*
	* \param[in] option Environment variable name (without prefix)
	* \param[in] value New value (or NULL to unset)
	* \param[in] compat If false and \p value is not \c NULL, set only
	* \c "PCMK_<option>"; otherwise, set (or unset) both
	* \c "PCMK_<option>" and \c "HA_<option>"
	*
	* \note \p compat is ignored when \p value is \c NULL. A \c NULL \p value
	* means we're unsetting \p option. \c pcmk__get_env_option() checks for
	* both prefixes, so we want to clear them both.
	*/
	void
	pcmk__set_env_option(const char option, const char value, bool compat)
	{
	// @COMPAT Drop support for "HA_" options eventually
	const char *const prefixes[] = {"PCMK_", "HA_"};
	char env_name[NAME_MAX];

	CRM_CHECK(!pcmk__str_empty(option) && (strchr(option, '=') == NULL),
	return);

	for (int i = 0; i < PCMK__NELEM(prefixes); i++) {
	int rv = snprintf(env_name, NAME_MAX, "%s%s", prefixes[i], option);

	if (rv < 0) {
	crm_err("Failed to write %s%s to buffer: %s", prefixes[i], option,
	strerror(errno));
	return;
	}

	if (rv >= sizeof(env_name)) {
	crm_trace("\"%s%s\" is too long", prefixes[i], option);
	continue;
	}

	if (value != NULL) {
	crm_trace("Setting %s to %s", env_name, value);
	rv = setenv(env_name, value, 1);
	} else {
	crm_trace("Unsetting %s", env_name);
	rv = unsetenv(env_name);
	}

	if (rv < 0) {
	crm_err("Failed to %sset %s: %s", (value != NULL)? "" : "un",
	env_name, strerror(errno));
	}

	if (!compat && (value != NULL)) {
	// For set, don't proceed to HA_<option> unless compat is enabled
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Check whether Pacemaker environment variable option is enabled
	*
	* Given a Pacemaker environment variable option that can either be boolean
	* or a list of daemon names, return true if the option is enabled for a given
	* daemon.
	*
	* \param[in] daemon Daemon name (can be NULL)
	* \param[in] option Pacemaker environment variable name
	*
	* \return true if variable is enabled for daemon, otherwise false
	*/
	bool
	pcmk__env_option_enabled(const char daemon, const char option)
	{
	const char *value = pcmk__env_option(option);

	return (value != NULL)
	&& (crm_is_true(value)
	\|\| ((daemon != NULL) && (strstr(value, daemon) != NULL)));
	}


	/*
	* Cluster option handling
	*/

	/*!
	* \internal
	* \brief Check whether a string represents a valid interval specification
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid interval specification, or \c false
	* otherwise
	*/
	bool
	pcmk__valid_interval_spec(const char *value)
	{
	return pcmk_parse_interval_spec(value, NULL) == pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Check whether a string represents a valid boolean value
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid boolean value, or \c false otherwise
	*/
	bool
	pcmk__valid_boolean(const char *value)
	{
	return crm_str_to_boolean(value, NULL) == 1;
	}

	/*!
	* \internal
	* \brief Check whether a string represents a valid integer
	*
	* Valid values include \c INFINITY, \c -INFINITY, and all 64-bit integers.
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid integer, or \c false otherwise
	*/
	bool
	pcmk__valid_int(const char *value)
	{
	return (value != NULL)
	&& (pcmk_str_is_infinity(value)
	\|\| pcmk_str_is_minus_infinity(value)
	\|\| (pcmk__scan_ll(value, NULL, 0LL) == pcmk_rc_ok));
	}

	/*!
	* \internal
	* \brief Check whether a string represents a valid positive integer
	*
	* Valid values include \c INFINITY and all 64-bit positive integers.
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid positive integer, or \c false
	* otherwise
	*/
	bool
	pcmk__valid_positive_int(const char *value)
	{
	long long num = 0LL;

	return pcmk_str_is_infinity(value)
	\|\| ((pcmk__scan_ll(value, &num, 0LL) == pcmk_rc_ok)
	&& (num > 0));
	}

	/*!
	* \internal
	* \brief Check whether a string represents a valid
	* \c PCMK__OPT_NO_QUORUM_POLICY value
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid \c PCMK__OPT_NO_QUORUM_POLICY value,
	* or \c false otherwise
	*/
	bool
	pcmk__valid_no_quorum_policy(const char *value)
	{
	return pcmk__strcase_any_of(value,
	"stop", "freeze", "ignore", "demote", "suicide",
	NULL);
	}

	/*!
	* \internal
	* \brief Check whether a string represents a valid percentage
	*
	* Valid values include long integers, with an optional trailing string
	* beginning with '%'.
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid percentage value, or \c false
	* otherwise
	*/
	bool
	pcmk__valid_percentage(const char *value)
	{
	char *end = NULL;
	float number = strtof(value, &end);

	return ((end == NULL) \|\| (end[0] == '%')) && (number >= 0);
	}

	/*!
	* \internal
	* \brief Check whether a string represents a valid script
	*
	* Valid values include \c /dev/null and paths of executable regular files
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid script, or \c false otherwise
	*/
	bool
	pcmk__valid_script(const char *value)
	{
	struct stat st;

	if (pcmk__str_eq(value, "/dev/null", pcmk__str_none)) {
	return true;
	}

	if (stat(value, &st) != 0) {
	crm_err("Script %s does not exist", value);
	return false;
	}

	if (S_ISREG(st.st_mode) == 0) {
	crm_err("Script %s is not a regular file", value);
	return false;
	}

	if ((st.st_mode & (S_IXUSR \| S_IXGRP)) == 0) {
	crm_err("Script %s is not executable", value);
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Check whether a string represents a valid placement strategy
	*
	* \param[in] value String to validate
	*
	* \return \c true if \p value is a valid placement strategy, or \c false
	* otherwise
	*/
	bool
	pcmk__valid_placement_strategy(const char *value)
	{
	return pcmk__strcase_any_of(value, "default", "utilization", "minimal",
	"balanced", NULL);
	}

	/*!
	* \internal
	* \brief Check a table of configured options for a particular option
	*
	* \param[in,out] options Name/value pairs for configured options
	* \param[in] validate If not NULL, validator function for option value
	* \param[in] name Option name to look for
	* \param[in] old_name Alternative option name to look for
	* \param[in] def_value Default to use if option not configured
	*
	* \return Option value (from supplied options table or default value)
	*/
	static const char *
	cluster_option_value(GHashTable options, bool (validate)(const char *),
	const char name, const char old_name,
	const char *def_value)
	{
	const char *value = NULL;
	char *new_value = NULL;

	CRM_ASSERT(name != NULL);

	if (options) {
	value = g_hash_table_lookup(options, name);

	if ((value == NULL) && old_name) {
	value = g_hash_table_lookup(options, old_name);
	if (value != NULL) {
	pcmk__config_warn("Support for legacy name '%s' for cluster "
	"option '%s' is deprecated and will be "
	"removed in a future release",
	old_name, name);

	// Inserting copy with current name ensures we only warn once
	new_value = strdup(value);
	g_hash_table_insert(options, strdup(name), new_value);
	value = new_value;
	}
	}

	if (value && validate && (validate(value) == FALSE)) {
	pcmk__config_err("Using default value for cluster option '%s' "
	"because '%s' is invalid", name, value);
	value = NULL;
	}

	if (value) {
	return value;
	}
	}

	// No value found, use default
	value = def_value;

	if (value == NULL) {
	crm_trace("No value or default provided for cluster option '%s'",
	name);
	return NULL;
	}

	if (validate) {
	CRM_CHECK(validate(value) != FALSE,
	crm_err("Bug: default value for cluster option '%s' is invalid", name);
	return NULL);
	}

	crm_trace("Using default value '%s' for cluster option '%s'",
	value, name);
	if (options) {
	new_value = strdup(value);
	g_hash_table_insert(options, strdup(name), new_value);
	value = new_value;
	}
	return value;
	}

	/*!
	* \internal
	* \brief Get the value of a cluster option
	*
	* \param[in,out] options Name/value pairs for configured options
	* \param[in] option_list Possible cluster options
	* \param[in] len Length of \p option_list
	* \param[in] name (Primary) option name to look for
	*
	* \return Option value
	*/
	const char *
	pcmk__cluster_option(GHashTable *options,
	const pcmk__cluster_option_t *option_list,
	int len, const char *name)
	{
	const char *value = NULL;

	for (int lpc = 0; lpc < len; lpc++) {
	if (pcmk__str_eq(name, option_list[lpc].name, pcmk__str_casei)) {
	value = cluster_option_value(options, option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name,
	option_list[lpc].default_value);
	return value;
	}
	}
	CRM_CHECK(FALSE, crm_err("Bug: looking for unknown option '%s'", name));
	return NULL;
	}

	/*!
	* \internal
	* \brief Add a description element to a meta-data string
	*
	* \param[in,out] s Meta-data string to add to
	* \param[in] tag Name of element to add (\c PCMK_XE_LONGDESC or
	* \c PCMK_XE_SHORTDESC)
	* \param[in] desc Textual description to add
	* \param[in] values If not \p NULL, the allowed values for the parameter
	* \param[in] spaces If not \p NULL, spaces to insert at the beginning of
	* each line
	*/
	static void
	add_desc(GString s, const char tag, const char desc, const char values,
	const char *spaces)
	{
	char *escaped_en = crm_xml_escape(desc);

	if (spaces != NULL) {
	g_string_append(s, spaces);
	}
	pcmk__g_strcat(s,
	"<", tag, " " PCMK_XA_LANG "=\"" PCMK__VALUE_EN "\">",
	escaped_en, NULL);

	if (values != NULL) {
	// Append a period if desc doesn't end in "." or ".)"
	if (!pcmk__str_empty(escaped_en)
	&& (s->str[s->len - 1] != '.')
	&& ((s->str[s->len - 2] != '.') \|\| (s->str[s->len - 1] != ')'))) {

	g_string_append_c(s, '.');
	}
	pcmk__g_strcat(s, " Allowed values: ", values, NULL);
	g_string_append_c(s, '.');
	}
	pcmk__g_strcat(s, "</", tag, ">\n", NULL);

	#ifdef ENABLE_NLS
	{
	static const char *locale = NULL;

	char *localized = crm_xml_escape(_(desc));

	if (strcmp(escaped_en, localized) != 0) {
	if (locale == NULL) {
	locale = strtok(setlocale(LC_ALL, NULL), "_");
	}

	if (spaces != NULL) {
	g_string_append(s, spaces);
	}
	pcmk__g_strcat(s,
	"<", tag, " " PCMK_XA_LANG "=\"", locale, "\">",
	localized, NULL);

	if (values != NULL) {
	pcmk__g_strcat(s, _(" Allowed values: "), _(values), NULL);
	}
	pcmk__g_strcat(s, "</", tag, ">\n", NULL);
	}
	free(localized);
	}
	#endif

	free(escaped_en);
	}

	/*!
	* \internal
	* \brief Format option metadata as an OCF-like XML string
	*
	* \param[in] name Daemon name
	* \param[in] desc_short Short description of the daemon
	* \param[in] desc_long Long description of the daemon
	* \param[in] filter If not \c pcmk__opt_context_none, include only
	* those options whose \c context field is equal to
	* \p filter
	* \param[in] option_list Options whose metadata to format
	* \param[in] len Number of items in \p option_list
	*
	* \return A string containing OCF-like option metadata XML
	*
	* \note The caller is responsible for freeing the return value using
	* \c g_free().
	*/
	gchar *
	pcmk__format_option_metadata(const char name, const char desc_short,
	const char *desc_long,
	enum pcmk__opt_context filter,
	pcmk__cluster_option_t *option_list, int len)
	{
	// Large enough to hold current cluster options with room for growth (2^15)
	GString *s = g_string_sized_new(32768);

	pcmk__g_strcat(s,
	"<?xml " PCMK_XA_VERSION "=\"1.0\"?>\n"
	"<" PCMK_XE_RESOURCE_AGENT " "
	PCMK_XA_NAME "=\"", name, "\" "
	PCMK_XA_VERSION "=\"" PACEMAKER_VERSION "\">\n"

	" <" PCMK_XE_VERSION ">" PCMK_OCF_VERSION
	"</" PCMK_XE_VERSION ">\n", NULL);

	add_desc(s, PCMK_XE_LONGDESC, desc_long, NULL, " ");
	add_desc(s, PCMK_XE_SHORTDESC, desc_short, NULL, " ");

	g_string_append(s, " <" PCMK_XE_PARAMETERS ">\n");

	for (int lpc = 0; lpc < len; lpc++) {
	const char *opt_name = option_list[lpc].name;
	const char *opt_type = option_list[lpc].type;
	const char *opt_values = option_list[lpc].values;
	const char *opt_default = option_list[lpc].default_value;
	const char *opt_desc_short = option_list[lpc].description_short;
	const char *opt_desc_long = option_list[lpc].description_long;

	if ((filter != pcmk__opt_context_none)
	&& (filter != option_list[lpc].context)) {
	continue;
	}

	// The standard requires long and short parameter descriptions
	CRM_ASSERT((opt_desc_short != NULL) \|\| (opt_desc_long != NULL));

	if (opt_desc_short == NULL) {
	opt_desc_short = opt_desc_long;
	} else if (opt_desc_long == NULL) {
	opt_desc_long = opt_desc_short;
	}

	// The standard requires a parameter type
	CRM_ASSERT(opt_type != NULL);

	pcmk__g_strcat(s,
	" <" PCMK_XE_PARAMETER " "
	PCMK_XA_NAME "=\"", opt_name, "\">\n", NULL);

	add_desc(s, PCMK_XE_LONGDESC, opt_desc_long, opt_values, " ");
	add_desc(s, PCMK_XE_SHORTDESC, opt_desc_short, NULL, " ");

	pcmk__g_strcat(s, " <" PCMK_XE_CONTENT " "
	PCMK_XA_TYPE "=\"", opt_type, "\"", NULL);
	if (opt_default != NULL) {
	pcmk__g_strcat(s,
	" " PCMK_XA_DEFAULT "=\"", opt_default, "\"", NULL);
	}

	if ((opt_values != NULL) && (strcmp(opt_type, "select") == 0)) {
	char *str = strdup(opt_values);
	const char *delim = ", ";
	char *ptr = strtok(str, delim);

	g_string_append(s, ">\n");

	while (ptr != NULL) {
	pcmk__g_strcat(s,
	" <" PCMK_XE_OPTION " "
	PCMK_XA_VALUE "=\"", ptr, "\" />\n",
	NULL);
	ptr = strtok(NULL, delim);
	}
	g_string_append(s, " </" PCMK_XE_CONTENT ">\n");
	free(str);

	} else {
	g_string_append(s, "/>\n");
	}

	g_string_append(s, " </" PCMK_XE_PARAMETER ">\n");
	}
	g_string_append(s,
	" </" PCMK_XE_PARAMETERS ">\n"
	"</" PCMK_XE_RESOURCE_AGENT ">\n");

	return g_string_free(s, FALSE);
	}

	+/*!
	+ * \internal
	+ * \brief Format cluster option metadata as an OCF-like XML string
	+ *
	+ * \param[in] name Daemon name
	+ * \param[in] desc_short Short description of the daemon
	+ * \param[in] desc_long Long description of the daemon
	+ * \param[in] filter If not \c pcmk__opt_context_none, include only
	+ * those options whose \c context field is equal to
	+ * \p filter
	+ *
	+ * \return A string containing OCF-like cluster option metadata XML
	+ *
	+ * \note The caller is responsible for freeing the return value using
	+ * \c g_free().
	+ */
	+gchar *
	+pcmk__cluster_option_metadata(const char name, const char desc_short,
	+ const char *desc_long,
	+ enum pcmk__opt_context filter)
	+{
	+ return pcmk__format_option_metadata(name, desc_short, desc_long, filter,
	+ cluster_options,
	+ PCMK__NELEM(cluster_options));
	+}
	+
	void
	pcmk__validate_cluster_options(GHashTable *options,
	pcmk__cluster_option_t *option_list, int len)
	{
	for (int lpc = 0; lpc < len; lpc++) {
	cluster_option_value(options, option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name,
	option_list[lpc].default_value);
	}
	}
	diff --git a/lib/pengine/common.c b/lib/pengine/common.c
	index ac5cb62661..5585e5b22b 100644
	--- a/lib/pengine/common.c
	+++ b/lib/pengine/common.c
	@@ -1,621 +1,621 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>

	#include <glib.h>

	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>

	static pcmk__cluster_option_t pe_opts[] = {
	/* name, old name, type, allowed values,
	* default value, validator,
	* context,
	* short description,
	* long description
	*/
	{
	PCMK_OPT_NO_QUORUM_POLICY, NULL, "select",
	"stop, freeze, ignore, demote, suicide",
	"stop", pcmk__valid_no_quorum_policy,
	pcmk__opt_context_schedulerd,
	N_("What to do when the cluster does not have quorum"),
	NULL
	},
	{
	PCMK_OPT_SYMMETRIC_CLUSTER, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether resources can run on any node by default"),
	NULL
	},
	{
	PCMK_OPT_MAINTENANCE_MODE, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether the cluster should refrain from monitoring, starting, "
	"and stopping resources"),
	NULL
	},
	{
	PCMK_OPT_START_FAILURE_IS_FATAL, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether a start failure should prevent a resource from being "
	"recovered on the same node"),
	N_("When true, the cluster will immediately ban a resource from a node "
	"if it fails to start there. When false, the cluster will instead "
	"check the resource's fail count against its migration-threshold.")
	},
	{
	PCMK_OPT_ENABLE_STARTUP_PROBES, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether the cluster should check for active resources during start-up"),
	NULL
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether to lock resources to a cleanly shut down node"),
	N_("When true, resources active on a node when it is cleanly shut down "
	"are kept \"locked\" to that node (not allowed to run elsewhere) "
	"until they start again on that node after it rejoins (or for at "
	"most shutdown-lock-limit, if set). Stonith resources and "
	"Pacemaker Remote connections are never locked. Clone and bundle "
	"instances and the promoted role of promotable clones are "
	"currently never locked, though support could be added in a future "
	"release.")
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("Do not lock resources to a cleanly shut down node longer than "
	"this"),
	N_("If shutdown-lock is true and this is set to a nonzero time "
	"duration, shutdown locks will expire after this much time has "
	"passed since the shutdown was initiated, even if the node has not "
	"rejoined.")
	},

	// Fencing-related options
	{
	PCMK_OPT_STONITH_ENABLED, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("* Advanced Use Only * "
	"Whether nodes may be fenced as part of recovery"),
	N_("If false, unresponsive nodes are immediately assumed to be harmless, "
	"and resources that were active on them may be recovered "
	"elsewhere. This can result in a \"split-brain\" situation, "
	"potentially leading to data loss and/or service unavailability.")
	},
	{
	PCMK_OPT_STONITH_ACTION, NULL, "select", "reboot, off, poweroff",
	PCMK_ACTION_REBOOT, pcmk__is_fencing_action,
	pcmk__opt_context_schedulerd,
	N_("Action to send to fence device when a node needs to be fenced "
	"(\"poweroff\" is a deprecated alias for \"off\")"),
	NULL
	},
	{
	PCMK_OPT_STONITH_TIMEOUT, NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("How long to wait for on, off, and reboot fence actions to complete "
	"by default"),
	NULL
	},
	{
	PCMK_OPT_HAVE_WATCHDOG, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether watchdog integration is enabled"),
	N_("This is set automatically by the cluster according to whether SBD "
	"is detected to be in use. User-configured values are ignored. "
	"The value `true` is meaningful if diskless SBD is used and "
	"`stonith-watchdog-timeout` is nonzero. In that case, if fencing "
	"is required, watchdog-based self-fencing will be performed via "
	"SBD without requiring a fencing resource explicitly configured.")
	},
	{
	PCMK_OPT_CONCURRENT_FENCING, NULL, "boolean", NULL,
	PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Allow performing fencing operations in parallel"),
	NULL
	},
	{
	PCMK_OPT_STARTUP_FENCING, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("* Advanced Use Only * Whether to fence unseen nodes at start-up"),
	N_("Setting this to false may lead to a \"split-brain\" situation, "
	"potentially leading to data loss and/or service unavailability.")
	},
	{
	PCMK_OPT_PRIORITY_FENCING_DELAY, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("Apply fencing delay targeting the lost nodes with the highest total resource priority"),
	N_("Apply specified delay for the fencings that are targeting the lost "
	"nodes with the highest total resource priority in case we don't "
	"have the majority of the nodes in our cluster partition, so that "
	"the more significant nodes potentially win any fencing match, "
	"which is especially meaningful under split-brain of 2-node "
	"cluster. A promoted resource instance takes the base priority + 1 "
	"on calculation if the base priority is not 0. Any static/random "
	"delays that are introduced by `pcmk_delay_base/max` configured "
	"for the corresponding fencing resources will be added to this "
	"delay. This delay should be significantly greater than, safely "
	"twice, the maximum `pcmk_delay_base/max`. By default, priority "
	"fencing delay is disabled.")
	},
	{
	PCMK_OPT_NODE_PENDING_TIMEOUT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("How long to wait for a node that has joined the cluster to join "
	"the controller process group"),
	N_("Fence nodes that do not join the controller process group within "
	"this much time after joining the cluster, to allow the cluster "
	"to continue managing resources. A value of 0 means never fence "
	"pending nodes. Setting the value to 2h means fence nodes after "
	"2 hours.")
	},
	{
	PCMK_OPT_CLUSTER_DELAY, NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	pcmk__opt_context_schedulerd,
	N_("Maximum time for node-to-node communication"),
	N_("The node elected Designated Controller (DC) will consider an action "
	"failed if it does not get a response from the node executing the "
	"action within this time (after considering the action's own "
	"timeout). The \"correct\" value will depend on the speed and "
	"load of your network and cluster nodes.")
	},
	{
	PCMK_OPT_BATCH_LIMIT, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("Maximum number of jobs that the cluster may execute in parallel "
	"across all nodes"),
	N_("The \"correct\" value will depend on the speed and load of your "
	"network and cluster nodes. If set to 0, the cluster will "
	"impose a dynamically calculated limit when any node has a "
	"high load.")
	},
	{
	PCMK_OPT_MIGRATION_LIMIT, NULL, "integer", NULL,
	"-1", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of live migration actions that the cluster is allowed "
	"to execute in parallel on a node (-1 means no limit)")
	},

	/* Orphans and stopping */
	{
	PCMK_OPT_STOP_ALL_RESOURCES, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether the cluster should stop all active resources"),
	NULL
	},
	{
	PCMK_OPT_STOP_ORPHAN_RESOURCES, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether to stop resources that were removed from the configuration"),
	NULL
	},
	{
	PCMK_OPT_STOP_ORPHAN_ACTIONS, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("Whether to cancel recurring actions removed from the configuration"),
	NULL
	},
	{
	PCMK__OPT_REMOVE_AFTER_STOP, NULL, "boolean", NULL,
	PCMK_VALUE_FALSE, pcmk__valid_boolean,
	pcmk__opt_context_schedulerd,
	N_("* Deprecated * Whether to remove stopped resources from "
	"the executor"),
	N_("Values other than default are poorly tested and potentially dangerous."
	" This option will be removed in a future release.")
	},

	/* Storing inputs */
	{
	PCMK_OPT_PE_ERROR_SERIES_MAX, NULL, "integer", NULL,
	"-1", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of scheduler inputs resulting in errors to save"),
	N_("Zero to disable, -1 to store unlimited.")
	},
	{
	PCMK_OPT_PE_WARN_SERIES_MAX, NULL, "integer", NULL,
	"5000", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of scheduler inputs resulting in warnings to save"),
	N_("Zero to disable, -1 to store unlimited.")
	},
	{
	PCMK_OPT_PE_INPUT_SERIES_MAX, NULL, "integer", NULL,
	"4000", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The number of scheduler inputs without errors or warnings to save"),
	N_("Zero to disable, -1 to store unlimited.")
	},

	/* Node health */
	{
	PCMK_OPT_NODE_HEALTH_STRATEGY, NULL, "select",
	PCMK__VALUE_NONE ", " PCMK__VALUE_MIGRATE_ON_RED ", "
	PCMK__VALUE_ONLY_GREEN ", " PCMK__VALUE_PROGRESSIVE ", "
	PCMK__VALUE_CUSTOM,
	PCMK__VALUE_NONE, pcmk__validate_health_strategy,
	pcmk__opt_context_schedulerd,
	N_("How cluster should react to node health attributes"),
	N_("Requires external entities to create node attributes (named with "
	"the prefix \"#health\") with values \"red\", "
	"\"yellow\", or \"green\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_BASE, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("Base health score assigned to a node"),
	N_("Only used when \"node-health-strategy\" is set to \"progressive\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_GREEN, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The score to use for a node health attribute whose value is \"green\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_YELLOW, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The score to use for a node health attribute whose value is \"yellow\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_RED, NULL, "integer", NULL,
	"-INFINITY", pcmk__valid_int,
	pcmk__opt_context_schedulerd,
	N_("The score to use for a node health attribute whose value is \"red\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".")
	},

	/Placement Strategy/
	{
	PCMK_OPT_PLACEMENT_STRATEGY, NULL, "select",
	"default, utilization, minimal, balanced",
	"default", pcmk__valid_placement_strategy,
	pcmk__opt_context_schedulerd,
	N_("How the cluster should allocate resources to nodes"),
	NULL
	},
	};

	void
	pe_metadata(pcmk__output_t *out)
	{
	const char *name = "pacemaker-schedulerd";
	const char *desc_short = "Pacemaker scheduler options";
	const char *desc_long = "Cluster options used by Pacemaker's scheduler";

	- gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long,
	- pcmk__opt_context_schedulerd,
	- pe_opts, PCMK__NELEM(pe_opts));
	+ gchar *s = pcmk__cluster_option_metadata(name, desc_short, desc_long,
	+ pcmk__opt_context_schedulerd);
	+
	out->output_xml(out, "metadata", s);
	g_free(s);
	}

	void
	verify_pe_options(GHashTable * options)
	{
	pcmk__validate_cluster_options(options, pe_opts, PCMK__NELEM(pe_opts));
	}

	const char *
	pe_pref(GHashTable * options, const char *name)
	{
	return pcmk__cluster_option(options, pe_opts, PCMK__NELEM(pe_opts), name);
	}

	const char *
	fail2text(enum action_fail_response fail)
	{
	const char *result = "<unknown>";

	switch (fail) {
	case pcmk_on_fail_ignore:
	result = "ignore";
	break;
	case pcmk_on_fail_demote:
	result = "demote";
	break;
	case pcmk_on_fail_block:
	result = "block";
	break;
	case pcmk_on_fail_restart:
	result = "recover";
	break;
	case pcmk_on_fail_ban:
	result = "migrate";
	break;
	case pcmk_on_fail_stop:
	result = "stop";
	break;
	case pcmk_on_fail_fence_node:
	result = "fence";
	break;
	case pcmk_on_fail_standby_node:
	result = "standby";
	break;
	case pcmk_on_fail_restart_container:
	result = "restart-container";
	break;
	case pcmk_on_fail_reset_remote:
	result = "reset-remote";
	break;
	}
	return result;
	}

	enum action_tasks
	text2task(const char *task)
	{
	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)) {
	return pcmk_action_stop;

	} else if (pcmk__str_eq(task, PCMK_ACTION_STOPPED, pcmk__str_casei)) {
	return pcmk_action_stopped;

	} else if (pcmk__str_eq(task, PCMK_ACTION_START, pcmk__str_casei)) {
	return pcmk_action_start;

	} else if (pcmk__str_eq(task, PCMK_ACTION_RUNNING, pcmk__str_casei)) {
	return pcmk_action_started;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_casei)) {
	return pcmk_action_shutdown;

	} else if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) {
	return pcmk_action_fence;

	} else if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	return pcmk_action_monitor;

	} else if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
	return pcmk_action_notify;

	} else if (pcmk__str_eq(task, PCMK_ACTION_NOTIFIED, pcmk__str_casei)) {
	return pcmk_action_notified;

	} else if (pcmk__str_eq(task, PCMK_ACTION_PROMOTE, pcmk__str_casei)) {
	return pcmk_action_promote;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DEMOTE, pcmk__str_casei)) {
	return pcmk_action_demote;

	} else if (pcmk__str_eq(task, PCMK_ACTION_PROMOTED, pcmk__str_casei)) {
	return pcmk_action_promoted;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DEMOTED, pcmk__str_casei)) {
	return pcmk_action_demoted;
	}
	return pcmk_action_unspecified;
	}

	const char *
	task2text(enum action_tasks task)
	{
	const char *result = "<unknown>";

	switch (task) {
	case pcmk_action_unspecified:
	result = "no_action";
	break;
	case pcmk_action_stop:
	result = PCMK_ACTION_STOP;
	break;
	case pcmk_action_stopped:
	result = PCMK_ACTION_STOPPED;
	break;
	case pcmk_action_start:
	result = PCMK_ACTION_START;
	break;
	case pcmk_action_started:
	result = PCMK_ACTION_RUNNING;
	break;
	case pcmk_action_shutdown:
	result = PCMK_ACTION_DO_SHUTDOWN;
	break;
	case pcmk_action_fence:
	result = PCMK_ACTION_STONITH;
	break;
	case pcmk_action_monitor:
	result = PCMK_ACTION_MONITOR;
	break;
	case pcmk_action_notify:
	result = PCMK_ACTION_NOTIFY;
	break;
	case pcmk_action_notified:
	result = PCMK_ACTION_NOTIFIED;
	break;
	case pcmk_action_promote:
	result = PCMK_ACTION_PROMOTE;
	break;
	case pcmk_action_promoted:
	result = PCMK_ACTION_PROMOTED;
	break;
	case pcmk_action_demote:
	result = PCMK_ACTION_DEMOTE;
	break;
	case pcmk_action_demoted:
	result = PCMK_ACTION_DEMOTED;
	break;
	}

	return result;
	}

	void
	add_hash_param(GHashTable * hash, const char name, const char value)
	{
	CRM_CHECK(hash != NULL, return);

	crm_trace("Adding name='%s' value='%s' to hash table",
	pcmk__s(name, "<null>"), pcmk__s(value, "<null>"));
	if (name == NULL \|\| value == NULL) {
	return;

	} else if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
	return;

	} else if (g_hash_table_lookup(hash, name) == NULL) {
	g_hash_table_insert(hash, strdup(name), strdup(value));
	}
	}

	/*!
	* \internal
	* \brief Look up an attribute value on the appropriate node
	*
	* If \p node is a guest node and either the \c PCMK_META_CONTAINER_ATTR_TARGET
	* meta attribute is set to "host" for \p rsc or \p force_host is \c true, query
	* the attribute on the node's host. Otherwise, query the attribute on \p node
	* itself.
	*
	* \param[in] node Node to query attribute value on by default
	* \param[in] name Name of attribute to query
	* \param[in] rsc Resource on whose behalf we're querying
	* \param[in] node_type Type of resource location lookup
	* \param[in] force_host Force a lookup on the guest node's host, regardless of
	* the \c PCMK_META_CONTAINER_ATTR_TARGET value
	*
	* \return Value of the attribute on \p node or on the host of \p node
	*
	* \note If \p force_host is \c true, \p node \e must be a guest node.
	*/
	const char *
	pe__node_attribute_calculated(const pcmk_node_t node, const char name,
	const pcmk_resource_t *rsc,
	enum pcmk__rsc_node node_type,
	bool force_host)
	{
	// @TODO: Use pe__is_guest_node() after merging libpe_{rules,status}
	bool is_guest = (node != NULL)
	&& (node->details->type == pcmk_node_variant_remote)
	&& (node->details->remote_rsc != NULL)
	&& (node->details->remote_rsc->container != NULL);
	const char *source = NULL;
	const char *node_type_s = NULL;
	const char *reason = NULL;

	const pcmk_resource_t *container = NULL;
	const pcmk_node_t *host = NULL;

	CRM_ASSERT((node != NULL) && (name != NULL) && (rsc != NULL)
	&& (!force_host \|\| is_guest));

	/* Ignore PCMK_META_CONTAINER_ATTR_TARGET if node is not a guest node. This
	* represents a user configuration error.
	*/
	source = g_hash_table_lookup(rsc->meta, PCMK_META_CONTAINER_ATTR_TARGET);
	if (!force_host
	&& (!is_guest \|\| !pcmk__str_eq(source, "host", pcmk__str_casei))) {

	return g_hash_table_lookup(node->details->attrs, name);
	}

	container = node->details->remote_rsc->container;

	switch (node_type) {
	case pcmk__rsc_node_assigned:
	node_type_s = "assigned";
	host = container->allocated_to;
	if (host == NULL) {
	reason = "not assigned";
	}
	break;

	case pcmk__rsc_node_current:
	node_type_s = "current";

	if (container->running_on != NULL) {
	host = container->running_on->data;
	}
	if (host == NULL) {
	reason = "inactive";
	}
	break;

	default:
	// Add support for other enum pcmk__rsc_node values if needed
	CRM_ASSERT(false);
	break;
	}

	if (host != NULL) {
	const char *value = g_hash_table_lookup(host->details->attrs, name);

	pcmk__rsc_trace(rsc,
	"%s: Value lookup for %s on %s container host %s %s%s",
	rsc->id, name, node_type_s, pcmk__node_name(host),
	((value != NULL)? "succeeded: " : "failed"),
	pcmk__s(value, ""));
	return value;
	}
	pcmk__rsc_trace(rsc,
	"%s: Not looking for %s on %s container host: %s is %s",
	rsc->id, name, node_type_s, container->id, reason);
	return NULL;
	}

	const char *
	pe_node_attribute_raw(const pcmk_node_t node, const char name)
	{
	if(node == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(node->details->attrs, name);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/pengine/common_compat.h>

	const char *
	role2text(enum rsc_role_e role)
	{
	return pcmk_role_text(role);
	}

	enum rsc_role_e
	text2role(const char *role)
	{
	return pcmk_parse_role(role);
	}

	// LCOV_EXCL_STOP
	// End deprecated API

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 11:20 AM (1 d, 18 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018674
Default Alt Text: (141 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions