No OneTemporary
Actions

Size

741 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
	index 7f6993b0e5..f0dfec7f9f 100644
	--- a/daemons/controld/controld_control.c
	+++ b/daemons/controld/controld_control.c
	@@ -1,872 +1,872 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election_internal.h>
	#include <crm/common/ipc_internal.h>

	#include <pacemaker-controld.h>

	static qb_ipcs_service_t *ipcs = NULL;

	static crm_trigger_t *config_read_trigger = NULL;

	#if SUPPORT_COROSYNC
	extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
	#endif

	static void crm_shutdown(int nsig);
	static gboolean crm_read_options(gpointer user_data);

	/* A_HA_CONNECT */
	void
	do_ha_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean registered = FALSE;
	static crm_cluster_t *cluster = NULL;

	if (cluster == NULL) {
	cluster = pcmk_cluster_new();
	}

	if (action & A_HA_DISCONNECT) {
	crm_cluster_disconnect(cluster);
	crm_info("Disconnected from the cluster");

	controld_set_fsa_input_flags(R_HA_DISCONNECTED);
	}

	if (action & A_HA_CONNECT) {
	crm_set_status_callback(&peer_update_callback);
	crm_set_autoreap(FALSE);

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	registered = crm_connect_corosync(cluster);
	}
	#endif // SUPPORT_COROSYNC

	if (registered) {
	controld_election_init(cluster->uname);
	controld_globals.our_nodename = cluster->uname;
	controld_globals.our_uuid = cluster->uuid;
	if(cluster->uuid == NULL) {
	crm_err("Could not obtain local uuid");
	registered = FALSE;
	}
	}

	if (!registered) {
	controld_set_fsa_input_flags(R_HA_DISCONNECTED);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	populate_cib_nodes(node_update_none, __func__);
	controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
	crm_info("Connected to the cluster");
	}

	if (action & ~(A_HA_CONNECT \| A_HA_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action),
	__func__);
	}
	}

	/* A_SHUTDOWN */
	void
	do_shutdown(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* just in case */
	controld_set_fsa_input_flags(R_SHUTDOWN);
	controld_disconnect_fencer(FALSE);
	}

	/* A_SHUTDOWN_REQ */
	void
	do_shutdown_req(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *msg = NULL;

	controld_set_fsa_input_flags(R_SHUTDOWN);
	//controld_set_fsa_input_flags(R_STAYDOWN);
	crm_info("Sending shutdown request to all peers (DC is %s)",
	pcmk__s(controld_globals.dc_name, "not set"));
	msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	free_xml(msg);
	}

	void
	crmd_fast_exit(crm_exit_t exit_code)
	{
	if (pcmk_is_set(controld_globals.fsa_input_register, R_STAYDOWN)) {
	crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d",
	exit_code, CRM_EX_FATAL);
	exit_code = CRM_EX_FATAL;

	} else if ((exit_code == CRM_EX_OK)
	&& pcmk_is_set(controld_globals.fsa_input_register,
	R_IN_RECOVERY)) {
	crm_err("Could not recover from internal error");
	exit_code = CRM_EX_ERROR;
	}

	if (controld_globals.logger_out != NULL) {
	controld_globals.logger_out->finish(controld_globals.logger_out,
	exit_code, true, NULL);
	pcmk__output_free(controld_globals.logger_out);
	controld_globals.logger_out = NULL;
	}

	crm_exit(exit_code);
	}

	crm_exit_t
	crmd_exit(crm_exit_t exit_code)
	{
	GMainLoop *mloop = controld_globals.mainloop;

	static bool in_progress = FALSE;

	if (in_progress && (exit_code == CRM_EX_OK)) {
	crm_debug("Exit is already in progress");
	return exit_code;

	} else if(in_progress) {
	crm_notice("Error during shutdown process, exiting now with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	crm_write_blackbox(SIGTRAP, NULL);
	crmd_fast_exit(exit_code);
	}

	in_progress = TRUE;
	crm_trace("Preparing to exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));

	/* Suppress secondary errors resulting from us disconnecting everything */
	controld_set_fsa_input_flags(R_HA_DISCONNECTED);

	/* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */

	if(ipcs) {
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs);
	ipcs = NULL;
	}

	controld_close_attrd_ipc();
	controld_shutdown_schedulerd_ipc();
	controld_disconnect_fencer(TRUE);

	if ((exit_code == CRM_EX_OK) && (controld_globals.mainloop == NULL)) {
	crm_debug("No mainloop detected");
	exit_code = CRM_EX_ERROR;
	}

	/* On an error, just get out.
	*
	* Otherwise, make the effort to have mainloop exit gracefully so
	* that it (mostly) cleans up after itself and valgrind has less
	* to report on - allowing real errors stand out
	*/
	if (exit_code != CRM_EX_OK) {
	crm_notice("Forcing immediate exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	crm_write_blackbox(SIGTRAP, NULL);
	crmd_fast_exit(exit_code);
	}

	/* Clean up as much memory as possible for valgrind */

	for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
	iter = iter->next) {
	fsa_data_t fsa_data = (fsa_data_t ) iter->data;

	crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
	fsa_input2string(fsa_data->fsa_input),
	fsa_state2string(controld_globals.fsa_state),
	fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
	delete_fsa_input(fsa_data);
	}

	controld_clear_fsa_input_flags(R_MEMBERSHIP);

	g_list_free(controld_globals.fsa_message_queue);
	controld_globals.fsa_message_queue = NULL;

	controld_free_node_pending_timers();
	controld_election_fini();

	/* Tear down the CIB manager connection, but don't free it yet -- it could
	* be used when we drain the mainloop later.
	*/

	controld_disconnect_cib_manager();

	verify_stopped(controld_globals.fsa_state, LOG_WARNING);
	controld_clear_fsa_input_flags(R_LRM_CONNECTED);
	lrm_state_destroy_all();

	mainloop_destroy_trigger(config_read_trigger);
	config_read_trigger = NULL;

	controld_destroy_fsa_trigger();
	controld_destroy_transition_trigger();

	pcmk__client_cleanup();
	crm_peer_destroy();

	controld_free_fsa_timers();
	te_cleanup_stonith_history_sync(NULL, TRUE);
	controld_free_sched_timer();

	free(controld_globals.our_nodename);
	controld_globals.our_nodename = NULL;

	free(controld_globals.our_uuid);
	controld_globals.our_uuid = NULL;

	free(controld_globals.dc_name);
	controld_globals.dc_name = NULL;

	free(controld_globals.dc_version);
	controld_globals.dc_version = NULL;

	free(controld_globals.cluster_name);
	controld_globals.cluster_name = NULL;

	free(controld_globals.te_uuid);
	controld_globals.te_uuid = NULL;

	free_max_generation();
	controld_destroy_failed_sync_table();
	controld_destroy_outside_events_table();

	mainloop_destroy_signal(SIGPIPE);
	mainloop_destroy_signal(SIGUSR1);
	mainloop_destroy_signal(SIGTERM);
	mainloop_destroy_signal(SIGTRAP);
	/* leave SIGCHLD engaged as we might still want to drain some service-actions */

	if (mloop) {
	GMainContext *ctx = g_main_loop_get_context(controld_globals.mainloop);

	/* Don't re-enter this block */
	controld_globals.mainloop = NULL;

	/* no signals on final draining anymore */
	mainloop_destroy_signal(SIGCHLD);

	crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));

	{
	int lpc = 0;

	while((g_main_context_pending(ctx) && lpc < 10)) {
	lpc++;
	crm_trace("Iteration %d", lpc);
	g_main_context_dispatch(ctx);
	}
	}

	crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
	g_main_loop_quit(mloop);

	/* Won't do anything yet, since we're inside it now */
	g_main_loop_unref(mloop);
	} else {
	mainloop_destroy_signal(SIGCHLD);
	}

	cib_delete(controld_globals.cib_conn);
	controld_globals.cib_conn = NULL;

	throttle_fini();

	/* Graceful */
	crm_trace("Done preparing for exit with status %d (%s)",
	exit_code, crm_exit_str(exit_code));
	return exit_code;
	}

	/* A_EXIT_0, A_EXIT_1 */
	void
	do_exit(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_exit_t exit_code = CRM_EX_OK;

	if (pcmk_is_set(action, A_EXIT_1)) {
	exit_code = CRM_EX_ERROR;
	crm_err("Exiting now due to errors");
	}
	verify_stopped(cur_state, LOG_ERR);
	crmd_exit(exit_code);
	}

	static void sigpipe_ignore(int nsig) { return; }

	/* A_STARTUP */
	void
	do_startup(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_debug("Registering Signal Handlers");
	mainloop_add_signal(SIGTERM, crm_shutdown);
	mainloop_add_signal(SIGPIPE, sigpipe_ignore);

	config_read_trigger = mainloop_add_trigger(G_PRIORITY_HIGH,
	crm_read_options, NULL);

	controld_init_fsa_trigger();
	controld_init_transition_trigger();

	crm_debug("Creating CIB manager and executor objects");
	controld_globals.cib_conn = cib_new();

	lrm_state_init_local();
	if (controld_init_fsa_timers() == FALSE) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	// \return libqb error code (0 on success, -errno on error)
	static int32_t
	accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
	{
	crm_trace("Accepting new IPC client connection");
	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	// \return libqb error code (0 on success, -errno on error)
	static int32_t
	dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	pcmk__client_t *client = pcmk__find_client(c);

	xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);

	if (msg == NULL) {
	pcmk__ipc_send_ack(client, id, flags, "ack", NULL, CRM_EX_PROTOCOL);
	return 0;
	}
	pcmk__ipc_send_ack(client, id, flags, "ack", NULL, CRM_EX_INDETERMINATE);

	CRM_ASSERT(client->user != NULL);
	pcmk__update_acl_user(msg, F_CRM_USER, client->user);

	crm_xml_add(msg, F_CRM_SYS_FROM, client->id);
	if (controld_authorize_ipc_message(msg, client, NULL)) {
	crm_trace("Processing IPC message from client %s",
	pcmk__client_name(client));
	route_message(C_IPC_MESSAGE, msg);
	}

	controld_trigger_fsa();
	free_xml(msg);
	return 0;
	}

	static int32_t
	ipc_client_disconnected(qb_ipcs_connection_t *c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client) {
	crm_trace("Disconnecting %sregistered client %s (%p/%p)",
	(client->userdata? "" : "un"), pcmk__client_name(client),
	c, client);
	free(client->userdata);
	pcmk__free_client(client);
	controld_trigger_fsa();
	}
	return 0;
	}

	static void
	ipc_connection_destroyed(qb_ipcs_connection_t *c)
	{
	crm_trace("Connection %p", c);
	ipc_client_disconnected(c);
	}

	/* A_STOP */
	void
	do_stop(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs); ipcs = NULL;
	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	/* A_STARTED */
	void
	do_started(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static struct qb_ipcs_service_handlers crmd_callbacks = {
	.connection_accept = accept_controller_client,
	.connection_created = NULL,
	.msg_process = dispatch_controller_ipc,
	.connection_closed = ipc_client_disconnected,
	.connection_destroyed = ipc_connection_destroyed
	};

	if (cur_state != S_STARTING) {
	crm_err("Start cancelled... %s", fsa_state2string(cur_state));
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_MEMBERSHIP)) {
	crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_LRM_CONNECTED)) {
	crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_CIB_CONNECTED)) {
	crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_READ_CONFIG)) {
	crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);

	crmd_fsa_stall(TRUE);
	return;

	} else if (!pcmk_is_set(controld_globals.fsa_input_register, R_PEER_DATA)) {

	crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
	crmd_fsa_stall(TRUE);
	return;
	}

	crm_debug("Init server comms");
	ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
	if (ipcs == NULL) {
	crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	} else {
	crm_notice("Pacemaker controller successfully started and accepting connections");
	}
	controld_set_fsa_input_flags(R_ST_REQUIRED);
	controld_timer_fencer_connect(GINT_TO_POINTER(TRUE));

	controld_clear_fsa_input_flags(R_STARTING);
	register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
	}

	/* A_RECOVER */
	void
	do_recover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	controld_set_fsa_input_flags(R_IN_RECOVERY);
	crm_warn("Fast-tracking shutdown in response to errors");

	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	static pcmk__cluster_option_t controller_options[] = {
	/* name, old name, type, allowed values,
	* default value, validator,
	* short description,
	* long description
	*/
	{
	PCMK_OPT_DC_VERSION, NULL, "string", NULL,
	PCMK__VALUE_NONE, NULL,
	N_("Pacemaker version on cluster node elected Designated Controller (DC)"),
	N_("Includes a hash which identifies the exact changeset the code was "
	"built from. Used for diagnostic purposes.")
	},
	{
	PCMK_OPT_CLUSTER_INFRASTRUCTURE, NULL, "string", NULL,
	"corosync", NULL,
	N_("The messaging stack on which Pacemaker is currently running"),
	N_("Used for informational and diagnostic purposes.")
	},
	{
	PCMK_OPT_CLUSTER_NAME, NULL, "string", NULL,
	NULL, NULL,
	N_("An arbitrary name for the cluster"),
	N_("This optional value is mostly for users' convenience as desired "
	"in administration, but may also be used in Pacemaker "
	"configuration rules via the #cluster-name node attribute, and "
	"by higher-level tools and resource agents.")
	},
	{
	PCMK_OPT_DC_DEADTIME, NULL, "time", NULL,
	"20s", pcmk__valid_interval_spec,
	N_("How long to wait for a response from other nodes during start-up"),
	N_("The optimal value will depend on the speed and load of your network "
	"and the type of switches used.")
	},
	{
	PCMK_OPT_CLUSTER_RECHECK_INTERVAL, NULL, "time",
	N_("Zero disables polling, while positive values are an interval in "
	"seconds (unless other units are specified, for example \"5min\")"),
	"15min", pcmk__valid_interval_spec,
	N_("Polling interval to recheck cluster state and evaluate rules "
	"with date specifications"),
	N_("Pacemaker is primarily event-driven, and looks ahead to know when to "
	"recheck cluster state for failure timeouts and most time-based "
	"rules. However, it will also recheck the cluster after this "
	"amount of inactivity, to evaluate rules with date specifications "
	"and serve as a fail-safe for certain types of scheduler bugs.")
	},
	{
	PCMK_OPT_LOAD_THRESHOLD, NULL, "percentage", NULL,
	"80%", pcmk__valid_percentage,
	N_("Maximum amount of system load that should be used by cluster nodes"),
	N_("The cluster will slow down its recovery process when the amount of "
	"system resources used (currently CPU) approaches this limit"),
	},
	{
	PCMK_OPT_NODE_ACTION_LIMIT, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	N_("Maximum number of jobs that can be scheduled per node "
	"(defaults to 2x cores)")
	},
	{
	PCMK_OPT_FENCE_REACTION, NULL, "select", "stop, panic",
	"stop", NULL,
	N_("How a cluster node should react if notified of its own fencing"),
	N_("A cluster node may receive notification of its own fencing if fencing "
	"is misconfigured, or if fabric fencing is in use that doesn't cut "
	"cluster communication. Use \"stop\" to attempt to immediately "
	"stop Pacemaker and stay stopped, or \"panic\" to attempt to "
	"immediately reboot the local node, falling back to stop on "
	"failure.")
	},
	{
	PCMK_OPT_ELECTION_TIMEOUT, NULL, "time", NULL,
	"2min", pcmk__valid_interval_spec,
	"* Advanced Use Only *",
	N_("Declare an election failed if it is not decided within this much "
	"time. If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_SHUTDOWN_ESCALATION, NULL, "time", NULL,
	"20min", pcmk__valid_interval_spec,
	"* Advanced Use Only *",
	N_("Exit immediately if shutdown does not complete within this much "
	"time. If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_JOIN_INTEGRATION_TIMEOUT, "crmd-integration-timeout", "time",
	NULL,
	"3min", pcmk__valid_interval_spec,
	"* Advanced Use Only *",
	N_("If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_JOIN_FINALIZATION_TIMEOUT, "crmd-finalization-timeout",
	"time", NULL,
	"30min", pcmk__valid_interval_spec,
	"* Advanced Use Only *",
	N_("If you need to adjust this value, it probably indicates "
	"the presence of a bug.")
	},
	{
	PCMK_OPT_TRANSITION_DELAY, "crmd-transition-delay", "time", NULL,
	"0s", pcmk__valid_interval_spec,
	N_("* Advanced Use Only * Enabling this option will slow down "
	"cluster recovery under all conditions"),
	N_("Delay cluster recovery for this much time to allow for additional "
	"events to occur. Useful if your configuration is sensitive to "
	"the order in which ping updates arrive.")
	},
	{
	PCMK_OPT_STONITH_WATCHDOG_TIMEOUT, NULL, "time", NULL,
	"0", controld_verify_stonith_watchdog_timeout,
	N_("How long before nodes can be assumed to be safely down when "
	"watchdog-based self-fencing via SBD is in use"),
	N_("If this is set to a positive value, lost nodes are assumed to "
	"self-fence using watchdog-based SBD within this much time. This "
	"does not require a fencing resource to be explicitly configured, "
	"though a fence_watchdog resource can be configured, to limit use "
	"to specific nodes. If this is set to 0 (the default), the cluster "
	"will never assume watchdog-based self-fencing. If this is set to a "
	"negative value, the cluster will use twice the local value of the "
	"`SBD_WATCHDOG_TIMEOUT` environment variable if that is positive, "
	"or otherwise treat this as 0. WARNING: When used, this timeout "
	"must be larger than `SBD_WATCHDOG_TIMEOUT` on all nodes that use "
	"watchdog-based SBD, and Pacemaker will refuse to start on any of "
	"those nodes where this is not true for the local value or SBD is "
	"not active. When this is set to a negative value, "
	"`SBD_WATCHDOG_TIMEOUT` must be set to the same value on all nodes "
	"that use SBD, otherwise data corruption or loss could occur.")
	},
	{
	PCMK_OPT_STONITH_MAX_ATTEMPTS, NULL, "integer", NULL,
	"10", pcmk__valid_positive_int,
	N_("How many times fencing can fail before it will no longer be "
	"immediately re-attempted on a target")
	},

	// Already documented in libpe_status (other values must be kept identical)
	{
	PCMK_OPT_NO_QUORUM_POLICY, NULL, "select",
	"stop, freeze, ignore, demote, suicide",
	"stop", pcmk__valid_no_quorum_policy,
	N_("What to do when the cluster does not have quorum"), NULL
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK, NULL, "boolean", NULL,
	- "false", pcmk__valid_boolean,
	+ PCMK_VALUE_FALSE, pcmk__valid_boolean,
	N_("Whether to lock resources to a cleanly shut down node"),
	N_("When true, resources active on a node when it is cleanly shut down "
	"are kept \"locked\" to that node (not allowed to run elsewhere) "
	"until they start again on that node after it rejoins (or for at "
	"most shutdown-lock-limit, if set). Stonith resources and "
	"Pacemaker Remote connections are never locked. Clone and bundle "
	"instances and the promoted role of promotable clones are "
	"currently never locked, though support could be added in a future "
	"release.")
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	N_("Do not lock resources to a cleanly shut down node longer than "
	"this"),
	N_("If shutdown-lock is true and this is set to a nonzero time "
	"duration, shutdown locks will expire after this much time has "
	"passed since the shutdown was initiated, even if the node has not "
	"rejoined.")
	},
	{
	PCMK_OPT_NODE_PENDING_TIMEOUT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	N_("How long to wait for a node that has joined the cluster to join "
	"the controller process group"),
	N_("Fence nodes that do not join the controller process group within "
	"this much time after joining the cluster, to allow the cluster "
	"to continue managing resources. A value of 0 means never fence "
	"pending nodes. Setting the value to 2h means fence nodes after "
	"2 hours.")
	},
	};

	void
	crmd_metadata(void)
	{
	const char *desc_short = "Pacemaker controller options";
	const char *desc_long = "Cluster options used by Pacemaker's controller";

	gchar *s = pcmk__format_option_metadata("pacemaker-controld", desc_short,
	desc_long, controller_options,
	PCMK__NELEM(controller_options));
	printf("%s", s);
	g_free(s);
	}

	static void
	config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	const char *value = NULL;
	GHashTable *config_hash = NULL;
	crm_time_t *now = crm_time_new(NULL);
	xmlNode *crmconfig = NULL;
	xmlNode *alerts = NULL;

	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL;

	crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	if (rc == -EACCES \|\| rc == -pcmk_err_schema_validation) {
	crm_err("The cluster is mis-configured - shutting down and staying down");
	controld_set_fsa_input_flags(R_STAYDOWN);
	}
	goto bail;
	}

	crmconfig = output;
	if ((crmconfig != NULL)
	&& !pcmk__xe_is(crmconfig, XML_CIB_TAG_CRMCONFIG)) {
	crmconfig = first_named_child(crmconfig, XML_CIB_TAG_CRMCONFIG);
	}
	if (!crmconfig) {
	fsa_data_t *msg_data = NULL;

	crm_err("Local CIB query for " XML_CIB_TAG_CRMCONFIG " section failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	goto bail;
	}

	crm_debug("Call %d : Parsing CIB options", call_id);
	config_hash = pcmk__strkey_table(free, free);
	pe_unpack_nvpairs(crmconfig, crmconfig, XML_CIB_TAG_PROPSET, NULL,
	config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);

	// Validate all options, and use defaults if not already present in hash
	pcmk__validate_cluster_options(config_hash, controller_options,
	PCMK__NELEM(controller_options));

	value = g_hash_table_lookup(config_hash, PCMK_OPT_NO_QUORUM_POLICY);
	if (pcmk__str_eq(value, "suicide", pcmk__str_casei) && pcmk__locate_sbd()) {
	controld_set_global_flags(controld_no_quorum_suicide);
	}

	value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK);
	if (crm_is_true(value)) {
	controld_set_global_flags(controld_shutdown_lock_enabled);
	} else {
	controld_clear_global_flags(controld_shutdown_lock_enabled);
	}

	value = g_hash_table_lookup(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
	pcmk_parse_interval_spec(value, &controld_globals.shutdown_lock_limit);
	controld_globals.shutdown_lock_limit /= 1000;

	value = g_hash_table_lookup(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
	pcmk_parse_interval_spec(value, &controld_globals.node_pending_timeout);
	controld_globals.node_pending_timeout /= 1000;

	value = g_hash_table_lookup(config_hash, PCMK_OPT_CLUSTER_NAME);
	pcmk__str_update(&(controld_globals.cluster_name), value);

	// Let subcomponents initialize their own static variables
	controld_configure_election(config_hash);
	controld_configure_fencing(config_hash);
	controld_configure_fsa_timers(config_hash);
	controld_configure_throttle(config_hash);

	alerts = first_named_child(output, XML_CIB_TAG_ALERTS);
	crmd_unpack_alerts(alerts);

	controld_set_fsa_input_flags(R_READ_CONFIG);
	controld_trigger_fsa();

	g_hash_table_destroy(config_hash);
	bail:
	crm_time_free(now);
	}

	/*!
	* \internal
	* \brief Trigger read and processing of the configuration
	*
	* \param[in] fn Calling function name
	* \param[in] line Line number where call occurred
	*/
	void
	controld_trigger_config_as(const char *fn, int line)
	{
	if (config_read_trigger != NULL) {
	crm_trace("%s:%d - Triggered config processing", fn, line);
	mainloop_set_trigger(config_read_trigger);
	}
	}

	gboolean
	crm_read_options(gpointer user_data)
	{
	cib_t *cib_conn = controld_globals.cib_conn;
	int call_id = cib_conn->cmds->query(cib_conn,
	"//" XML_CIB_TAG_CRMCONFIG
	" \| //" XML_CIB_TAG_ALERTS,
	NULL, cib_xpath\|cib_scope_local);

	fsa_register_cib_callback(call_id, NULL, config_query_callback);
	crm_trace("Querying the CIB... call %d", call_id);
	return TRUE;
	}

	/* A_READCONFIG */
	void
	do_read_config(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	throttle_init();
	controld_trigger_config();
	}

	static void
	crm_shutdown(int nsig)
	{
	const char *value = NULL;
	guint default_period_ms = 0;

	if ((controld_globals.mainloop == NULL)
	\|\| !g_main_loop_is_running(controld_globals.mainloop)) {
	crmd_exit(CRM_EX_OK);
	return;
	}

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	crm_err("Escalating shutdown");
	register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
	return;
	}

	controld_set_fsa_input_flags(R_SHUTDOWN);
	register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);

	/* If shutdown timer doesn't have a period set, use the default
	*
	* @TODO: Evaluate whether this is still necessary. As long as
	* config_query_callback() has been run at least once, it doesn't look like
	* anything could have changed the timer period since then.
	*/
	value = pcmk__cluster_option(NULL, controller_options,
	PCMK__NELEM(controller_options),
	PCMK_OPT_SHUTDOWN_ESCALATION);
	pcmk_parse_interval_spec(value, &default_period_ms);
	controld_shutdown_start_countdown(default_period_ms);
	}
	diff --git a/include/crm/common/options.h b/include/crm/common/options.h
	index 83ebd9877c..32f77c7832 100644
	--- a/include/crm/common/options.h
	+++ b/include/crm/common/options.h
	@@ -1,132 +1,133 @@
	/*
	* Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_OPTIONS__H
	# define PCMK__CRM_COMMON_OPTIONS__H

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief API related to options
	* \ingroup core
	*/

	/*
	* Cluster options
	*/

	#define PCMK_OPT_BATCH_LIMIT "batch-limit"
	#define PCMK_OPT_CLUSTER_DELAY "cluster-delay"
	#define PCMK_OPT_CLUSTER_INFRASTRUCTURE "cluster-infrastructure"
	#define PCMK_OPT_CLUSTER_IPC_LIMIT "cluster-ipc-limit"
	#define PCMK_OPT_CLUSTER_NAME "cluster-name"
	#define PCMK_OPT_CLUSTER_RECHECK_INTERVAL "cluster-recheck-interval"
	#define PCMK_OPT_CONCURRENT_FENCING "concurrent-fencing"
	#define PCMK_OPT_DC_DEADTIME "dc-deadtime"
	#define PCMK_OPT_DC_VERSION "dc-version"
	#define PCMK_OPT_ELECTION_TIMEOUT "election-timeout"
	#define PCMK_OPT_ENABLE_ACL "enable-acl"
	#define PCMK_OPT_ENABLE_STARTUP_PROBES "enable-startup-probes"
	#define PCMK_OPT_FENCE_REACTION "fence-reaction"
	#define PCMK_OPT_HAVE_WATCHDOG "have-watchdog"
	#define PCMK_OPT_JOIN_FINALIZATION_TIMEOUT "join-finalization-timeout"
	#define PCMK_OPT_JOIN_INTEGRATION_TIMEOUT "join-integration-timeout"
	#define PCMK_OPT_LOAD_THRESHOLD "load-threshold"
	#define PCMK_OPT_MAINTENANCE_MODE "maintenance-mode"
	#define PCMK_OPT_MIGRATION_LIMIT "migration-limit"
	#define PCMK_OPT_NO_QUORUM_POLICY "no-quorum-policy"
	#define PCMK_OPT_NODE_ACTION_LIMIT "node-action-limit"
	#define PCMK_OPT_NODE_HEALTH_BASE "node-health-base"
	#define PCMK_OPT_NODE_HEALTH_GREEN "node-health-green"
	#define PCMK_OPT_NODE_HEALTH_RED "node-health-red"
	#define PCMK_OPT_NODE_HEALTH_STRATEGY "node-health-strategy"
	#define PCMK_OPT_NODE_HEALTH_YELLOW "node-health-yellow"
	#define PCMK_OPT_NODE_PENDING_TIMEOUT "node-pending-timeout"
	#define PCMK_OPT_PE_ERROR_SERIES_MAX "pe-error-series-max"
	#define PCMK_OPT_PE_INPUT_SERIES_MAX "pe-input-series-max"
	#define PCMK_OPT_PE_WARN_SERIES_MAX "pe-warn-series-max"
	#define PCMK_OPT_PLACEMENT_STRATEGY "placement-strategy"
	#define PCMK_OPT_PRIORITY_FENCING_DELAY "priority-fencing-delay"
	#define PCMK_OPT_SHUTDOWN_ESCALATION "shutdown-escalation"
	#define PCMK_OPT_SHUTDOWN_LOCK "shutdown-lock"
	#define PCMK_OPT_SHUTDOWN_LOCK_LIMIT "shutdown-lock-limit"
	#define PCMK_OPT_START_FAILURE_IS_FATAL "start-failure-is-fatal"
	#define PCMK_OPT_STARTUP_FENCING "startup-fencing"
	#define PCMK_OPT_STONITH_ACTION "stonith-action"
	#define PCMK_OPT_STONITH_ENABLED "stonith-enabled"
	#define PCMK_OPT_STONITH_MAX_ATTEMPTS "stonith-max-attempts"
	#define PCMK_OPT_STONITH_TIMEOUT "stonith-timeout"
	#define PCMK_OPT_STONITH_WATCHDOG_TIMEOUT "stonith-watchdog-timeout"
	#define PCMK_OPT_STOP_ALL_RESOURCES "stop-all-resources"
	#define PCMK_OPT_STOP_ORPHAN_ACTIONS "stop-orphan-actions"
	#define PCMK_OPT_STOP_ORPHAN_RESOURCES "stop-orphan-resources"
	#define PCMK_OPT_SYMMETRIC_CLUSTER "symmetric-cluster"
	#define PCMK_OPT_TRANSITION_DELAY "transition-delay"


	/*
	* Meta-attributes
	*/

	#define PCMK_META_ALLOW_MIGRATE "allow-migrate"
	#define PCMK_META_ALLOW_UNHEALTHY_NODES "allow-unhealthy-nodes"
	#define PCMK_META_CLONE_MAX "clone-max"
	#define PCMK_META_CLONE_MIN "clone-min"
	#define PCMK_META_CLONE_NODE_MAX "clone-node-max"
	#define PCMK_META_CONTAINER_ATTR_TARGET "container-attribute-target"
	#define PCMK_META_CRITICAL "critical"
	#define PCMK_META_ENABLED "enabled"
	#define PCMK_META_FAILURE_TIMEOUT "failure-timeout"
	#define PCMK_META_GLOBALLY_UNIQUE "globally-unique"
	#define PCMK_META_INTERLEAVE "interleave"
	#define PCMK_META_IS_MANAGED "is-managed"
	#define PCMK_META_MAINTENANCE "maintenance"
	#define PCMK_META_MIGRATION_THRESHOLD "migration-threshold"
	#define PCMK_META_MULTIPLE_ACTIVE "multiple-active"
	#define PCMK_META_NOTIFY "notify"
	#define PCMK_META_ORDERED "ordered"
	#define PCMK_META_PRIORITY "priority"
	#define PCMK_META_PROMOTABLE "promotable"
	#define PCMK_META_PROMOTED_MAX "promoted-max"
	#define PCMK_META_PROMOTED_NODE_MAX "promoted-node-max"
	#define PCMK_META_REMOTE_ADDR "remote-addr"
	#define PCMK_META_REMOTE_ALLOW_MIGRATE "remote-allow-migrate"
	#define PCMK_META_REMOTE_CONNECT_TIMEOUT "remote-connect-timeout"
	#define PCMK_META_REMOTE_NODE "remote-node"
	#define PCMK_META_REMOTE_PORT "remote-port"
	#define PCMK_META_REQUIRES "requires"
	#define PCMK_META_RESOURCE_STICKINESS "resource-stickiness"
	#define PCMK_META_TARGET_ROLE "target-role"


	/*
	* Remote resource instance attributes
	*/

	#define PCMK_REMOTE_RA_ADDR "addr"
	#define PCMK_REMOTE_RA_PORT "port"
	#define PCMK_REMOTE_RA_RECONNECT_INTERVAL "reconnect_interval"
	#define PCMK_REMOTE_RA_SERVER "server"


	/*
	* Enumerated values
	*/

	+#define PCMK_VALUE_FALSE "false"
	#define PCMK_VALUE_TRUE "true"


	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_OPTIONS__H
	diff --git a/include/crm/common/strings_internal.h b/include/crm/common/strings_internal.h
	index ecb4a981e6..3558cc1734 100644
	--- a/include/crm/common/strings_internal.h
	+++ b/include/crm/common/strings_internal.h
	@@ -1,214 +1,214 @@
	/*
	* Copyright 2015-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__STRINGS_INTERNAL__H
	#define PCMK__STRINGS_INTERNAL__H

	#include <stdbool.h> // bool

	#include <glib.h> // guint, GList, GHashTable

	/* internal constants for generic string functions (from strings.c) */

	#define PCMK__PARSE_INT_DEFAULT -1
	#define PCMK__PARSE_DBL_DEFAULT -1.0

	/* internal generic string functions (from strings.c) */

	enum pcmk__str_flags {
	pcmk__str_none = 0,
	pcmk__str_casei = 1 << 0,
	pcmk__str_null_matches = 1 << 1,
	pcmk__str_regex = 1 << 2,
	pcmk__str_star_matches = 1 << 3,
	};

	int pcmk__scan_double(const char text, double result,
	const char default_text, char *end_text);
	int pcmk__guint_from_hash(GHashTable table, const char key, guint default_val,
	guint *result);
	bool pcmk__starts_with(const char str, const char prefix);
	bool pcmk__ends_with(const char s, const char match);
	bool pcmk__ends_with_ext(const char s, const char match);
	char pcmk__trim(char str);
	void pcmk__add_separated_word(GString **list, size_t init_size,
	const char word, const char separator);
	int pcmk__compress(const char *data, unsigned int length, unsigned int max,
	char *result, unsigned int result_len);

	int pcmk__scan_ll(const char text, long long result, long long default_value);
	int pcmk__scan_min_int(const char text, int result, int minimum);
	int pcmk__scan_port(const char text, int port);
	int pcmk__parse_ll_range(const char srcstring, long long start, long long *end);

	GHashTable *pcmk__strkey_table(GDestroyNotify key_destroy_func,
	GDestroyNotify value_destroy_func);
	GHashTable *pcmk__strikey_table(GDestroyNotify key_destroy_func,
	GDestroyNotify value_destroy_func);
	GHashTable pcmk__str_table_dup(GHashTable old_table);

	/*!
	* \internal
	* \brief Get a string value with a default if NULL
	*
	* \param[in] s String to return if non-NULL
	* \param[in] default_value String (or NULL) to return if \p s is NULL
	*
	* \return \p s if \p s is non-NULL, otherwise \p default_value
	*/
	static inline const char *
	pcmk__s(const char s, const char default_value)
	{
	return (s == NULL)? default_value : s;
	}

	/*!
	* \internal
	* \brief Create a hash table with integer keys
	*
	* \param[in] value_destroy_func Function to free a value
	*
	* \return Newly allocated hash table
	* \note It is the caller's responsibility to free the result, using
	* g_hash_table_destroy().
	*/
	static inline GHashTable *
	pcmk__intkey_table(GDestroyNotify value_destroy_func)
	{
	return g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL,
	value_destroy_func);
	}

	/*!
	* \internal
	* \brief Insert a value into a hash table with integer keys
	*
	* \param[in,out] hash_table Table to insert into
	* \param[in] key Integer key to insert
	* \param[in] value Value to insert
	*
	* \return Whether the key/value was already in the table
	* \note This has the same semantics as g_hash_table_insert(). If the key
	* already exists in the table, the old value is freed and replaced.
	*/
	static inline gboolean
	pcmk__intkey_table_insert(GHashTable *hash_table, int key, gpointer value)
	{
	return g_hash_table_insert(hash_table, GINT_TO_POINTER(key), value);
	}

	/*!
	* \internal
	* \brief Look up a value in a hash table with integer keys
	*
	* \param[in] hash_table Table to check
	* \param[in] key Integer key to look for
	*
	* \return Value in table for \key (or NULL if not found)
	*/
	static inline gpointer
	pcmk__intkey_table_lookup(GHashTable *hash_table, int key)
	{
	return g_hash_table_lookup(hash_table, GINT_TO_POINTER(key));
	}

	/*!
	* \internal
	* \brief Remove a key/value from a hash table with integer keys
	*
	* \param[in,out] hash_table Table to modify
	* \param[in] key Integer key of entry to remove
	*
	* \return Whether \p key was found and removed from \p hash_table
	*/
	static inline gboolean
	pcmk__intkey_table_remove(GHashTable *hash_table, int key)
	{
	return g_hash_table_remove(hash_table, GINT_TO_POINTER(key));
	}

	gboolean pcmk__str_in_list(const gchar s, const GList lst, uint32_t flags);

	bool pcmk__strcase_any_of(const char *s, ...) G_GNUC_NULL_TERMINATED;
	bool pcmk__str_any_of(const char *s, ...) G_GNUC_NULL_TERMINATED;
	bool pcmk__char_in_any_str(int ch, ...) G_GNUC_NULL_TERMINATED;

	int pcmk__strcmp(const char s1, const char s2, uint32_t flags);
	int pcmk__numeric_strcasecmp(const char s1, const char s2);
	void pcmk__str_update(char *str, const char value);
	void pcmk__g_strcat(GString *buffer, ...) G_GNUC_NULL_TERMINATED;

	static inline bool
	pcmk__str_eq(const char s1, const char s2, uint32_t flags)
	{
	return pcmk__strcmp(s1, s2, flags) == 0;
	}

	// Like pcmk__add_separated_word() but using a space as separator
	static inline void
	pcmk__add_word(GString *list, size_t init_size, const char word)
	{
	return pcmk__add_separated_word(list, init_size, word, " ");
	}

	/* Correctly displaying singular or plural is complicated; consider "1 node has"
	* vs. "2 nodes have". A flexible solution is to pluralize entire strings, e.g.
	*
	* if (a == 1) {
	* crm_info("singular message"):
	* } else {
	* crm_info("plural message");
	* }
	*
	* though even that's not sufficient for all languages besides English (if we
	* ever desire to do translations of output and log messages). But the following
	* convenience macros are "good enough" and more concise for many cases.
	*/

	/* Example:
	* crm_info("Found %d %s", nentries,
	* pcmk__plural_alt(nentries, "entry", "entries"));
	*/
	#define pcmk__plural_alt(i, s1, s2) (((i) == 1)? (s1) : (s2))

	// Example: crm_info("Found %d node%s", nnodes, pcmk__plural_s(nnodes));
	#define pcmk__plural_s(i) pcmk__plural_alt(i, "", "s")

	static inline int
	pcmk__str_empty(const char *s)
	{
	return (s == NULL) \|\| (s[0] == '\0');
	}

	static inline char *
	pcmk__itoa(int an_int)
	{
	return crm_strdup_printf("%d", an_int);
	}

	static inline char *
	pcmk__ftoa(double a_float)
	{
	return crm_strdup_printf("%f", a_float);
	}

	static inline char *
	pcmk__ttoa(time_t epoch_time)
	{
	return crm_strdup_printf("%lld", (long long) epoch_time);
	}

	// note this returns const not allocated
	static inline const char *
	pcmk__btoa(bool condition)
	{
	- return condition? PCMK_VALUE_TRUE : "false";
	+ return condition? PCMK_VALUE_TRUE : PCMK_VALUE_FALSE;
	}

	#endif /* PCMK__STRINGS_INTERNAL__H */
	diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
	index 3ddfd055cb..eca7ffeaaa 100644
	--- a/include/crm/msg_xml.h
	+++ b/include/crm/msg_xml.h
	@@ -1,383 +1,383 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_MSG_XML__H
	# define PCMK__CRM_MSG_XML__H

	# include <crm/common/xml.h>

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	#include <crm/msg_xml_compat.h>
	#endif

	#ifdef __cplusplus
	extern "C" {
	#endif

	/* This file defines constants for various XML syntax (mainly element and
	* attribute names).
	*
	* For consistency, new constants should start with "PCMK_", followed by "XE"
	* for XML element names, "XA" for XML attribute names, and "META" for meta
	* attribute names. Old names that don't follow this policy should eventually be
	* deprecated and replaced with names that do.
	*
	* Symbols should be public if the user may specify them somewhere (especially
	* the CIB). They should be internal if they're used only internally to
	* Pacemaker (such as daemon IPC/CPG message XML).
	*
	* For meta-attributes that can be specified as either XML attributes or nvpair
	* names, use "META" unless using both "XA" and "META" constants adds clarity.
	*/

	/*
	* XML elements
	*/

	#define PCMK_XE_DATE_EXPRESSION "date_expression"
	#define PCMK_XE_OP "op"
	#define PCMK_XE_OP_EXPRESSION "op_expression"
	#define PCMK_XE_RSC_EXPRESSION "rsc_expression"

	/* This has been deprecated as a CIB element (an alias for <clone> with
	* PCMK_META_PROMOTABLE set to "true") since 2.0.0.
	*/
	#define PCMK_XE_PROMOTABLE_LEGACY "master"


	/*
	* XML attributes
	*/

	#define PCMK_XA_ADMIN_EPOCH "admin_epoch"
	#define PCMK_XA_CIB_LAST_WRITTEN "cib-last-written"
	#define PCMK_XA_CLASS "class"
	#define PCMK_XA_CRM_DEBUG_ORIGIN "crm-debug-origin"
	#define PCMK_XA_CRM_FEATURE_SET "crm_feature_set"
	#define PCMK_XA_CRM_TIMESTAMP "crm-timestamp"
	#define PCMK_XA_DC_UUID "dc-uuid"
	#define PCMK_XA_DESCRIPTION "description"
	#define PCMK_XA_DEVICES "devices"
	#define PCMK_XA_EPOCH "epoch"
	#define PCMK_XA_FORMAT "format"
	#define PCMK_XA_HAVE_QUORUM "have-quorum"
	#define PCMK_XA_ID "id"
	#define PCMK_XA_ID_REF "id-ref"
	#define PCMK_XA_INDEX "index"
	#define PCMK_XA_NAME "name"
	#define PCMK_XA_NO_QUORUM_PANIC "no-quorum-panic"
	#define PCMK_XA_NUM_UPDATES "num_updates"
	#define PCMK_XA_OP "op"
	#define PCMK_XA_PROVIDER "provider"
	#define PCMK_XA_REFERENCE "reference"
	#define PCMK_XA_REQUEST "request"
	#define PCMK_XA_TARGET "target"
	#define PCMK_XA_TARGET_ATTRIBUTE "target-attribute"
	#define PCMK_XA_TARGET_PATTERN "target-pattern"
	#define PCMK_XA_TARGET_VALUE "target-value"
	#define PCMK_XA_TYPE "type"
	#define PCMK_XA_UNAME "uname"
	#define PCMK_XA_UPDATE_CLIENT "update-client"
	#define PCMK_XA_UPDATE_ORIGIN "update-origin"
	#define PCMK_XA_UPDATE_USER "update-user"
	#define PCMK_XA_VALIDATE_WITH "validate-with"
	#define PCMK_XA_VALUE "value"
	#define PCMK_XA_VERSION "version"


	/*
	* Older constants that don't follow current naming
	*/

	# ifndef T_CRM
	# define T_CRM "crmd"
	# endif

	# ifndef T_ATTRD
	# define T_ATTRD "attrd"
	# endif

	# define CIB_OPTIONS_FIRST "cib-bootstrap-options"

	# define F_CRM_DATA "crm_xml"
	# define F_CRM_TASK "crm_task"
	# define F_CRM_HOST_TO "crm_host_to"
	# define F_CRM_SYS_TO "crm_sys_to"
	# define F_CRM_SYS_FROM "crm_sys_from"
	# define F_CRM_VERSION PCMK_XA_VERSION
	# define F_CRM_ORIGIN "origin"
	# define F_CRM_USER "crm_user"
	# define F_CRM_JOIN_ID "join_id"
	# define F_CRM_DC_LEAVING "dc-leaving"
	# define F_CRM_ELECTION_ID "election-id"
	# define F_CRM_ELECTION_AGE_S "election-age-sec"
	# define F_CRM_ELECTION_AGE_US "election-age-nano-sec"
	# define F_CRM_ELECTION_OWNER "election-owner"
	# define F_CRM_TGRAPH "crm-tgraph-file"
	# define F_CRM_TGRAPH_INPUT "crm-tgraph-in"

	# define F_CRM_THROTTLE_MODE "crm-limit-mode"
	# define F_CRM_THROTTLE_MAX "crm-limit-max"

	/---- Common tags/attrs /
	# define XML_DIFF_MARKER "__crm_diff_marker__"
	# define XML_TAG_CIB "cib"
	# define XML_TAG_FAILED "failed"

	# define XML_ATTR_TIMEOUT "timeout"

	-# define XML_BOOLEAN_FALSE "false"
	+# define XML_BOOLEAN_FALSE PCMK_VALUE_FALSE

	# define XML_TAG_OPTIONS "options"

	/---- top level tags/attrs /
	# define XML_CRM_TAG_PING "ping_response"
	# define XML_PING_ATTR_STATUS "result"
	# define XML_PING_ATTR_SYSFROM "crm_subsystem"
	# define XML_PING_ATTR_CRMDSTATE "crmd_state"
	# define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state"
	# define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init"
	# define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons"
	# define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping"
	# define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running"
	# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down"
	# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete"
	# define XML_PING_ATTR_PACEMAKERDSTATE_REMOTE "remote"

	# define XML_FAIL_TAG_CIB "failed_update"

	# define XML_FAILCIB_ATTR_OBJTYPE "object_type"
	# define XML_FAILCIB_ATTR_OP "operation"
	# define XML_FAILCIB_ATTR_REASON "reason"

	/---- CIB specific tags/attrs /
	# define XML_CIB_TAG_SECTION_ALL "all"
	# define XML_CIB_TAG_CONFIGURATION "configuration"
	# define XML_CIB_TAG_STATUS "status"
	# define XML_CIB_TAG_RESOURCES "resources"
	# define XML_CIB_TAG_NODES "nodes"
	# define XML_CIB_TAG_CONSTRAINTS "constraints"
	# define XML_CIB_TAG_CRMCONFIG "crm_config"
	# define XML_CIB_TAG_OPCONFIG "op_defaults"
	# define XML_CIB_TAG_RSCCONFIG "rsc_defaults"
	# define XML_CIB_TAG_ACLS "acls"
	# define XML_CIB_TAG_ALERTS "alerts"
	# define XML_CIB_TAG_ALERT "alert"
	# define XML_CIB_TAG_ALERT_RECIPIENT "recipient"
	# define XML_CIB_TAG_ALERT_SELECT "select"
	# define XML_CIB_TAG_ALERT_ATTRIBUTES "select_attributes"
	# define XML_CIB_TAG_ALERT_FENCING "select_fencing"
	# define XML_CIB_TAG_ALERT_NODES "select_nodes"
	# define XML_CIB_TAG_ALERT_RESOURCES "select_resources"
	# define XML_CIB_TAG_ALERT_ATTR "attribute"

	# define XML_CIB_TAG_STATE "node_state"
	# define XML_CIB_TAG_NODE "node"
	# define XML_CIB_TAG_NVPAIR "nvpair"

	# define XML_CIB_TAG_PROPSET "cluster_property_set"
	# define XML_TAG_ATTR_SETS "instance_attributes"
	# define XML_TAG_META_SETS "meta_attributes"
	# define XML_TAG_ATTRS "attributes"
	# define XML_TAG_PARAMS "parameters"
	# define XML_TAG_PARAM "param"
	# define XML_TAG_UTILIZATION "utilization"

	# define XML_TAG_RESOURCE_REF "resource_ref"
	# define XML_CIB_TAG_RESOURCE "primitive"
	# define XML_CIB_TAG_GROUP "group"
	# define XML_CIB_TAG_INCARNATION "clone"
	# define XML_CIB_TAG_CONTAINER "bundle"

	# define XML_CIB_TAG_RSC_TEMPLATE "template"

	# define XML_OP_ATTR_ON_FAIL "on-fail"
	# define XML_OP_ATTR_START_DELAY "start-delay"
	# define XML_OP_ATTR_ORIGIN "interval-origin"
	# define XML_OP_ATTR_PENDING "record-pending"
	# define XML_OP_ATTR_DIGESTS_ALL "digests-all"
	# define XML_OP_ATTR_DIGESTS_SECURE "digests-secure"

	# define XML_CIB_TAG_LRM "lrm"
	# define XML_LRM_TAG_RESOURCES "lrm_resources"
	# define XML_LRM_TAG_RESOURCE "lrm_resource"
	# define XML_LRM_TAG_RSC_OP "lrm_rsc_op"

	//! \deprecated Do not use (will be removed in a future release)
	# define XML_CIB_ATTR_REPLACE "replace"

	# define XML_CIB_ATTR_PRIORITY "priority"

	# define XML_NODE_IS_REMOTE "remote_node"
	# define XML_NODE_IS_FENCED "node_fenced"
	# define XML_NODE_IS_MAINTENANCE "node_in_maintenance"

	# define XML_CIB_ATTR_SHUTDOWN "shutdown"

	/* Aside from being an old name for the executor, LRM is a misnomer here because
	* the controller and scheduler use these to track actions, which are not always
	* executor operations.
	*/

	// XML attribute that takes interval specification (user-facing configuration)
	# define XML_LRM_ATTR_INTERVAL "interval"

	// XML attribute that takes interval in milliseconds (daemon APIs)
	// (identical value as above, but different constant allows clearer code intent)
	# define XML_LRM_ATTR_INTERVAL_MS XML_LRM_ATTR_INTERVAL

	# define XML_LRM_ATTR_TASK "operation"
	# define XML_LRM_ATTR_TASK_KEY "operation_key"
	# define XML_LRM_ATTR_TARGET "on_node"
	# define XML_LRM_ATTR_TARGET_UUID "on_node_uuid"
	/*! Actions to be executed on Pacemaker Remote nodes are routed through the
	* controller on the cluster node hosting the remote connection. That cluster
	* node is considered the router node for the action.
	*/
	# define XML_LRM_ATTR_ROUTER_NODE "router_node"
	# define XML_LRM_ATTR_RSCID "rsc-id"
	# define XML_LRM_ATTR_OPSTATUS "op-status"
	# define XML_LRM_ATTR_RC "rc-code"
	# define XML_LRM_ATTR_CALLID "call-id"
	# define XML_LRM_ATTR_OP_DIGEST "op-digest"
	# define XML_LRM_ATTR_OP_RESTART "op-force-restart"
	# define XML_LRM_ATTR_OP_SECURE "op-secure-params"
	# define XML_LRM_ATTR_RESTART_DIGEST "op-restart-digest"
	# define XML_LRM_ATTR_SECURE_DIGEST "op-secure-digest"
	# define XML_LRM_ATTR_EXIT_REASON "exit-reason"

	# define XML_RSC_OP_LAST_CHANGE "last-rc-change"
	# define XML_RSC_OP_T_EXEC "exec-time"
	# define XML_RSC_OP_T_QUEUE "queue-time"

	# define XML_LRM_ATTR_MIGRATE_SOURCE "migrate_source"
	# define XML_LRM_ATTR_MIGRATE_TARGET "migrate_target"

	# define XML_TAG_GRAPH "transition_graph"
	# define XML_GRAPH_TAG_RSC_OP "rsc_op"
	# define XML_GRAPH_TAG_PSEUDO_EVENT "pseudo_event"
	# define XML_GRAPH_TAG_CRM_EVENT "crm_event"
	# define XML_GRAPH_TAG_DOWNED "downed"
	# define XML_GRAPH_TAG_MAINTENANCE "maintenance"

	# define XML_TAG_RULE "rule"
	# define XML_RULE_ATTR_SCORE "score"
	# define XML_RULE_ATTR_SCORE_ATTRIBUTE "score-attribute"
	# define XML_RULE_ATTR_ROLE "role"
	# define XML_RULE_ATTR_BOOLEAN_OP "boolean-op"

	# define XML_TAG_EXPRESSION "expression"
	# define XML_EXPR_ATTR_ATTRIBUTE "attribute"
	# define XML_EXPR_ATTR_OPERATION "operation"
	# define XML_EXPR_ATTR_VALUE_SOURCE "value-source"

	# define XML_CONS_TAG_RSC_DEPEND "rsc_colocation"
	# define XML_CONS_TAG_RSC_ORDER "rsc_order"
	# define XML_CONS_TAG_RSC_LOCATION "rsc_location"
	# define XML_CONS_TAG_RSC_TICKET "rsc_ticket"
	# define XML_CONS_TAG_RSC_SET "resource_set"
	# define XML_CONS_ATTR_SYMMETRICAL "symmetrical"

	# define XML_LOCATION_ATTR_DISCOVERY "resource-discovery"

	# define XML_COLOC_ATTR_SOURCE "rsc"
	# define XML_COLOC_ATTR_SOURCE_ROLE "rsc-role"
	# define XML_COLOC_ATTR_TARGET "with-rsc"
	# define XML_COLOC_ATTR_TARGET_ROLE "with-rsc-role"
	# define XML_COLOC_ATTR_NODE_ATTR "node-attribute"
	# define XML_COLOC_ATTR_INFLUENCE "influence"

	//! \deprecated Deprecated since 2.1.5
	# define XML_COLOC_ATTR_SOURCE_INSTANCE "rsc-instance"

	//! \deprecated Deprecated since 2.1.5
	# define XML_COLOC_ATTR_TARGET_INSTANCE "with-rsc-instance"

	# define XML_LOC_ATTR_SOURCE "rsc"
	# define XML_LOC_ATTR_SOURCE_PATTERN "rsc-pattern"

	# define XML_ORDER_ATTR_FIRST "first"
	# define XML_ORDER_ATTR_THEN "then"
	# define XML_ORDER_ATTR_FIRST_ACTION "first-action"
	# define XML_ORDER_ATTR_THEN_ACTION "then-action"
	# define XML_ORDER_ATTR_KIND "kind"

	//! \deprecated Deprecated since 2.1.5
	# define XML_ORDER_ATTR_FIRST_INSTANCE "first-instance"

	//! \deprecated Deprecated since 2.1.5
	# define XML_ORDER_ATTR_THEN_INSTANCE "then-instance"

	# define XML_TICKET_ATTR_TICKET "ticket"
	# define XML_TICKET_ATTR_LOSS_POLICY "loss-policy"

	# define XML_NODE_ATTR_RSC_DISCOVERY "resource-discovery-enabled"

	# define XML_ALERT_ATTR_PATH "path"
	# define XML_ALERT_ATTR_TIMEOUT "timeout"
	# define XML_ALERT_ATTR_TSTAMP_FORMAT "timestamp-format"

	# define XML_CIB_TAG_GENERATION_TUPPLE "generation_tuple"

	# define XML_ATTR_TE_NOWAIT "op_no_wait"
	# define XML_ATTR_TE_TARGET_RC "op_target_rc"
	# define XML_TAG_TRANSIENT_NODEATTRS "transient_attributes"

	//! \deprecated Do not use (will be removed in a future release)
	# define XML_TAG_DIFF_ADDED "diff-added"

	//! \deprecated Do not use (will be removed in a future release)
	# define XML_TAG_DIFF_REMOVED "diff-removed"

	# define XML_ACL_TAG_USER "acl_target"
	# define XML_ACL_TAG_USERv1 "acl_user"
	# define XML_ACL_TAG_GROUP "acl_group"
	# define XML_ACL_TAG_ROLE "acl_role"
	# define XML_ACL_TAG_PERMISSION "acl_permission"
	# define XML_ACL_TAG_ROLE_REF "role"
	# define XML_ACL_TAG_ROLE_REFv1 "role_ref"
	# define XML_ACL_ATTR_KIND "kind"
	# define XML_ACL_TAG_READ "read"
	# define XML_ACL_TAG_WRITE "write"
	# define XML_ACL_TAG_DENY "deny"
	# define XML_ACL_ATTR_REFv1 "ref"
	# define XML_ACL_ATTR_TAG "object-type"
	# define XML_ACL_ATTR_TAGv1 "tag"
	# define XML_ACL_ATTR_XPATH "xpath"
	# define XML_ACL_ATTR_ATTRIBUTE "attribute"

	# define XML_CIB_TAG_TICKETS "tickets"
	# define XML_CIB_TAG_TICKET_STATE "ticket_state"

	# define XML_CIB_TAG_TAGS "tags"
	# define XML_CIB_TAG_TAG "tag"
	# define XML_CIB_TAG_OBJ_REF "obj_ref"

	# define XML_TAG_FENCING_TOPOLOGY "fencing-topology"
	# define XML_TAG_FENCING_LEVEL "fencing-level"

	# define XML_TAG_DIFF "diff"
	# define XML_DIFF_VERSION "version"
	# define XML_DIFF_VSOURCE "source"
	# define XML_DIFF_VTARGET "target"
	# define XML_DIFF_CHANGE "change"
	# define XML_DIFF_LIST "change-list"
	# define XML_DIFF_ATTR "change-attr"
	# define XML_DIFF_RESULT "change-result"
	# define XML_DIFF_OP "operation"
	# define XML_DIFF_PATH "path"
	# define XML_DIFF_POSITION "position"

	# define ID(x) crm_element_value(x, PCMK_XA_ID)

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/include/crm/msg_xml_compat.h b/include/crm/msg_xml_compat.h
	index a20ac5dfff..384ad32f86 100644
	--- a/include/crm/msg_xml_compat.h
	+++ b/include/crm/msg_xml_compat.h
	@@ -1,407 +1,407 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_MSG_XML_COMPAT__H
	# define PCMK__CRM_MSG_XML_COMPAT__H

	#include <crm/common/agents.h> // PCMK_STONITH_PROVIDES

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Deprecated Pacemaker XML constants API
	* \ingroup core
	* \deprecated Do not include this header directly. The XML constants in this
	* header, and the header itself, will be removed in a future
	* release.
	*/

	//! \deprecated Use PCMK_META_CLONE_MAX instead
	#define XML_RSC_ATTR_INCARNATION_MAX PCMK_META_CLONE_MAX

	//! \deprecated Use PCMK_META_CLONE_MIN instead
	#define XML_RSC_ATTR_INCARNATION_MIN PCMK_META_CLONE_MIN

	//! \deprecated Use PCMK_META_CLONE_NODE_MAX instead
	#define XML_RSC_ATTR_INCARNATION_NODEMAX PCMK_META_CLONE_NODE_MAX

	//! \deprecated Use PCMK_META_PROMOTED_MAX instead
	#define XML_RSC_ATTR_PROMOTED_MAX PCMK_META_PROMOTED_MAX

	//! \deprecated Use PCMK_META_PROMOTED_NODE_MAX instead
	#define XML_RSC_ATTR_PROMOTED_NODEMAX PCMK_META_PROMOTED_NODE_MAX

	//! \deprecated Use PCMK_STONITH_PROVIDES instead
	#define XML_RSC_ATTR_PROVIDES PCMK_STONITH_PROVIDES

	//! \deprecated Use PCMK_XE_PROMOTABLE_LEGACY instead
	#define XML_CIB_TAG_MASTER PCMK_XE_PROMOTABLE_LEGACY

	//! \deprecated Do not use
	#define PCMK_XA_PROMOTED_MAX_LEGACY "master-max"

	//! \deprecated Do not use
	#define PCMK_XE_PROMOTED_MAX_LEGACY PCMK_XA_PROMOTED_MAX_LEGACY

	//! \deprecated Do not use
	#define XML_RSC_ATTR_MASTER_MAX PCMK_XA_PROMOTED_MAX_LEGACY

	//! \deprecated Do not use
	#define PCMK_XA_PROMOTED_NODE_MAX_LEGACY "master-node-max"

	//! \deprecated Do not use
	#define PCMK_XE_PROMOTED_NODE_MAX_LEGACY PCMK_XA_PROMOTED_NODE_MAX_LEGACY

	//! \deprecated Do not use
	#define XML_RSC_ATTR_MASTER_NODEMAX PCMK_XA_PROMOTED_NODE_MAX_LEGACY

	//! \deprecated Use PCMK_META_MIGRATION_THRESHOLD instead
	#define XML_RSC_ATTR_FAIL_STICKINESS PCMK_META_MIGRATION_THRESHOLD

	//! \deprecated Use PCMK_META_FAILURE_TIMEOUT instead
	#define XML_RSC_ATTR_FAIL_TIMEOUT PCMK_META_FAILURE_TIMEOUT

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_ATTR_RA_VERSION "ra-version"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_TAG_FRAGMENT "cib_fragment"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_TAG_RSC_VER_ATTRS "rsc_versioned_attrs"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_TAG_OP_VER_ATTRS "op_versioned_attrs"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_TAG_OP_VER_META "op_versioned_meta"

	//! \deprecated Use \p PCMK_XA_ID instead
	#define XML_ATTR_UUID "id"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_ATTR_VERBOSE "verbose"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_CIB_TAG_DOMAINS "domains"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_CIB_ATTR_SOURCE "source"

	//! \deprecated Do not use
	#define XML_NODE_EXPECTED "expected"

	//! \deprecated Do not use
	#define XML_NODE_IN_CLUSTER "in_ccm"

	//! \deprecated Do not use
	#define XML_NODE_IS_PEER "crmd"

	//! \deprecated Do not use
	#define XML_NODE_JOIN_STATE "join"

	//! \deprecated Do not use (will be removed in a future release)
	#define XML_RSC_OP_LAST_RUN "last-run"

	//! \deprecated Use name member directly
	#define TYPE(x) (((x) == NULL)? NULL : (const char *) ((x)->name))

	//! \deprecated Use \c PCMK_OPT_CLUSTER_RECHECK_INTERVAL instead
	#define XML_CONFIG_ATTR_RECHECK PCMK_OPT_CLUSTER_RECHECK_INTERVAL

	//! \deprecated Use \c PCMK_OPT_DC_DEADTIME instead
	#define XML_CONFIG_ATTR_DC_DEADTIME PCMK_OPT_DC_DEADTIME

	//! \deprecated Use \c PCMK_OPT_ELECTION_TIMEOUT instead
	#define XML_CONFIG_ATTR_ELECTION_FAIL PCMK_OPT_ELECTION_TIMEOUT

	//! \deprecated Use \c PCMK_OPT_FENCE_REACTION instead
	#define XML_CONFIG_ATTR_FENCE_REACTION PCMK_OPT_FENCE_REACTION

	//! \deprecated Use \c PCMK_OPT_HAVE_WATCHDOG instead
	#define XML_ATTR_HAVE_WATCHDOG PCMK_OPT_HAVE_WATCHDOG

	//! \deprecated Use \c PCMK_OPT_NODE_PENDING_TIMEOUT instead
	#define XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT PCMK_OPT_NODE_PENDING_TIMEOUT

	//! \deprecated Use \c PCMK_OPT_PRIORITY_FENCING_DELAY instead
	#define XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY PCMK_OPT_PRIORITY_FENCING_DELAY

	//! \deprecated Use \c PCMK_OPT_SHUTDOWN_ESCALATION instead
	#define XML_CONFIG_ATTR_FORCE_QUIT PCMK_OPT_SHUTDOWN_ESCALATION

	//! \deprecated Use \c PCMK_OPT_SHUTDOWN_LOCK instead
	#define XML_CONFIG_ATTR_SHUTDOWN_LOCK PCMK_OPT_SHUTDOWN_LOCK

	//! \deprecated Use \c PCMK_OPT_SHUTDOWN_LOCK_LIMIT instead
	#define XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT PCMK_OPT_SHUTDOWN_LOCK_LIMIT

	//! \deprecated Use \c PCMK_XA_CRM_FEATURE_SET instead
	#define XML_ATTR_CRM_VERSION PCMK_XA_CRM_FEATURE_SET

	//! \deprecated Do not use
	#define XML_ATTR_DIGEST "digest"

	//! \deprecated Use \c PCMK_XA_VALIDATE_WITH instead
	#define XML_ATTR_VALIDATION PCMK_XA_VALIDATE_WITH

	//! \deprecated Use \c PCMK_XA_NO_QUORUM_PANIC instead
	#define XML_ATTR_QUORUM_PANIC PCMK_XA_NO_QUORUM_PANIC

	//! \deprecated Use \c PCMK_XA_HAVE_QUORUM instead
	#define XML_ATTR_HAVE_QUORUM PCMK_XA_HAVE_QUORUM

	//! \deprecated Use \c PCMK_XA_EPOCH instead
	#define XML_ATTR_GENERATION PCMK_XA_EPOCH

	//! \deprecated Use \c PCMK_XA_ADMIN_EPOCH instead
	#define XML_ATTR_GENERATION_ADMIN PCMK_XA_ADMIN_EPOCH

	//! \deprecated Use \c PCMK_XA_NUM_UPDATES instead
	#define XML_ATTR_NUMUPDATES PCMK_XA_NUM_UPDATES

	//! \deprecated Use \c PCMK_XA_CRM_DEBUG_ORIGIN instead
	#define XML_ATTR_ORIGIN PCMK_XA_CRM_DEBUG_ORIGIN

	//! \deprecated Use \c PCMK_XA_CRM_TIMESTAMP instead
	#define XML_ATTR_TSTAMP PCMK_XA_CRM_TIMESTAMP

	//! \deprecated Use \c PCMK_XA_CIB_LAST_WRITTEN instead
	#define XML_CIB_ATTR_WRITTEN PCMK_XA_CIB_LAST_WRITTEN

	//! \deprecated Use \c PCMK_XA_VERSION instead
	#define XML_ATTR_VERSION PCMK_XA_VERSION

	//! \deprecated Use \c PCMK_XA_DESCRIPTION instead
	#define XML_ATTR_DESC PCMK_XA_DESCRIPTION

	//! \deprecated Use \c PCMK_XA_ID instead
	#define XML_ATTR_ID PCMK_XA_ID

	//! \deprecated Use \c PCMK_XA_ID instead
	#define XML_FAILCIB_ATTR_ID PCMK_XA_ID

	//! \deprecated Use \c PCMK_META_CONTAINER_ATTR_TARGET instead
	#define XML_RSC_ATTR_TARGET PCMK_META_CONTAINER_ATTR_TARGET

	//! \deprecated Do not use
	#define XML_RSC_ATTR_RESTART "restart-type"

	//! \deprecated Use \c PCMK_META_ORDERED instead
	#define XML_RSC_ATTR_ORDERED PCMK_META_ORDERED

	//! \deprecated Use \c PCMK_META_INTERLEAVE instead
	#define XML_RSC_ATTR_INTERLEAVE PCMK_META_INTERLEAVE

	//! \deprecated Do not use
	#define XML_RSC_ATTR_INCARNATION "clone"

	//! \deprecated Use \c PCMK_META_PROMOTABLE instead
	#define XML_RSC_ATTR_PROMOTABLE PCMK_META_PROMOTABLE

	//! \deprecated Use \c PCMK_META_IS_MANAGED instead
	#define XML_RSC_ATTR_MANAGED PCMK_META_IS_MANAGED

	//! \deprecated Use \c PCMK_META_TARGET_ROLE instead
	#define XML_RSC_ATTR_TARGET_ROLE PCMK_META_TARGET_ROLE

	//! \deprecated Use \c PCMK_META_GLOBALLY_UNIQUE instead
	#define XML_RSC_ATTR_UNIQUE PCMK_META_GLOBALLY_UNIQUE

	//! \deprecated Use \c PCMK_META_NOTIFY instead
	#define XML_RSC_ATTR_NOTIFY PCMK_META_NOTIFY

	//! \deprecated Use \c PCMK_META_RESOURCE_STICKINESS instead
	#define XML_RSC_ATTR_STICKINESS PCMK_META_RESOURCE_STICKINESS

	//! \deprecated Use \c PCMK_META_MULTIPLE_ACTIVE instead
	#define XML_RSC_ATTR_MULTIPLE PCMK_META_MULTIPLE_ACTIVE

	//! \deprecated Use \c PCMK_META_REQUIRES instead
	#define XML_RSC_ATTR_REQUIRES PCMK_META_REQUIRES

	//! \deprecated Do not use
	#define XML_RSC_ATTR_CONTAINER "container"

	//! \deprecated Do not use
	#define XML_RSC_ATTR_INTERNAL_RSC "internal_rsc"

	//! \deprecated Use \c PCMK_META_MAINTENANCE instead
	#define XML_RSC_ATTR_MAINTENANCE PCMK_META_MAINTENANCE

	//! \deprecated Use \c PCMK_META_REMOTE_NODE instead
	#define XML_RSC_ATTR_REMOTE_NODE PCMK_META_REMOTE_NODE

	//! \deprecated Do not use
	#define XML_RSC_ATTR_CLEAR_OP "clear_failure_op"

	//! \deprecated Do not use
	#define XML_RSC_ATTR_CLEAR_INTERVAL "clear_failure_interval"

	//! \deprecated Use \c PCMK_META_CRITICAL instead
	#define XML_RSC_ATTR_CRITICAL PCMK_META_CRITICAL

	//! \deprecated Use \c PCMK_META_ALLOW_MIGRATE instead
	#define XML_OP_ATTR_ALLOW_MIGRATE "allow-migrate"

	//! \deprecated Use \c PCMK_VALUE_TRUE instead
	#define XML_BOOLEAN_YES PCMK_VALUE_TRUE

	-//! \deprecated Use \c XML_BOOLEAN_FALSE instead
	-#define XML_BOOLEAN_NO XML_BOOLEAN_FALSE
	+//! \deprecated Use \c PCMK_VALUE_FALSE instead
	+#define XML_BOOLEAN_NO PCMK_VALUE_FALSE

	//! \deprecated Use \c PCMK_REMOTE_RA_ADDR instead
	#define XML_RSC_ATTR_REMOTE_RA_ADDR "addr"

	//! \deprecated Use \c PCMK_REMOTE_RA_SERVER instead
	#define XML_RSC_ATTR_REMOTE_RA_SERVER "server"

	//! \deprecated Use \c PCMK_REMOTE_RA_PORT instead
	#define XML_RSC_ATTR_REMOTE_RA_PORT "port"

	//! \deprecated Use \c PCMK_REMOTE_RA_RECONNECT_INTERVAL instead
	#define XML_REMOTE_ATTR_RECONNECT_INTERVAL "reconnect_interval"

	//! \deprecated Use \c PCMK_XA_NAME instead
	#define XML_ATTR_NAME PCMK_XA_NAME

	//! \deprecated Use \c PCMK_XA_NAME instead
	#define XML_NVPAIR_ATTR_NAME PCMK_XA_NAME

	//! \deprecated Use \c PCMK_XA_VALUE instead
	#define XML_EXPR_ATTR_VALUE PCMK_XA_VALUE

	//! \deprecated Use \c PCMK_XA_VALUE instead
	#define XML_NVPAIR_ATTR_VALUE PCMK_XA_VALUE

	//! \deprecated Use \c PCMK_XA_VALUE instead
	#define XML_ALERT_ATTR_REC_VALUE PCMK_XA_VALUE

	//! \deprecated Use \c PCMK_XA_ID_REF instead
	#define XML_ATTR_IDREF PCMK_XA_ID_REF

	//! \deprecated Do not use
	#define XML_ATTR_ID_LONG "long-id"

	//! \deprecated Use \c PCMK_XA_TYPE instead
	#define XML_ATTR_TYPE PCMK_XA_TYPE

	//! \deprecated Use \c PCMK_XA_TYPE instead
	#define XML_EXPR_ATTR_TYPE PCMK_XA_TYPE

	//! \deprecated Use \c PCMK_XA_PROVIDER instead
	#define XML_AGENT_ATTR_PROVIDER PCMK_XA_PROVIDER

	//! \deprecated Use \c PCMK_XA_CLASS instead
	#define XML_AGENT_ATTR_CLASS PCMK_XA_CLASS

	//! \deprecated Use \c PCMK_XE_OP instead
	#define XML_ATTR_OP PCMK_XE_OP

	//! \deprecated Use \c PCMK_XA_DC_UUID instead
	#define XML_ATTR_DC_UUID PCMK_XA_DC_UUID

	//! \deprecated Use \c PCMK_XA_UPDATE_ORIGIN instead
	#define XML_ATTR_UPDATE_ORIG PCMK_XA_UPDATE_ORIGIN

	//! \deprecated Use \c PCMK_XA_UPDATE_CLIENT instead
	#define XML_ATTR_UPDATE_CLIENT PCMK_XA_UPDATE_CLIENT

	//! \deprecated Use \c PCMK_XA_UPDATE_USER instead
	#define XML_ATTR_UPDATE_USER PCMK_XA_UPDATE_USER

	//! \deprecated Use \c PCMK_XA_REQUEST instead
	#define XML_ATTR_REQUEST PCMK_XA_REQUEST

	//! \deprecated Do not use
	#define XML_ATTR_RESPONSE "response"

	//! \deprecated Use \c PCMK_XA_UNAME instead
	#define XML_ATTR_UNAME PCMK_XA_UNAME

	//! \deprecated Use \c PCMK_XA_REFERENCE instead
	#define XML_ATTR_REFERENCE PCMK_XA_REFERENCE

	//! \deprecated Use \c PCMK_XA_REFERENCE instead
	#define XML_ACL_ATTR_REF PCMK_XA_REFERENCE

	//! \deprecated Use \c PCMK_XA_REFERENCE instead
	#define F_CRM_REFERENCE PCMK_XA_REFERENCE

	//! \deprecated Do not use
	#define XML_ATTR_TRANSITION_MAGIC "transition-magic"

	//! \deprecated Do not use
	#define XML_ATTR_TRANSITION_KEY "transition-key"

	//! \deprecated Use \c PCMK_XA_INDEX instead
	#define XML_ATTR_STONITH_INDEX PCMK_XA_INDEX

	//! \deprecated Use \c PCMK_XA_TARGET instead
	#define XML_ATTR_STONITH_TARGET PCMK_XA_TARGET

	//! \deprecated Use \c PCMK_XA_TARGET_VALUE instead
	#define XML_ATTR_STONITH_TARGET_VALUE PCMK_XA_TARGET_VALUE

	//! \deprecated Use \c PCMK_XA_TARGET_PATTERN instead
	#define XML_ATTR_STONITH_TARGET_PATTERN PCMK_XA_TARGET_PATTERN

	//! \deprecated Use \c PCMK_XA_TARGET_ATTRIBUTE instead
	#define XML_ATTR_STONITH_TARGET_ATTRIBUTE PCMK_XA_TARGET_ATTRIBUTE

	//! \deprecated Use \c PCMK_XA_DEVICES instead
	#define XML_ATTR_STONITH_DEVICES PCMK_XA_DEVICES

	#ifndef F_ORIG
	//! \deprecated Do not use
	#define F_ORIG "src"
	#endif

	//! \deprecated Do not use
	#define F_CRM_HOST_FROM F_ORIG

	#ifndef F_SEQ
	//! \deprecated Do not use
	#define F_SEQ "seq"
	#endif

	#ifndef F_SUBTYPE
	//! \deprecated Do not use
	#define F_SUBTYPE "subt"
	#endif

	//! \deprecated Do not use
	#define F_CRM_MSG_TYPE F_SUBTYPE

	#ifndef F_TYPE
	//! \deprecated Do not use
	#define F_TYPE "t"
	#endif

	#ifndef F_CLIENTNAME
	//! \deprecated Do not use
	#define F_CLIENTNAME "cn"
	#endif

	#ifndef F_XML_TAGNAME
	//! \deprecated Do not use
	#define F_XML_TAGNAME "__name__"
	#endif

	//! \deprecated Use \c PCMK_VALUE_TRUE instead
	#define XML_BOOLEAN_TRUE PCMK_VALUE_TRUE

	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_MSG_XML_COMPAT__H
	diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
	index 1401ddd1c1..d7fd9a3224 100644
	--- a/lib/cib/cib_utils.c
	+++ b/lib/cib/cib_utils.c
	@@ -1,1113 +1,1113 @@
	/*
	* Original copyright 2004 International Business Machines
	* Later changes copyright 2008-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <sys/utsname.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/cib_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/rules.h>

	xmlNode *
	cib_get_generation(cib_t * cib)
	{
	xmlNode *the_cib = NULL;
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	cib->cmds->query(cib, NULL, &the_cib, cib_scope_local \| cib_sync_call);
	if (the_cib != NULL) {
	copy_in_properties(generation, the_cib);
	free_xml(the_cib);
	}

	return generation;
	}

	gboolean
	cib_version_details(xmlNode * cib, int admin_epoch, int epoch, int *updates)
	{
	*epoch = -1;
	*updates = -1;
	*admin_epoch = -1;

	if (cib == NULL) {
	return FALSE;

	} else {
	crm_element_value_int(cib, PCMK_XA_EPOCH, epoch);
	crm_element_value_int(cib, PCMK_XA_NUM_UPDATES, updates);
	crm_element_value_int(cib, PCMK_XA_ADMIN_EPOCH, admin_epoch);
	}
	return TRUE;
	}

	gboolean
	cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(diff, add, del);

	*admin_epoch = add[0];
	*epoch = add[1];
	*updates = add[2];

	*_admin_epoch = del[0];
	*_epoch = del[1];
	*_updates = del[2];

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get the XML patchset from a CIB diff notification
	*
	* \param[in] msg CIB diff notification
	* \param[out] patchset Where to store XML patchset
	*
	* \return Standard Pacemaker return code
	*/
	int
	cib__get_notify_patchset(const xmlNode msg, const xmlNode *patchset)
	{
	int rc = pcmk_err_generic;

	CRM_ASSERT(patchset != NULL);
	*patchset = NULL;

	if (msg == NULL) {
	crm_err("CIB diff notification received with no XML");
	return ENOMSG;
	}

	if ((crm_element_value_int(msg, F_CIB_RC, &rc) != 0) \|\| (rc != pcmk_ok)) {
	crm_warn("Ignore failed CIB update: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	return pcmk_legacy2rc(rc);
	}

	*patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	if (*patchset == NULL) {
	crm_err("CIB diff notification received with no patchset");
	return ENOMSG;
	}
	return pcmk_rc_ok;
	}

	#define XPATH_DIFF_V1 "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset (v1)
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c XML_TAG_CIB)
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	static bool
	element_in_patchset_v1(const xmlNode patchset, const char element)
	{
	char *xpath = crm_strdup_printf(XPATH_DIFF_V1 "//%s",
	pcmk__s(element, XML_TAG_CIB));
	xmlXPathObject *xpath_obj = xpath_search(patchset, xpath);

	free(xpath);

	if (xpath_obj == NULL) {
	return false;
	}
	freeXpathObject(xpath_obj);
	return true;
	}

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset (v2)
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c XML_TAG_CIB). Supported values include any CIB
	* element supported by \c pcmk__cib_abs_xpath_for().
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	static bool
	element_in_patchset_v2(const xmlNode patchset, const char element)
	{
	const char *element_xpath = pcmk__cib_abs_xpath_for(element);
	const char *parent_xpath = pcmk_cib_parent_name_for(element);
	char *element_regex = NULL;
	bool rc = false;

	CRM_CHECK(element_xpath != NULL, return false); // Unsupported element

	// Matches if and only if element_xpath is part of a changed path
	element_regex = crm_strdup_printf("^%s(/\|$)", element_xpath);

	for (const xmlNode *change = first_named_child(patchset, XML_DIFF_CHANGE);
	change != NULL; change = crm_next_same_xml(change)) {

	const char *op = crm_element_value(change, F_CIB_OPERATION);
	const char *diff_xpath = crm_element_value(change, XML_DIFF_PATH);

	if (pcmk__str_eq(diff_xpath, element_regex, pcmk__str_regex)) {
	// Change to an existing element
	rc = true;
	break;
	}

	if (pcmk__str_eq(op, "create", pcmk__str_none)
	&& pcmk__str_eq(diff_xpath, parent_xpath, pcmk__str_none)
	&& pcmk__xe_is(pcmk__xml_first_child(change), element)) {

	// Newly added element
	rc = true;
	break;
	}
	}

	free(element_regex);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether a given CIB element was modified in a CIB patchset
	*
	* \param[in] patchset CIB XML patchset
	* \param[in] element XML tag of CIB element to check (\c NULL is equivalent
	* to \c XML_TAG_CIB). Supported values include any CIB
	* element supported by \c pcmk__cib_abs_xpath_for().
	*
	* \return \c true if \p element was modified, or \c false otherwise
	*/
	bool
	cib__element_in_patchset(const xmlNode patchset, const char element)
	{
	int format = 1;

	CRM_ASSERT(patchset != NULL);

	crm_element_value_int(patchset, PCMK_XA_FORMAT, &format);
	switch (format) {
	case 1:
	return element_in_patchset_v1(patchset, element);

	case 2:
	return element_in_patchset_v2(patchset, element);

	default:
	crm_warn("Unknown patch format: %d", format);
	return false;
	}
	}

	/*!
	* \brief Create XML for a new (empty) CIB
	*
	* \param[in] cib_epoch What to use as \c PCMK_XA_EPOCH CIB attribute
	*
	* \return Newly created XML for empty CIB
	* \note It is the caller's responsibility to free the result with free_xml().
	*/
	xmlNode *
	createEmptyCib(int cib_epoch)
	{
	xmlNode cib_root = NULL, config = NULL;

	cib_root = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add(cib_root, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
	crm_xml_add(cib_root, PCMK_XA_VALIDATE_WITH, xml_latest_schema());

	crm_xml_add_int(cib_root, PCMK_XA_EPOCH, cib_epoch);
	crm_xml_add_int(cib_root, PCMK_XA_NUM_UPDATES, 0);
	crm_xml_add_int(cib_root, PCMK_XA_ADMIN_EPOCH, 0);

	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
	create_xml_node(cib_root, XML_CIB_TAG_STATUS);

	create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
	create_xml_node(config, XML_CIB_TAG_NODES);
	create_xml_node(config, XML_CIB_TAG_RESOURCES);
	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);

	#if PCMK__RESOURCE_STICKINESS_DEFAULT != 0
	{
	xmlNode *rsc_defaults = create_xml_node(config, XML_CIB_TAG_RSCCONFIG);
	xmlNode *meta = create_xml_node(rsc_defaults, XML_TAG_META_SETS);
	xmlNode *nvpair = create_xml_node(meta, XML_CIB_TAG_NVPAIR);

	crm_xml_add(meta, PCMK_XA_ID, "build-resource-defaults");
	crm_xml_add(nvpair, PCMK_XA_ID, "build-" PCMK_META_RESOURCE_STICKINESS);
	crm_xml_add(nvpair, PCMK_XA_NAME, PCMK_META_RESOURCE_STICKINESS);
	crm_xml_add_int(nvpair, PCMK_XA_VALUE,
	PCMK__RESOURCE_STICKINESS_DEFAULT);
	}
	#endif
	return cib_root;
	}

	static bool
	cib_acl_enabled(xmlNode xml, const char user)
	{
	bool rc = FALSE;

	if(pcmk_acl_required(user)) {
	const char *value = NULL;
	GHashTable *options = pcmk__strkey_table(free, free);

	cib_read_config(options, xml);
	value = cib_pref(options, PCMK_OPT_ENABLE_ACL);
	rc = crm_is_true(value);
	g_hash_table_destroy(options);
	}

	crm_trace("CIB ACL is %s", rc ? "enabled" : "disabled");
	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether to perform operations on a scratch copy of the CIB
	*
	* \param[in] op CIB operation
	* \param[in] section CIB section
	* \param[in] call_options CIB call options
	*
	* \return \p true if we should make a copy of the CIB, or \p false otherwise
	*/
	static bool
	should_copy_cib(const char op, const char section, int call_options)
	{
	if (pcmk_is_set(call_options, cib_dryrun)) {
	// cib_dryrun implies a scratch copy by definition; no side effects
	return true;
	}

	if (pcmk__str_eq(op, PCMK__CIB_REQUEST_COMMIT_TRANSACT, pcmk__str_none)) {
	/* Commit-transaction must make a copy for atomicity. We must revert to
	* the original CIB if the entire transaction cannot be applied
	* successfully.
	*/
	return true;
	}

	if (pcmk_is_set(call_options, cib_transaction)) {
	/* If cib_transaction is set, then we're in the process of committing a
	* transaction. The commit-transaction request already made a scratch
	* copy, and we're accumulating changes in that copy.
	*/
	return false;
	}

	if (pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_none)) {
	/* Copying large CIBs accounts for a huge percentage of our CIB usage,
	* and this avoids some of it.
	*
	* @TODO: Is this safe? See discussion at
	* https://github.com/ClusterLabs/pacemaker/pull/3094#discussion_r1211400690.
	*/
	return false;
	}

	// Default behavior is to operate on a scratch copy
	return true;
	}

	int
	cib_perform_op(const char *op, int call_options, cib__op_fn_t fn, bool is_query,
	const char section, xmlNode req, xmlNode *input,
	bool manage_counters, bool *config_changed,
	xmlNode current_cib, xmlNode result_cib, xmlNode **diff,
	xmlNode **output)
	{
	int rc = pcmk_ok;
	bool check_schema = true;
	bool make_copy = true;
	xmlNode *top = NULL;
	xmlNode *scratch = NULL;
	xmlNode *patchset_cib = NULL;
	xmlNode *local_diff = NULL;

	const char *new_version = NULL;
	const char *user = crm_element_value(req, F_CIB_USER);
	bool with_digest = false;

	crm_trace("Begin %s%s%s op",
	(pcmk_is_set(call_options, cib_dryrun)? "dry run of " : ""),
	(is_query? "read-only " : ""), op);

	CRM_CHECK(output != NULL, return -ENOMSG);
	CRM_CHECK(current_cib != NULL, return -ENOMSG);
	CRM_CHECK(result_cib != NULL, return -ENOMSG);
	CRM_CHECK(config_changed != NULL, return -ENOMSG);

	if(output) {
	*output = NULL;
	}

	*result_cib = NULL;
	*config_changed = false;

	if (fn == NULL) {
	return -EINVAL;
	}

	if (is_query) {
	xmlNode cib_ro = current_cib;
	xmlNode *cib_filtered = NULL;

	if (cib_acl_enabled(cib_ro, user)
	&& xml_acl_filtered_copy(user, current_cib, current_cib,
	&cib_filtered)) {

	if (cib_filtered == NULL) {
	crm_debug("Pre-filtered the entire cib");
	return -EACCES;
	}
	cib_ro = cib_filtered;
	crm_log_xml_trace(cib_ro, "filtered");
	}

	rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output);

	if(output == NULL \|\| *output == NULL) {
	/* nothing */

	} else if(cib_filtered == *output) {
	cib_filtered = NULL; /* Let them have this copy */

	} else if (output == current_cib) {
	/* They already know not to free it */

	} else if(cib_filtered && (*output)->doc == cib_filtered->doc) {
	/* We're about to free the document of which output is a part /
	output = copy_xml(output);

	} else if ((output)->doc == (current_cib)->doc) {
	/* Give them a copy they can free */
	output = copy_xml(output);
	}

	free_xml(cib_filtered);
	return rc;
	}

	make_copy = should_copy_cib(op, section, call_options);

	if (!make_copy) {
	/* Conditional on v2 patch style */

	scratch = *current_cib;

	// Make a copy of the top-level element to store version details
	top = create_xml_node(NULL, (const char *) scratch->name);
	copy_in_properties(top, scratch);
	patchset_cib = top;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output);

	/* If scratch points to a new object now (for example, after an erase
	* operation), then *current_cib should point to the same object.
	*/
	*current_cib = scratch;

	} else {
	scratch = copy_xml(*current_cib);
	patchset_cib = *current_cib;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (fn) (op, call_options, section, req, input, current_cib,
	&scratch, output);

	if ((scratch != NULL) && !xml_tracking_changes(scratch)) {
	crm_trace("Inferring changes after %s op", op);
	xml_track_changes(scratch, user, *current_cib,
	cib_acl_enabled(*current_cib, user));
	xml_calculate_changes(*current_cib, scratch);
	}
	CRM_CHECK(*current_cib != scratch, return -EINVAL);
	}

	xml_acl_disable(scratch); /* Allow the system to make any additional changes */

	if (rc == pcmk_ok && scratch == NULL) {
	rc = -EINVAL;
	goto done;

	} else if(rc == pcmk_ok && xml_acl_denied(scratch)) {
	crm_trace("ACL rejected part or all of the proposed changes");
	rc = -EACCES;
	goto done;

	} else if (rc != pcmk_ok) {
	goto done;
	}

	if (scratch) {
	new_version = crm_element_value(scratch, PCMK_XA_CRM_FEATURE_SET);

	if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
	crm_err("Discarding update with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	rc = -EPROTONOSUPPORT;
	goto done;
	}
	}

	if (patchset_cib != NULL) {
	int old = 0;
	int new = 0;

	crm_element_value_int(scratch, PCMK_XA_ADMIN_EPOCH, &new);
	crm_element_value_int(patchset_cib, PCMK_XA_ADMIN_EPOCH, &old);

	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: %#x)",
	PCMK_XA_ADMIN_EPOCH, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;

	} else if (old == new) {
	crm_element_value_int(scratch, PCMK_XA_EPOCH, &new);
	crm_element_value_int(patchset_cib, PCMK_XA_EPOCH, &old);
	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: %#x)",
	PCMK_XA_EPOCH, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;
	}
	}
	}

	crm_trace("Massaging CIB contents");
	pcmk__strip_xml_text(scratch);
	fix_plus_plus_recursive(scratch);

	if (!make_copy) {
	/* At this point, patchset_cib is just the "cib" tag and its properties.
	*
	* The v1 format would barf on this, but we know the v2 patch
	* format only needs it for the top-level version fields
	*/
	local_diff = xml_create_patchset(2, patchset_cib, scratch,
	config_changed, manage_counters);

	} else {
	static time_t expires = 0;
	time_t tm_now = time(NULL);

	if (expires < tm_now) {
	expires = tm_now + 60; /* Validate clients are correctly applying v2-style diffs at most once a minute */
	with_digest = true;
	}

	local_diff = xml_create_patchset(0, patchset_cib, scratch,
	config_changed, manage_counters);
	}

	pcmk__log_xml_changes(LOG_TRACE, scratch);
	xml_accept_changes(scratch);

	if(local_diff) {
	patchset_process_digest(local_diff, patchset_cib, scratch, with_digest);
	pcmk__log_xml_patchset(LOG_INFO, local_diff);
	crm_log_xml_trace(local_diff, "raw patch");
	}

	if (make_copy && (local_diff != NULL)) {
	// Original to compare against doesn't exist
	pcmk__if_tracing(
	{
	// Validate the calculated patch set
	int test_rc = pcmk_ok;
	int format = 1;
	xmlNode *cib_copy = copy_xml(patchset_cib);

	crm_element_value_int(local_diff, PCMK_XA_FORMAT, &format);
	test_rc = xml_apply_patchset(cib_copy, local_diff,
	manage_counters);

	if (test_rc != pcmk_ok) {
	save_xml_to_file(cib_copy, "PatchApply:calculated", NULL);
	save_xml_to_file(patchset_cib, "PatchApply:input", NULL);
	save_xml_to_file(scratch, "PatchApply:actual", NULL);
	save_xml_to_file(local_diff, "PatchApply:diff", NULL);
	crm_err("v%d patchset error, patch failed to apply: %s "
	"(%d)",
	format, pcmk_rc_str(pcmk_legacy2rc(test_rc)),
	test_rc);
	}
	free_xml(cib_copy);
	},
	{}
	);
	}

	if (pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
	/* Throttle the amount of costly validation we perform due to status updates
	* a) we don't really care whats in the status section
	* b) we don't validate any of its contents at the moment anyway
	*/
	check_schema = false;
	}

	/* === scratch must not be modified after this point ===
	* Exceptions, anything in:

	static filter_t filter[] = {
	{ 0, PCMK_XA_CRM_DEBUG_ORIGIN },
	{ 0, PCMK_XA_CIB_LAST_WRITTEN },
	{ 0, PCMK_XA_UPDATE_ORIGIN },
	{ 0, PCMK_XA_UPDATE_CLIENT },
	{ 0, PCMK_XA_UPDATE_USER },
	};
	*/

	if (*config_changed && !pcmk_is_set(call_options, cib_no_mtime)) {
	const char *schema = crm_element_value(scratch, PCMK_XA_VALIDATE_WITH);

	pcmk__xe_add_last_written(scratch);
	if (schema) {
	static int minimum_schema = 0;
	int current_schema = get_schema_version(schema);

	if (minimum_schema == 0) {
	minimum_schema = get_schema_version("pacemaker-1.2");
	}

	/* Does the CIB support the "update-" attributes... /
	if (current_schema >= minimum_schema) {
	/* Ensure values of origin, client, and user in scratch match
	* the values in req
	*/
	const char *origin = crm_element_value(req, PCMK__XA_SRC);
	const char *client = crm_element_value(req, F_CIB_CLIENTNAME);

	if (origin != NULL) {
	crm_xml_add(scratch, PCMK_XA_UPDATE_ORIGIN, origin);
	} else {
	xml_remove_prop(scratch, PCMK_XA_UPDATE_ORIGIN);
	}

	if (client != NULL) {
	crm_xml_add(scratch, PCMK_XA_UPDATE_CLIENT, user);
	} else {
	xml_remove_prop(scratch, PCMK_XA_UPDATE_CLIENT);
	}

	if (user != NULL) {
	crm_xml_add(scratch, PCMK_XA_UPDATE_USER, user);
	} else {
	xml_remove_prop(scratch, PCMK_XA_UPDATE_USER);
	}
	}
	}
	}

	crm_trace("Perform validation: %s", pcmk__btoa(check_schema));
	if ((rc == pcmk_ok) && check_schema && !validate_xml(scratch, NULL, true)) {
	const char *current_schema = crm_element_value(scratch,
	PCMK_XA_VALIDATE_WITH);

	crm_warn("Updated CIB does not validate against %s schema",
	pcmk__s(current_schema, "unspecified"));
	rc = -pcmk_err_schema_validation;
	}

	done:

	*result_cib = scratch;

	/* @TODO: This may not work correctly with !make_copy, since we don't
	* keep the original CIB.
	*/
	if ((rc != pcmk_ok) && cib_acl_enabled(patchset_cib, user)
	&& xml_acl_filtered_copy(user, patchset_cib, scratch, result_cib)) {

	if (*result_cib == NULL) {
	crm_debug("Pre-filtered the entire cib result");
	}
	free_xml(scratch);
	}

	if(diff) {
	*diff = local_diff;
	} else {
	free_xml(local_diff);
	}

	free_xml(top);
	crm_trace("Done");
	return rc;
	}

	int
	cib__create_op(cib_t cib, const char op, const char *host,
	const char section, xmlNode data, int call_options,
	const char user_name, const char client_name,
	xmlNode **op_msg)
	{
	CRM_CHECK((cib != NULL) && (op_msg != NULL), return -EPROTO);

	*op_msg = create_xml_node(NULL, T_CIB_COMMAND);
	if (*op_msg == NULL) {
	return -EPROTO;
	}

	cib->call_id++;
	if (cib->call_id < 1) {
	cib->call_id = 1;
	}

	crm_xml_add(*op_msg, PCMK__XA_T, T_CIB);
	crm_xml_add(*op_msg, F_CIB_OPERATION, op);
	crm_xml_add(*op_msg, F_CIB_HOST, host);
	crm_xml_add(*op_msg, F_CIB_SECTION, section);
	crm_xml_add(*op_msg, F_CIB_USER, user_name);
	crm_xml_add(*op_msg, F_CIB_CLIENTNAME, client_name);
	crm_xml_add_int(*op_msg, F_CIB_CALLID, cib->call_id);

	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(*op_msg, F_CIB_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(*op_msg, F_CIB_CALLDATA, data);
	}

	if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
	CRM_CHECK(pcmk_is_set(call_options, cib_scope_local),
	free_xml(*op_msg); return -EPROTO);
	}
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Check whether a CIB request is supported in a transaction
	*
	* \param[in] request CIB request
	*
	* \return Standard Pacemaker return code
	*/
	static int
	validate_transaction_request(const xmlNode *request)
	{
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *host = crm_element_value(request, F_CIB_HOST);
	const cib__operation_t *operation = NULL;
	int rc = cib__get_operation(op, &operation);

	if (rc != pcmk_rc_ok) {
	// cib__get_operation() logs error
	return rc;
	}

	if (!pcmk_is_set(operation->flags, cib__op_attr_transaction)) {
	crm_err("Operation %s is not supported in CIB transactions", op);
	return EOPNOTSUPP;
	}

	if (host != NULL) {
	crm_err("Operation targeting a specific node (%s) is not supported in "
	"a CIB transaction",
	host);
	return EOPNOTSUPP;
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Append a CIB request to a CIB transaction
	*
	* \param[in,out] cib CIB client whose transaction to extend
	* \param[in,out] request Request to add to transaction
	*
	* \return Legacy Pacemaker return code
	*/
	int
	cib__extend_transaction(cib_t cib, xmlNode request)
	{
	int rc = pcmk_rc_ok;

	CRM_ASSERT((cib != NULL) && (request != NULL));

	rc = validate_transaction_request(request);

	if ((rc == pcmk_rc_ok) && (cib->transaction == NULL)) {
	rc = pcmk_rc_no_transaction;
	}

	if (rc == pcmk_rc_ok) {
	add_node_copy(cib->transaction, request);

	} else {
	const char *op = crm_element_value(request, F_CIB_OPERATION);
	const char *client_id = NULL;

	cib->cmds->client_id(cib, NULL, &client_id);
	crm_err("Failed to add '%s' operation to transaction for client %s: %s",
	op, pcmk__s(client_id, "(unidentified)"), pcmk_rc_str(rc));
	crm_log_xml_info(request, "failed");
	}
	return pcmk_rc2legacy(rc);
	}

	void
	cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	cib_callback_client_t *blob = NULL;

	if (msg != NULL) {
	crm_element_value_int(msg, F_CIB_RC, &rc);
	crm_element_value_int(msg, F_CIB_CALLID, &call_id);
	output = get_message_xml(msg, F_CIB_CALLDATA);
	}

	blob = cib__lookup_id(call_id);

	if (blob == NULL) {
	crm_trace("No callback found for call %d", call_id);
	}

	if (cib == NULL) {
	crm_debug("No cib object supplied");
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (blob && blob->callback && (rc == pcmk_ok \|\| blob->only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d",
	pcmk__s(blob->id, "without ID"), call_id);
	blob->callback(msg, call_id, rc, output, blob->user_data);

	} else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("CIB command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed CIB Update");
	}

	/* This may free user_data, so do it after the callback */
	if (blob) {
	remove_cib_op_callback(call_id, FALSE);
	}

	if (cib && cib->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	cib->op_callback(msg, call_id, rc, output);
	}
	crm_trace("OP callback activated for %d", call_id);
	}

	void
	cib_native_notify(gpointer data, gpointer user_data)
	{
	xmlNode *msg = user_data;
	cib_notify_client_t *entry = data;
	const char *event = NULL;

	if (msg == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(msg, PCMK__XA_SUBT);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->callback == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (!pcmk__str_eq(entry->event, event, pcmk__str_casei)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->callback(event, msg);
	crm_trace("Callback invoked...");
	}

	static pcmk__cluster_option_t cib_opts[] = {
	/* name, legacy name, type, allowed values,
	* default value, validator,
	* short description,
	* long description
	*/
	{
	PCMK_OPT_ENABLE_ACL, NULL, "boolean", NULL,
	- "false", pcmk__valid_boolean,
	+ PCMK_VALUE_FALSE, pcmk__valid_boolean,
	N_("Enable Access Control Lists (ACLs) for the CIB"),
	NULL
	},
	{
	PCMK_OPT_CLUSTER_IPC_LIMIT, NULL, "integer", NULL,
	"500", pcmk__valid_positive_int,
	N_("Maximum IPC message backlog before disconnecting a cluster daemon"),
	N_("Raise this if log has \"Evicting client\" messages for cluster daemon"
	" PIDs (a good value is the number of resources in the cluster"
	" multiplied by the number of nodes).")
	},
	};

	void
	cib_metadata(void)
	{
	const char *desc_short = "Cluster Information Base manager options";
	const char *desc_long = "Cluster options used by Pacemaker's Cluster "
	"Information Base manager";

	gchar *s = pcmk__format_option_metadata("pacemaker-based", desc_short,
	desc_long, cib_opts,
	PCMK__NELEM(cib_opts));
	printf("%s", s);
	g_free(s);
	}

	static void
	verify_cib_options(GHashTable *options)
	{
	pcmk__validate_cluster_options(options, cib_opts, PCMK__NELEM(cib_opts));
	}

	const char *
	cib_pref(GHashTable * options, const char *name)
	{
	return pcmk__cluster_option(options, cib_opts, PCMK__NELEM(cib_opts),
	name);
	}

	gboolean
	cib_read_config(GHashTable * options, xmlNode * current_cib)
	{
	xmlNode *config = NULL;
	crm_time_t *now = NULL;

	if (options == NULL \|\| current_cib == NULL) {
	return FALSE;
	}

	now = crm_time_new(NULL);

	g_hash_table_remove_all(options);

	config = pcmk_find_cib_element(current_cib, XML_CIB_TAG_CRMCONFIG);
	if (config) {
	pe_unpack_nvpairs(current_cib, config, XML_CIB_TAG_PROPSET, NULL,
	options, CIB_OPTIONS_FIRST, TRUE, now, NULL);
	}

	verify_cib_options(options);

	crm_time_free(now);

	return TRUE;
	}

	int
	cib_internal_op(cib_t * cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name)
	{
	int (delegate) (cib_t cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name) =
	cib->delegate_fn;

	if(user_name == NULL) {
	user_name = getenv("CIB_user");
	}

	return delegate(cib, op, host, section, data, output_data, call_options, user_name);
	}

	/*!
	* \brief Apply a CIB update patch to a given CIB
	*
	* \param[in] event CIB update patch
	* \param[in] input CIB to patch
	* \param[out] output Resulting CIB after patch
	* \param[in] level Log the patch at this log level (unless LOG_CRIT)
	*
	* \return Legacy Pacemaker return code
	* \note sbd calls this function
	*/
	int
	cib_apply_patch_event(xmlNode event, xmlNode input, xmlNode **output,
	int level)
	{
	int rc = pcmk_err_generic;

	xmlNode *diff = NULL;

	CRM_ASSERT(event);
	CRM_ASSERT(input);
	CRM_ASSERT(output);

	crm_element_value_int(event, F_CIB_RC, &rc);
	diff = get_message_xml(event, F_CIB_UPDATE_RESULT);

	if (rc < pcmk_ok \|\| diff == NULL) {
	return rc;
	}

	if (level > LOG_CRIT) {
	pcmk__log_xml_patchset(level, diff);
	}

	if (input != NULL) {
	rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output,
	NULL);

	if (rc != pcmk_ok) {
	crm_debug("Update didn't apply: %s (%d) %p",
	pcmk_strerror(rc), rc, *output);

	if (rc == -pcmk_err_old_data) {
	crm_trace("Masking error, we already have the supplied update");
	return pcmk_ok;
	}
	free_xml(*output);
	*output = NULL;
	return rc;
	}
	}
	return rc;
	}

	#define log_signon_query_err(out, fmt, args...) do { \
	if (out != NULL) { \
	out->err(out, fmt, ##args); \
	} else { \
	crm_err(fmt, ##args); \
	} \
	} while (0)

	int
	cib__signon_query(pcmk__output_t out, cib_t cib, xmlNode *cib_object)
	{
	int rc = pcmk_rc_ok;
	cib_t *cib_conn = NULL;

	CRM_ASSERT(cib_object != NULL);

	if (cib == NULL) {
	cib_conn = cib_new();
	} else {
	if (*cib == NULL) {
	*cib = cib_new();
	}
	cib_conn = *cib;
	}

	if (cib_conn == NULL) {
	return ENOMEM;
	}

	if (cib_conn->state == cib_disconnected) {
	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);
	}

	if (rc != pcmk_rc_ok) {
	log_signon_query_err(out, "Could not connect to the CIB: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	if (out != NULL) {
	out->transient(out, "Querying CIB...");
	}
	rc = cib_conn->cmds->query(cib_conn, NULL, cib_object,
	cib_scope_local\|cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	log_signon_query_err(out, "CIB query failed: %s", pcmk_rc_str(rc));
	}

	done:
	if (cib == NULL) {
	cib__clean_up_connection(&cib_conn);
	}

	if ((rc == pcmk_rc_ok) && (*cib_object == NULL)) {
	return pcmk_rc_no_input;
	}
	return rc;
	}

	int
	cib__clean_up_connection(cib_t **cib)
	{
	int rc;

	if (*cib == NULL) {
	return pcmk_rc_ok;
	}

	rc = (cib)->cmds->signoff(cib);
	cib_delete(*cib);
	*cib = NULL;
	return pcmk_legacy2rc(rc);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/cib/util_compat.h>

	const char *
	get_object_path(const char *object_type)
	{
	return pcmk_cib_xpath_for(object_type);
	}

	const char *
	get_object_parent(const char *object_type)
	{
	return pcmk_cib_parent_name_for(object_type);
	}

	xmlNode *
	get_object_root(const char object_type, xmlNode the_root)
	{
	return pcmk_find_cib_element(the_root, object_type);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/common/nvpair.c b/lib/common/nvpair.c
	index f01ad1e418..02a4ee51a0 100644
	--- a/lib/common/nvpair.c
	+++ b/lib/common/nvpair.c
	@@ -1,982 +1,982 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <string.h>
	#include <ctype.h>
	#include <glib.h>
	#include <libxml/tree.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include "crmcommon_private.h"

	/*
	* This file isolates handling of three types of name/value pairs:
	*
	* - pcmk_nvpair_t data type
	* - XML attributes (<TAG ... NAME=VALUE ...>)
	* - XML nvpair elements (<nvpair id=ID name=NAME value=VALUE>)
	*/

	// pcmk_nvpair_t handling

	/*!
	* \internal
	* \brief Allocate a new name/value pair
	*
	* \param[in] name New name (required)
	* \param[in] value New value
	*
	* \return Newly allocated name/value pair
	* \note The caller is responsible for freeing the result with
	* \c pcmk__free_nvpair().
	*/
	static pcmk_nvpair_t *
	pcmk__new_nvpair(const char name, const char value)
	{
	pcmk_nvpair_t *nvpair = NULL;

	CRM_ASSERT(name);

	nvpair = calloc(1, sizeof(pcmk_nvpair_t));
	CRM_ASSERT(nvpair);

	pcmk__str_update(&nvpair->name, name);
	pcmk__str_update(&nvpair->value, value);
	return nvpair;
	}

	/*!
	* \internal
	* \brief Free a name/value pair
	*
	* \param[in,out] nvpair Name/value pair to free
	*/
	static void
	pcmk__free_nvpair(gpointer data)
	{
	if (data) {
	pcmk_nvpair_t *nvpair = data;

	free(nvpair->name);
	free(nvpair->value);
	free(nvpair);
	}
	}

	/*!
	* \brief Prepend a name/value pair to a list
	*
	* \param[in,out] nvpairs List to modify
	* \param[in] name New entry's name
	* \param[in] value New entry's value
	*
	* \return New head of list
	* \note The caller is responsible for freeing the list with
	* \c pcmk_free_nvpairs().
	*/
	GSList *
	pcmk_prepend_nvpair(GSList nvpairs, const char name, const char *value)
	{
	return g_slist_prepend(nvpairs, pcmk__new_nvpair(name, value));
	}

	/*!
	* \brief Free a list of name/value pairs
	*
	* \param[in,out] list List to free
	*/
	void
	pcmk_free_nvpairs(GSList *nvpairs)
	{
	g_slist_free_full(nvpairs, pcmk__free_nvpair);
	}

	/*!
	* \internal
	* \brief Compare two name/value pairs
	*
	* \param[in] a First name/value pair to compare
	* \param[in] b Second name/value pair to compare
	*
	* \return 0 if a == b, 1 if a > b, -1 if a < b
	*/
	static gint
	pcmk__compare_nvpair(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const pcmk_nvpair_t *pair_a = a;
	const pcmk_nvpair_t *pair_b = b;

	CRM_ASSERT(a != NULL);
	CRM_ASSERT(pair_a->name != NULL);

	CRM_ASSERT(b != NULL);
	CRM_ASSERT(pair_b->name != NULL);

	rc = strcmp(pair_a->name, pair_b->name);
	if (rc < 0) {
	return -1;
	} else if (rc > 0) {
	return 1;
	}
	return 0;
	}

	/*!
	* \brief Sort a list of name/value pairs
	*
	* \param[in,out] list List to sort
	*
	* \return New head of list
	*/
	GSList *
	pcmk_sort_nvpairs(GSList *list)
	{
	return g_slist_sort(list, pcmk__compare_nvpair);
	}

	/*!
	* \brief Create a list of name/value pairs from an XML node's attributes
	*
	* \param[in] XML to parse
	*
	* \return New list of name/value pairs
	* \note It is the caller's responsibility to free the list with
	* \c pcmk_free_nvpairs().
	*/
	GSList *
	pcmk_xml_attrs2nvpairs(const xmlNode *xml)
	{
	GSList *result = NULL;

	for (xmlAttrPtr iter = pcmk__xe_first_attr(xml); iter != NULL;
	iter = iter->next) {

	result = pcmk_prepend_nvpair(result,
	(const char *) iter->name,
	(const char *) pcmk__xml_attr_value(iter));
	}
	return result;
	}

	/*!
	* \internal
	* \brief Add an XML attribute corresponding to a name/value pair
	*
	* Suitable for glib list iterators, this function adds a NAME=VALUE
	* XML attribute based on a given name/value pair.
	*
	* \param[in] data Name/value pair
	* \param[out] user_data XML node to add attributes to
	*/
	static void
	pcmk__nvpair_add_xml_attr(gpointer data, gpointer user_data)
	{
	pcmk_nvpair_t *pair = data;
	xmlNode *parent = user_data;

	crm_xml_add(parent, pair->name, pair->value);
	}

	/*!
	* \brief Add XML attributes based on a list of name/value pairs
	*
	* \param[in,out] list List of name/value pairs
	* \param[in,out] xml XML node to add attributes to
	*/
	void
	pcmk_nvpairs2xml_attrs(GSList list, xmlNode xml)
	{
	g_slist_foreach(list, pcmk__nvpair_add_xml_attr, xml);
	}

	// convenience function for name=value strings

	/*!
	* \internal
	* \brief Extract the name and value from an input string formatted as "name=value".
	* If unable to extract them, they are returned as NULL.
	*
	* \param[in] input The input string, likely from the command line
	* \param[out] name Everything before the first '=' in the input string
	* \param[out] value Everything after the first '=' in the input string
	*
	* \return 2 if both name and value could be extracted, 1 if only one could, and
	* and error code otherwise
	*/
	int
	pcmk__scan_nvpair(const char input, char name, char *value)
	{
	#ifdef HAVE_SSCANF_M
	*name = NULL;
	*value = NULL;
	if (sscanf(input, "%m[^=]=%m[^\n]", name, value) <= 0) {
	return -pcmk_err_bad_nvpair;
	}
	#else
	char *sep = NULL;
	*name = NULL;
	*value = NULL;

	sep = strstr(optarg, "=");
	if (sep == NULL) {
	return -pcmk_err_bad_nvpair;
	}

	*name = strndup(input, sep-input);

	if (*name == NULL) {
	return -ENOMEM;
	}

	/* If the last char in optarg is =, the user gave no
	* value for the option. Leave it as NULL.
	*/
	if (*(sep+1) != '\0') {
	*value = strdup(sep+1);

	if (*value == NULL) {
	return -ENOMEM;
	}
	}
	#endif

	if (name != NULL && value != NULL) {
	return 2;
	} else if (name != NULL \|\| value != NULL) {
	return 1;
	} else {
	return -pcmk_err_bad_nvpair;
	}
	}

	/*!
	* \internal
	* \brief Format a name/value pair.
	*
	* Units can optionally be provided for the value. Note that unlike most
	* formatting functions, this one returns the formatted string. It is
	* assumed that the most common use of this function will be to build up
	* a string to be output as part of other functions.
	*
	* \note The caller is responsible for freeing the return value after use.
	*
	* \param[in] name The name of the nvpair.
	* \param[in] value The value of the nvpair.
	* \param[in] units Optional units for the value, or NULL.
	*
	* \return Newly allocated string with name/value pair
	*/
	char *
	pcmk__format_nvpair(const char name, const char value, const char *units)
	{
	return crm_strdup_printf("%s=\"%s%s\"", name, value, units ? units : "");
	}

	// XML attribute handling

	/*!
	* \brief Create an XML attribute with specified name and value
	*
	* \param[in,out] node XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] value Attribute value to set
	*
	* \return New value on success, \c NULL otherwise
	* \note This does nothing if node, name, or value are \c NULL or empty.
	*/
	const char *
	crm_xml_add(xmlNode node, const char name, const char *value)
	{
	bool dirty = FALSE;
	xmlAttr *attr = NULL;

	CRM_CHECK(node != NULL, return NULL);
	CRM_CHECK(name != NULL, return NULL);

	if (value == NULL) {
	return NULL;
	}

	if (pcmk__tracking_xml_changes(node, FALSE)) {
	const char *old = crm_element_value(node, name);

	if (old == NULL \|\| value == NULL \|\| strcmp(old, value) != 0) {
	dirty = TRUE;
	}
	}

	if (dirty && (pcmk__check_acl(node, name, pcmk__xf_acl_create) == FALSE)) {
	crm_trace("Cannot add %s=%s to %s", name, value, node->name);
	return NULL;
	}

	attr = xmlSetProp(node, (pcmkXmlStr) name, (pcmkXmlStr) value);
	if (dirty) {
	pcmk__mark_xml_attr_dirty(attr);
	}

	CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
	return (char *)attr->children->content;
	}

	/*!
	* \brief Create an XML attribute with specified name and integer value
	*
	* This is like \c crm_xml_add() but taking an integer value.
	*
	* \param[in,out] node XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] value Attribute value to set
	*
	* \return New value as string on success, \c NULL otherwise
	* \note This does nothing if node or name are \c NULL or empty.
	*/
	const char *
	crm_xml_add_int(xmlNode node, const char name, int value)
	{
	char *number = pcmk__itoa(value);
	const char *added = crm_xml_add(node, name, number);

	free(number);
	return added;
	}

	/*!
	* \brief Create an XML attribute with specified name and unsigned value
	*
	* This is like \c crm_xml_add() but taking a guint value.
	*
	* \param[in,out] node XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] ms Attribute value to set
	*
	* \return New value as string on success, \c NULL otherwise
	* \note This does nothing if node or name are \c NULL or empty.
	*/
	const char *
	crm_xml_add_ms(xmlNode node, const char name, guint ms)
	{
	char *number = crm_strdup_printf("%u", ms);
	const char *added = crm_xml_add(node, name, number);

	free(number);
	return added;
	}

	// Maximum size of null-terminated string representation of 64-bit integer
	// -9223372036854775808
	#define LLSTRSIZE 21

	/*!
	* \brief Create an XML attribute with specified name and long long int value
	*
	* This is like \c crm_xml_add() but taking a long long int value. It is a
	* useful equivalent for defined types like time_t, etc.
	*
	* \param[in,out] xml XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] value Attribute value to set
	*
	* \return New value as string on success, \c NULL otherwise
	* \note This does nothing if xml or name are \c NULL or empty.
	* This does not support greater than 64-bit values.
	*/
	const char *
	crm_xml_add_ll(xmlNode xml, const char name, long long value)
	{
	char s[LLSTRSIZE] = { '\0', };

	if (snprintf(s, LLSTRSIZE, "%lld", (long long) value) == LLSTRSIZE) {
	return NULL;
	}
	return crm_xml_add(xml, name, s);
	}

	/*!
	* \brief Create XML attributes for seconds and microseconds
	*
	* This is like \c crm_xml_add() but taking a struct timeval.
	*
	* \param[in,out] xml XML node to modify
	* \param[in] name_sec Name of XML attribute for seconds
	* \param[in] name_usec Name of XML attribute for microseconds (or NULL)
	* \param[in] value Time value to set
	*
	* \return New seconds value as string on success, \c NULL otherwise
	* \note This does nothing if xml, name_sec, or value is \c NULL.
	*/
	const char *
	crm_xml_add_timeval(xmlNode xml, const char name_sec, const char *name_usec,
	const struct timeval *value)
	{
	const char *added = NULL;

	if (xml && name_sec && value) {
	added = crm_xml_add_ll(xml, name_sec, (long long) value->tv_sec);
	if (added && name_usec) {
	// Any error is ignored (we successfully added seconds)
	crm_xml_add_ll(xml, name_usec, (long long) value->tv_usec);
	}
	}
	return added;
	}

	/*!
	* \brief Retrieve the value of an XML attribute
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	*
	* \return Value of specified attribute (may be \c NULL)
	*/
	const char *
	crm_element_value(const xmlNode data, const char name)
	{
	xmlAttr *attr = NULL;

	if (data == NULL) {
	crm_err("Couldn't find %s in NULL", name ? name : "<null>");
	CRM_LOG_ASSERT(data != NULL);
	return NULL;

	} else if (name == NULL) {
	crm_err("Couldn't find NULL in %s", data->name);
	return NULL;
	}

	/* The first argument to xmlHasProp() has always been const,
	* but libxml2 <2.9.2 didn't declare that, so cast it
	*/
	attr = xmlHasProp((xmlNode *) data, (pcmkXmlStr) name);
	if (!attr \|\| !attr->children) {
	return NULL;
	}
	return (const char *) attr->children->content;
	}

	/*!
	* \brief Retrieve the integer value of an XML attribute
	*
	* This is like \c crm_element_value() but getting the value as an integer.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store element value
	*
	* \return 0 on success, -1 otherwise
	*/
	int
	crm_element_value_int(const xmlNode data, const char name, int *dest)
	{
	const char *value = NULL;

	CRM_CHECK(dest != NULL, return -1);
	value = crm_element_value(data, name);
	if (value) {
	long long value_ll;

	if ((pcmk__scan_ll(value, &value_ll, 0LL) != pcmk_rc_ok)
	\|\| (value_ll < INT_MIN) \|\| (value_ll > INT_MAX)) {
	*dest = PCMK__PARSE_INT_DEFAULT;
	} else {
	*dest = (int) value_ll;
	return 0;
	}
	}
	return -1;
	}

	/*!
	* \brief Retrieve the long long integer value of an XML attribute
	*
	* This is like \c crm_element_value() but getting the value as a long long int.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store element value
	*
	* \return 0 on success, -1 otherwise
	*/
	int
	crm_element_value_ll(const xmlNode data, const char name, long long *dest)
	{
	const char *value = NULL;

	CRM_CHECK(dest != NULL, return -1);
	value = crm_element_value(data, name);
	if ((value != NULL)
	&& (pcmk__scan_ll(value, dest, PCMK__PARSE_INT_DEFAULT) == pcmk_rc_ok)) {
	return 0;
	}
	return -1;
	}

	/*!
	* \brief Retrieve the millisecond value of an XML attribute
	*
	* This is like \c crm_element_value() but returning the value as a guint.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store attribute value
	*
	* \return \c pcmk_ok on success, -1 otherwise
	*/
	int
	crm_element_value_ms(const xmlNode data, const char name, guint *dest)
	{
	const char *value = NULL;
	long long value_ll;

	CRM_CHECK(dest != NULL, return -1);
	*dest = 0;
	value = crm_element_value(data, name);
	if ((pcmk__scan_ll(value, &value_ll, 0LL) != pcmk_rc_ok)
	\|\| (value_ll < 0) \|\| (value_ll > G_MAXUINT)) {
	return -1;
	}
	*dest = (guint) value_ll;
	return pcmk_ok;
	}

	/*!
	* \brief Retrieve the seconds-since-epoch value of an XML attribute
	*
	* This is like \c crm_element_value() but returning the value as a time_t.
	*
	* \param[in] xml XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store attribute value
	*
	* \return \c pcmk_ok on success, -1 otherwise
	*/
	int
	crm_element_value_epoch(const xmlNode xml, const char name, time_t *dest)
	{
	long long value_ll = 0;

	if (crm_element_value_ll(xml, name, &value_ll) < 0) {
	return -1;
	}

	/* Unfortunately, we can't do any bounds checking, since time_t has neither
	* standardized bounds nor constants defined for them.
	*/
	*dest = (time_t) value_ll;
	return pcmk_ok;
	}

	/*!
	* \brief Retrieve the value of XML second/microsecond attributes as time
	*
	* This is like \c crm_element_value() but returning value as a struct timeval.
	*
	* \param[in] xml XML to parse
	* \param[in] name_sec Name of XML attribute for seconds
	* \param[in] name_usec Name of XML attribute for microseconds
	* \param[out] dest Where to store result
	*
	* \return \c pcmk_ok on success, -errno on error
	* \note Values default to 0 if XML or XML attribute does not exist
	*/
	int
	crm_element_value_timeval(const xmlNode xml, const char name_sec,
	const char name_usec, struct timeval dest)
	{
	long long value_i = 0;

	CRM_CHECK(dest != NULL, return -EINVAL);
	dest->tv_sec = 0;
	dest->tv_usec = 0;

	if (xml == NULL) {
	return pcmk_ok;
	}

	/* Unfortunately, we can't do any bounds checking, since there are no
	* constants provided for the bounds of time_t and suseconds_t, and
	* calculating them isn't worth the effort. If there are XML values
	* beyond the native sizes, there will probably be worse problems anyway.
	*/

	// Parse seconds
	errno = 0;
	if (crm_element_value_ll(xml, name_sec, &value_i) < 0) {
	return -errno;
	}
	dest->tv_sec = (time_t) value_i;

	// Parse microseconds
	if (crm_element_value_ll(xml, name_usec, &value_i) < 0) {
	return -errno;
	}
	dest->tv_usec = (suseconds_t) value_i;

	return pcmk_ok;
	}

	/*!
	* \brief Retrieve a copy of the value of an XML attribute
	*
	* This is like \c crm_element_value() but allocating new memory for the result.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	*
	* \return Value of specified attribute (may be \c NULL)
	* \note The caller is responsible for freeing the result.
	*/
	char *
	crm_element_value_copy(const xmlNode data, const char name)
	{
	char *value_copy = NULL;

	pcmk__str_update(&value_copy, crm_element_value(data, name));
	return value_copy;
	}

	/*!
	* \brief Add hash table entry to XML as (possibly legacy) name/value
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as user data, and adds an XML attribute
	* with the specified name and value if it does not already exist. If the key
	* name starts with a digit, this will instead add a \<param name=NAME
	* value=VALUE/> child to the XML (for legacy compatibility with heartbeat).
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2smartfield(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if (isdigit(name[0])) {
	xmlNode *tmp = create_xml_node(xml_node, XML_TAG_PARAM);

	crm_xml_add(tmp, PCMK_XA_NAME, name);
	crm_xml_add(tmp, PCMK_XA_VALUE, s_value);

	} else if (crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);
	crm_trace("dumped: %s=%s", name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	/*!
	* \brief Set XML attribute based on hash table entry
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as user data, and adds an XML attribute
	* with the specified name and value if it does not already exist.
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2field(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if (crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	/*!
	* \brief Set XML attribute based on hash table entry, as meta-attribute name
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as user data, and adds an XML attribute
	* with the meta-attribute version of the specified name and value if it does
	* not already exist and if the name does not appear to be cluster-internal.
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2metafield(gpointer key, gpointer value, gpointer user_data)
	{
	char *crm_name = NULL;

	if (key == NULL \|\| value == NULL) {
	return;
	}

	/* Filter out cluster-generated attributes that contain a '#' or ':'
	* (like fail-count and last-failure).
	*/
	for (crm_name = key; *crm_name; ++crm_name) {
	if ((crm_name == '#') \|\| (crm_name == ':')) {
	return;
	}
	}

	crm_name = crm_meta_name(key);
	hash2field(crm_name, value, user_data);
	free(crm_name);
	}

	// nvpair handling

	/*!
	* \brief Create an XML name/value pair
	*
	* \param[in,out] parent If not \c NULL, make new XML node a child of this one
	* \param[in] id Set this as XML ID (or NULL to auto-generate)
	* \param[in] name Name to use
	* \param[in] value Value to use
	*
	* \return New XML object on success, \c NULL otherwise
	*/
	xmlNode *
	crm_create_nvpair_xml(xmlNode parent, const char id, const char *name,
	const char *value)
	{
	xmlNode *nvp;

	/* id can be NULL so we auto-generate one, and name can be NULL if this
	* will be used to delete a name/value pair by ID, but both can't be NULL
	*/
	CRM_CHECK(id \|\| name, return NULL);

	nvp = create_xml_node(parent, XML_CIB_TAG_NVPAIR);
	CRM_CHECK(nvp, return NULL);

	if (id) {
	crm_xml_add(nvp, PCMK_XA_ID, id);
	} else {
	const char *parent_id = ID(parent);

	crm_xml_set_id(nvp, "%s-%s",
	(parent_id? parent_id : XML_CIB_TAG_NVPAIR), name);
	}
	crm_xml_add(nvp, PCMK_XA_NAME, name);
	crm_xml_add(nvp, PCMK_XA_VALUE, value);
	return nvp;
	}

	/*!
	* \brief Add XML nvpair element based on hash table entry
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as the user data, and adds an \c nvpair
	* XML element with the specified name and value.
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2nvpair(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;
	xmlNode *xml_node = user_data;

	crm_create_nvpair_xml(xml_node, name, name, s_value);
	crm_trace("dumped: name=%s value=%s", name, s_value);
	}

	/*!
	* \brief Retrieve XML attributes as a hash table
	*
	* Given an XML element, this will look for any \<attributes> element child,
	* creating a hash table of (newly allocated string) name/value pairs taken
	* first from the attributes element's NAME=VALUE XML attributes, and then
	* from any \<param name=NAME value=VALUE> children of attributes.
	*
	* \param[in] XML node to parse
	*
	* \return Hash table with name/value pairs
	* \note It is the caller's responsibility to free the result using
	* \c g_hash_table_destroy().
	*/
	GHashTable *
	xml2list(const xmlNode *parent)
	{
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	xmlNode *nvpair_list = NULL;
	GHashTable *nvpair_hash = pcmk__strkey_table(free, free);

	CRM_CHECK(parent != NULL, return nvpair_hash);

	nvpair_list = find_xml_node(parent, XML_TAG_ATTRS, FALSE);
	if (nvpair_list == NULL) {
	crm_trace("No attributes in %s", parent->name);
	crm_log_xml_trace(parent, "No attributes for resource op");
	}

	crm_log_xml_trace(nvpair_list, "Unpacking");

	for (pIter = pcmk__xe_first_attr(nvpair_list); pIter != NULL;
	pIter = pIter->next) {

	const char p_name = (const char )pIter->name;
	const char *p_value = pcmk__xml_attr_value(pIter);

	crm_trace("Added %s=%s", p_name, p_value);

	g_hash_table_insert(nvpair_hash, strdup(p_name), strdup(p_value));
	}

	for (child = pcmk__xml_first_child(nvpair_list); child != NULL;
	child = pcmk__xml_next(child)) {

	if (strcmp((const char *)child->name, XML_TAG_PARAM) == 0) {
	const char *key = crm_element_value(child, PCMK_XA_NAME);
	const char *value = crm_element_value(child, PCMK_XA_VALUE);

	crm_trace("Added %s=%s", key, value);
	if (key != NULL && value != NULL) {
	g_hash_table_insert(nvpair_hash, strdup(key), strdup(value));
	}
	}
	}

	return nvpair_hash;
	}

	void
	pcmk__xe_set_bool_attr(xmlNodePtr node, const char *name, bool value)
	{
	- crm_xml_add(node, name, value ? PCMK_VALUE_TRUE : XML_BOOLEAN_FALSE);
	+ crm_xml_add(node, name, value ? PCMK_VALUE_TRUE : PCMK_VALUE_FALSE);
	}

	int
	pcmk__xe_get_bool_attr(const xmlNode node, const char name, bool *value)
	{
	const char *xml_value = NULL;
	int ret, rc;

	if (node == NULL) {
	return ENODATA;
	} else if (name == NULL \|\| value == NULL) {
	return EINVAL;
	}

	xml_value = crm_element_value(node, name);

	if (xml_value == NULL) {
	return ENODATA;
	}

	rc = crm_str_to_boolean(xml_value, &ret);
	if (rc == 1) {
	*value = ret;
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_bad_input;
	}
	}

	bool
	pcmk__xe_attr_is_true(const xmlNode node, const char name)
	{
	bool value = false;
	int rc;

	rc = pcmk__xe_get_bool_attr(node, name, &value);
	return rc == pcmk_rc_ok && value == true;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/common/util_compat.h>

	int
	pcmk_scan_nvpair(const char input, char name, char *value)
	{
	return pcmk__scan_nvpair(input, name, value);
	}

	char *
	pcmk_format_nvpair(const char name, const char value,
	const char *units)
	{
	return pcmk__format_nvpair(name, value, units);
	}

	char *
	pcmk_format_named_time(const char *name, time_t epoch_time)
	{
	char *now_s = pcmk__epoch2str(&epoch_time, 0);
	char *result = crm_strdup_printf("%s=\"%s\"", name, pcmk__s(now_s, ""));

	free(now_s);
	return result;
	}

	const char *
	crm_xml_replace(xmlNode node, const char name, const char *value)
	{
	bool dirty = FALSE;
	xmlAttr *attr = NULL;
	const char *old_value = NULL;

	CRM_CHECK(node != NULL, return NULL);
	CRM_CHECK(name != NULL && name[0] != 0, return NULL);

	old_value = crm_element_value(node, name);

	/* Could be re-setting the same value */
	CRM_CHECK(old_value != value, return value);

	if (pcmk__check_acl(node, name, pcmk__xf_acl_write) == FALSE) {
	/* Create a fake object linked to doc->_private instead? */
	crm_trace("Cannot replace %s=%s to %s", name, value, node->name);
	return NULL;

	} else if (old_value && !value) {
	xml_remove_prop(node, name);
	return NULL;
	}

	if (pcmk__tracking_xml_changes(node, FALSE)) {
	if (!old_value \|\| !value \|\| !strcmp(old_value, value)) {
	dirty = TRUE;
	}
	}

	attr = xmlSetProp(node, (pcmkXmlStr) name, (pcmkXmlStr) value);
	if (dirty) {
	pcmk__mark_xml_attr_dirty(attr);
	}
	CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
	return (char *) attr->children->content;
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/common/strings.c b/lib/common/strings.c
	index 4e2b35ec58..9a89841036 100644
	--- a/lib/common/strings.c
	+++ b/lib/common/strings.c
	@@ -1,1419 +1,1417 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include "crm/common/results.h"
	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <regex.h>
	#include <stdio.h>
	#include <string.h>
	#include <stdlib.h>
	#include <ctype.h>
	#include <float.h> // DBL_MIN
	#include <limits.h>
	#include <bzlib.h>
	#include <sys/types.h>

	/*!
	* \internal
	* \brief Scan a long long integer from a string
	*
	* \param[in] text String to scan
	* \param[out] result If not NULL, where to store scanned value
	* \param[in] default_value Value to use if text is NULL or invalid
	* \param[out] end_text If not NULL, where to store pointer to first
	* non-integer character
	*
	* \return Standard Pacemaker return code (\c pcmk_rc_ok on success,
	* \c EINVAL on failed string conversion due to invalid input,
	* or \c EOVERFLOW on arithmetic overflow)
	* \note Sets \c errno on error
	*/
	static int
	scan_ll(const char text, long long result, long long default_value,
	char **end_text)
	{
	long long local_result = default_value;
	char *local_end_text = NULL;
	int rc = pcmk_rc_ok;

	errno = 0;
	if (text != NULL) {
	local_result = strtoll(text, &local_end_text, 10);
	if (errno == ERANGE) {
	rc = EOVERFLOW;
	crm_warn("Integer parsed from '%s' was clipped to %lld",
	text, local_result);

	} else if (errno != 0) {
	rc = errno;
	local_result = default_value;
	crm_warn("Could not parse integer from '%s' (using %lld instead): "
	"%s", text, default_value, pcmk_rc_str(rc));

	} else if (local_end_text == text) {
	rc = EINVAL;
	local_result = default_value;
	crm_warn("Could not parse integer from '%s' (using %lld instead): "
	"No digits found", text, default_value);
	}

	if ((end_text == NULL) && !pcmk__str_empty(local_end_text)) {
	crm_warn("Characters left over after parsing '%s': '%s'",
	text, local_end_text);
	}
	errno = rc;
	}
	if (end_text != NULL) {
	*end_text = local_end_text;
	}
	if (result != NULL) {
	*result = local_result;
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Scan a long long integer value from a string
	*
	* \param[in] text The string to scan (may be NULL)
	* \param[out] result Where to store result (or NULL to ignore)
	* \param[in] default_value Value to use if text is NULL or invalid
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__scan_ll(const char text, long long result, long long default_value)
	{
	long long local_result = default_value;
	int rc = pcmk_rc_ok;

	if (text != NULL) {
	rc = scan_ll(text, &local_result, default_value, NULL);
	if (rc != pcmk_rc_ok) {
	local_result = default_value;
	}
	}
	if (result != NULL) {
	*result = local_result;
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Scan an integer value from a string, constrained to a minimum
	*
	* \param[in] text The string to scan (may be NULL)
	* \param[out] result Where to store result (or NULL to ignore)
	* \param[in] minimum Value to use as default and minimum
	*
	* \return Standard Pacemaker return code
	* \note If the value is larger than the maximum integer, EOVERFLOW will be
	* returned and \p result will be set to the maximum integer.
	*/
	int
	pcmk__scan_min_int(const char text, int result, int minimum)
	{
	int rc;
	long long result_ll;

	rc = pcmk__scan_ll(text, &result_ll, (long long) minimum);

	if (result_ll < (long long) minimum) {
	crm_warn("Clipped '%s' to minimum acceptable value %d", text, minimum);
	result_ll = (long long) minimum;

	} else if (result_ll > INT_MAX) {
	crm_warn("Clipped '%s' to maximum integer %d", text, INT_MAX);
	result_ll = (long long) INT_MAX;
	rc = EOVERFLOW;
	}

	if (result != NULL) {
	*result = (int) result_ll;
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Scan a TCP port number from a string
	*
	* \param[in] text The string to scan
	* \param[out] port Where to store result (or NULL to ignore)
	*
	* \return Standard Pacemaker return code
	* \note \p port will be -1 if \p text is NULL or invalid
	*/
	int
	pcmk__scan_port(const char text, int port)
	{
	long long port_ll;
	int rc = pcmk__scan_ll(text, &port_ll, -1LL);

	if ((text != NULL) && (rc == pcmk_rc_ok) // wasn't default or invalid
	&& ((port_ll < 0LL) \|\| (port_ll > 65535LL))) {
	crm_warn("Ignoring port specification '%s' "
	"not in valid range (0-65535)", text);
	rc = (port_ll < 0LL)? pcmk_rc_before_range : pcmk_rc_after_range;
	port_ll = -1LL;
	}
	if (port != NULL) {
	*port = (int) port_ll;
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Scan a double-precision floating-point value from a string
	*
	* \param[in] text The string to parse
	* \param[out] result Parsed value on success, or
	* \c PCMK__PARSE_DBL_DEFAULT on error
	* \param[in] default_text Default string to parse if \p text is
	* \c NULL
	* \param[out] end_text If not \c NULL, where to store a pointer
	* to the position immediately after the
	* value
	*
	* \return Standard Pacemaker return code (\c pcmk_rc_ok on success,
	* \c EINVAL on failed string conversion due to invalid input,
	* \c EOVERFLOW on arithmetic overflow, \c pcmk_rc_underflow
	* on arithmetic underflow, or \c errno from \c strtod() on
	* other parse errors)
	*/
	int
	pcmk__scan_double(const char text, double result, const char *default_text,
	char **end_text)
	{
	int rc = pcmk_rc_ok;
	char *local_end_text = NULL;

	CRM_ASSERT(result != NULL);
	*result = PCMK__PARSE_DBL_DEFAULT;

	text = (text != NULL) ? text : default_text;

	if (text == NULL) {
	rc = EINVAL;
	crm_debug("No text and no default conversion value supplied");

	} else {
	errno = 0;
	*result = strtod(text, &local_end_text);

	if (errno == ERANGE) {
	/*
	* Overflow: strtod() returns +/- HUGE_VAL and sets errno to
	* ERANGE
	*
	* Underflow: strtod() returns "a value whose magnitude is
	* no greater than the smallest normalized
	* positive" double. Whether ERANGE is set is
	* implementation-defined.
	*/
	const char *over_under;

	if (QB_ABS(*result) > DBL_MIN) {
	rc = EOVERFLOW;
	over_under = "over";
	} else {
	rc = pcmk_rc_underflow;
	over_under = "under";
	}

	crm_debug("Floating-point value parsed from '%s' would %sflow "
	"(using %g instead)", text, over_under, *result);

	} else if (errno != 0) {
	rc = errno;
	// strtod() set *result = 0 on parse failure
	*result = PCMK__PARSE_DBL_DEFAULT;

	crm_debug("Could not parse floating-point value from '%s' (using "
	"%.1f instead): %s", text, PCMK__PARSE_DBL_DEFAULT,
	pcmk_rc_str(rc));

	} else if (local_end_text == text) {
	// errno == 0, but nothing was parsed
	rc = EINVAL;
	*result = PCMK__PARSE_DBL_DEFAULT;

	crm_debug("Could not parse floating-point value from '%s' (using "
	"%.1f instead): No digits found", text,
	PCMK__PARSE_DBL_DEFAULT);

	} else if (QB_ABS(*result) <= DBL_MIN) {
	/*
	* errno == 0 and text was parsed, but value might have
	* underflowed.
	*
	* ERANGE might not be set for underflow. Check magnitude
	* of *result, but also make sure the input number is not
	* actually zero (0 <= DBL_MIN is not underflow).
	*
	* This check must come last. A parse failure in strtod()
	* also sets *result == 0, so a parse failure would match
	* this test condition prematurely.
	*/
	for (const char *p = text; p != local_end_text; p++) {
	if (strchr("0.eE", *p) == NULL) {
	rc = pcmk_rc_underflow;
	crm_debug("Floating-point value parsed from '%s' would "
	"underflow (using %g instead)", text, *result);
	break;
	}
	}

	} else {
	crm_trace("Floating-point value parsed successfully from "
	"'%s': %g", text, *result);
	}

	if ((end_text == NULL) && !pcmk__str_empty(local_end_text)) {
	crm_debug("Characters left over after parsing '%s': '%s'",
	text, local_end_text);
	}
	}

	if (end_text != NULL) {
	*end_text = local_end_text;
	}

	return rc;
	}

	/*!
	* \internal
	* \brief Parse a guint from a string stored in a hash table
	*
	* \param[in] table Hash table to search
	* \param[in] key Hash table key to use to retrieve string
	* \param[in] default_val What to use if key has no entry in table
	* \param[out] result If not NULL, where to store parsed integer
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__guint_from_hash(GHashTable table, const char key, guint default_val,
	guint *result)
	{
	const char *value;
	long long value_ll;
	int rc = pcmk_rc_ok;

	CRM_CHECK((table != NULL) && (key != NULL), return EINVAL);

	if (result != NULL) {
	*result = default_val;
	}

	value = g_hash_table_lookup(table, key);
	if (value == NULL) {
	return pcmk_rc_ok;
	}

	rc = pcmk__scan_ll(value, &value_ll, 0LL);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((value_ll < 0) \|\| (value_ll > G_MAXUINT)) {
	crm_warn("Could not parse non-negative integer from %s", value);
	return ERANGE;
	}

	if (result != NULL) {
	*result = (guint) value_ll;
	}
	return pcmk_rc_ok;
	}

	#ifndef NUMCHARS
	# define NUMCHARS "0123456789."
	#endif

	#ifndef WHITESPACE
	# define WHITESPACE " \t\n\r\f"
	#endif

	/*!
	* \brief Parse a time+units string and return milliseconds equivalent
	*
	* \param[in] input String with a nonnegative number and optional unit
	* (optionally with whitespace before and/or after the
	* number). If missing, the unit defaults to seconds.
	*
	* \return Milliseconds corresponding to string expression, or
	* \c PCMK__PARSE_INT_DEFAULT on error
	*/
	long long
	crm_get_msec(const char *input)
	{
	const char *num_start = NULL;
	const char *units;
	long long multiplier = 1000;
	long long divisor = 1;
	long long msec = PCMK__PARSE_INT_DEFAULT;
	size_t num_len = 0;
	char *end_text = NULL;

	if (input == NULL) {
	return PCMK__PARSE_INT_DEFAULT;
	}

	num_start = input + strspn(input, WHITESPACE);
	num_len = strspn(num_start, NUMCHARS);
	if (num_len < 1) {
	return PCMK__PARSE_INT_DEFAULT;
	}
	units = num_start + num_len;
	units += strspn(units, WHITESPACE);

	if (!strncasecmp(units, "ms", 2) \|\| !strncasecmp(units, "msec", 4)) {
	multiplier = 1;
	divisor = 1;
	} else if (!strncasecmp(units, "us", 2) \|\| !strncasecmp(units, "usec", 4)) {
	multiplier = 1;
	divisor = 1000;
	} else if (!strncasecmp(units, "s", 1) \|\| !strncasecmp(units, "sec", 3)) {
	multiplier = 1000;
	divisor = 1;
	} else if (!strncasecmp(units, "m", 1) \|\| !strncasecmp(units, "min", 3)) {
	multiplier = 60 * 1000;
	divisor = 1;
	} else if (!strncasecmp(units, "h", 1) \|\| !strncasecmp(units, "hr", 2)) {
	multiplier = 60 * 60 * 1000;
	divisor = 1;
	} else if ((units != '\0') && (units != '\n') && (*units != '\r')) {
	return PCMK__PARSE_INT_DEFAULT;
	}

	scan_ll(num_start, &msec, PCMK__PARSE_INT_DEFAULT, &end_text);
	if (msec > (LLONG_MAX / multiplier)) {
	// Arithmetics overflow while multiplier/divisor mutually exclusive
	return LLONG_MAX;
	}
	msec *= multiplier;
	msec /= divisor;
	return msec;
	}

	/*!
	* \brief Parse milliseconds from a Pacemaker interval specification
	*
	* \param[in] input Pacemaker time interval specification (a bare number
	* of seconds; a number with a unit, optionally with
	* whitespace before and/or after the number; or an ISO
	* 8601 duration)
	* \param[out] result_ms Where to store milliseconds equivalent of \p input on
	* success (limited to the range of an unsigned integer),
	* or 0 if \p input is \c NULL or invalid
	*
	* \return Standard Pacemaker return code (specifically, \c pcmk_rc_ok if
	* \p input is valid or \c NULL, and \c EINVAL otherwise)
	*/
	int
	pcmk_parse_interval_spec(const char input, guint result_ms)
	{
	long long msec = PCMK__PARSE_INT_DEFAULT;
	int rc = pcmk_rc_ok;

	if (input == NULL) {
	msec = 0;
	goto done;
	}

	if (input[0] == 'P') {
	crm_time_t *period_s = crm_time_parse_duration(input);

	if (period_s != NULL) {
	msec = 1000 * crm_time_get_seconds(period_s);
	crm_time_free(period_s);
	}

	} else {
	msec = crm_get_msec(input);
	}

	if (msec == PCMK__PARSE_INT_DEFAULT) {
	crm_warn("Using 0 instead of invalid interval specification '%s'",
	input);
	msec = 0;
	rc = EINVAL;
	}

	done:
	if (result_ms != NULL) {
	*result_ms = (msec >= G_MAXUINT)? G_MAXUINT : (guint) msec;
	}
	return rc;
	}

	gboolean
	crm_is_true(const char *s)
	{
	gboolean ret = FALSE;

	return (crm_str_to_boolean(s, &ret) < 0)? FALSE : ret;
	}

	int
	crm_str_to_boolean(const char s, int ret)
	{
	if (s == NULL) {
	return -1;
	}

	if (pcmk__strcase_any_of(s, PCMK_VALUE_TRUE, "on", "yes", "y", "1", NULL)) {
	if (ret != NULL) {
	*ret = TRUE;
	}
	return 1;
	}

	- if (strcasecmp(s, "false") == 0
	- \|\| strcasecmp(s, "off") == 0
	- \|\| strcasecmp(s, "no") == 0 \|\| strcasecmp(s, "n") == 0 \|\| strcasecmp(s, "0") == 0) {
	-
	+ if (pcmk__strcase_any_of(s, PCMK_VALUE_FALSE, "off", "no", "n", "0",
	+ NULL)) {
	if (ret != NULL) {
	*ret = FALSE;
	}
	return 1;
	}
	return -1;
	}

	/*!
	* \internal
	* \brief Replace any trailing newlines in a string with \0's
	*
	* \param[in,out] str String to trim
	*
	* \return \p str
	*/
	char *
	pcmk__trim(char *str)
	{
	int len;

	if (str == NULL) {
	return str;
	}

	for (len = strlen(str) - 1; len >= 0 && str[len] == '\n'; len--) {
	str[len] = '\0';
	}

	return str;
	}

	/*!
	* \brief Check whether a string starts with a certain sequence
	*
	* \param[in] str String to check
	* \param[in] prefix Sequence to match against beginning of \p str
	*
	* \return \c true if \p str begins with match, \c false otherwise
	* \note This is equivalent to !strncmp(s, prefix, strlen(prefix))
	* but is likely less efficient when prefix is a string literal
	* if the compiler optimizes away the strlen() at compile time,
	* and more efficient otherwise.
	*/
	bool
	pcmk__starts_with(const char str, const char prefix)
	{
	const char *s = str;
	const char *p = prefix;

	if (!s \|\| !p) {
	return false;
	}
	while (s && p) {
	if (s++ != p++) {
	return false;
	}
	}
	return (*p == 0);
	}

	static inline bool
	ends_with(const char s, const char match, bool as_extension)
	{
	if (pcmk__str_empty(match)) {
	return true;
	} else if (s == NULL) {
	return false;
	} else {
	size_t slen, mlen;

	/* Besides as_extension, we could also check
	!strchr(&match[1], match[0]) but that would be inefficient.
	*/
	if (as_extension) {
	s = strrchr(s, match[0]);
	return (s == NULL)? false : !strcmp(s, match);
	}

	mlen = strlen(match);
	slen = strlen(s);
	return ((slen >= mlen) && !strcmp(s + slen - mlen, match));
	}
	}

	/*!
	* \internal
	* \brief Check whether a string ends with a certain sequence
	*
	* \param[in] s String to check
	* \param[in] match Sequence to match against end of \p s
	*
	* \return \c true if \p s ends case-sensitively with match, \c false otherwise
	* \note pcmk__ends_with_ext() can be used if the first character of match
	* does not recur in match.
	*/
	bool
	pcmk__ends_with(const char s, const char match)
	{
	return ends_with(s, match, false);
	}

	/*!
	* \internal
	* \brief Check whether a string ends with a certain "extension"
	*
	* \param[in] s String to check
	* \param[in] match Extension to match against end of \p s, that is,
	* its first character must not occur anywhere
	* in the rest of that very sequence (example: file
	* extension where the last dot is its delimiter,
	* e.g., ".html"); incorrect results may be
	* returned otherwise.
	*
	* \return \c true if \p s ends (verbatim, i.e., case sensitively)
	* with "extension" designated as \p match (including empty
	* string), \c false otherwise
	*
	* \note Main incentive to prefer this function over \c pcmk__ends_with()
	* where possible is the efficiency (at the cost of added
	* restriction on \p match as stated; the complexity class
	* remains the same, though: BigO(M+N) vs. BigO(M+2N)).
	*/
	bool
	pcmk__ends_with_ext(const char s, const char match)
	{
	return ends_with(s, match, true);
	}

	/*!
	* \internal
	* \brief Create a hash of a string suitable for use with GHashTable
	*
	* \param[in] v String to hash
	*
	* \return A hash of \p v compatible with g_str_hash() before glib 2.28
	* \note glib changed their hash implementation:
	*
	* https://gitlab.gnome.org/GNOME/glib/commit/354d655ba8a54b754cb5a3efb42767327775696c
	*
	* Note that the new g_str_hash is presumably a better hash (it's actually
	* a correct implementation of DJB's hash), but we need to preserve existing
	* behaviour, because the hash key ultimately determines the "sort" order
	* when iterating through GHashTables, which affects allocation of scores to
	* clone instances when iterating through rsc->allowed_nodes. It (somehow)
	* also appears to have some minor impact on the ordering of a few
	* pseudo_event IDs in the transition graph.
	*/
	static guint
	pcmk__str_hash(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + *p;

	return h;
	}

	/*!
	* \internal
	* \brief Create a hash table with case-sensitive strings as keys
	*
	* \param[in] key_destroy_func Function to free a key
	* \param[in] value_destroy_func Function to free a value
	*
	* \return Newly allocated hash table
	* \note It is the caller's responsibility to free the result, using
	* g_hash_table_destroy().
	*/
	GHashTable *
	pcmk__strkey_table(GDestroyNotify key_destroy_func,
	GDestroyNotify value_destroy_func)
	{
	return g_hash_table_new_full(pcmk__str_hash, g_str_equal,
	key_destroy_func, value_destroy_func);
	}

	/* used with hash tables where case does not matter */
	static gboolean
	pcmk__strcase_equal(gconstpointer a, gconstpointer b)
	{
	return pcmk__str_eq((const char )a, (const char )b, pcmk__str_casei);
	}

	static guint
	pcmk__strcase_hash(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + g_ascii_tolower(*p);

	return h;
	}

	/*!
	* \internal
	* \brief Create a hash table with case-insensitive strings as keys
	*
	* \param[in] key_destroy_func Function to free a key
	* \param[in] value_destroy_func Function to free a value
	*
	* \return Newly allocated hash table
	* \note It is the caller's responsibility to free the result, using
	* g_hash_table_destroy().
	*/
	GHashTable *
	pcmk__strikey_table(GDestroyNotify key_destroy_func,
	GDestroyNotify value_destroy_func)
	{
	return g_hash_table_new_full(pcmk__strcase_hash, pcmk__strcase_equal,
	key_destroy_func, value_destroy_func);
	}

	static void
	copy_str_table_entry(gpointer key, gpointer value, gpointer user_data)
	{
	if (key && value && user_data) {
	g_hash_table_insert((GHashTable*)user_data, strdup(key), strdup(value));
	}
	}

	/*!
	* \internal
	* \brief Copy a hash table that uses dynamically allocated strings
	*
	* \param[in,out] old_table Hash table to duplicate
	*
	* \return New hash table with copies of everything in \p old_table
	* \note This assumes the hash table uses dynamically allocated strings -- that
	* is, both the key and value free functions are free().
	*/
	GHashTable *
	pcmk__str_table_dup(GHashTable *old_table)
	{
	GHashTable *new_table = NULL;

	if (old_table) {
	new_table = pcmk__strkey_table(free, free);
	g_hash_table_foreach(old_table, copy_str_table_entry, new_table);
	}
	return new_table;
	}

	/*!
	* \internal
	* \brief Add a word to a string list of words
	*
	* \param[in,out] list Pointer to current string list (may not be \p NULL)
	* \param[in] init_size \p list will be initialized to at least this size,
	* if it needs initialization (if 0, use GLib's default
	* initial string size)
	* \param[in] word String to add to \p list (\p list will be
	* unchanged if this is \p NULL or the empty string)
	* \param[in] separator String to separate words in \p list
	* (a space will be used if this is NULL)
	*
	* \note \p word may contain \p separator, though that would be a bad idea if
	* the string needs to be parsed later.
	*/
	void
	pcmk__add_separated_word(GString *list, size_t init_size, const char word,
	const char *separator)
	{
	CRM_ASSERT(list != NULL);

	if (pcmk__str_empty(word)) {
	return;
	}

	if (*list == NULL) {
	if (init_size > 0) {
	*list = g_string_sized_new(init_size);
	} else {
	*list = g_string_new(NULL);
	}
	}

	if ((*list)->len == 0) {
	// Don't add a separator before the first word in the list
	separator = "";

	} else if (separator == NULL) {
	// Default to space-separated
	separator = " ";
	}

	g_string_append(*list, separator);
	g_string_append(*list, word);
	}

	/*!
	* \internal
	* \brief Compress data
	*
	* \param[in] data Data to compress
	* \param[in] length Number of characters of data to compress
	* \param[in] max Maximum size of compressed data (or 0 to estimate)
	* \param[out] result Where to store newly allocated compressed result
	* \param[out] result_len Where to store actual compressed length of result
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__compress(const char *data, unsigned int length, unsigned int max,
	char *result, unsigned int result_len)
	{
	int rc;
	char *compressed = NULL;
	char *uncompressed = strdup(data);
	#ifdef CLOCK_MONOTONIC
	struct timespec after_t;
	struct timespec before_t;
	#endif

	if (max == 0) {
	max = (length * 1.01) + 601; // Size guaranteed to hold result
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &before_t);
	#endif

	compressed = calloc((size_t) max, sizeof(char));
	CRM_ASSERT(compressed);

	*result_len = max;
	rc = BZ2_bzBuffToBuffCompress(compressed, result_len, uncompressed, length,
	CRM_BZ2_BLOCKS, 0, CRM_BZ2_WORK);
	rc = pcmk__bzlib2rc(rc);

	free(uncompressed);

	if (rc != pcmk_rc_ok) {
	crm_err("Compression of %d bytes failed: %s " CRM_XS " rc=%d",
	length, pcmk_rc_str(rc), rc);
	free(compressed);
	return rc;
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &after_t);

	crm_trace("Compressed %d bytes into %d (ratio %d:1) in %.0fms",
	length, result_len, length / (result_len),
	(after_t.tv_sec - before_t.tv_sec) * 1000 +
	(after_t.tv_nsec - before_t.tv_nsec) / 1e6);
	#else
	crm_trace("Compressed %d bytes into %d (ratio %d:1)",
	length, result_len, length / (result_len));
	#endif

	*result = compressed;
	return pcmk_rc_ok;
	}

	char *
	crm_strdup_printf(char const *format, ...)
	{
	va_list ap;
	int len = 0;
	char *string = NULL;

	va_start(ap, format);
	len = vasprintf (&string, format, ap);
	CRM_ASSERT(len > 0);
	va_end(ap);
	return string;
	}

	int
	pcmk__parse_ll_range(const char srcstring, long long start, long long *end)
	{
	char *remainder = NULL;
	int rc = pcmk_rc_ok;

	CRM_ASSERT(start != NULL && end != NULL);

	*start = PCMK__PARSE_INT_DEFAULT;
	*end = PCMK__PARSE_INT_DEFAULT;

	crm_trace("Attempting to decode: [%s]", srcstring);
	if (pcmk__str_eq(srcstring, "", pcmk__str_null_matches)) {
	return ENODATA;
	} else if (pcmk__str_eq(srcstring, "-", pcmk__str_none)) {
	return pcmk_rc_bad_input;
	}

	/* String starts with a dash, so this is either a range with
	* no beginning or garbage.
	* */
	if (*srcstring == '-') {
	int rc = scan_ll(srcstring+1, end, PCMK__PARSE_INT_DEFAULT, &remainder);

	if (rc != pcmk_rc_ok \|\| *remainder != '\0') {
	return pcmk_rc_bad_input;
	} else {
	return pcmk_rc_ok;
	}
	}

	rc = scan_ll(srcstring, start, PCMK__PARSE_INT_DEFAULT, &remainder);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if (remainder && remainder == '-') {
	if (*(remainder+1)) {
	char *more_remainder = NULL;
	int rc = scan_ll(remainder+1, end, PCMK__PARSE_INT_DEFAULT,
	&more_remainder);

	if (rc != pcmk_rc_ok) {
	return rc;
	} else if (*more_remainder != '\0') {
	return pcmk_rc_bad_input;
	}
	}
	} else if (remainder && remainder != '-') {
	*start = PCMK__PARSE_INT_DEFAULT;
	return pcmk_rc_bad_input;
	} else {
	/* The input string contained only one number. Set start and end
	* to the same value and return pcmk_rc_ok. This gives the caller
	* a way to tell this condition apart from a range with no end.
	*/
	end = start;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Find a string in a list of strings
	*
	* \note This function takes the same flags and has the same behavior as
	* pcmk__str_eq().
	*
	* \note No matter what input string or flags are provided, an empty
	* list will always return FALSE.
	*
	* \param[in] s String to search for
	* \param[in] lst List to search
	* \param[in] flags A bitfield of pcmk__str_flags to modify operation
	*
	* \return \c TRUE if \p s is in \p lst, or \c FALSE otherwise
	*/
	gboolean
	pcmk__str_in_list(const gchar s, const GList lst, uint32_t flags)
	{
	for (const GList *ele = lst; ele != NULL; ele = ele->next) {
	if (pcmk__str_eq(s, ele->data, flags)) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static bool
	str_any_of(const char *s, va_list args, uint32_t flags)
	{
	if (s == NULL) {
	return pcmk_is_set(flags, pcmk__str_null_matches);
	}

	while (1) {
	const char ele = va_arg(args, const char );

	if (ele == NULL) {
	break;
	} else if (pcmk__str_eq(s, ele, flags)) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \internal
	* \brief Is a string a member of a list of strings?
	*
	* \param[in] s String to search for in \p ...
	* \param[in] ... Strings to compare \p s against. The final string
	* must be NULL.
	*
	* \note The comparison is done case-insensitively. The function name is
	* meant to be reminiscent of strcasecmp.
	*
	* \return \c true if \p s is in \p ..., or \c false otherwise
	*/
	bool
	pcmk__strcase_any_of(const char *s, ...)
	{
	va_list ap;
	bool rc;

	va_start(ap, s);
	rc = str_any_of(s, ap, pcmk__str_casei);
	va_end(ap);
	return rc;
	}

	/*!
	* \internal
	* \brief Is a string a member of a list of strings?
	*
	* \param[in] s String to search for in \p ...
	* \param[in] ... Strings to compare \p s against. The final string
	* must be NULL.
	*
	* \note The comparison is done taking case into account.
	*
	* \return \c true if \p s is in \p ..., or \c false otherwise
	*/
	bool
	pcmk__str_any_of(const char *s, ...)
	{
	va_list ap;
	bool rc;

	va_start(ap, s);
	rc = str_any_of(s, ap, pcmk__str_none);
	va_end(ap);
	return rc;
	}

	/*!
	* \internal
	* \brief Check whether a character is in any of a list of strings
	*
	* \param[in] ch Character (ASCII) to search for
	* \param[in] ... Strings to search. Final argument must be
	* \c NULL.
	*
	* \return \c true if any of \p ... contain \p ch, \c false otherwise
	* \note \p ... must contain at least one argument (\c NULL).
	*/
	bool
	pcmk__char_in_any_str(int ch, ...)
	{
	bool rc = false;
	va_list ap;

	/*
	* Passing a char to va_start() can generate compiler warnings,
	* so ch is declared as an int.
	*/
	va_start(ap, ch);

	while (1) {
	const char ele = va_arg(ap, const char );

	if (ele == NULL) {
	break;
	} else if (strchr(ele, ch) != NULL) {
	rc = true;
	break;
	}
	}

	va_end(ap);
	return rc;
	}

	/*!
	* \internal
	* \brief Sort strings, with numeric portions sorted numerically
	*
	* Sort two strings case-insensitively like strcasecmp(), but with any numeric
	* portions of the string sorted numerically. This is particularly useful for
	* node names (for example, "node10" will sort higher than "node9" but lower
	* than "remotenode9").
	*
	* \param[in] s1 First string to compare (must not be NULL)
	* \param[in] s2 Second string to compare (must not be NULL)
	*
	* \retval -1 \p s1 comes before \p s2
	* \retval 0 \p s1 and \p s2 are equal
	* \retval 1 \p s1 comes after \p s2
	*/
	int
	pcmk__numeric_strcasecmp(const char s1, const char s2)
	{
	CRM_ASSERT((s1 != NULL) && (s2 != NULL));

	while (s1 && s2) {
	if (isdigit(s1) && isdigit(s2)) {
	// If node names contain a number, sort numerically

	char *end1 = NULL;
	char *end2 = NULL;
	long num1 = strtol(s1, &end1, 10);
	long num2 = strtol(s2, &end2, 10);

	// allow ordering e.g. 007 > 7
	size_t len1 = end1 - s1;
	size_t len2 = end2 - s2;

	if (num1 < num2) {
	return -1;
	} else if (num1 > num2) {
	return 1;
	} else if (len1 < len2) {
	return -1;
	} else if (len1 > len2) {
	return 1;
	}
	s1 = end1;
	s2 = end2;
	} else {
	// Compare non-digits case-insensitively
	int lower1 = tolower(*s1);
	int lower2 = tolower(*s2);

	if (lower1 < lower2) {
	return -1;
	} else if (lower1 > lower2) {
	return 1;
	}
	++s1;
	++s2;
	}
	}
	if (!s1 && s2) {
	return -1;
	} else if (s1 && !s2) {
	return 1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Sort strings.
	*
	* This is your one-stop function for string comparison. By default, this
	* function works like \p g_strcmp0. That is, like \p strcmp but a \p NULL
	* string sorts before a non-<tt>NULL</tt> string.
	*
	* The \p pcmk__str_none flag produces the default behavior. Behavior can be
	* changed with various flags:
	*
	* - \p pcmk__str_regex - The second string is a regular expression that the
	* first string will be matched against.
	* - \p pcmk__str_casei - By default, comparisons are done taking case into
	* account. This flag makes comparisons case-
	* insensitive. This can be combined with
	* \p pcmk__str_regex.
	* - \p pcmk__str_null_matches - If one string is \p NULL and the other is not,
	* still return \p 0.
	* - \p pcmk__str_star_matches - If one string is \p "*" and the other is not,
	* still return \p 0.
	*
	* \param[in] s1 First string to compare
	* \param[in] s2 Second string to compare, or a regular expression to
	* match if \p pcmk__str_regex is set
	* \param[in] flags A bitfield of \p pcmk__str_flags to modify operation
	*
	* \retval negative \p s1 is \p NULL or comes before \p s2
	* \retval 0 \p s1 and \p s2 are equal, or \p s1 is found in \p s2 if
	* \c pcmk__str_regex is set
	* \retval positive \p s2 is \p NULL or \p s1 comes after \p s2, or \p s2
	* is an invalid regular expression, or \p s1 was not found
	* in \p s2 if \p pcmk__str_regex is set.
	*/
	int
	pcmk__strcmp(const char s1, const char s2, uint32_t flags)
	{
	/* If this flag is set, the second string is a regex. */
	if (pcmk_is_set(flags, pcmk__str_regex)) {
	regex_t r_patt;
	int reg_flags = REG_EXTENDED \| REG_NOSUB;
	int regcomp_rc = 0;
	int rc = 0;

	if (s1 == NULL \|\| s2 == NULL) {
	return 1;
	}

	if (pcmk_is_set(flags, pcmk__str_casei)) {
	reg_flags \|= REG_ICASE;
	}
	regcomp_rc = regcomp(&r_patt, s2, reg_flags);
	if (regcomp_rc != 0) {
	rc = 1;
	crm_err("Bad regex '%s' for update: %s", s2, strerror(regcomp_rc));
	} else {
	rc = regexec(&r_patt, s1, 0, NULL, 0);
	regfree(&r_patt);
	if (rc != 0) {
	rc = 1;
	}
	}
	return rc;
	}

	/* If the strings are the same pointer, return 0 immediately. */
	if (s1 == s2) {
	return 0;
	}

	/* If this flag is set, return 0 if either (or both) of the input strings
	* are NULL. If neither one is NULL, we need to continue and compare
	* them normally.
	*/
	if (pcmk_is_set(flags, pcmk__str_null_matches)) {
	if (s1 == NULL \|\| s2 == NULL) {
	return 0;
	}
	}

	/* Handle the cases where one is NULL and the str_null_matches flag is not set.
	* A NULL string always sorts to the beginning.
	*/
	if (s1 == NULL) {
	return -1;
	} else if (s2 == NULL) {
	return 1;
	}

	/* If this flag is set, return 0 if either (or both) of the input strings
	* are "*". If neither one is, we need to continue and compare them
	* normally.
	*/
	if (pcmk_is_set(flags, pcmk__str_star_matches)) {
	if (strcmp(s1, "") == 0 \|\| strcmp(s2, "") == 0) {
	return 0;
	}
	}

	if (pcmk_is_set(flags, pcmk__str_casei)) {
	return strcasecmp(s1, s2);
	} else {
	return strcmp(s1, s2);
	}
	}

	/*!
	* \internal
	* \brief Update a dynamically allocated string with a new value
	*
	* Given a dynamically allocated string and a new value for it, if the string
	* is different from the new value, free the string and replace it with either a
	* newly allocated duplicate of the value or NULL as appropriate.
	*
	* \param[in,out] str Pointer to dynamically allocated string
	* \param[in] value New value to duplicate (or NULL)
	*
	* \note The caller remains responsibile for freeing \p *str.
	*/
	void
	pcmk__str_update(char *str, const char value)
	{
	if ((str != NULL) && !pcmk__str_eq(*str, value, pcmk__str_none)) {
	free(*str);
	if (value == NULL) {
	*str = NULL;
	} else {
	*str = strdup(value);
	CRM_ASSERT(*str != NULL);
	}
	}
	}

	/*!
	* \internal
	* \brief Append a list of strings to a destination \p GString
	*
	* \param[in,out] buffer Where to append the strings (must not be \p NULL)
	* \param[in] ... A <tt>NULL</tt>-terminated list of strings
	*
	* \note This tends to be more efficient than a single call to
	* \p g_string_append_printf().
	*/
	void
	pcmk__g_strcat(GString *buffer, ...)
	{
	va_list ap;

	CRM_ASSERT(buffer != NULL);
	va_start(ap, buffer);

	while (true) {
	const char ele = va_arg(ap, const char );

	if (ele == NULL) {
	break;
	}
	g_string_append(buffer, ele);
	}
	va_end(ap);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/common/util_compat.h>

	gboolean
	safe_str_neq(const char a, const char b)
	{
	if (a == b) {
	return FALSE;

	} else if (a == NULL \|\| b == NULL) {
	return TRUE;

	} else if (strcasecmp(a, b) == 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	crm_str_eq(const char a, const char b, gboolean use_case)
	{
	if (use_case) {
	return g_strcmp0(a, b) == 0;

	/* TODO - Figure out which calls, if any, really need to be case independent */
	} else if (a == b) {
	return TRUE;

	} else if (a == NULL \|\| b == NULL) {
	/* shouldn't be comparing NULLs */
	return FALSE;

	} else if (strcasecmp(a, b) == 0) {
	return TRUE;
	}
	return FALSE;
	}

	char *
	crm_itoa_stack(int an_int, char *buffer, size_t len)
	{
	if (buffer != NULL) {
	snprintf(buffer, len, "%d", an_int);
	}
	return buffer;
	}

	guint
	g_str_hash_traditional(gconstpointer v)
	{
	return pcmk__str_hash(v);
	}

	gboolean
	crm_strcase_equal(gconstpointer a, gconstpointer b)
	{
	return pcmk__strcase_equal(a, b);
	}

	guint
	crm_strcase_hash(gconstpointer v)
	{
	return pcmk__strcase_hash(v);
	}

	GHashTable *
	crm_str_table_dup(GHashTable *old_table)
	{
	return pcmk__str_table_dup(old_table);
	}

	long long
	crm_parse_ll(const char text, const char default_text)
	{
	long long result;

	if (text == NULL) {
	text = default_text;
	if (text == NULL) {
	crm_err("No default conversion value supplied");
	errno = EINVAL;
	return PCMK__PARSE_INT_DEFAULT;
	}
	}
	scan_ll(text, &result, PCMK__PARSE_INT_DEFAULT, NULL);
	return result;
	}

	int
	crm_parse_int(const char text, const char default_text)
	{
	long long result = crm_parse_ll(text, default_text);

	if (result < INT_MIN) {
	// If errno is ERANGE, crm_parse_ll() has already logged a message
	if (errno != ERANGE) {
	crm_err("Conversion of %s was clipped: %lld", text, result);
	errno = ERANGE;
	}
	return INT_MIN;

	} else if (result > INT_MAX) {
	// If errno is ERANGE, crm_parse_ll() has already logged a message
	if (errno != ERANGE) {
	crm_err("Conversion of %s was clipped: %lld", text, result);
	errno = ERANGE;
	}
	return INT_MAX;
	}

	return (int) result;
	}

	char *
	crm_strip_trailing_newline(char *str)
	{
	return pcmk__trim(str);
	}

	int
	pcmk_numeric_strcasecmp(const char s1, const char s2)
	{
	return pcmk__numeric_strcasecmp(s1, s2);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c b/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c
	index 8a05d4fb8f..5422e4ec16 100644
	--- a/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c
	+++ b/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c
	@@ -1,31 +1,31 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>
	#include <crm/common/xml_internal.h>
	#include <crm/msg_xml.h>

	static void
	set_attr(void **state)
	{
	xmlNode *node = string2xml("<node/>");

	pcmk__xe_set_bool_attr(node, "a", true);
	pcmk__xe_set_bool_attr(node, "b", false);

	assert_string_equal(crm_element_value(node, "a"), PCMK_VALUE_TRUE);
	- assert_string_equal(crm_element_value(node, "b"), XML_BOOLEAN_FALSE);
	+ assert_string_equal(crm_element_value(node, "b"), PCMK_VALUE_FALSE);

	free_xml(node);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(set_attr))
	diff --git a/lib/pacemaker/pcmk_injections.c b/lib/pacemaker/pcmk_injections.c
	index d1a1a31801..c28a249e27 100644
	--- a/lib/pacemaker/pcmk_injections.c
	+++ b/lib/pacemaker/pcmk_injections.c
	@@ -1,788 +1,788 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>

	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/util.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/xml_internal.h>
	#include <crm/lrmd_events.h> // lrmd_event_data_t, etc.
	#include <crm/lrmd_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	bool pcmk__simulate_node_config = false;

	#define XPATH_NODE_CONFIG "//" XML_CIB_TAG_NODE "[@" PCMK_XA_UNAME "='%s']"
	#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']"
	#define XPATH_NODE_STATE_BY_ID "//" XML_CIB_TAG_STATE "[@" PCMK_XA_ID "='%s']"
	#define XPATH_RSC_HISTORY XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']"


	/*!
	* \internal
	* \brief Inject a fictitious transient node attribute into scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node node_state XML to inject attribute into
	* \param[in] name Transient node attribute name to inject
	* \param[in] value Transient node attribute value to inject
	*/
	static void
	inject_transient_attr(pcmk__output_t out, xmlNode cib_node,
	const char name, const char value)
	{
	xmlNode *attrs = NULL;
	xmlNode *instance_attrs = NULL;
	const char *node_uuid = ID(cib_node);

	out->message(out, "inject-attr", name, value, cib_node);

	attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	if (attrs == NULL) {
	attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	crm_xml_add(attrs, PCMK_XA_ID, node_uuid);
	}

	instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
	if (instance_attrs == NULL) {
	instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
	crm_xml_add(instance_attrs, PCMK_XA_ID, node_uuid);
	}

	crm_create_nvpair_xml(instance_attrs, NULL, name, value);
	}

	/*!
	* \internal
	* \brief Inject a fictitious fail count into a scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node Node state XML to inject into
	* \param[in] resource ID of resource for fail count to inject
	* \param[in] task Action name for fail count to inject
	* \param[in] interval_ms Action interval (in milliseconds) for fail count
	* \param[in] rc Action result for fail count to inject (if 0, or
	* 7 when interval_ms is 0, inject nothing)
	*/
	void
	pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char task,
	guint interval_ms, int rc)
	{
	if (rc == 0) {
	return;

	} else if ((rc == 7) && (interval_ms == 0)) {
	return;

	} else {
	char *name = NULL;
	char *now = pcmk__ttoa(time(NULL));

	name = pcmk__failcount_name(resource, task, interval_ms);
	inject_transient_attr(out, cib_node, name, "value++");
	free(name);

	name = pcmk__lastfailure_name(resource, task, interval_ms);
	inject_transient_attr(out, cib_node, name, now);
	free(name);

	free(now);
	}
	}

	/*!
	* \internal
	* \brief Create a CIB configuration entry for a fictitious node
	*
	* \param[in,out] cib_conn CIB object to use
	* \param[in] node Node name to use
	*/
	static void
	create_node_entry(cib_t cib_conn, const char node)
	{
	int rc = pcmk_ok;
	char *xpath = crm_strdup_printf(XPATH_NODE_CONFIG, node);

	rc = cib_conn->cmds->query(cib_conn, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if (rc == -ENXIO) { // Only add if not already existing
	xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);

	crm_xml_add(cib_object, PCMK_XA_ID, node); // Use node name as ID
	crm_xml_add(cib_object, PCMK_XA_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
	cib_sync_call\|cib_scope_local);
	/* Not bothering with subsequent query to see if it exists,
	we'll bomb out later in the call to query_node_uuid()... */

	free_xml(cib_object);
	}

	free(xpath);
	}

	/*!
	* \internal
	* \brief Synthesize a fake executor event for an action
	*
	* \param[in] cib_resource XML for any existing resource action history
	* \param[in] task Name of action to synthesize
	* \param[in] interval_ms Interval of action to synthesize
	* \param[in] outcome Result of action to synthesize
	*
	* \return Newly allocated executor event
	* \note It is the caller's responsibility to free the result with
	* lrmd_free_event().
	*/
	static lrmd_event_data_t *
	create_op(const xmlNode cib_resource, const char task, guint interval_ms,
	int outcome)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *xop = NULL;

	op = lrmd_new_event(ID(cib_resource), task, interval_ms);
	lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
	op->params = NULL; // Not needed for simulation purposes
	op->t_run = (unsigned int) time(NULL);
	op->t_rcchange = op->t_run;

	// Use a call ID higher than any existing history entries
	op->call_id = 0;
	for (xop = pcmk__xe_first_child(cib_resource); xop != NULL;
	xop = pcmk__xe_next(xop)) {

	int tmp = 0;

	crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
	if (tmp > op->call_id) {
	op->call_id = tmp;
	}
	}
	op->call_id++;

	return op;
	}

	/*!
	* \internal
	* \brief Inject a fictitious resource history entry into a scheduler input
	*
	* \param[in,out] cib_resource Resource history XML to inject entry into
	* \param[in,out] op Action result to inject
	* \param[in] target_rc Expected result for action to inject
	*
	* \return XML of injected resource history entry
	*/
	xmlNode *
	pcmk__inject_action_result(xmlNode cib_resource, lrmd_event_data_t op,
	int target_rc)
	{
	return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
	target_rc, NULL, crm_system_name);
	}

	/*!
	* \internal
	* \brief Inject a fictitious node into a scheduler input
	*
	* \param[in,out] cib_conn Scheduler input CIB to inject node into
	* \param[in] node Name of node to inject
	* \param[in] uuid UUID of node to inject
	*
	* \return XML of node_state entry for new node
	* \note If the global pcmk__simulate_node_config has been set to true, a
	* node entry in the configuration section will be added, as well as a
	* node state entry in the status section.
	*/
	xmlNode *
	pcmk__inject_node(cib_t cib_conn, const char node, const char *uuid)
	{
	int rc = pcmk_ok;
	xmlNode *cib_object = NULL;
	char *xpath = crm_strdup_printf(XPATH_NODE_STATE, node);
	bool duplicate = false;
	char *found_uuid = NULL;

	if (pcmk__simulate_node_config) {
	create_node_entry(cib_conn, node);
	}

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
	crm_err("Detected multiple node_state entries for xpath=%s, bailing",
	xpath);
	duplicate = true;
	goto done;
	}

	if (rc == -ENXIO) {
	if (uuid == NULL) {
	query_node_uuid(cib_conn, node, &found_uuid, NULL);
	} else {
	found_uuid = strdup(uuid);
	}

	if (found_uuid) {
	char *xpath_by_uuid = crm_strdup_printf(XPATH_NODE_STATE_BY_ID,
	found_uuid);

	// It's possible that a node_state entry doesn't have an uname yet.
	rc = cib_conn->cmds->query(cib_conn, xpath_by_uuid, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
	crm_err("Can't inject node state for %s because multiple "
	"state entries found for ID %s", node, found_uuid);
	duplicate = true;
	free(xpath_by_uuid);
	goto done;

	} else if (cib_object != NULL) {
	crm_xml_add(cib_object, PCMK_XA_UNAME, node);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS,
	cib_object,
	cib_sync_call\|cib_scope_local);
	}

	free(xpath_by_uuid);
	}
	}

	if (rc == -ENXIO) {
	cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
	crm_xml_add(cib_object, PCMK_XA_ID, found_uuid);
	crm_xml_add(cib_object, PCMK_XA_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
	cib_sync_call\|cib_scope_local);
	free_xml(cib_object);

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	crm_trace("Injecting node state for %s (rc=%d)", node, rc);
	}

	done:
	free(found_uuid);
	free(xpath);

	if (duplicate) {
	crm_log_xml_warn(cib_object, "Duplicates");
	crm_exit(CRM_EX_SOFTWARE);
	return NULL; // not reached, but makes static analysis happy
	}

	CRM_ASSERT(rc == pcmk_ok);
	return cib_object;
	}

	/*!
	* \internal
	* \brief Inject a fictitious node state change into a scheduler input
	*
	* \param[in,out] cib_conn Scheduler input CIB to inject into
	* \param[in] node Name of node to inject change for
	* \param[in] up If true, change state to online, otherwise offline
	*
	* \return XML of changed (or added) node state entry
	*/
	xmlNode *
	pcmk__inject_node_state_change(cib_t cib_conn, const char node, bool up)
	{
	xmlNode *cib_node = pcmk__inject_node(cib_conn, node, NULL);

	if (up) {
	pcmk__xe_set_props(cib_node,
	PCMK__XA_IN_CCM, PCMK_VALUE_TRUE,
	PCMK__XA_CRMD, ONLINESTATUS,
	PCMK__XA_JOIN, CRMD_JOINSTATE_MEMBER,
	PCMK__XA_EXPECTED, CRMD_JOINSTATE_MEMBER,
	NULL);
	} else {
	pcmk__xe_set_props(cib_node,
	- PCMK__XA_IN_CCM, XML_BOOLEAN_FALSE,
	+ PCMK__XA_IN_CCM, PCMK_VALUE_FALSE,
	PCMK__XA_CRMD, OFFLINESTATUS,
	PCMK__XA_JOIN, CRMD_JOINSTATE_DOWN,
	PCMK__XA_EXPECTED, CRMD_JOINSTATE_DOWN,
	NULL);
	}
	crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, crm_system_name);
	return cib_node;
	}

	/*!
	* \internal
	* \brief Check whether a node has history for a given resource
	*
	* \param[in,out] cib_node Node state XML to check
	* \param[in] resource Resource name to check for
	*
	* \return Resource's lrm_resource XML entry beneath \p cib_node if found,
	* otherwise NULL
	*/
	static xmlNode *
	find_resource_xml(xmlNode cib_node, const char resource)
	{
	const char *node = crm_element_value(cib_node, PCMK_XA_UNAME);
	char *xpath = crm_strdup_printf(XPATH_RSC_HISTORY, node, resource);
	xmlNode *match = get_xpath_object(xpath, cib_node, LOG_TRACE);

	free(xpath);
	return match;
	}

	/*!
	* \internal
	* \brief Inject a resource history element into a scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node Node state XML to inject resource history entry into
	* \param[in] resource ID (in configuration) of resource to inject
	* \param[in] lrm_name ID as used in history (could be clone instance)
	* \param[in] rclass Resource agent class of resource to inject
	* \param[in] rtype Resource agent type of resource to inject
	* \param[in] rprovider Resource agent provider of resource to inject
	*
	* \return XML of injected resource history element
	* \note If a history element already exists under either \p resource or
	* \p lrm_name, this will return it rather than injecting a new one.
	*/
	xmlNode *
	pcmk__inject_resource_history(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char lrm_name,
	const char rclass, const char rtype,
	const char *rprovider)
	{
	xmlNode *lrm = NULL;
	xmlNode *container = NULL;
	xmlNode *cib_resource = NULL;

	cib_resource = find_resource_xml(cib_node, resource);
	if (cib_resource != NULL) {
	/* If an existing LRM history entry uses the resource name,
	* continue using it, even if lrm_name is different.
	*/
	return cib_resource;
	}

	// Check for history entry under preferred name
	if (strcmp(resource, lrm_name) != 0) {
	cib_resource = find_resource_xml(cib_node, lrm_name);
	if (cib_resource != NULL) {
	return cib_resource;
	}
	}

	if ((rclass == NULL) \|\| (rtype == NULL)) {
	// @TODO query configuration for class, provider, type
	out->err(out,
	"Resource %s not found in the status section of %s "
	"(supply class and type to continue)",
	resource, ID(cib_node));
	return NULL;

	} else if (!pcmk__strcase_any_of(rclass,
	PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_STONITH,
	PCMK_RESOURCE_CLASS_SERVICE,
	PCMK_RESOURCE_CLASS_UPSTART,
	PCMK_RESOURCE_CLASS_SYSTEMD,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	out->err(out, "Invalid class for %s: %s", resource, rclass);
	return NULL;

	} else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
	&& (rprovider == NULL)) {
	// @TODO query configuration for provider
	out->err(out, "Please specify the provider for resource %s", resource);
	return NULL;
	}

	crm_info("Injecting new resource %s into node state '%s'",
	lrm_name, ID(cib_node));

	lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
	if (lrm == NULL) {
	const char *node_uuid = ID(cib_node);

	lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
	crm_xml_add(lrm, PCMK_XA_ID, node_uuid);
	}

	container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
	if (container == NULL) {
	container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
	}

	cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);

	// If we're creating a new entry, use the preferred name
	crm_xml_add(cib_resource, PCMK_XA_ID, lrm_name);

	crm_xml_add(cib_resource, PCMK_XA_CLASS, rclass);
	crm_xml_add(cib_resource, PCMK_XA_PROVIDER, rprovider);
	crm_xml_add(cib_resource, PCMK_XA_TYPE, rtype);

	return cib_resource;
	}

	static int
	find_ticket_state(pcmk__output_t out, cib_t the_cib, const char *ticket_id,
	xmlNode **ticket_state_xml)
	{
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;

	GString *xpath = g_string_sized_new(256);

	CRM_ASSERT(ticket_state_xml != NULL);
	*ticket_state_xml = NULL;

	g_string_append(xpath,
	"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS
	"/" XML_CIB_TAG_TICKETS);

	if (ticket_id) {
	pcmk__g_strcat(xpath,
	"/" XML_CIB_TAG_TICKET_STATE
	"[@" PCMK_XA_ID "=\"", ticket_id, "\"]", NULL);
	}
	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call\|cib_scope_local\|cib_xpath);
	g_string_free(xpath, TRUE);

	if (rc != pcmk_ok) {
	return rc;
	}

	crm_log_xml_debug(xml_search, "Match");
	if ((xml_search->children != NULL) && (ticket_id != NULL)) {
	out->err(out, "Multiple ticket_states match ticket_id=%s", ticket_id);
	}
	*ticket_state_xml = xml_search;

	return rc;
	}

	/*!
	* \internal
	* \brief Inject a ticket attribute into ticket state
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in] ticket_id Ticket whose state should be changed
	* \param[in] attr_name Ticket attribute name to inject
	* \param[in] attr_value Boolean value of ticket attribute to inject
	* \param[in,out] cib CIB object to use
	*
	* \return Standard Pacemaker return code
	*/
	static int
	set_ticket_state_attr(pcmk__output_t out, const char ticket_id,
	const char attr_name, bool attr_value, cib_t cib)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_top = NULL;
	xmlNode *ticket_state_xml = NULL;

	// Check for an existing ticket state entry
	rc = find_ticket_state(out, cib, ticket_id, &ticket_state_xml);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) { // Ticket state found, use it
	crm_debug("Injecting attribute into existing ticket state %s",
	ticket_id);
	xml_top = ticket_state_xml;

	} else if (rc == ENXIO) { // No ticket state, create it
	xmlNode *xml_obj = NULL;

	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
	ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
	crm_xml_add(ticket_state_xml, PCMK_XA_ID, ticket_id);

	} else { // Error
	return rc;
	}

	// Add the attribute to the ticket state
	pcmk__xe_set_bool_attr(ticket_state_xml, attr_name, attr_value);
	crm_log_xml_debug(xml_top, "Update");

	// Commit the change to the CIB
	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top,
	cib_sync_call\|cib_scope_local);
	rc = pcmk_legacy2rc(rc);

	free_xml(xml_top);
	return rc;
	}

	/*!
	* \internal
	* \brief Inject a fictitious action into the cluster
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in] spec Action specification to inject
	* \param[in,out] cib CIB object for scheduler input
	* \param[in] scheduler Scheduler data
	*/
	static void
	inject_action(pcmk__output_t out, const char spec, cib_t *cib,
	const pcmk_scheduler_t *scheduler)
	{
	int rc;
	int outcome = PCMK_OCF_OK;
	guint interval_ms = 0;

	char *key = NULL;
	char *node = NULL;
	char *task = NULL;
	char *resource = NULL;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *rprovider = NULL;

	xmlNode *cib_op = NULL;
	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	const pcmk_resource_t *rsc = NULL;
	lrmd_event_data_t *op = NULL;

	out->message(out, "inject-spec", spec);

	key = calloc(1, strlen(spec) + 1);
	node = calloc(1, strlen(spec) + 1);
	rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
	if (rc != 3) {
	out->err(out, "Invalid operation spec: %s. Only found %d fields",
	spec, rc);
	goto done;
	}

	parse_op_key(key, &resource, &task, &interval_ms);

	rsc = pe_find_resource(scheduler->resources, resource);
	if (rsc == NULL) {
	out->err(out, "Invalid resource name: %s", resource);
	goto done;
	}

	rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	rprovider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);

	cib_node = pcmk__inject_node(cib, node, NULL);
	CRM_ASSERT(cib_node != NULL);

	pcmk__inject_failcount(out, cib_node, resource, task, interval_ms, outcome);

	cib_resource = pcmk__inject_resource_history(out, cib_node,
	resource, resource,
	rclass, rtype, rprovider);
	CRM_ASSERT(cib_resource != NULL);

	op = create_op(cib_resource, task, interval_ms, outcome);
	CRM_ASSERT(op != NULL);

	cib_op = pcmk__inject_action_result(cib_resource, op, 0);
	CRM_ASSERT(cib_op != NULL);
	lrmd_free_event(op);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(task);
	free(node);
	free(key);
	}

	/*!
	* \internal
	* \brief Inject fictitious scheduler inputs
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] cib CIB object for scheduler input to modify
	* \param[in] injections Injections to apply
	*/
	void
	pcmk__inject_scheduler_input(pcmk_scheduler_t scheduler, cib_t cib,
	const pcmk_injections_t *injections)
	{
	int rc = pcmk_ok;
	const GList *iter = NULL;
	xmlNode *cib_node = NULL;
	pcmk__output_t *out = scheduler->priv;

	out->message(out, "inject-modify-config", injections->quorum,
	injections->watchdog);
	if (injections->quorum != NULL) {
	xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);

	/* crm_xml_add(top, PCMK_XA_DC_UUID, dc_uuid); */
	crm_xml_add(top, PCMK_XA_HAVE_QUORUM, injections->quorum);

	rc = cib->cmds->modify(cib, NULL, top, cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	}

	if (injections->watchdog != NULL) {
	rc = cib__update_node_attr(out, cib, cib_sync_call\|cib_scope_local,
	XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL,
	NULL, PCMK_OPT_HAVE_WATCHDOG,
	injections->watchdog, NULL, NULL);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->node_up; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;

	out->message(out, "inject-modify-node", "Online", node);

	cib_node = pcmk__inject_node_state_change(cib, node, true);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (iter = injections->node_down; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;
	char *xpath = NULL;

	out->message(out, "inject-modify-node", "Offline", node);

	cib_node = pcmk__inject_node_state_change(cib, node, false);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);

	xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
	node, XML_CIB_TAG_LRM);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	free(xpath);

	xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
	node, XML_TAG_TRANSIENT_NODEATTRS);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	free(xpath);
	}

	for (iter = injections->node_fail; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;

	out->message(out, "inject-modify-node", "Failing", node);

	cib_node = pcmk__inject_node_state_change(cib, node, true);
	- crm_xml_add(cib_node, PCMK__XA_IN_CCM, XML_BOOLEAN_FALSE);
	+ crm_xml_add(cib_node, PCMK__XA_IN_CCM, PCMK_VALUE_FALSE);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (iter = injections->ticket_grant; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Granting", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "granted", true, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_revoke; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Revoking", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "granted", false, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_standby; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Standby", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "standby", true, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_activate; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Activating", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "standby", false, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->op_inject; iter != NULL; iter = iter->next) {
	inject_action(out, (const char *) iter->data, cib, scheduler);
	}

	if (!out->is_quiet(out)) {
	out->end_list(out);
	}
	}

	void
	pcmk_free_injections(pcmk_injections_t *injections)
	{
	if (injections == NULL) {
	return;
	}

	g_list_free_full(injections->node_up, g_free);
	g_list_free_full(injections->node_down, g_free);
	g_list_free_full(injections->node_fail, g_free);
	g_list_free_full(injections->op_fail, g_free);
	g_list_free_full(injections->op_inject, g_free);
	g_list_free_full(injections->ticket_grant, g_free);
	g_list_free_full(injections->ticket_revoke, g_free);
	g_list_free_full(injections->ticket_standby, g_free);
	g_list_free_full(injections->ticket_activate, g_free);
	free(injections->quorum);
	free(injections->watchdog);

	free(injections);
	}
	diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c
	index 185a4d0048..381ddfe5ac 100644
	--- a/lib/pacemaker/pcmk_sched_constraints.c
	+++ b/lib/pacemaker/pcmk_sched_constraints.c
	@@ -1,424 +1,424 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <stdbool.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/iso8601.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/pengine/rules.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	static bool
	evaluate_lifetime(xmlNode lifetime, pcmk_scheduler_t scheduler)
	{
	bool result = FALSE;
	crm_time_t *next_change = crm_time_new_undefined();

	result = pe_evaluate_rules(lifetime, NULL, scheduler->now, next_change);
	if (crm_time_is_defined(next_change)) {
	time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);

	pe__update_recheck_time(recheck, scheduler, "constraint lifetime");
	}
	crm_time_free(next_change);
	return result;
	}

	/*!
	* \internal
	* \brief Unpack constraints from XML
	*
	* Given scheduler data, unpack all constraints from its input XML into
	* data structures.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__unpack_constraints(pcmk_scheduler_t *scheduler)
	{
	xmlNode *xml_constraints = pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_CONSTRAINTS);

	for (xmlNode *xml_obj = pcmk__xe_first_child(xml_constraints);
	xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj)) {

	xmlNode *lifetime = NULL;
	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
	const char tag = (const char ) xml_obj->name;

	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without "
	PCMK_XA_ID, tag);
	continue;
	}

	crm_trace("Unpacking %s constraint '%s'", tag, id);

	lifetime = first_named_child(xml_obj, "lifetime");
	if (lifetime != NULL) {
	pcmk__config_warn("Support for 'lifetime' attribute (in %s) is "
	"deprecated (the rules it contains should "
	"instead be direct descendants of the "
	"constraint object)", id);
	}

	if ((lifetime != NULL) && !evaluate_lifetime(lifetime, scheduler)) {
	crm_info("Constraint %s %s is not active", tag, id);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_ORDER, tag, pcmk__str_none)) {
	pcmk__unpack_ordering(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_DEPEND, tag, pcmk__str_none)) {
	pcmk__unpack_colocation(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_LOCATION, tag,
	pcmk__str_none)) {
	pcmk__unpack_location(xml_obj, scheduler);

	} else if (pcmk__str_eq(XML_CONS_TAG_RSC_TICKET, tag, pcmk__str_none)) {
	pcmk__unpack_rsc_ticket(xml_obj, scheduler);

	} else {
	pcmk__config_err("Unsupported constraint type: %s", tag);
	}
	}
	}

	pcmk_resource_t *
	pcmk__find_constraint_resource(GList rsc_list, const char id)
	{
	if (id == NULL) {
	return NULL;
	}
	for (GList *iter = rsc_list; iter != NULL; iter = iter->next) {
	pcmk_resource_t *parent = iter->data;
	pcmk_resource_t *match = parent->fns->find_rsc(parent, id, NULL,
	pcmk_rsc_match_history);

	if (match != NULL) {
	if (!pcmk__str_eq(match->id, id, pcmk__str_none)) {
	/* We found an instance of a clone instead */
	match = uber_parent(match);
	crm_debug("Found %s for %s", match->id, id);
	}
	return match;
	}
	}
	crm_trace("No match for %s", id);
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether an ID references a resource tag
	*
	* \param[in] scheduler Scheduler data
	* \param[in] id Tag ID to search for
	* \param[out] tag Where to store tag, if found
	*
	* \return true if ID refers to a tagged resource or resource set template,
	* otherwise false
	*/
	static bool
	find_constraint_tag(const pcmk_scheduler_t scheduler, const char id,
	pcmk_tag_t **tag)
	{
	*tag = NULL;

	// Check whether id refers to a resource set template
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, (gpointer *) tag)) {
	if (*tag == NULL) {
	crm_notice("No resource is derived from template '%s'", id);
	return false;
	}
	return true;
	}

	// If not, check whether id refers to a tag
	if (g_hash_table_lookup_extended(scheduler->tags, id,
	NULL, (gpointer *) tag)) {
	if (*tag == NULL) {
	crm_notice("No resource is tagged with '%s'", id);
	return false;
	}
	return true;
	}

	pcmk__config_warn("No resource, template, or tag named '%s'", id);
	return false;
	}

	/*!
	* \brief
	* \internal Check whether an ID refers to a valid resource or tag
	*
	* \param[in] scheduler Scheduler data
	* \param[in] id ID to search for
	* \param[out] rsc Where to store resource, if found
	* (or NULL to skip searching resources)
	* \param[out] tag Where to store tag, if found
	* (or NULL to skip searching tags)
	*
	* \return true if id refers to a resource (possibly indirectly via a tag)
	*/
	bool
	pcmk__valid_resource_or_tag(const pcmk_scheduler_t scheduler, const char id,
	pcmk_resource_t rsc, pcmk_tag_t tag)
	{
	if (rsc != NULL) {
	*rsc = pcmk__find_constraint_resource(scheduler->resources, id);
	if (*rsc != NULL) {
	return true;
	}
	}

	if ((tag != NULL) && find_constraint_tag(scheduler, id, tag)) {
	return true;
	}

	return false;
	}

	/*!
	* \internal
	* \brief Replace any resource tags with equivalent resource_ref entries
	*
	* If a given constraint has resource sets, check each set for resource_ref
	* entries that list tags rather than resource IDs, and replace any found with
	* resource_ref entries for the corresponding resource IDs.
	*
	* \param[in,out] xml_obj Constraint XML
	* \param[in] scheduler Scheduler data
	*
	* \return Equivalent XML with resource tags replaced (or NULL if none)
	* \note It is the caller's responsibility to free the result with free_xml().
	*/
	xmlNode *
	pcmk__expand_tags_in_sets(xmlNode xml_obj, const pcmk_scheduler_t scheduler)
	{
	xmlNode *new_xml = NULL;
	bool any_refs = false;

	// Short-circuit if there are no sets
	if (first_named_child(xml_obj, XML_CONS_TAG_RSC_SET) == NULL) {
	return NULL;
	}

	new_xml = copy_xml(xml_obj);

	for (xmlNode *set = first_named_child(new_xml, XML_CONS_TAG_RSC_SET);
	set != NULL; set = crm_next_same_xml(set)) {

	GList *tag_refs = NULL;
	GList *iter = NULL;

	for (xmlNode *xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
	xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {

	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;

	if (!pcmk__valid_resource_or_tag(scheduler, ID(xml_rsc), &rsc,
	&tag)) {
	pcmk__config_err("Ignoring resource sets for constraint '%s' "
	"because '%s' is not a valid resource or tag",
	ID(xml_obj), ID(xml_rsc));
	free_xml(new_xml);
	return NULL;

	} else if (rsc) {
	continue;

	} else if (tag) {
	// resource_ref under resource_set references template or tag
	xmlNode *last_ref = xml_rsc;

	/* For example, given the original XML:
	*
	* <resource_set id="tag1-colocation-0" sequential="true">
	* <resource_ref id="rsc1"/>
	* <resource_ref id="tag1"/>
	* <resource_ref id="rsc4"/>
	* </resource_set>
	*
	* If rsc2 and rsc3 are tagged with tag1, we add them after it:
	*
	* <resource_set id="tag1-colocation-0" sequential="true">
	* <resource_ref id="rsc1"/>
	* <resource_ref id="tag1"/>
	* <resource_ref id="rsc2"/>
	* <resource_ref id="rsc3"/>
	* <resource_ref id="rsc4"/>
	* </resource_set>
	*/

	for (iter = tag->refs; iter != NULL; iter = iter->next) {
	const char *obj_ref = iter->data;
	xmlNode *new_rsc_ref = NULL;

	new_rsc_ref = xmlNewDocRawNode(set->doc, NULL,
	(pcmkXmlStr)
	XML_TAG_RESOURCE_REF,
	NULL);
	crm_xml_add(new_rsc_ref, PCMK_XA_ID, obj_ref);
	xmlAddNextSibling(last_ref, new_rsc_ref);

	last_ref = new_rsc_ref;
	}

	any_refs = true;

	/* Freeing the resource_ref now would break the XML child
	* iteration, so just remember it for freeing later.
	*/
	tag_refs = g_list_append(tag_refs, xml_rsc);
	}
	}

	/* Now free '<resource_ref id="tag1"/>', and finally get:

	<resource_set id="tag1-colocation-0" sequential="true">
	<resource_ref id="rsc1"/>
	<resource_ref id="rsc2"/>
	<resource_ref id="rsc3"/>
	<resource_ref id="rsc4"/>
	</resource_set>

	*/
	for (iter = tag_refs; iter != NULL; iter = iter->next) {
	xmlNode *tag_ref = iter->data;

	free_xml(tag_ref);
	}
	g_list_free(tag_refs);
	}

	if (!any_refs) {
	free_xml(new_xml);
	new_xml = NULL;
	}
	return new_xml;
	}

	/*!
	* \internal
	* \brief Convert a tag into a resource set of tagged resources
	*
	* \param[in,out] xml_obj Constraint XML
	* \param[out] rsc_set Where to store resource set XML
	* \param[in] attr Name of XML attribute with resource or tag ID
	* \param[in] convert_rsc If true, convert to set even if \p attr
	* references a resource
	* \param[in] scheduler Scheduler data
	*/
	bool
	pcmk__tag_to_set(xmlNode xml_obj, xmlNode rsc_set, const char attr,
	bool convert_rsc, const pcmk_scheduler_t *scheduler)
	{
	const char *cons_id = NULL;
	const char *id = NULL;

	pcmk_resource_t *rsc = NULL;
	pcmk_tag_t *tag = NULL;

	*rsc_set = NULL;

	CRM_CHECK((xml_obj != NULL) && (attr != NULL), return false);

	cons_id = ID(xml_obj);
	if (cons_id == NULL) {
	pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
	xml_obj->name);
	return false;
	}

	id = crm_element_value(xml_obj, attr);
	if (id == NULL) {
	return true;
	}

	if (!pcmk__valid_resource_or_tag(scheduler, id, &rsc, &tag)) {
	pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
	"valid resource or tag", cons_id, id);
	return false;

	} else if (tag) {
	/* The "attr" attribute (for a resource in a constraint) specifies a
	* template or tag. Add the corresponding resource_set containing the
	* resources derived from or tagged with it.
	*/
	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, PCMK_XA_ID, id);

	for (GList *iter = tag->refs; iter != NULL; iter = iter->next) {
	const char *obj_ref = iter->data;
	xmlNode *rsc_ref = NULL;

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, PCMK_XA_ID, obj_ref);
	}

	- /* Set sequential="false" for the resource_set */
	+ /* Set sequential=PCMK_VALUE_FALSE for the resource_set */
	pcmk__xe_set_bool_attr(*rsc_set, "sequential", false);

	} else if ((rsc != NULL) && convert_rsc) {
	/* Even if a regular resource is referenced by "attr", convert it into a
	* resource_set, because the other resource reference in the constraint
	* could be a template or tag.
	*/
	xmlNode *rsc_ref = NULL;

	*rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
	crm_xml_add(*rsc_set, PCMK_XA_ID, id);

	rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
	crm_xml_add(rsc_ref, PCMK_XA_ID, id);

	} else {
	return true;
	}

	/* Remove the "attr" attribute referencing the template/tag */
	if (*rsc_set != NULL) {
	xml_remove_prop(xml_obj, attr);
	}

	return true;
	}

	/*!
	* \internal
	* \brief Create constraints inherent to resource types
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__create_internal_constraints(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Create internal constraints");
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->internal_constraints(rsc);
	}
	}
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index eeb98c08e7..b3ee9f3775 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2229 +1,2229 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pcmk_resource_t *child;

	GList replicas; // pcmk__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) \
	CRM_ASSERT(rsc != NULL); \
	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle); \
	CRM_ASSERT(rsc->variant_opaque != NULL); \
	data = (pe__bundle_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__bundled_resource(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pcmk_resource_t *
	pe__get_rsc_in_container(const pcmk_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pcmk_resource_t *top = pe__const_top_resource(instance, true);

	if ((top == NULL) \|\| (top->variant != pcmk_rsc_variant_bundle)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pcmk__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (pe__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pcmk_resource_t *
	pe__first_container(const pcmk_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pcmk__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pcmk__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	crm_xml_add(rsc, PCMK_XA_ID, name);
	crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(rsc, PCMK_XA_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	pcmk__config_err("Specifying the 'control-port' for %s requires "
	"'replicas-per-host=1'", data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	crm_xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d",
	data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = create_xml_node(xml_ip, "operations");
	crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->children = g_list_append(parent->children, replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pcmk_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	crm_xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
	- crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
	- crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
	+ crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
	+ crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
	env_opt, DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network, "host",
	pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = create_xml_node(xml_container, "operations");
	crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
	parent->children = g_list_append(parent->children, replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pcmk_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);

	if (match) {
	((pcmk_node_t *) match)->weight = -INFINITY;
	((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
	}
	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
	}
	}

	static int
	create_remote_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pcmk_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->cluster->resources, id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->cluster->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during scheduler data cleanup to
	* use as the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = ID(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pe_find_node(parent->cluster->nodes, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, "remote", "-INFINITY",
	parent->cluster);
	} else {
	node->weight = -INFINITY;
	}
	node->rsc_discover_mode = pcmk_probe_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pcmk_node_t entry. Ideally, it would do the same for bundle
	* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
	* obvious what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
	(gpointer) uname);

	replica->node = pe__copy_node(node);
	replica->node->weight = 500;
	replica->node->rsc_discover_mode = pcmk_probe_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->allowed_nodes);
	}
	replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->allowed_nodes,
	(gpointer) replica->node->details->id,
	pe__copy_node(replica->node));

	{
	pcmk_node_t *copy = pe__copy_node(replica->node);
	copy->weight = -INFINITY;
	g_hash_table_insert(replica->child->parent->allowed_nodes,
	(gpointer) replica->node->details->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pe__is_guest_or_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	node->weight = -INFINITY;
	}
	}

	replica->node->details->remote_rsc = replica->remote;

	// Ensure pe__is_guest_node() functions correctly immediately
	replica->remote->container = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	g_hash_table_insert(replica->node->details->attrs,
	strdup(CRM_ATTR_KIND), strdup("container"));

	/* One effect of this is that setup_container() will add
	* replica->remote to replica->container's fillers, which will make
	* pe__resource_contains_guest_node() true for replica->container.
	*
	* replica->child does NOT get added to replica->container's fillers.
	* The only noticeable effect if it did would be for its fail count to
	* be taken into account when checking replica->container's migration
	* threshold.
	*/
	parent->children = g_list_append(parent->children, replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pcmk_resource_t *parent,
	pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));

	CRM_ASSERT(mount != NULL);
	mount->source = strdup(source);
	mount->target = strdup(target);
	pcmk__str_update(&mount->options, options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pcmk__bundle_replica_t *
	replica_for_remote(pcmk_resource_t *remote)
	{
	pcmk_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}

	while (top->parent != NULL) {
	top = top->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->cluster);
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->xml);
	}

	const char *
	pe__add_bundle_remote_name(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	xmlNode xml, const char field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pcmk_node_t *node = NULL;
	pcmk__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->allocated_to;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pe__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pe__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->details->uname);
	}

	return node->details->uname;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", ID(mount_xml), flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid PCMK_META_PROMOTED_MAX
	value = crm_element_value(xml_obj, PCMK_META_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, "masters");
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	/* Default replicas to PCMK_META_PROMOTED_MAX if it was specified and 1
	* otherwise
	*/
	value = crm_element_value(xml_obj, "replicas");
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, "replicas-per-host");
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}

	bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
	bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
	bundle_data->image = crm_element_value_copy(xml_obj, "image");
	bundle_data->container_network = crm_element_value_copy(xml_obj, "network");

	xml_obj = first_named_child(rsc->xml, "network");
	if(xml_obj) {

	bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
	bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
	bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
	bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
	value = crm_element_value(xml_obj, "add-host");
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
	port->source = crm_element_value_copy(xml_child, "port");

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, "range");
	} else {
	port->target = crm_element_value_copy(xml_child, "internal-port");
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	pcmk__config_err("Invalid port directive %s", ID(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = first_named_child(rsc->xml, "storage");
	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	const char *source = crm_element_value(xml_child, "source-dir");
	const char *target = crm_element_value(xml_child, "target-dir");
	const char *options = crm_element_value(xml_child, "options");
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, "source-dir-root");
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	pcmk__config_err("Invalid mount directive %s", ID(xml_child));
	}
	}

	xml_obj = first_named_child(rsc->xml, "primitive");
	if (xml_obj && valid_network(bundle_data)) {
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
	(bundle_data->promoted_max? "master"
	: (const char *)xml_resource->name));

	xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
	crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_ORDERED, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
	add_node_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	pcmk__config_err("Cannot control %s inside %s without either "
	"ip-range-start or control-port",
	rsc->id, ID(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	scheduler) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = calloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting control-port.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->children; childIter != NULL;
	childIter = childIter->next) {

	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->child = childIter->data;
	replica->child->exclusive_discover = TRUE;
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
	pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target =
	g_hash_table_lookup(replica->child->meta,
	PCMK_META_CONTAINER_ATTR_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	g_hash_table_replace(rsc->meta,
	strdup(PCMK_META_CONTAINER_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	g_hash_table_replace(bundle_data->child->meta,
	strdup(PCMK_META_CONTAINER_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	pcmk__config_err("Failed unpacking resource %s", rsc->id);
	rsc->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->utilization;

	replica->container->utilization = replica->child->utilization;
	replica->child->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->children = g_list_append(rsc->children, bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pcmk_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pcmk_resource_t *
	pe__find_bundle_replica(const pcmk_resource_t bundle, const pcmk_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (replica->node->details == node->details) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_rsc_in_list(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	if (rsc != NULL) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	rsc->fns->print(rsc, pre_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	bundle_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (pre_text == NULL) {
	pre_text = "";
	}
	child_text = crm_strdup_printf("%s ", pre_text);

	get_bundle_variant_data(bundle_data, rsc);

	status_print("%s<bundle ", pre_text);
	status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
	status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
	status_print("image=\"%s\" ", bundle_data->image);
	status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print(">\n");

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	status_print("%s <replica " PCMK_XA_ID "=\"%d\">\n",
	pre_text, replica->offset);
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	status_print("%s </replica>\n", pre_text);
	}
	status_print("%s</bundle>\n", pre_text);
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_TYPE, container_agent_str(bundle_data->agent_type),
	"image", bundle_data->image,
	"unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
	"maintenance",
	pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	PCMK_XA_DESCRIPTION, desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, PCMK_XA_ID, id);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
	replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml), show_opts,
	replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml), show_opts,
	replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
	replica->remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	const pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_bundle_variant_data(bundle_data, rsc);

	CRM_ASSERT(rsc != NULL);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_bundle_replica(pcmk__bundle_replica_t replica, const char pre_text,
	long options, void *print_data)
	{
	pcmk_node_t *node = NULL;
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	node = pe__current_node(replica->container);
	common_print(rsc, pre_text, buffer, node, options, print_data);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (options & pe_print_xml) {
	bundle_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_bundle_variant_data(bundle_data, rsc);

	if (pre_text == NULL) {
	pre_text = " ";
	}

	status_print("%sContainer bundle%s: %s [%s]%s%s\n",
	pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}


	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (options & pe_print_html) {
	status_print("<li>");
	}

	if (pcmk_is_set(options, pe_print_implicit)) {
	child_text = crm_strdup_printf(" %s", pre_text);
	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	status_print(" %sReplica[%d]\n", pre_text, replica->offset);
	}
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	} else {
	child_text = crm_strdup_printf("%s ", pre_text);
	print_bundle_replica(replica, child_text, options, print_data);
	}
	free(child_text);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	}

	static void
	free_bundle_replica(pcmk__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	free_xml(replica->ip->xml);
	replica->ip->xml = NULL;
	replica->ip->fns->free(replica->ip);
	replica->ip = NULL;
	}
	if (replica->container) {
	free_xml(replica->container->xml);
	replica->container->xml = NULL;
	replica->container->fns->free(replica->container);
	replica->container = NULL;
	}
	if (replica->remote) {
	free_xml(replica->remote->xml);
	replica->remote->xml = NULL;
	replica->remote->fns->free(replica->remote);
	replica->remote = NULL;
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->children);

	if(bundle_data->child) {
	free_xml(bundle_data->child->xml);
	bundle_data->child->xml = NULL;
	bundle_data->child->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pcmk_resource_t *rsc)
	{
	if ((rsc == NULL) \|\| (rsc->variant != pcmk_rsc_variant_bundle)) {
	return 0;
	} else {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	}

	void
	pe__count_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pcmk__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pcmk_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of pcmk_rsc_methods_t:active_node()
	pcmk_node_t *
	pe__bundle_active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;
	pcmk_node_t *node = NULL;
	pcmk_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (replica->container->running_on != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->fns->active_node(container, count_all, count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;

	for (GList *node_iter = container->running_on; node_iter != NULL;
	node_iter = node_iter->next) {
	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/common.c b/lib/pengine/common.c
	index e28f02dafb..19f5fe1adf 100644
	--- a/lib/pengine/common.c
	+++ b/lib/pengine/common.c
	@@ -1,624 +1,624 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>

	#include <glib.h>

	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>

	static bool
	check_placement_strategy(const char *value)
	{
	return pcmk__strcase_any_of(value, "default", "utilization", "minimal",
	"balanced", NULL);
	}

	static pcmk__cluster_option_t pe_opts[] = {
	/* name, old name, type, allowed values,
	* default value, validator,
	* short description,
	* long description
	*/
	{
	PCMK_OPT_NO_QUORUM_POLICY, NULL, "select",
	"stop, freeze, ignore, demote, suicide",
	"stop", pcmk__valid_no_quorum_policy,
	N_("What to do when the cluster does not have quorum"),
	NULL
	},
	{
	PCMK_OPT_SYMMETRIC_CLUSTER, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	N_("Whether resources can run on any node by default"),
	NULL
	},
	{
	PCMK_OPT_MAINTENANCE_MODE, NULL, "boolean", NULL,
	- "false", pcmk__valid_boolean,
	+ PCMK_VALUE_FALSE, pcmk__valid_boolean,
	N_("Whether the cluster should refrain from monitoring, starting, "
	"and stopping resources"),
	NULL
	},
	{
	PCMK_OPT_START_FAILURE_IS_FATAL, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	N_("Whether a start failure should prevent a resource from being "
	"recovered on the same node"),
	N_("When true, the cluster will immediately ban a resource from a node "
	"if it fails to start there. When false, the cluster will instead "
	"check the resource's fail count against its migration-threshold.")
	},
	{
	PCMK_OPT_ENABLE_STARTUP_PROBES, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	N_("Whether the cluster should check for active resources during start-up"),
	NULL
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK, NULL, "boolean", NULL,
	- "false", pcmk__valid_boolean,
	+ PCMK_VALUE_FALSE, pcmk__valid_boolean,
	N_("Whether to lock resources to a cleanly shut down node"),
	N_("When true, resources active on a node when it is cleanly shut down "
	"are kept \"locked\" to that node (not allowed to run elsewhere) "
	"until they start again on that node after it rejoins (or for at "
	"most shutdown-lock-limit, if set). Stonith resources and "
	"Pacemaker Remote connections are never locked. Clone and bundle "
	"instances and the promoted role of promotable clones are "
	"currently never locked, though support could be added in a future "
	"release.")
	},
	{
	PCMK_OPT_SHUTDOWN_LOCK_LIMIT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	N_("Do not lock resources to a cleanly shut down node longer than "
	"this"),
	N_("If shutdown-lock is true and this is set to a nonzero time "
	"duration, shutdown locks will expire after this much time has "
	"passed since the shutdown was initiated, even if the node has not "
	"rejoined.")
	},

	// Fencing-related options
	{
	PCMK_OPT_STONITH_ENABLED, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	N_("* Advanced Use Only * "
	"Whether nodes may be fenced as part of recovery"),
	N_("If false, unresponsive nodes are immediately assumed to be harmless, "
	"and resources that were active on them may be recovered "
	"elsewhere. This can result in a \"split-brain\" situation, "
	"potentially leading to data loss and/or service unavailability.")
	},
	{
	PCMK_OPT_STONITH_ACTION, NULL, "select", "reboot, off, poweroff",
	PCMK_ACTION_REBOOT, pcmk__is_fencing_action,
	N_("Action to send to fence device when a node needs to be fenced "
	"(\"poweroff\" is a deprecated alias for \"off\")"),
	NULL
	},
	{
	PCMK_OPT_STONITH_TIMEOUT, NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	N_("How long to wait for on, off, and reboot fence actions to complete "
	"by default"),
	NULL
	},
	{
	PCMK_OPT_HAVE_WATCHDOG, NULL, "boolean", NULL,
	- "false", pcmk__valid_boolean,
	+ PCMK_VALUE_FALSE, pcmk__valid_boolean,
	N_("Whether watchdog integration is enabled"),
	N_("This is set automatically by the cluster according to whether SBD "
	"is detected to be in use. User-configured values are ignored. "
	"The value `true` is meaningful if diskless SBD is used and "
	"`stonith-watchdog-timeout` is nonzero. In that case, if fencing "
	"is required, watchdog-based self-fencing will be performed via "
	"SBD without requiring a fencing resource explicitly configured.")
	},
	{
	PCMK_OPT_CONCURRENT_FENCING, NULL, "boolean", NULL,
	PCMK__CONCURRENT_FENCING_DEFAULT, pcmk__valid_boolean,
	N_("Allow performing fencing operations in parallel"),
	NULL
	},
	{
	PCMK_OPT_STARTUP_FENCING, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	N_("* Advanced Use Only * Whether to fence unseen nodes at start-up"),
	N_("Setting this to false may lead to a \"split-brain\" situation, "
	"potentially leading to data loss and/or service unavailability.")
	},
	{
	PCMK_OPT_PRIORITY_FENCING_DELAY, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	N_("Apply fencing delay targeting the lost nodes with the highest total resource priority"),
	N_("Apply specified delay for the fencings that are targeting the lost "
	"nodes with the highest total resource priority in case we don't "
	"have the majority of the nodes in our cluster partition, so that "
	"the more significant nodes potentially win any fencing match, "
	"which is especially meaningful under split-brain of 2-node "
	"cluster. A promoted resource instance takes the base priority + 1 "
	"on calculation if the base priority is not 0. Any static/random "
	"delays that are introduced by `pcmk_delay_base/max` configured "
	"for the corresponding fencing resources will be added to this "
	"delay. This delay should be significantly greater than, safely "
	"twice, the maximum `pcmk_delay_base/max`. By default, priority "
	"fencing delay is disabled.")
	},
	{
	PCMK_OPT_NODE_PENDING_TIMEOUT, NULL, "time", NULL,
	"0", pcmk__valid_interval_spec,
	N_("How long to wait for a node that has joined the cluster to join "
	"the controller process group"),
	N_("Fence nodes that do not join the controller process group within "
	"this much time after joining the cluster, to allow the cluster "
	"to continue managing resources. A value of 0 means never fence "
	"pending nodes. Setting the value to 2h means fence nodes after "
	"2 hours.")
	},
	{
	PCMK_OPT_CLUSTER_DELAY, NULL, "time", NULL,
	"60s", pcmk__valid_interval_spec,
	N_("Maximum time for node-to-node communication"),
	N_("The node elected Designated Controller (DC) will consider an action "
	"failed if it does not get a response from the node executing the "
	"action within this time (after considering the action's own "
	"timeout). The \"correct\" value will depend on the speed and "
	"load of your network and cluster nodes.")
	},
	{
	PCMK_OPT_BATCH_LIMIT, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	N_("Maximum number of jobs that the cluster may execute in parallel "
	"across all nodes"),
	N_("The \"correct\" value will depend on the speed and load of your "
	"network and cluster nodes. If set to 0, the cluster will "
	"impose a dynamically calculated limit when any node has a "
	"high load.")
	},
	{
	PCMK_OPT_MIGRATION_LIMIT, NULL, "integer", NULL,
	"-1", pcmk__valid_int,
	N_("The number of live migration actions that the cluster is allowed "
	"to execute in parallel on a node (-1 means no limit)")
	},

	/* Orphans and stopping */
	{
	PCMK_OPT_STOP_ALL_RESOURCES, NULL, "boolean", NULL,
	- "false", pcmk__valid_boolean,
	+ PCMK_VALUE_FALSE, pcmk__valid_boolean,
	N_("Whether the cluster should stop all active resources"),
	NULL
	},
	{
	PCMK_OPT_STOP_ORPHAN_RESOURCES, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	N_("Whether to stop resources that were removed from the configuration"),
	NULL
	},
	{
	PCMK_OPT_STOP_ORPHAN_ACTIONS, NULL, "boolean", NULL,
	PCMK_VALUE_TRUE, pcmk__valid_boolean,
	N_("Whether to cancel recurring actions removed from the configuration"),
	NULL
	},
	{
	PCMK__OPT_REMOVE_AFTER_STOP, NULL, "boolean", NULL,
	- "false", pcmk__valid_boolean,
	+ PCMK_VALUE_FALSE, pcmk__valid_boolean,
	N_("* Deprecated * Whether to remove stopped resources from "
	"the executor"),
	N_("Values other than default are poorly tested and potentially dangerous."
	" This option will be removed in a future release.")
	},

	/* Storing inputs */
	{
	PCMK_OPT_PE_ERROR_SERIES_MAX, NULL, "integer", NULL,
	"-1", pcmk__valid_int,
	N_("The number of scheduler inputs resulting in errors to save"),
	N_("Zero to disable, -1 to store unlimited.")
	},
	{
	PCMK_OPT_PE_WARN_SERIES_MAX, NULL, "integer", NULL,
	"5000", pcmk__valid_int,
	N_("The number of scheduler inputs resulting in warnings to save"),
	N_("Zero to disable, -1 to store unlimited.")
	},
	{
	PCMK_OPT_PE_INPUT_SERIES_MAX, NULL, "integer", NULL,
	"4000", pcmk__valid_int,
	N_("The number of scheduler inputs without errors or warnings to save"),
	N_("Zero to disable, -1 to store unlimited.")
	},

	/* Node health */
	{
	PCMK_OPT_NODE_HEALTH_STRATEGY, NULL, "select",
	PCMK__VALUE_NONE ", " PCMK__VALUE_MIGRATE_ON_RED ", "
	PCMK__VALUE_ONLY_GREEN ", " PCMK__VALUE_PROGRESSIVE ", "
	PCMK__VALUE_CUSTOM,
	PCMK__VALUE_NONE, pcmk__validate_health_strategy,
	N_("How cluster should react to node health attributes"),
	N_("Requires external entities to create node attributes (named with "
	"the prefix \"#health\") with values \"red\", "
	"\"yellow\", or \"green\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_BASE, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	N_("Base health score assigned to a node"),
	N_("Only used when \"node-health-strategy\" is set to \"progressive\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_GREEN, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	N_("The score to use for a node health attribute whose value is \"green\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_YELLOW, NULL, "integer", NULL,
	"0", pcmk__valid_int,
	N_("The score to use for a node health attribute whose value is \"yellow\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".")
	},
	{
	PCMK_OPT_NODE_HEALTH_RED, NULL, "integer", NULL,
	"-INFINITY", pcmk__valid_int,
	N_("The score to use for a node health attribute whose value is \"red\""),
	N_("Only used when \"node-health-strategy\" is set to \"custom\" or \"progressive\".")
	},

	/Placement Strategy/
	{
	PCMK_OPT_PLACEMENT_STRATEGY, NULL, "select",
	"default, utilization, minimal, balanced",
	"default", check_placement_strategy,
	N_("How the cluster should allocate resources to nodes"),
	NULL
	},
	};

	void
	pe_metadata(pcmk__output_t *out)
	{
	const char *desc_short = "Pacemaker scheduler options";
	const char *desc_long = "Cluster options used by Pacemaker's scheduler";

	gchar *s = pcmk__format_option_metadata("pacemaker-schedulerd", desc_short,
	desc_long, pe_opts,
	PCMK__NELEM(pe_opts));
	out->output_xml(out, "metadata", s);
	g_free(s);
	}

	void
	verify_pe_options(GHashTable * options)
	{
	pcmk__validate_cluster_options(options, pe_opts, PCMK__NELEM(pe_opts));
	}

	const char *
	pe_pref(GHashTable * options, const char *name)
	{
	return pcmk__cluster_option(options, pe_opts, PCMK__NELEM(pe_opts), name);
	}

	const char *
	fail2text(enum action_fail_response fail)
	{
	const char *result = "<unknown>";

	switch (fail) {
	case pcmk_on_fail_ignore:
	result = "ignore";
	break;
	case pcmk_on_fail_demote:
	result = "demote";
	break;
	case pcmk_on_fail_block:
	result = "block";
	break;
	case pcmk_on_fail_restart:
	result = "recover";
	break;
	case pcmk_on_fail_ban:
	result = "migrate";
	break;
	case pcmk_on_fail_stop:
	result = "stop";
	break;
	case pcmk_on_fail_fence_node:
	result = "fence";
	break;
	case pcmk_on_fail_standby_node:
	result = "standby";
	break;
	case pcmk_on_fail_restart_container:
	result = "restart-container";
	break;
	case pcmk_on_fail_reset_remote:
	result = "reset-remote";
	break;
	}
	return result;
	}

	enum action_tasks
	text2task(const char *task)
	{
	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)) {
	return pcmk_action_stop;

	} else if (pcmk__str_eq(task, PCMK_ACTION_STOPPED, pcmk__str_casei)) {
	return pcmk_action_stopped;

	} else if (pcmk__str_eq(task, PCMK_ACTION_START, pcmk__str_casei)) {
	return pcmk_action_start;

	} else if (pcmk__str_eq(task, PCMK_ACTION_RUNNING, pcmk__str_casei)) {
	return pcmk_action_started;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_casei)) {
	return pcmk_action_shutdown;

	} else if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) {
	return pcmk_action_fence;

	} else if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	return pcmk_action_monitor;

	} else if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
	return pcmk_action_notify;

	} else if (pcmk__str_eq(task, PCMK_ACTION_NOTIFIED, pcmk__str_casei)) {
	return pcmk_action_notified;

	} else if (pcmk__str_eq(task, PCMK_ACTION_PROMOTE, pcmk__str_casei)) {
	return pcmk_action_promote;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DEMOTE, pcmk__str_casei)) {
	return pcmk_action_demote;

	} else if (pcmk__str_eq(task, PCMK_ACTION_PROMOTED, pcmk__str_casei)) {
	return pcmk_action_promoted;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DEMOTED, pcmk__str_casei)) {
	return pcmk_action_demoted;
	}
	return pcmk_action_unspecified;
	}

	const char *
	task2text(enum action_tasks task)
	{
	const char *result = "<unknown>";

	switch (task) {
	case pcmk_action_unspecified:
	result = "no_action";
	break;
	case pcmk_action_stop:
	result = PCMK_ACTION_STOP;
	break;
	case pcmk_action_stopped:
	result = PCMK_ACTION_STOPPED;
	break;
	case pcmk_action_start:
	result = PCMK_ACTION_START;
	break;
	case pcmk_action_started:
	result = PCMK_ACTION_RUNNING;
	break;
	case pcmk_action_shutdown:
	result = PCMK_ACTION_DO_SHUTDOWN;
	break;
	case pcmk_action_fence:
	result = PCMK_ACTION_STONITH;
	break;
	case pcmk_action_monitor:
	result = PCMK_ACTION_MONITOR;
	break;
	case pcmk_action_notify:
	result = PCMK_ACTION_NOTIFY;
	break;
	case pcmk_action_notified:
	result = PCMK_ACTION_NOTIFIED;
	break;
	case pcmk_action_promote:
	result = PCMK_ACTION_PROMOTE;
	break;
	case pcmk_action_promoted:
	result = PCMK_ACTION_PROMOTED;
	break;
	case pcmk_action_demote:
	result = PCMK_ACTION_DEMOTE;
	break;
	case pcmk_action_demoted:
	result = PCMK_ACTION_DEMOTED;
	break;
	}

	return result;
	}

	const char *
	role2text(enum rsc_role_e role)
	{
	switch (role) {
	case pcmk_role_stopped:
	return PCMK__ROLE_STOPPED;

	case pcmk_role_started:
	return PCMK__ROLE_STARTED;

	case pcmk_role_unpromoted:
	#ifdef PCMK__COMPAT_2_0
	return PCMK__ROLE_UNPROMOTED_LEGACY;
	#else
	return PCMK__ROLE_UNPROMOTED;
	#endif

	case pcmk_role_promoted:
	#ifdef PCMK__COMPAT_2_0
	return PCMK__ROLE_PROMOTED_LEGACY;
	#else
	return PCMK__ROLE_PROMOTED;
	#endif

	default: // pcmk_role_unknown
	return PCMK__ROLE_UNKNOWN;
	}
	}

	enum rsc_role_e
	text2role(const char *role)
	{
	if (pcmk__str_eq(role, PCMK__ROLE_UNKNOWN,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	return pcmk_role_unknown;
	} else if (pcmk__str_eq(role, PCMK__ROLE_STOPPED, pcmk__str_casei)) {
	return pcmk_role_stopped;
	} else if (pcmk__str_eq(role, PCMK__ROLE_STARTED, pcmk__str_casei)) {
	return pcmk_role_started;
	} else if (pcmk__strcase_any_of(role, PCMK__ROLE_UNPROMOTED,
	PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
	return pcmk_role_unpromoted;
	} else if (pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
	return pcmk_role_promoted;
	}
	return pcmk_role_unknown; // Invalid role given
	}

	void
	add_hash_param(GHashTable * hash, const char name, const char value)
	{
	CRM_CHECK(hash != NULL, return);

	crm_trace("Adding name='%s' value='%s' to hash table",
	pcmk__s(name, "<null>"), pcmk__s(value, "<null>"));
	if (name == NULL \|\| value == NULL) {
	return;

	} else if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
	return;

	} else if (g_hash_table_lookup(hash, name) == NULL) {
	g_hash_table_insert(hash, strdup(name), strdup(value));
	}
	}

	/*!
	* \internal
	* \brief Look up an attribute value on the appropriate node
	*
	* If \p node is a guest node and either the \c PCMK_META_CONTAINER_ATTR_TARGET
	* meta attribute is set to "host" for \p rsc or \p force_host is \c true, query
	* the attribute on the node's host. Otherwise, query the attribute on \p node
	* itself.
	*
	* \param[in] node Node to query attribute value on by default
	* \param[in] name Name of attribute to query
	* \param[in] rsc Resource on whose behalf we're querying
	* \param[in] node_type Type of resource location lookup
	* \param[in] force_host Force a lookup on the guest node's host, regardless of
	* the \c PCMK_META_CONTAINER_ATTR_TARGET value
	*
	* \return Value of the attribute on \p node or on the host of \p node
	*
	* \note If \p force_host is \c true, \p node \e must be a guest node.
	*/
	const char *
	pe__node_attribute_calculated(const pcmk_node_t node, const char name,
	const pcmk_resource_t *rsc,
	enum pcmk__rsc_node node_type,
	bool force_host)
	{
	// @TODO: Use pe__is_guest_node() after merging libpe_{rules,status}
	bool is_guest = (node != NULL)
	&& (node->details->type == pcmk_node_variant_remote)
	&& (node->details->remote_rsc != NULL)
	&& (node->details->remote_rsc->container != NULL);
	const char *source = NULL;
	const char *node_type_s = NULL;
	const char *reason = NULL;

	const pcmk_resource_t *container = NULL;
	const pcmk_node_t *host = NULL;

	CRM_ASSERT((node != NULL) && (name != NULL) && (rsc != NULL)
	&& (!force_host \|\| is_guest));

	/* Ignore PCMK_META_CONTAINER_ATTR_TARGET if node is not a guest node. This
	* represents a user configuration error.
	*/
	source = g_hash_table_lookup(rsc->meta, PCMK_META_CONTAINER_ATTR_TARGET);
	if (!force_host
	&& (!is_guest \|\| !pcmk__str_eq(source, "host", pcmk__str_casei))) {

	return g_hash_table_lookup(node->details->attrs, name);
	}

	container = node->details->remote_rsc->container;

	switch (node_type) {
	case pcmk__rsc_node_assigned:
	node_type_s = "assigned";
	host = container->allocated_to;
	if (host == NULL) {
	reason = "not assigned";
	}
	break;

	case pcmk__rsc_node_current:
	node_type_s = "current";

	if (container->running_on != NULL) {
	host = container->running_on->data;
	}
	if (host == NULL) {
	reason = "inactive";
	}
	break;

	default:
	// Add support for other enum pcmk__rsc_node values if needed
	CRM_ASSERT(false);
	break;
	}

	if (host != NULL) {
	const char *value = g_hash_table_lookup(host->details->attrs, name);

	pcmk__rsc_trace(rsc,
	"%s: Value lookup for %s on %s container host %s %s%s",
	rsc->id, name, node_type_s, pe__node_name(host),
	((value != NULL)? "succeeded: " : "failed"),
	pcmk__s(value, ""));
	return value;
	}
	pcmk__rsc_trace(rsc,
	"%s: Not looking for %s on %s container host: %s is %s",
	rsc->id, name, node_type_s, container->id, reason);
	return NULL;
	}

	const char *
	pe_node_attribute_raw(const pcmk_node_t node, const char name)
	{
	if(node == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(node->details->attrs, name);
	}
	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index 970cb5df22..9346974f22 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,1468 +1,1468 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/output.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <pe_status_private.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROVIDER_SEP "::"
	#else
	#define PROVIDER_SEP ":"
	#endif

	/*!
	* \internal
	* \brief Check whether a resource is active on multiple nodes
	*/
	static bool
	is_multiply_active(const pcmk_resource_t *rsc)
	{
	unsigned int count = 0;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	pe__find_active_requires(rsc, &count);
	}
	return count > 1;
	}

	static void
	native_priority_to_node(pcmk_resource_t rsc, pcmk_node_t node,
	gboolean failed)
	{
	int priority = 0;

	if ((rsc->priority == 0) \|\| (failed == TRUE)) {
	return;
	}

	if (rsc->role == pcmk_role_promoted) {
	// Promoted instance takes base priority + 1
	priority = rsc->priority + 1;

	} else {
	priority = rsc->priority;
	}

	node->details->priority += priority;
	pcmk__rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s)",
	pe__node_name(node), node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "");

	/* Priority of a resource running on a guest node is added to the cluster
	* node as well. */
	if (node->details->remote_rsc
	&& node->details->remote_rsc->container) {
	GList *gIter = node->details->remote_rsc->container->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *a_node = gIter->data;

	a_node->details->priority += priority;
	pcmk__rsc_trace(rsc,
	"%s now has priority %d with %s'%s' "
	"(priority: %d%s) from guest node %s",
	pe__node_name(a_node), a_node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "",
	pe__node_name(node));
	}
	}
	}

	void
	native_add_running(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler, gboolean failed)
	{
	GList *gIter = rsc->running_on;

	CRM_CHECK(node != NULL, return);
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	CRM_CHECK(a_node != NULL, return);
	if (pcmk__str_eq(a_node->details->id, node->details->id, pcmk__str_casei)) {
	return;
	}
	}

	pcmk__rsc_trace(rsc, "Adding %s to %s %s", rsc->id, pe__node_name(node),
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : "(unmanaged)");

	rsc->running_on = g_list_append(rsc->running_on, node);
	if (rsc->variant == pcmk_rsc_variant_primitive) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);

	native_priority_to_node(rsc, node, failed);
	}

	if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& node->details->maintenance) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_maintenance);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk_resource_t *p = rsc->parent;

	pcmk__rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, INFINITY, "not_managed_default",
	scheduler);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->running_on = g_list_append(p->running_on, node);
	p = p->parent;
	}
	return;
	}

	if (is_multiply_active(rsc)) {
	switch (rsc->recovery_type) {
	case pcmk_multiply_active_stop:
	{
	GHashTableIter gIter;
	pcmk_node_t *local_node = NULL;

	/* make sure it doesn't come up again */
	if (rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->allowed_nodes);
	}
	rsc->allowed_nodes = pe__node_list2table(scheduler->nodes);
	g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	local_node->weight = -INFINITY;
	}
	}
	break;
	case pcmk_multiply_active_block:
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_blocked);

	/* If the resource belongs to a group or bundle configured with
	* PCMK_META_MULTIPLE_ACTIVE=block, block the entire entity.
	*/
	if (rsc->parent
	&& ((rsc->parent->variant == pcmk_rsc_variant_group)
	\|\| (rsc->parent->variant == pcmk_rsc_variant_bundle))
	&& (rsc->parent->recovery_type == pcmk_multiply_active_block)) {
	GList *gIter = rsc->parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *child = gIter->data;

	pcmk__clear_rsc_flags(child, pcmk_rsc_managed);
	pcmk__set_rsc_flags(child, pcmk_rsc_blocked);
	}
	}
	break;

	// pcmk_multiply_active_restart, pcmk_multiply_active_unexpected
	default:
	/* The scheduler will do the right thing because the relevant
	* variables and flags are set when unpacking the history.
	*/
	break;
	}
	crm_debug("%s is active on multiple nodes including %s: %s",
	rsc->id, pe__node_name(node),
	pcmk_multiply_active_text(rsc->recovery_type));

	} else {
	pcmk__rsc_trace(rsc, "Resource %s is active on %s",
	rsc->id, pe__node_name(node));
	}

	if (rsc->parent != NULL) {
	native_add_running(rsc->parent, node, scheduler, FALSE);
	}
	}

	static void
	recursive_clear_unique(pcmk_resource_t *rsc, gpointer user_data)
	{
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	- add_hash_param(rsc->meta, PCMK_META_GLOBALLY_UNIQUE, XML_BOOLEAN_FALSE);
	+ add_hash_param(rsc->meta, PCMK_META_GLOBALLY_UNIQUE, PCMK_VALUE_FALSE);
	g_list_foreach(rsc->children, (GFunc) recursive_clear_unique, NULL);
	}

	gboolean
	native_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *parent = uber_parent(rsc);
	const char *standard = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	uint32_t ra_caps = pcmk_get_ra_caps(standard);

	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	// Only some agent standards support unique and promotable clones
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& pe_rsc_is_clone(parent)) {

	/* @COMPAT We should probably reject this situation as an error (as we
	* do for promotable below) rather than warn and convert, but that would
	* be a backward-incompatible change that we should probably do with a
	* transform at a schema major version bump.
	*/
	pe__force_anon(standard, parent, rsc->id, scheduler);

	/* Clear PCMK_META_GLOBALLY_UNIQUE on the parent and all its descendants
	* unpacked so far (clearing the parent should make any future children
	* unpacking correct). We have to clear this resource explicitly because
	* it isn't hooked into the parent's children yet.
	*/
	recursive_clear_unique(parent, NULL);
	recursive_clear_unique(rsc, NULL);
	}
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
	&& pcmk_is_set(parent->flags, pcmk_rsc_promotable)) {

	pcmk__config_err("Resource %s is of type %s and therefore "
	"cannot be used as a promotable clone resource",
	rsc->id, standard);
	return FALSE;
	}
	return TRUE;
	}

	static bool
	rsc_is_on_node(pcmk_resource_t rsc, const pcmk_node_t node, int flags)
	{
	pcmk__rsc_trace(rsc, "Checking whether %s is on %s",
	rsc->id, pe__node_name(node));

	if (pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->running_on != NULL)) {

	for (GList *iter = rsc->running_on; iter; iter = iter->next) {
	pcmk_node_t loc = (pcmk_node_t ) iter->data;

	if (loc->details == node->details) {
	return true;
	}
	}

	} else if (pcmk_is_set(flags, pe_find_inactive) // @COMPAT deprecated
	&& (rsc->running_on == NULL)) {
	return true;

	} else if (!pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->allocated_to != NULL)
	&& (rsc->allocated_to->details == node->details)) {
	return true;
	}
	return false;
	}

	pcmk_resource_t *
	native_find_rsc(pcmk_resource_t rsc, const char id,
	const pcmk_node_t *on_node, int flags)
	{
	bool match = false;
	pcmk_resource_t *result = NULL;

	CRM_CHECK(id && rsc && rsc->id, return NULL);

	if (pcmk_is_set(flags, pcmk_rsc_match_clone_only)) {
	const char *rid = ID(rsc->xml);

	if (!pe_rsc_is_clone(pe__const_top_resource(rsc, false))) {
	match = false;

	} else if (!strcmp(id, rsc->id) \|\| pcmk__str_eq(id, rid, pcmk__str_none)) {
	match = true;
	}

	} else if (!strcmp(id, rsc->id)) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_history)
	&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_basename)
	\|\| (pcmk_is_set(flags, pcmk_rsc_match_anon_basename)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique))) {
	match = pe_base_name_eq(rsc, id);
	}

	if (match && on_node) {
	if (!rsc_is_on_node(rsc, on_node, flags)) {
	match = false;
	}
	}

	if (match) {
	return rsc;
	}

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	result = rsc->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	// create is ignored
	char *
	native_parameter(pcmk_resource_t rsc, pcmk_node_t node, gboolean create,
	const char name, pcmk_scheduler_t scheduler)
	{
	char *value_copy = NULL;
	const char *value = NULL;
	GHashTable *params = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pcmk__rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
	params = pe_rsc_params(rsc, node, scheduler);
	value = g_hash_table_lookup(params, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->meta, name);
	}
	pcmk__str_update(&value_copy, value);
	return value_copy;
	}

	gboolean
	native_active(pcmk_resource_t * rsc, gboolean all)
	{
	for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	if (a_node->details->unclean) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is unclean",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	} else if (!a_node->details->online
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is offline",
	rsc->id, pe__node_name(a_node));
	} else {
	pcmk__rsc_trace(rsc, "Resource %s active on %s",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	}
	}
	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static const char *
	native_pending_state(const pcmk_resource_t *rsc)
	{
	const char *pending_state = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_START, pcmk__str_casei)) {
	pending_state = "Starting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	pending_state = "Stopping";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_casei)) {
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_casei)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_PROMOTE,
	pcmk__str_casei)) {
	pending_state = "Promoting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_DEMOTE,
	pcmk__str_casei)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_task(const pcmk_resource_t *rsc)
	{
	const char *pending_task = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	pending_task = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	/*
	} else if (pcmk__str_eq(rsc->pending_task, "probe", pcmk__str_casei)) {
	pending_task = "Checking";
	*/
	}

	return pending_task;
	}

	static enum rsc_role_e
	native_displayable_role(const pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	if ((role == pcmk_role_started)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {

	role = pcmk_role_unpromoted;
	}
	return role;
	}

	static const char *
	native_displayable_state(const pcmk_resource_t *rsc, bool print_pending)
	{
	const char *rsc_state = NULL;

	if (print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(native_displayable_role(rsc));
	}
	return rsc_state;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	native_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, pcmk_is_set(options, pe_print_pending));
	const char *target_role = NULL;

	/* resource information. */
	status_print("%s<resource ", pre_text);
	status_print(PCMK_XA_ID "=\"%s\" ", rsc_printable_id(rsc));
	status_print("resource_agent=\"%s%s%s:%s\" ", class,
	((prov == NULL)? "" : PROVIDER_SEP),
	((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, PCMK_XA_TYPE));

	status_print("role=\"%s\" ", rsc_state);
	if (rsc->meta) {
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print("active=\"%s\" ", pcmk__btoa(rsc->fns->active(rsc, TRUE)));
	status_print("orphaned=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_removed));
	status_print("blocked=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_blocked));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure));
	status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	status_print("pending=\"%s\" ", pending_task);
	}
	}

	/* print out the nodes this resource is running on */
	if (options & pe_print_rsconly) {
	status_print("/>\n");
	/* do nothing */
	} else if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	status_print(">\n");
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	status_print("%s <node name=\"%s\" " PCMK_XA_ID "=\"%s\" "
	"cached=\"%s\"/>\n",
	pre_text, pcmk__s(node->details->uname, ""),
	node->details->id, pcmk__btoa(!node->details->online));
	}
	status_print("%s</resource>\n", pre_text);
	} else {
	status_print("/>\n");
	}
	}

	// Append a flag to resource description string's flags list
	static bool
	add_output_flag(GString s, const char flag_desc, bool have_flags)
	{
	g_string_append(s, (have_flags? ", " : " ("));
	g_string_append(s, flag_desc);
	return true;
	}

	// Append a node name to resource description string's node list
	static bool
	add_output_node(GString s, const char node, bool have_nodes)
	{
	g_string_append(s, (have_nodes? " " : " [ "));
	g_string_append(s, node);
	return true;
	}

	/*!
	* \internal
	* \brief Create a string description of a resource
	*
	* \param[in] rsc Resource to describe
	* \param[in] name Desired identifier for the resource
	* \param[in] node If not NULL, node that resource is "on"
	* \param[in] show_opts Bitmask of pcmk_show_opt_e.
	* \param[in] target_role Resource's target role
	* \param[in] show_nodes Whether to display nodes when multiply active
	*
	* \return Newly allocated string description of resource
	* \note Caller must free the result with g_free().
	*/
	gchar *
	pcmk__native_output_string(const pcmk_resource_t rsc, const char name,
	const pcmk_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes)
	{
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *provider = NULL;
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	GString *outstr = NULL;
	bool have_flags = false;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	return NULL;
	}

	CRM_CHECK(name != NULL, name = "unknown");
	CRM_CHECK(kind != NULL, kind = "unknown");
	CRM_CHECK(class != NULL, class = "unknown");

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	}

	if ((node == NULL) && (rsc->lock_node != NULL)) {
	node = rsc->lock_node;
	}
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	node = NULL;
	}

	outstr = g_string_sized_new(128);

	// Resource name and agent
	pcmk__g_strcat(outstr,
	name, "\t(", class, ((provider == NULL)? "" : PROVIDER_SEP),
	pcmk__s(provider, ""), ":", kind, "):\t", NULL);

	// State on node
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	g_string_append(outstr, " ORPHANED");
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	enum rsc_role_e role = native_displayable_role(rsc);

	g_string_append(outstr, " FAILED");
	if (role > pcmk_role_unpromoted) {
	pcmk__add_word(&outstr, 0, role2text(role));
	}
	} else {
	bool show_pending = pcmk_is_set(show_opts, pcmk_show_pending);

	pcmk__add_word(&outstr, 0, native_displayable_state(rsc, show_pending));
	}
	if (node) {
	pcmk__add_word(&outstr, 0, pe__node_name(node));
	}

	// Failed probe operation
	if (native_displayable_role(rsc) == pcmk_role_stopped) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__g_strcat(outstr, " (", services_ocf_exitcode_str(rc), ") ",
	NULL);
	}
	}

	// Flags, as: (<flag> [...])
	if (node && !(node->details->online) && node->details->unclean) {
	have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
	}
	if (node && (node == rsc->lock_node)) {
	have_flags = add_output_flag(outstr, "LOCKED", have_flags);
	}
	if (pcmk_is_set(show_opts, pcmk_show_pending)) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	have_flags = add_output_flag(outstr, pending_task, have_flags);
	}
	}
	if (target_role != NULL) {
	switch (text2role(target_role)) {
	case pcmk_role_unknown:
	pcmk__config_err("Invalid " PCMK_META_TARGET_ROLE
	" %s for resource %s", target_role, rsc->id);
	break;

	case pcmk_role_stopped:
	have_flags = add_output_flag(outstr, "disabled", have_flags);
	break;

	case pcmk_role_unpromoted:
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {
	have_flags = add_output_flag(outstr,
	PCMK_META_TARGET_ROLE ":",
	have_flags);
	g_string_append(outstr, target_role);
	}
	break;

	default:
	/* Only show target role if it limits our abilities (i.e. ignore
	* Started, as it is the default anyways, and doesn't prevent
	* the resource from becoming promoted).
	*/
	break;
	}
	}

	// Blocked or maintenance implies unmanaged
	if (pcmk_any_flags_set(rsc->flags,
	pcmk_rsc_blocked\|pcmk_rsc_maintenance)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	have_flags = add_output_flag(outstr, "blocked", have_flags);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	have_flags = add_output_flag(outstr, "maintenance", have_flags);
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	have_flags = add_output_flag(outstr, "unmanaged", have_flags);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	have_flags = add_output_flag(outstr, "failure ignored", have_flags);
	}


	if (have_flags) {
	g_string_append_c(outstr, ')');
	}

	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	const char *desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);

	if (desc) {
	g_string_append(outstr, " (");
	g_string_append(outstr, desc);
	g_string_append(outstr, ")");

	}
	}

	if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
	&& pcmk__list_of_multiple(rsc->running_on)) {
	bool have_nodes = false;

	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t n = (pcmk_node_t ) iter->data;

	have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
	}
	if (have_nodes) {
	g_string_append(outstr, " ]");
	}
	}

	return g_string_free(outstr, FALSE);
	}

	int
	pe__common_output_html(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	const char *target_role = NULL;

	xmlNodePtr list_node = NULL;
	const char *cl = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	CRM_ASSERT(kind != NULL);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	PCMK__META_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	cl = "rsc-managed";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	cl = "rsc-failed";

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& (rsc->running_on == NULL)) {
	cl = "rsc-failed";

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	cl = "rsc-multiple";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	cl = "rsc-failure-ignored";

	} else {
	cl = "rsc-ok";
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
	pcmk_create_html_node(list_node, "span", NULL, cl, s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	int
	pe__common_output_text(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	PCMK__META_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	out->list_item(out, NULL, "%s", s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	common_print(pcmk_resource_t rsc, const char pre_text, const char *name,
	const pcmk_node_t node, long options, void print_data)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	PCMK__META_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(options, pe_print_implicit)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return;
	}
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	if ((pre_text == NULL) && (options & pe_print_printf)) {
	pre_text = " ";
	}

	if (options & pe_print_html) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	status_print("<font color=\"yellow\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	status_print("<font color=\"red\">");

	} else if (rsc->running_on == NULL) {
	status_print("<font color=\"red\">");

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	status_print("<font color=\"orange\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	status_print("<font color=\"yellow\">");

	} else {
	status_print("<font color=\"green\">");
	}
	}

	{
	gchar *resource_s = pcmk__native_output_string(rsc, name, node, options,
	target_role, false);
	status_print("%s%s", (pre_text? pre_text : ""), resource_s);
	g_free(resource_s);
	}

	if (pcmk_is_set(options, pe_print_html)) {
	status_print(" </font> ");
	}

	if (!pcmk_is_set(options, pe_print_rsconly)
	&& pcmk__list_of_multiple(rsc->running_on)) {

	GList *gIter = rsc->running_on;
	int counter = 0;

	if (options & pe_print_html) {
	status_print("<ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print("[");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t n = (pcmk_node_t ) gIter->data;

	counter++;

	if (options & pe_print_html) {
	status_print("<li>\n%s", pe__node_name(n));

	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" %s", pe__node_name(n));

	} else if ((options & pe_print_log)) {
	status_print("\t%d : %s", counter, pe__node_name(n));

	} else {
	status_print("%s", pe__node_name(n));
	}
	if (options & pe_print_html) {
	status_print("</li>\n");

	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" ]");
	}
	}

	if (options & pe_print_html) {
	status_print("<br/>\n");
	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	native_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const pcmk_node_t *node = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	node = pe__current_node(rsc);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}

	common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, print_pending);

	const char *desc = NULL;
	char ra_name[LINE_MAX];
	char *nodes_running_on = NULL;
	const char *lock_node_name = NULL;
	int rc = pcmk_rc_no_output;
	const char *target_role = NULL;

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->meta != NULL) {
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	/* resource information. */
	snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
	((prov == NULL)? "" : PROVIDER_SEP), ((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, PCMK_XA_TYPE));

	nodes_running_on = pcmk__itoa(g_list_length(rsc->running_on));

	if (rsc->lock_node != NULL) {
	lock_node_name = rsc->lock_node->details->uname;
	}

	rc = pe__name_and_nvpairs_xml(out, true, "resource", 15,
	PCMK_XA_ID, rsc_printable_id(rsc),
	"resource_agent", ra_name,
	"role", rsc_state,
	"target_role", target_role,
	"active", pcmk__btoa(rsc->fns->active(rsc, TRUE)),
	"orphaned", pe__rsc_bool_str(rsc, pcmk_rsc_removed),
	"blocked", pe__rsc_bool_str(rsc, pcmk_rsc_blocked),
	"maintenance", pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"failure_ignored", pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure),
	"nodes_running_on", nodes_running_on,
	"pending", (print_pending? native_pending_task(rsc) : NULL),
	"locked_to", lock_node_name,
	PCMK_XA_DESCRIPTION, desc);
	free(nodes_running_on);

	CRM_ASSERT(rc == pcmk_rc_ok);

	if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	rc = pe__name_and_nvpairs_xml(out, false, "node", 3,
	PCMK_XA_NAME, node->details->uname,
	PCMK_XA_ID, node->details->id,
	"cached", pcmk__btoa(node->details->online));
	CRM_ASSERT(rc == pcmk_rc_ok);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return rc;
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pe__current_node(rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pe__current_node(rsc);

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	void
	native_free(pcmk_resource_t * rsc)
	{
	pcmk__rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const pcmk_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->next_role;

	if (current) {
	role = rsc->role;
	}
	pcmk__rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] current 0 = where allocated, 1 = where running,
	* 2 = where running or pending
	*
	* \return If list contains only one node, that node, or NULL otherwise
	*/
	pcmk_node_t *
	native_location(const pcmk_resource_t rsc, GList *list, int current)
	{
	// @COMPAT: Accept a pcmk__rsc_node argument instead of int current
	pcmk_node_t *one = NULL;
	GList *result = NULL;

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	child->fns->location(child, &result, current);
	}

	} else if (current) {

	if (rsc->running_on) {
	result = g_list_copy(rsc->running_on);
	}
	if ((current == 2) && rsc->pending_node
	&& !pe_find_node_id(result, rsc->pending_node->details->id)) {
	result = g_list_append(result, rsc->pending_node);
	}

	} else if (current == FALSE && rsc->allocated_to) {
	result = g_list_append(NULL, rsc->allocated_to);
	}

	if (result && (result->next == NULL)) {
	one = result->data;
	}

	if (list) {
	GList *gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GList rsc_list, GHashTable rsc_table, GHashTable * active_table)
	{
	GList *gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);

	if (prov != NULL) {
	offset += snprintf(buffer + offset, LINE_MAX - offset,
	PROVIDER_SEP "%s", prov);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = calloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	GList *gIter2 = rsc->running_on;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE &&
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->details->uname);
	if (node_table == NULL) {
	node_table = pcmk__strkey_table(free, free);
	g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = calloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void *print_data, gboolean print_all)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GHashTableIter hash_iter;
	char *type = NULL;
	int *rsc_counter = NULL;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	g_hash_table_iter_init(&hash_iter, rsc_table);
	while (g_hash_table_iter_next(&hash_iter, (gpointer )&type, (gpointer )&rsc_counter)) {
	GHashTableIter hash_iter2;
	char *node_name = NULL;
	GHashTable *node_table = NULL;
	int active_counter_all = 0;

	g_hash_table_iter_init(&hash_iter2, active_table);
	while (g_hash_table_iter_next(&hash_iter2, (gpointer )&node_name, (gpointer )&node_table)) {
	int *active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (options & pe_print_rsconly) {
	node_name = NULL;
	}

	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	if (print_all) {
	status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0,
	rsc_counter ? *rsc_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	} else {
	status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	}

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}

	if (print_all && active_counter_all == 0) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	}

	int
	pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, uint32_t show_opts)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GList *sorted_rscs;
	int rc = pcmk_rc_no_output;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	/* Make a list of the rsc_table keys so that it can be sorted. This is to make sure
	* output order stays consistent between systems.
	*/
	sorted_rscs = g_hash_table_get_keys(rsc_table);
	sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);

	for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
	char type = (char ) gIter->data;
	int *rsc_counter = g_hash_table_lookup(rsc_table, type);

	GList *sorted_nodes = NULL;
	int active_counter_all = 0;

	/* Also make a list of the active_table keys so it can be sorted. If there's
	* more than one instance of a type of resource running, we need the nodes to
	* be sorted to make sure output order stays consistent between systems.
	*/
	sorted_nodes = g_hash_table_get_keys(active_table);
	sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);

	for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
	char node_name = (char ) gIter2->data;
	GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
	int *active_counter = NULL;

	if (node_table == NULL) {
	continue;
	}

	active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
	node_name = NULL;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
	*active_counter,
	rsc_counter ? *rsc_counter : 0, type,
	(*active_counter > 0) && node_name ? node_name : "");
	} else {
	out->list_item(out, NULL, "%d\t(%s):\tActive %s",
	*active_counter, type,
	(*active_counter > 0) && node_name ? node_name : "");
	}

	rc = pcmk_rc_ok;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);
	rc = pcmk_rc_ok;
	}

	if (sorted_nodes) {
	g_list_free(sorted_nodes);
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	if (sorted_rscs) {
	g_list_free(sorted_rscs);
	}

	return rc;
	}

	gboolean
	pe__native_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	return FALSE;
	} else if (check_parent && rsc->parent) {
	const pcmk_resource_t *up = pe__const_top_resource(rsc, true);

	return up->fns->is_filtered(up, only_rsc, FALSE);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get maximum primitive resource instances per node
	*
	* \param[in] rsc Primitive resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__primitive_max_per_node(const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
	return 1U;
	}
	diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
	index 15116b49cc..fb07f3e211 100644
	--- a/lib/pengine/pe_output.c
	+++ b/lib/pengine/pe_output.c
	@@ -1,3182 +1,3182 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/xml_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/cib/util.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/internal.h>

	const char *
	pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
	{
	const char * desc = NULL;
	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)) {
	desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);
	}
	return desc;
	}

	/* Never display node attributes whose name starts with one of these prefixes */
	#define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX, \
	"shutdown", PCMK_NODE_ATTR_TERMINATE, "standby", "#", \
	NULL }

	static int
	compare_attribute(gconstpointer a, gconstpointer b)
	{
	int rc;

	rc = strcmp((const char )a, (const char )b);

	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether extended information about an attribute should be added.
	*
	* \param[in] node Node that ran this resource
	* \param[in,out] rsc_list List of resources for this node
	* \param[in,out] scheduler Scheduler data
	* \param[in] attrname Attribute to find
	* \param[out] expected_score Expected value for this attribute
	*
	* \return true if extended information should be printed, false otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static bool
	add_extra_info(const pcmk_node_t node, GList rsc_list,
	pcmk_scheduler_t scheduler, const char attrname,
	int *expected_score)
	{
	GList *gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->meta, PCMK_XA_TYPE);
	const char *name = NULL;
	GHashTable *params = NULL;

	if (rsc->children != NULL) {
	if (add_extra_info(node, rsc->children, scheduler, attrname,
	expected_score)) {
	return true;
	}
	}

	if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
	continue;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	name = g_hash_table_lookup(params, PCMK_XA_NAME);

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
	int host_list_num = 0;
	const char *hosts = g_hash_table_lookup(params, "host_list");
	const char *multiplier = g_hash_table_lookup(params, "multiplier");
	int multiplier_i;

	if (hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	if ((multiplier == NULL)
	\|\| (pcmk__scan_min_int(multiplier, &multiplier_i,
	INT_MIN) != pcmk_rc_ok)) {
	/* The ocf:pacemaker:ping resource agent defaults multiplier to
	* 1. The agent currently does not handle invalid text, but it
	* should, and this would be a reasonable choice ...
	*/
	multiplier_i = 1;
	}
	expected_score = host_list_num multiplier_i;

	return true;
	}
	}
	return false;
	}

	static GList *
	filter_attr_list(GList attr_list, char name)
	{
	int i;
	const char *filt_str[] = FILTER_STR;

	CRM_CHECK(name != NULL, return attr_list);

	/* filtering automatic attributes */
	for (i = 0; filt_str[i] != NULL; i++) {
	if (g_str_has_prefix(name, filt_str[i])) {
	return attr_list;
	}
	}

	return g_list_insert_sorted(attr_list, name, compare_attribute);
	}

	static GList *
	get_operation_list(xmlNode *rsc_entry) {
	GList *op_list = NULL;
	xmlNode *rsc_op = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
	rsc_op = pcmk__xe_next(rsc_op)) {
	const char *task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(rsc_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)
	\|\| (pcmk__str_eq(task, "probe", pcmk__str_none)
	&& (op_rc_i == CRM_EX_NOT_RUNNING))) {
	continue;
	}

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}

	op_list = g_list_sort(op_list, sort_op_by_callid);
	return op_list;
	}

	static void
	add_dump_node(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr node = user_data;
	pcmk_create_xml_text_node(node, (const char ) key, (const char ) value);
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	char *new_text = crm_strdup_printf("%s %s=%s",
	dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	#define XPATH_STACK "//" XML_CIB_TAG_NVPAIR \
	"[@" PCMK_XA_NAME "='" \
	PCMK_OPT_CLUSTER_INFRASTRUCTURE "']"

	static const char *
	get_cluster_stack(pcmk_scheduler_t *scheduler)
	{
	xmlNode *stack = get_xpath_object(XPATH_STACK, scheduler->input, LOG_DEBUG);
	return (stack != NULL)? crm_element_value(stack, PCMK_XA_VALUE) : "unknown";
	}

	static char *
	last_changed_string(const char last_written, const char user,
	const char client, const char origin) {
	if (last_written != NULL \|\| user != NULL \|\| client != NULL \|\| origin != NULL) {
	return crm_strdup_printf("%s%s%s%s%s%s%s",
	last_written ? last_written : "",
	user ? " by " : "",
	user ? user : "",
	client ? " via " : "",
	client ? client : "",
	origin ? " on " : "",
	origin ? origin : "");
	} else {
	return strdup("");
	}
	}

	static char *
	op_history_string(xmlNode xml_op, const char task, const char *interval_ms_s,
	int rc, bool print_timing) {
	const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	char *interval_str = NULL;
	char *buf = NULL;

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *pair = pcmk__format_nvpair("interval", interval_ms_s, "ms");
	interval_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	if (print_timing) {
	char *last_change_str = NULL;
	char *exec_str = NULL;
	char *queue_str = NULL;

	const char *value = NULL;

	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE, &epoch) == pcmk_ok)
	&& (epoch > 0)) {
	char *epoch_str = pcmk__epoch2str(&epoch, 0);

	last_change_str = crm_strdup_printf(" %s=\"%s\"",
	XML_RSC_OP_LAST_CHANGE,
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
	if (value) {
	char *pair = pcmk__format_nvpair(XML_RSC_OP_T_EXEC, value, "ms");
	exec_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	if (value) {
	char *pair = pcmk__format_nvpair(XML_RSC_OP_T_QUEUE, value, "ms");
	queue_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
	interval_str ? interval_str : "",
	last_change_str ? last_change_str : "",
	exec_str ? exec_str : "",
	queue_str ? queue_str : "",
	rc, services_ocf_exitcode_str(rc));

	if (last_change_str) {
	free(last_change_str);
	}

	if (exec_str) {
	free(exec_str);
	}

	if (queue_str) {
	free(queue_str);
	}
	} else {
	buf = crm_strdup_printf("(%s) %s%s%s", call, task,
	interval_str ? ":" : "",
	interval_str ? interval_str : "");
	}

	if (interval_str) {
	free(interval_str);
	}

	return buf;
	}

	static char *
	resource_history_string(pcmk_resource_t rsc, const char rsc_id, bool all,
	int failcount, time_t last_failure) {
	char *buf = NULL;

	if (rsc == NULL) {
	buf = crm_strdup_printf("%s: orphan", rsc_id);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *failcount_s = NULL;
	char *lastfail_s = NULL;

	if (failcount > 0) {
	failcount_s = crm_strdup_printf(" %s=%d", PCMK__FAIL_COUNT_PREFIX,
	failcount);
	} else {
	failcount_s = strdup("");
	}
	if (last_failure > 0) {
	buf = pcmk__epoch2str(&last_failure, 0);
	lastfail_s = crm_strdup_printf(" %s='%s'",
	PCMK__LAST_FAILURE_PREFIX, buf);
	free(buf);
	}

	buf = crm_strdup_printf("%s: " PCMK_META_MIGRATION_THRESHOLD "=%d%s%s",
	rsc_id, rsc->migration_threshold, failcount_s,
	lastfail_s? lastfail_s : "");
	free(failcount_s);
	free(lastfail_s);
	} else {
	buf = crm_strdup_printf("%s:", rsc_id);
	}

	return buf;
	}

	/*!
	* \internal
	* \brief Get a node's feature set for status display purposes
	*
	* \param[in] node Node to check
	*
	* \return String representation of feature set if the node is fully up (using
	* "<3.15.1" for older nodes that don't set the #feature-set attribute),
	* otherwise NULL
	*/
	static const char *
	get_node_feature_set(const pcmk_node_t *node)
	{
	if (node->details->online && node->details->expected_up
	&& !pe__is_guest_or_remote_node(node)) {

	const char *feature_set = g_hash_table_lookup(node->details->attrs,
	CRM_ATTR_FEATURE_SET);

	/* The feature set attribute is present since 3.15.1. If it is missing,
	* then the node must be running an earlier version.
	*/
	return pcmk__s(feature_set, "<3.15.1");
	}
	return NULL;
	}

	static bool
	is_mixed_version(pcmk_scheduler_t *scheduler)
	{
	const char *feature_set = NULL;
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;
	const char *node_feature_set = get_node_feature_set(node);
	if (node_feature_set != NULL) {
	if (feature_set == NULL) {
	feature_set = node_feature_set;
	} else if (strcmp(feature_set, node_feature_set) != 0) {
	return true;
	}
	}
	}
	return false;
	}

	static char *
	formatted_xml_buf(const pcmk_resource_t *rsc, bool raw)
	{
	if (raw) {
	return dump_xml_formatted(rsc->orig_xml ? rsc->orig_xml : rsc->xml);
	} else {
	return dump_xml_formatted(rsc->xml);
	}
	}

	#define XPATH_DC_VERSION "//" XML_CIB_TAG_NVPAIR \
	"[@" PCMK_XA_NAME "='" PCMK_OPT_DC_VERSION "']"

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	if (pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	/* Always print DC if none, even if not requested */
	if ((scheduler->dc_node == NULL)
	\|\| pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	/* Kind of a hack - close the list we may have opened earlier in this
	* function so we can put all the options into their own list. We
	* only want to do this on HTML output, though.
	*/
	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	out->begin_list(out, NULL, NULL, "Config Options");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	char *
	pe__node_display_name(pcmk_node_t *node, bool print_detail)
	{
	char *node_name;
	const char *node_host = NULL;
	const char *node_id = NULL;
	int name_len;

	CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));

	/* Host is displayed only if this is a guest node and detail is requested */
	if (print_detail && pe__is_guest_node(node)) {
	const pcmk_resource_t *container = node->details->remote_rsc->container;
	const pcmk_node_t *host_node = pe__current_node(container);

	if (host_node && host_node->details) {
	node_host = host_node->details->uname;
	}
	if (node_host == NULL) {
	node_host = ""; /* so we at least get "uname@" to indicate guest */
	}
	}

	/* Node ID is displayed if different from uname and detail is requested */
	if (print_detail && !pcmk__str_eq(node->details->uname, node->details->id, pcmk__str_casei)) {
	node_id = node->details->id;
	}

	/* Determine name length */
	name_len = strlen(node->details->uname) + 1;
	if (node_host) {
	name_len += strlen(node_host) + 1; /* "@node_host" */
	}
	if (node_id) {
	name_len += strlen(node_id) + 3; /* + " (node_id)" */
	}

	/* Allocate and populate display name */
	node_name = malloc(name_len);
	CRM_ASSERT(node_name != NULL);
	strcpy(node_name, node->details->uname);
	if (node_host) {
	strcat(node_name, "@");
	strcat(node_name, node_host);
	}
	if (node_id) {
	strcat(node_name, " (");
	strcat(node_name, node_id);
	strcat(node_name, ")");
	}
	return node_name;
	}

	int
	pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...)
	{
	xmlNodePtr xml_node = NULL;
	va_list args;

	CRM_ASSERT(tag_name != NULL);

	xml_node = pcmk__output_xml_peek_parent(out);
	CRM_ASSERT(xml_node != NULL);
	xml_node = create_xml_node(xml_node, tag_name);

	va_start(args, pairs_count);
	while(pairs_count--) {
	const char param_name = va_arg(args, const char );
	const char param_value = va_arg(args, const char );
	if (param_name && param_value) {
	crm_xml_add(xml_node, param_name, param_value);
	}
	};
	va_end(args);

	if (is_list) {
	pcmk__output_xml_push_parent(out, xml_node);
	}
	return pcmk_rc_ok;
	}

	static const char *
	role_desc(enum rsc_role_e role)
	{
	if (role == pcmk_role_promoted) {
	#ifdef PCMK__COMPAT_2_0
	return "as " PCMK__ROLE_PROMOTED_LEGACY " ";
	#else
	return "in " PCMK__ROLE_PROMOTED " role ";
	#endif
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	pcmk__output_create_html_node(out, "li", NULL, NULL, buf);

	free(node_name);
	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	free(node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
	char *weight_s = pcmk__itoa(pe_node->weight);

	pcmk__output_create_xml_node(out, "ban",
	PCMK_XA_ID, location->id,
	"resource", location->rsc->id,
	"node", pe_node->details->uname,
	"weight", weight_s,
	"promoted-only", promoted_only,
	/* This is a deprecated alias for
	* promoted_only. Removing it will break
	* backward compatibility of the API schema,
	* which will require an API schema major
	* version bump.
	*/
	"master_only", promoted_only,
	NULL);

	free(weight_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t ", "const char ", "GList *",
	"uint32_t", "bool")
	static int
	ban_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	const char prefix = va_arg(args, const char );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	GList gIter, gIter2;
	int rc = pcmk_rc_no_output;

	/* Print each ban */
	for (gIter = scheduler->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pcmk__location_t *location = gIter->data;
	const pcmk_resource_t *rsc = location->rsc;

	if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
	only_rsc, pcmk__str_star_matches)) {
	continue;
	}

	for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;

	if (node->weight < 0) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
	out->message(out, "ban", node, location, show_opts);
	}
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_html(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *nnodes_str = crm_strdup_printf("%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	pcmk_create_html_node(nodes_node, "span", NULL, NULL, nnodes_str);
	free(nnodes_str);

	if (ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");

	s = crm_strdup_printf(", %d ", nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else if (ndisabled && !nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL, ")");
	} else if (!ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else {
	char *s = crm_strdup_printf("%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_text(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	out->list_item(out, NULL, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED, %d BLOCKED from "
	"further action due to failure)",
	nresources, pcmk__plural_s(nresources), ndisabled,
	nblocked);
	} else if (ndisabled && !nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED)",
	nresources, pcmk__plural_s(nresources), ndisabled);
	} else if (!ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d BLOCKED from further action "
	"due to failure)",
	nresources, pcmk__plural_s(nresources), nblocked);
	} else {
	out->list_item(out, NULL, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_xml(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "nodes_configured", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "resources_configured", NULL);

	char *s = pcmk__itoa(nnodes);
	crm_xml_add(nodes_node, "number", s);
	free(s);

	s = pcmk__itoa(nresources);
	crm_xml_add(resources_node, "number", s);
	free(s);

	s = pcmk__itoa(ndisabled);
	crm_xml_add(resources_node, "disabled", s);
	free(s);

	s = pcmk__itoa(nblocked);
	crm_xml_add(resources_node, "blocked", s);
	free(s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Current DC: ");

	if (dc) {
	char *buf = crm_strdup_printf("%s (version %s) -", dc_name,
	dc_version_s ? dc_version_s : "unknown");
	pcmk_create_html_node(node, "span", NULL, NULL, buf);
	free(buf);

	if (mixed_version) {
	pcmk_create_html_node(node, "span", NULL, "warning",
	" MIXED-VERSION");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " partition");
	if (crm_is_true(quorum)) {
	pcmk_create_html_node(node, "span", NULL, NULL, " with");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", " WITHOUT");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " quorum");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	out->list_item(out, "Current DC",
	"%s (version %s) - %spartition %s quorum",
	dc_name, dc_version_s ? dc_version_s : "unknown",
	mixed_version ? "MIXED-VERSION " : "",
	crm_is_true(quorum) ? "with" : "WITHOUT");
	} else {
	out->list_item(out, "Current DC", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name G_GNUC_UNUSED = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	pcmk__output_create_xml_node(out, "current_dc",
	"present", PCMK_VALUE_TRUE,
	PCMK_XA_VERSION, pcmk__s(dc_version_s, ""),
	PCMK_XA_NAME, dc->details->uname,
	PCMK_XA_ID, dc->details->id,
	"with_quorum", pcmk__btoa(crm_is_true(quorum)),
	"mixed_version", pcmk__btoa(mixed_version),
	NULL);
	} else {
	pcmk__output_create_xml_node(out, "current_dc",
	- "present", "false",
	+ "present", PCMK_VALUE_FALSE,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
	static int
	cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
	unsigned long long flags = va_arg(args, unsigned long long);

	if (pcmk_is_set(flags, pcmk_sched_in_maintenance)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will not attempt to start, stop or recover services\n");
	return pcmk_rc_ok;
	} else if (pcmk_is_set(flags, pcmk_sched_stop_all)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will keep all resources stopped\n");
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will not attempt to start, stop, or recover services)");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "STOPPED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will keep all resources stopped)");
	} else {
	out->list_item(out, NULL, "Resource management: enabled");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_log(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	return out->info(out, "Resource management is DISABLED. The cluster will not attempt to start, stop or recover services.");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	return out->info(out, "Resource management is DISABLED. The cluster has stopped all resources.");
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_text(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	return pcmk_rc_ok;
	}

	#define bv(flag) pcmk__btoa(pcmk_is_set(scheduler->flags, (flag)))

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	const char *no_quorum_policy = NULL;
	char *stonith_timeout_str = pcmk__itoa(scheduler->stonith_timeout);
	char priority_fencing_delay_str = pcmk__itoa(scheduler->priority_fencing_delay 1000);

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	no_quorum_policy = "freeze";
	break;

	case pcmk_no_quorum_stop:
	no_quorum_policy = "stop";
	break;

	case pcmk_no_quorum_demote:
	no_quorum_policy = "demote";
	break;

	case pcmk_no_quorum_ignore:
	no_quorum_policy = "ignore";
	break;

	case pcmk_no_quorum_fence:
	no_quorum_policy = "suicide";
	break;
	}

	pcmk__output_create_xml_node(out, "cluster_options",
	PCMK_OPT_STONITH_ENABLED,
	bv(pcmk_sched_fencing_enabled),
	PCMK_OPT_SYMMETRIC_CLUSTER,
	bv(pcmk_sched_symmetric_cluster),
	PCMK_OPT_NO_QUORUM_POLICY, no_quorum_policy,
	PCMK_OPT_MAINTENANCE_MODE,
	bv(pcmk_sched_in_maintenance),
	PCMK_OPT_STOP_ALL_RESOURCES,
	bv(pcmk_sched_stop_all),
	PCMK_OPT_STONITH_TIMEOUT "-ms",
	stonith_timeout_str,
	PCMK_OPT_PRIORITY_FENCING_DELAY "-ms",
	priority_fencing_delay_str,
	NULL);
	free(stonith_timeout_str);
	free(priority_fencing_delay_str);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_html(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Stack: ");
	pcmk_create_html_node(node, "span", NULL, NULL, stack_s);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	pcmk_create_html_node(node, "span", NULL, NULL, " (");
	pcmk_create_html_node(node, "span", NULL, NULL,
	pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	pcmk_create_html_node(node, "span", NULL, NULL, ")");
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_text(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	out->list_item(out, "Stack", "%s (%s)",
	stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	} else {
	out->list_item(out, "Stack", "%s", stack_s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_xml(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	const char *state_s = NULL;

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
	}

	pcmk__output_create_xml_node(out, "stack",
	PCMK_XA_TYPE, stack_s,
	"pacemakerd-state", state_s,
	NULL);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_html(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk_create_html_node(updated_node, "span", NULL, "bold", "Last updated: ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, time_s);

	if (our_nodename != NULL) {
	pcmk_create_html_node(updated_node, "span", NULL, NULL, " on ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, our_nodename);
	}

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	pcmk_create_html_node(changed_node, "span", NULL, "bold", "Last change: ");
	pcmk_create_html_node(changed_node, "span", NULL, NULL, time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_xml(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk__output_create_xml_node(out, "last_update",
	"time", time_s,
	"origin", our_nodename,
	NULL);

	pcmk__output_create_xml_node(out, "last_change",
	"time", last_written ? last_written : "",
	"user", user ? user : "",
	"client", client ? client : "",
	"origin", origin ? origin : "",
	NULL);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_text(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	out->list_item(out, "Last updated", "%s%s%s",
	time_s, (our_nodename != NULL)? " on " : "",
	pcmk__s(our_nodename, ""));

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	out->list_item(out, "Last change", " %s", time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Display a failed action in less-technical natural language
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_friendly(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	char *rsc_id = NULL;
	char *task = NULL;
	guint interval_ms = 0;
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)
	\|\| !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {
	rsc_id = strdup("unknown resource");
	task = strdup("unknown action");
	interval_ms = 0;
	}
	CRM_ASSERT((rsc_id != NULL) && (task != NULL));

	str = g_string_sized_new(256); // Should be sufficient for most messages

	pcmk__g_strcat(str, rsc_id, " ", NULL);

	if (interval_ms != 0) {
	pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
	NULL);
	}
	pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
	node_name, NULL);

	if (status == PCMK_EXEC_DONE) {
	pcmk__g_strcat(str, " returned '", services_ocf_exitcode_str(rc), "'",
	NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
	}

	} else {
	pcmk__g_strcat(str, " could not be executed (",
	pcmk_exec_status_str(status), NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ": ", exit_reason, NULL);
	}
	g_string_append_c(str, ')');
	}


	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *s = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str, " at ", s, NULL);
	free(s);
	}
	if (!pcmk__str_empty(exec_time)) {
	int exec_time_ms = 0;

	if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
	&& (exec_time_ms > 0)) {

	pcmk__g_strcat(str, " after ",
	pcmk__readable_interval(exec_time_ms), NULL);
	}
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	free(rsc_id);
	free(task);
	}

	/*!
	* \internal
	* \brief Display a failed action with technical details
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_technical(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	const char *call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	const char *queue_time = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	const char *exit_status = services_ocf_exitcode_str(rc);
	const char *lrm_status = pcmk_exec_status_str(status);
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)) {
	op_key = "unknown operation";
	}
	if (pcmk__str_empty(exit_status)) {
	exit_status = "unknown exit status";
	}
	if (pcmk__str_empty(call_id)) {
	call_id = "unknown";
	}

	str = g_string_sized_new(256);

	g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
	op_key, node_name, exit_status, rc, call_id,
	lrm_status);

	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str,
	", " XML_RSC_OP_LAST_CHANGE "="
	"'", last_change_str, "'", NULL);
	free(last_change_str);
	}
	if (!pcmk__str_empty(queue_time)) {
	pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
	}
	if (!pcmk__str_empty(exec_time)) {
	pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_default(pcmk__output_t *out, va_list args)
	{
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *op_key = pe__xe_history_key(xml_op);
	const char *node_name = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *exit_reason = crm_element_value(xml_op,
	XML_LRM_ATTR_EXIT_REASON);
	const char *exec_time = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);

	int rc;
	int status;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, 0);

	if (pcmk__str_empty(node_name)) {
	node_name = "unknown node";
	}

	if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
	failed_action_technical(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	} else {
	failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *op_key = pe__xe_history_key(xml_op);
	const char *op_key_name = "op_key";
	int rc;
	int status;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);

	time_t epoch = 0;
	char *rc_s = NULL;
	char *reason_s = crm_xml_escape(exit_reason ? exit_reason : "none");
	xmlNodePtr node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, 0);

	rc_s = pcmk__itoa(rc);
	if (crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY) == NULL) {
	op_key_name = PCMK_XA_ID;
	}
	node = pcmk__output_create_xml_node(out, "failure",
	op_key_name, op_key,
	"node", uname,
	"exitstatus", services_ocf_exitcode_str(rc),
	"exitreason", pcmk__s(reason_s, ""),
	"exitcode", rc_s,
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"status", pcmk_exec_status_str(status),
	NULL);
	free(rc_s);

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	guint interval_ms = 0;
	char *interval_ms_s = NULL;
	char *rc_change = pcmk__epoch2str(&epoch,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);

	pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE, rc_change,
	"queued", crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
	"exec", crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
	"interval", interval_ms_s,
	"task", crm_element_value(xml_op, XML_LRM_ATTR_TASK),
	NULL);

	free(interval_ms_s);
	free(rc_change);
	}

	free(reason_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t ", "GList ",
	"GList *", "uint32_t", "bool")
	static int
	failed_action_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *xml_op = NULL;
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(scheduler->failed) == 0) {
	return rc;
	}

	for (xml_op = pcmk__xml_first_child(scheduler->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	char *rsc = NULL;

	if (!pcmk__str_in_list(crm_element_value(xml_op, PCMK_XA_UNAME),
	only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	if (pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	if (!parse_op_key(pe__xe_history_key(xml_op), &rsc, NULL, NULL)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
	free(rsc);
	continue;
	}

	free(rsc);

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
	out->message(out, "failed-action", xml_op, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
	{
	int health = pe__node_health(node);

	// Cluster membership
	if (node->details->online) {
	pcmk_create_html_node(parent, "span", NULL, "online", " online");
	} else {
	pcmk_create_html_node(parent, "span", NULL, "offline", " OFFLINE");
	}

	// Standby mode
	if (node->details->standby_onfail && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail,"
	" with active resources)");
	} else if (node->details->standby_onfail) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail)");
	} else if (node->details->standby && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby, with active resources)");
	} else if (node->details->standby) {
	pcmk_create_html_node(parent, "span", NULL, "standby", " (in standby)");
	}

	// Maintenance mode
	if (node->details->maintenance) {
	pcmk_create_html_node(parent, "span", NULL, "maint",
	" (in maintenance mode)");
	}

	// Node health
	if (health < 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_red",
	" (health is RED)");
	} else if (health == 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_yellow",
	" (health is YELLOW)");
	}

	// Feature set
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	char *buf = crm_strdup_printf(", feature set %s", feature_set);
	pcmk_create_html_node(parent, "span", NULL, NULL, buf);
	free(buf);
	}
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
	"GList ", "GList ")
	static int
	node_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (full) {
	xmlNodePtr item_node;

	if (pcmk_all_flags_set(show_opts, pcmk_show_brief \| pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "Resources");
	pe__rscs_brief_output(out, rscs, new_show_opts);
	out->end_list(out);
	}

	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc2 = NULL;
	int rc = pcmk_rc_no_output;

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc2->data;
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else {
	char *buf = crm_strdup_printf("%s:", node_name);

	item_node = pcmk__output_create_xml_node(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	status_node(node, item_node, show_opts);

	free(buf);
	}
	} else {
	out->begin_list(out, NULL, NULL, "%s:", node_name);
	}

	free(node_name);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get a human-friendly textual description of a node's status
	*
	* \param[in] node Node to check
	*
	* \return String representation of node's status
	*/
	static const char *
	node_text_status(const pcmk_node_t *node)
	{
	if (node->details->unclean) {
	if (node->details->online) {
	return "UNCLEAN (online)";

	} else if (node->details->pending) {
	return "UNCLEAN (pending)";

	} else {
	return "UNCLEAN (offline)";
	}

	} else if (node->details->pending) {
	return "pending";

	} else if (node->details->standby_onfail && node->details->online) {
	return "standby (on-fail)";

	} else if (node->details->standby) {
	if (node->details->online) {
	if (node->details->running_rsc) {
	return "standby (with active resources)";
	} else {
	return "standby";
	}
	} else {
	return "OFFLINE (standby)";
	}

	} else if (node->details->maintenance) {
	if (node->details->online) {
	return "maintenance";
	} else {
	return "OFFLINE (maintenance)";
	}

	} else if (node->details->online) {
	return "online";
	}

	return "OFFLINE";
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	GString *str = g_string_sized_new(64);
	int health = pe__node_health(node);

	// Create a summary line with node type, name, and status
	if (pe__is_guest_node(node)) {
	g_string_append(str, "GuestNode");
	} else if (pe__is_remote_node(node)) {
	g_string_append(str, "RemoteNode");
	} else {
	g_string_append(str, "Node");
	}
	pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);

	if (health < 0) {
	g_string_append(str, " (health is RED)");
	} else if (health == 0) {
	g_string_append(str, " (health is YELLOW)");
	}
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
	}
	}

	/* If we're grouping by node, print its resources */
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	pe__rscs_brief_output(out, rscs, new_show_opts);

	out->end_list(out);
	out->end_list(out);

	g_list_free(rscs);
	}

	} else {
	GList *gIter2 = NULL;

	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter2->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	out->end_list(out);
	out->end_list(out);
	}
	} else {
	out->list_item(out, NULL, "%s", str->str);
	}

	g_string_free(str, TRUE);
	free(node_name);
	} else {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	out->begin_list(out, NULL, NULL, "Node: %s", node_name);
	free(node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	const char *node_type = "unknown";
	char *length_s = pcmk__itoa(g_list_length(node->details->running_rsc));
	int health = pe__node_health(node);
	const char *health_s = NULL;
	const char *feature_set;

	switch (node->details->type) {
	case pcmk_node_variant_cluster:
	node_type = "member";
	break;
	case pcmk_node_variant_remote:
	node_type = "remote";
	break;
	case node_ping:
	node_type = "ping";
	break;
	}

	if (health < 0) {
	health_s = "red";
	} else if (health == 0) {
	health_s = "yellow";
	} else {
	health_s = "green";
	}

	feature_set = get_node_feature_set(node);

	pe__name_and_nvpairs_xml(out, true, "node", 15,
	PCMK_XA_NAME, node->details->uname,
	PCMK_XA_ID, node->details->id,
	"online", pcmk__btoa(node->details->online),
	"standby", pcmk__btoa(node->details->standby),
	"standby_onfail", pcmk__btoa(node->details->standby_onfail),
	"maintenance", pcmk__btoa(node->details->maintenance),
	"pending", pcmk__btoa(node->details->pending),
	"unclean", pcmk__btoa(node->details->unclean),
	"health", health_s,
	"feature_set", feature_set,
	"shutdown", pcmk__btoa(node->details->shutdown),
	"expected_up", pcmk__btoa(node->details->expected_up),
	"is_dc", pcmk__btoa(node->details->is_dc),
	"resources_running", length_s,
	PCMK_XA_TYPE, node_type);

	if (pe__is_guest_node(node)) {
	xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
	crm_xml_add(xml_node, "id_as_resource", node->details->remote_rsc->container->id);
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc = NULL;

	for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}
	}

	free(length_s);

	out->end_list(out);
	} else {
	pcmk__output_xml_create_parent(out, "node",
	PCMK_XA_NAME, node->details->uname,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_text(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}
	if (v <= 0) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
	} else if (v < expected_score) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_html(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;
	char *s = crm_strdup_printf("%s: %s", name, value);
	xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}

	pcmk_create_html_node(item_node, "span", NULL, NULL, s);
	free(s);

	if (v <= 0) {
	pcmk_create_html_node(item_node, "span", NULL, "bold", "(connectivity is lost)");
	} else if (v < expected_score) {
	char *buf = crm_strdup_printf("(connectivity is degraded -- expected %d", expected_score);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	free(buf);
	}
	} else {
	out->list_item(out, NULL, "%s: %s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	gchar *node_str = NULL;
	char *last_change_str = NULL;

	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, PCMK_EXEC_UNKNOWN);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const pcmk_node_t *node = pe__current_node(rsc);
	const char *target_role = g_hash_table_lookup(rsc->meta,
	PCMK_META_TARGET_ROLE);
	uint32_t show_opts = pcmk_show_rsc_only \| pcmk_show_pending;

	if (node == NULL) {
	node = rsc->pending_node;
	}

	node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
	show_opts, target_role, false);
	} else {
	node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change) == pcmk_ok) {
	last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
	XML_RSC_OP_LAST_CHANGE,
	pcmk__trim(ctime(&last_change)),
	crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
	}

	out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
	node_str, pe__xe_history_key(xml_op),
	crm_element_value(xml_op, PCMK_XA_UNAME),
	crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	crm_element_value(xml_op, XML_LRM_ATTR_RC),
	last_change_str ? last_change_str : "",
	pcmk_exec_status_str(status));

	g_free(node_str);
	free(last_change_str);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;
	xmlNode *node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, PCMK_EXEC_UNKNOWN);
	node = pcmk__output_create_xml_node(out, "operation",
	PCMK_XA_OP, pe__xe_history_key(xml_op),
	"node", uname,
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"rc", crm_element_value(xml_op, XML_LRM_ATTR_RC),
	"status", pcmk_exec_status_str(status),
	NULL);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	bool has_provider = pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_provider);

	char *agent_tuple = crm_strdup_printf("%s:%s:%s",
	class,
	(has_provider? provider : ""),
	kind);

	pcmk__xe_set_props(node, "rsc", rsc_printable_id(rsc),
	"agent", agent_tuple,
	NULL);
	free(agent_tuple);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change) == pcmk_ok) {
	pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE,
	pcmk__trim(ctime(&last_change)),
	XML_RSC_OP_T_EXEC, crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_xml(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "attribute",
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (add_extra) {
	char *buf = pcmk__itoa(expected_score);
	crm_xml_add(node, "expected", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
	"bool", "GList ", "GList ")
	static int
	node_attribute_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	int rc = pcmk_rc_no_output;

	/* Display each node's attributes */
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;

	GList *attr_list = NULL;
	GHashTableIter iter;
	gpointer key;

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	g_hash_table_iter_init(&iter, node->details->attrs);
	while (g_hash_table_iter_next (&iter, &key, NULL)) {
	attr_list = filter_attr_list(attr_list, key);
	}

	if (attr_list == NULL) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node, pcmk__str_star_matches\|pcmk__str_casei)) {
	g_list_free(attr_list);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");

	out->message(out, "node", node, show_opts, false, only_node, only_rsc);

	for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
	const char *name = aIter->data;
	const char *value = NULL;
	int expected_score = 0;
	bool add_extra = false;

	value = pe_node_attribute_raw(node, name);

	add_extra = add_extra_info(node, node->details->running_rsc,
	scheduler, name, &expected_score);

	/* Print attribute name and value */
	out->message(out, "node-attribute", name, value, add_extra,
	expected_score);
	}

	g_list_free(attr_list);
	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s capacity:",
	comment, pe__node_name(node));

	g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "capacity",
	"node", node->details->uname,
	"comment", comment,
	NULL);
	g_hash_table_foreach(node->details->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t ", "pcmk_node_t ",
	"xmlNodePtr", "GList ", "GList ", "uint32_t", "uint32_t")
	static int
	node_history_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	xmlNode node_state = va_arg(args, xmlNode );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;
	int rc = pcmk_rc_no_output;

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	/* Print history of each of the node's resources */
	for (rsc_entry = first_named_child(lrm_rsc, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {
	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	/* We can't use is_filtered here to filter group resources. For is_filtered,
	* we have to decide whether to check the parent or not. If we check the
	* parent, all elements of a group will always be printed because that's how
	* is_filtered works for groups. If we do not check the parent, sometimes
	* this will filter everything out.
	*
	* For other resource types, is_filtered is okay.
	*/
	if (parent->variant == pcmk_rsc_variant_group) {
	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
	pcmk__str_star_matches)) {
	continue;
	}
	} else {
	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	continue;
	}
	}

	if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	if (failcount <= 0) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-history", rsc, rsc_id, false,
	failcount, last_failure, false);
	} else {
	GList *op_list = get_operation_list(rsc_entry);
	pcmk_resource_t *rsc = NULL;

	if (op_list == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources,
	crm_element_value(rsc_entry, PCMK_XA_ID));

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-operation-list", scheduler, rsc, node,
	op_list, show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_html(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_text(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	/* space-separated lists of node names */
	GString *online_nodes = NULL;
	GString *online_remote_nodes = NULL;
	GString *online_guest_nodes = NULL;
	GString *offline_nodes = NULL;
	GString *offline_remote_nodes = NULL;

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	free(node_name);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");

	// Determine whether to display node individually or in a list
	if (node->details->unclean \|\| node->details->pending
	\|\| (node->details->standby_onfail && node->details->online)
	\|\| node->details->standby \|\| node->details->maintenance
	\|\| pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
	\|\| pcmk_is_set(show_opts, pcmk_show_feature_set)
	\|\| (pe__node_health(node) <= 0)) {
	// Display node individually

	} else if (node->details->online) {
	// Display online node in a list
	if (pe__is_guest_node(node)) {
	pcmk__add_word(&online_guest_nodes, 1024, node_name);

	} else if (pe__is_remote_node(node)) {
	pcmk__add_word(&online_remote_nodes, 1024, node_name);

	} else {
	pcmk__add_word(&online_nodes, 1024, node_name);
	}
	free(node_name);
	continue;

	} else {
	// Display offline node in a list
	if (pe__is_remote_node(node)) {
	pcmk__add_word(&offline_remote_nodes, 1024, node_name);

	} else if (pe__is_guest_node(node)) {
	/* ignore offline guest nodes */

	} else {
	pcmk__add_word(&offline_nodes, 1024, node_name);
	}
	free(node_name);
	continue;
	}

	/* If we get here, node is in bad state, or we're grouping by node */
	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	free(node_name);
	}

	/* If we're not grouping by node, summarize nodes by status */
	if (online_nodes != NULL) {
	out->list_item(out, "Online", "[ %s ]",
	(const char *) online_nodes->str);
	g_string_free(online_nodes, TRUE);
	}
	if (offline_nodes != NULL) {
	out->list_item(out, "OFFLINE", "[ %s ]",
	(const char *) offline_nodes->str);
	g_string_free(offline_nodes, TRUE);
	}
	if (online_remote_nodes) {
	out->list_item(out, "RemoteOnline", "[ %s ]",
	(const char *) online_remote_nodes->str);
	g_string_free(online_remote_nodes, TRUE);
	}
	if (offline_remote_nodes) {
	out->list_item(out, "RemoteOFFLINE", "[ %s ]",
	(const char *) offline_remote_nodes->str);
	g_string_free(offline_remote_nodes, TRUE);
	}
	if (online_guest_nodes != NULL) {
	out->list_item(out, "GuestOnline", "[ %s ]",
	(const char *) online_guest_nodes->str);
	g_string_free(online_guest_nodes, TRUE);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_xml(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	out->begin_list(out, NULL, NULL, "nodes");
	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}
	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t ", "GList ", "GList *",
	"uint32_t", "uint32_t", "bool")
	static int
	node_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *node_state = NULL;
	xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_STATUS);
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(cib_status) == 0) {
	return rc;
	}

	for (node_state = first_named_child(cib_status, XML_CIB_TAG_STATE);
	node_state != NULL; node_state = crm_next_same_xml(node_state)) {
	pcmk_node_t *node = pe_find_node_id(scheduler->nodes, ID(node_state));

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
	pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");

	out->message(out, "node-history-list", scheduler, node, node_state,
	only_node, only_rsc, section_opts, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	if (rsc) {
	out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
	prefix, rsc->id, uname, score);
	} else {
	out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "node_weight",
	"function", prefix,
	"node", uname,
	"score", score,
	NULL);

	if (rsc) {
	crm_xml_add(node, PCMK_XA_ID, rsc->id);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_text(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
	pcmk_is_set(show_opts, pcmk_show_timing));

	out->list_item(out, NULL, "%s", buf);

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *rc_s = pcmk__itoa(rc);
	xmlNodePtr node = pcmk__output_create_xml_node(out, "operation_history",
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"task", task,
	"rc", rc_s,
	"rc_text", services_ocf_exitcode_str(rc),
	NULL);
	free(rc_s);

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *s = crm_strdup_printf("%sms", interval_ms_s);
	crm_xml_add(node, "interval", s);
	free(s);
	}

	if (pcmk_is_set(show_opts, pcmk_show_timing)) {
	const char *value = NULL;
	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	char *s = pcmk__epoch2str(&epoch, 0);
	crm_xml_add(node, XML_RSC_OP_LAST_CHANGE, s);
	free(s);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, XML_RSC_OP_T_EXEC, s);
	free(s);
	}
	value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, XML_RSC_OP_T_QUEUE, s);
	free(s);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	out->list_item(out, NULL, "%s promotion score on %s: %s",
	child_rsc->id,
	chosen? chosen->details->uname : "none",
	score);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "promotion_score",
	PCMK_XA_ID, child_rsc->id,
	"score", score,
	NULL);

	if (chosen) {
	crm_xml_add(node, "node", chosen->details->uname);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config_text(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	pcmk__formatted_printf(out, "Resource XML:\n");
	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);

	if (as_header) {
	out->begin_list(out, NULL, NULL, "%s", buf);
	} else {
	out->list_item(out, NULL, "%s", buf);
	}

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource_history",
	PCMK_XA_ID, rsc_id,
	NULL);

	if (rsc == NULL) {
	pcmk__xe_set_bool_attr(node, "orphan", true);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *migration_s = pcmk__itoa(rsc->migration_threshold);

	- pcmk__xe_set_props(node, "orphan", "false",
	+ pcmk__xe_set_props(node, "orphan", PCMK_VALUE_FALSE,
	PCMK_META_MIGRATION_THRESHOLD, migration_s,
	NULL);
	free(migration_s);

	if (failcount > 0) {
	char *s = pcmk__itoa(failcount);

	crm_xml_add(node, PCMK__FAIL_COUNT_PREFIX, s);
	free(s);
	}

	if (last_failure > 0) {
	char *s = pcmk__epoch2str(&last_failure, 0);

	crm_xml_add(node, PCMK__LAST_FAILURE_PREFIX, s);
	free(s);
	}
	}

	if (!as_header) {
	pcmk__output_xml_pop_parent(out);
	}

	return pcmk_rc_ok;
	}

	static void
	print_resource_header(pcmk__output_t *out, uint32_t show_opts)
	{
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	/* Active resources have already been printed by node */
	out->begin_list(out, NULL, NULL, "Inactive Resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->begin_list(out, NULL, NULL, "Full List of Resources");
	} else {
	out->begin_list(out, NULL, NULL, "Active Resources");
	}
	}


	PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
	"GList ", "GList ", "bool")
	static int
	resource_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_summary = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	bool print_spacer = va_arg(args, int);

	GList *rsc_iter;
	int rc = pcmk_rc_no_output;
	bool printed_header = false;

	/* If we already showed active resources by node, and
	* we're not showing inactive resources, we have nothing to do
	*/
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
	!pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	return rc;
	}

	/* If we haven't already printed resources grouped by node,
	* and brief output was requested, print resource summary */
	if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(scheduler->resources, only_rsc);

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;

	rc = pe__rscs_brief_output(out, rscs, show_opts);
	g_list_free(rscs);
	}

	/* For each resource, display it if appropriate */
	for (rsc_iter = scheduler->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rsc_iter->data;
	int x;

	/* Complex resources may have some sub-resources active and some inactive */
	gboolean is_active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	/* Skip inactive orphans (deleted but still in CIB) */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed) && !is_active) {
	continue;

	/* Skip active resources if we already displayed them by node */
	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (is_active) {
	continue;
	}

	/* Skip primitives already counted in a brief summary */
	} else if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& (rsc->variant == pcmk_rsc_variant_primitive)) {
	continue;

	/* Skip resources that aren't at least partially active,
	* unless we're displaying inactive resources
	*/
	} else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	continue;

	} else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
	continue;
	}

	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	/* Print this resource */
	x = out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc,
	only_node, only_rsc);
	if (x == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	if (print_summary && rc != pcmk_rc_ok) {
	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	out->list_item(out, NULL, "No inactive resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "No resources");
	} else {
	out->list_item(out, NULL, "No active resources");
	}
	}

	if (printed_header) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
	"pcmk_resource_t ", "pcmk_node_t ", "GList *", "uint32_t")
	static int
	resource_operation_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
	pcmk_scheduler_t *);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	GList op_list = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);

	GList *gIter = NULL;
	int rc = pcmk_rc_no_output;

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(xml_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* If this is the first printed operation, print heading for resource */
	if (rc == pcmk_rc_no_output) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
	failcount, last_failure, true);
	rc = pcmk_rc_ok;
	}

	/* Print the operation */
	out->message(out, "op-history", xml_op, task, interval_ms_s,
	op_rc_i, show_opts);
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
	fn, rsc->id, pe__node_name(node));

	g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "utilization",
	"resource", rsc->id,
	"node", node->details->uname,
	"function", fn,
	NULL);
	g_hash_table_foreach(rsc->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_html(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, NULL, "%s:\t%s%s %s=\"%s\"", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, NULL, "%s:\t%s%s", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_text(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, ticket->id, "%s%s %s=\"%s\"",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, ticket->id, "%s%s",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_xml(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, "ticket",
	PCMK_XA_ID, ticket->id,
	"status", ticket->granted ? "granted" : "revoked",
	"standby", pcmk__btoa(ticket->standby),
	NULL);

	if (ticket->last_granted > -1) {
	char *buf = pcmk__epoch2str(&ticket->last_granted, 0);

	crm_xml_add(node, "last-granted", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-list", "pcmk_scheduler_t *", "bool")
	static int
	ticket_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	bool print_spacer = va_arg(args, int);

	GHashTableIter iter;
	gpointer key, value;

	if (g_hash_table_size(scheduler->tickets) == 0) {
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);

	/* Print section heading */
	out->begin_list(out, NULL, NULL, "Tickets");

	/* Print each ticket */
	g_hash_table_iter_init(&iter, scheduler->tickets);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	pcmk_ticket_t ticket = (pcmk_ticket_t ) value;
	out->message(out, "ticket", ticket);
	}

	/* Close section */
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "ban", "default", ban_text },
	{ "ban", "html", ban_html },
	{ "ban", "xml", ban_xml },
	{ "ban-list", "default", ban_list },
	{ "bundle", "default", pe__bundle_text },
	{ "bundle", "xml", pe__bundle_xml },
	{ "bundle", "html", pe__bundle_html },
	{ "clone", "default", pe__clone_default },
	{ "clone", "xml", pe__clone_xml },
	{ "cluster-counts", "default", cluster_counts_text },
	{ "cluster-counts", "html", cluster_counts_html },
	{ "cluster-counts", "xml", cluster_counts_xml },
	{ "cluster-dc", "default", cluster_dc_text },
	{ "cluster-dc", "html", cluster_dc_html },
	{ "cluster-dc", "xml", cluster_dc_xml },
	{ "cluster-options", "default", cluster_options_text },
	{ "cluster-options", "html", cluster_options_html },
	{ "cluster-options", "log", cluster_options_log },
	{ "cluster-options", "xml", cluster_options_xml },
	{ "cluster-summary", "default", cluster_summary },
	{ "cluster-summary", "html", cluster_summary_html },
	{ "cluster-stack", "default", cluster_stack_text },
	{ "cluster-stack", "html", cluster_stack_html },
	{ "cluster-stack", "xml", cluster_stack_xml },
	{ "cluster-times", "default", cluster_times_text },
	{ "cluster-times", "html", cluster_times_html },
	{ "cluster-times", "xml", cluster_times_xml },
	{ "failed-action", "default", failed_action_default },
	{ "failed-action", "xml", failed_action_xml },
	{ "failed-action-list", "default", failed_action_list },
	{ "group", "default", pe__group_default},
	{ "group", "xml", pe__group_xml },
	{ "maint-mode", "text", cluster_maint_mode_text },
	{ "node", "default", node_text },
	{ "node", "html", node_html },
	{ "node", "xml", node_xml },
	{ "node-and-op", "default", node_and_op },
	{ "node-and-op", "xml", node_and_op_xml },
	{ "node-capacity", "default", node_capacity },
	{ "node-capacity", "xml", node_capacity_xml },
	{ "node-history-list", "default", node_history_list },
	{ "node-list", "default", node_list_text },
	{ "node-list", "html", node_list_html },
	{ "node-list", "xml", node_list_xml },
	{ "node-weight", "default", node_weight },
	{ "node-weight", "xml", node_weight_xml },
	{ "node-attribute", "default", node_attribute_text },
	{ "node-attribute", "html", node_attribute_html },
	{ "node-attribute", "xml", node_attribute_xml },
	{ "node-attribute-list", "default", node_attribute_list },
	{ "node-summary", "default", node_summary },
	{ "op-history", "default", op_history_text },
	{ "op-history", "xml", op_history_xml },
	{ "primitive", "default", pe__resource_text },
	{ "primitive", "xml", pe__resource_xml },
	{ "primitive", "html", pe__resource_html },
	{ "promotion-score", "default", promotion_score },
	{ "promotion-score", "xml", promotion_score_xml },
	{ "resource-config", "default", resource_config },
	{ "resource-config", "text", resource_config_text },
	{ "resource-history", "default", resource_history_text },
	{ "resource-history", "xml", resource_history_xml },
	{ "resource-list", "default", resource_list },
	{ "resource-operation-list", "default", resource_operation_list },
	{ "resource-util", "default", resource_util },
	{ "resource-util", "xml", resource_util_xml },
	{ "ticket", "default", ticket_text },
	{ "ticket", "html", ticket_html },
	{ "ticket", "xml", ticket_xml },
	{ "ticket-list", "default", ticket_list },

	{ NULL, NULL, NULL }
	};

	void
	pe__register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index 6857d41447..57a9d4fa77 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,5129 +1,5129 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	// A (parsed) resource action history entry
	struct action_history {
	pcmk_resource_t *rsc; // Resource that history is for
	pcmk_node_t *node; // Node that history is for
	xmlNode *xml; // History entry XML

	// Parsed from entry XML
	const char *id; // XML ID of history entry
	const char *key; // Operation key of action
	const char *task; // Action name
	const char *exit_reason; // Exit reason given for result
	guint interval_ms; // Action interval
	int call_id; // Call ID of action
	int expected_exit_status; // Expected exit status of action
	int exit_status; // Actual exit status of action
	int execution_status; // Execution status of action
	};

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	* use pcmk__set_scheduler_flags()/pcmk__clear_scheduler_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(scheduler, option, flag) do { \
	const char *scf_value = pe_pref((scheduler)->config_hash, (option)); \
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} else { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node,
	xmlNode xml_op, xmlNode *last_failure,
	enum action_fail_response *failed);
	static void determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node);
	static void add_node_attrs(const xmlNode xml_obj, pcmk_node_t node,
	bool overwrite, pcmk_scheduler_t *scheduler);
	static void determine_online_status(const xmlNode *node_state,
	pcmk_node_t *this_node,
	pcmk_scheduler_t *scheduler);

	static void unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler);


	static gboolean
	is_dangling_guest_node(pcmk_node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (pe__is_guest_or_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_removed_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider
	* \c PCMK_OPT_PRIORITY_FENCING_DELAY
	*/
	void
	pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	/* A guest node is fenced by marking its container as failed */
	if (pe__is_guest_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc->container;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	pe__node_name(node), reason, rsc->id);
	} else {
	pcmk__sched_warn("Guest node %s will be fenced "
	"(by recovering its guest resource %s): %s",
	pe__node_name(node), rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	node->details->remote_requires_reset = TRUE;
	pcmk__set_rsc_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	pe__node_name(node), reason);
	pcmk__set_rsc_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);

	} else if (pe__is_remote_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	pe__node_name(node), reason);
	} else if(node->details->remote_requires_reset == FALSE) {
	node->details->remote_requires_reset = TRUE;
	pcmk__sched_warn("Remote node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	}
	node->details->unclean = TRUE;
	// No need to apply PCMK_OPT_PRIORITY_FENCING_DELAY for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, scheduler);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	pcmk__sched_warn("Cluster node %s %s: %s",
	pe__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, scheduler);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
	"[(@" PCMK_XA_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" PCMK_XA_NAME "='" PCMK_META_REQUIRES "') " \
	"and @" PCMK_XA_VALUE "='" PCMK__VALUE_UNFENCING "']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES \
	"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
	"\|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG \
	"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pcmk_scheduler_t scheduler)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(scheduler->flags, flag)) {
	result = xpath_search(scheduler->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	pcmk__set_scheduler_flags(scheduler, flag);
	}
	freeXpathObject(result);
	}
	}

	gboolean
	unpack_config(xmlNode config, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	guint interval_ms = 0U;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	scheduler->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
	CIB_OPTIONS_FIRST, FALSE, scheduler);

	verify_pe_options(scheduler->config_hash);

	set_config_flag(scheduler, PCMK_OPT_ENABLE_STARTUP_PROBES,
	pcmk_sched_probe_resources);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and " PCMK_OPT_STONITH_WATCHDOG_TIMEOUT
	" is nonzero");
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_fencing);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING,
	scheduler);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_STONITH_TIMEOUT);
	pcmk_parse_interval_spec(value, &interval_ms);

	if (interval_ms >= INT_MAX) {
	scheduler->stonith_timeout = INT_MAX;
	} else {
	scheduler->stonith_timeout = (int) interval_ms;
	}
	crm_debug("STONITH timeout: %d", scheduler->stonith_timeout);

	set_config_flag(scheduler, PCMK_OPT_STONITH_ENABLED,
	pcmk_sched_fencing_enabled);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	crm_debug("STONITH of failed nodes is enabled");
	} else {
	crm_debug("STONITH of failed nodes is disabled");
	}

	scheduler->stonith_action = pe_pref(scheduler->config_hash,
	PCMK_OPT_STONITH_ACTION);
	if (!strcmp(scheduler->stonith_action, "poweroff")) {
	pcmk__warn_once(pcmk__wo_poweroff,
	"Support for " PCMK_OPT_STONITH_ACTION " of "
	"'poweroff' is deprecated and will be removed in a "
	"future release (use 'off' instead)");
	scheduler->stonith_action = PCMK_ACTION_OFF;
	}
	crm_trace("STONITH will %s nodes", scheduler->stonith_action);

	set_config_flag(scheduler, PCMK_OPT_CONCURRENT_FENCING,
	pcmk_sched_concurrent_fencing);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	crm_debug("Concurrent fencing is enabled");
	} else {
	crm_debug("Concurrent fencing is disabled");
	}

	value = pe_pref(scheduler->config_hash,
	PCMK_OPT_PRIORITY_FENCING_DELAY);
	if (value) {
	pcmk_parse_interval_spec(value, &interval_ms);
	scheduler->priority_fencing_delay = (int) (interval_ms / 1000);
	crm_trace("Priority fencing delay is %ds",
	scheduler->priority_fencing_delay);
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ALL_RESOURCES,
	pcmk_sched_stop_all);
	crm_debug("Stop all active resources: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)));

	set_config_flag(scheduler, PCMK_OPT_SYMMETRIC_CLUSTER,
	pcmk_sched_symmetric_cluster);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_NO_QUORUM_POLICY);

	if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_ignore;

	} else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_freeze;

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_demote;

	} else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	int do_panic = 0;

	crm_element_value_int(scheduler->input, PCMK_XA_NO_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_fence;
	} else {
	crm_notice("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop': cluster has never had quorum");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop' because fencing is disabled");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	} else {
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case pcmk_no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case pcmk_no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case pcmk_no_quorum_fence:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case pcmk_no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_RESOURCES,
	pcmk_sched_stop_removed_resources);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	crm_trace("Orphan resources are stopped");
	} else {
	crm_trace("Orphan resources are ignored");
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_ACTIONS,
	pcmk_sched_cancel_removed_actions);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_cancel_removed_actions)) {
	crm_trace("Orphan resource actions are stopped");
	} else {
	crm_trace("Orphan resource actions are ignored");
	}

	value = pe_pref(scheduler->config_hash, PCMK__OPT_REMOVE_AFTER_STOP);
	if (value != NULL) {
	if (crm_is_true(value)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_remove_after_stop);
	#ifndef PCMK__COMPAT_2_0
	pcmk__warn_once(pcmk__wo_remove_after,
	"Support for the " PCMK__OPT_REMOVE_AFTER_STOP
	" cluster property is deprecated and will be "
	"removed in a future release");
	#endif
	} else {
	pcmk__clear_scheduler_flags(scheduler,
	pcmk_sched_remove_after_stop);
	}
	}

	set_config_flag(scheduler, PCMK_OPT_MAINTENANCE_MODE,
	pcmk_sched_in_maintenance);
	crm_trace("Maintenance mode: %s",
	pcmk__btoa(pcmk_is_set(scheduler->flags,
	pcmk_sched_in_maintenance)));

	set_config_flag(scheduler, PCMK_OPT_START_FAILURE_IS_FATAL,
	pcmk_sched_start_failure_fatal);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_trace("Start failures are always fatal");
	} else {
	crm_trace("Start failures are handled by failcount");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	set_config_flag(scheduler, PCMK_OPT_STARTUP_FENCING,
	pcmk_sched_startup_fencing);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pcmk__warn_once(pcmk__wo_blind,
	"Blind faith: not fencing unseen nodes");
	}

	pe__unpack_node_health_scores(scheduler);

	scheduler->placement_strategy = pe_pref(scheduler->config_hash,
	PCMK_OPT_PLACEMENT_STRATEGY);
	crm_trace("Placement strategy: %s", scheduler->placement_strategy);

	set_config_flag(scheduler, PCMK_OPT_SHUTDOWN_LOCK,
	pcmk_sched_shutdown_lock);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	value = pe_pref(scheduler->config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
	pcmk_parse_interval_spec(value, &(scheduler->shutdown_lock));
	scheduler->shutdown_lock /= 1000;
	crm_trace("Resources will be locked to nodes that were cleanly "
	"shut down (locks expire after %s)",
	pcmk__readable_interval(scheduler->shutdown_lock));
	} else {
	crm_trace("Resources will not be locked to nodes that were cleanly "
	"shut down");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
	pcmk_parse_interval_spec(value, &(scheduler->node_pending_timeout));
	scheduler->node_pending_timeout /= 1000;
	if (scheduler->node_pending_timeout == 0) {
	crm_trace("Do not fence pending nodes");
	} else {
	crm_trace("Fence pending nodes after %s",
	pcmk__readable_interval(scheduler->node_pending_timeout
	* 1000));
	}

	return TRUE;
	}

	pcmk_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pcmk_scheduler_t scheduler)
	{
	pcmk_node_t *new_node = NULL;

	if (pe_find_node(scheduler->nodes, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pcmk_node_t));
	if (new_node == NULL) {
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->data_set = scheduler;

	if (pcmk__str_eq(type, "member", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_cluster;

	} else if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_remote;
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_remote_nodes);

	} else {
	/* @COMPAT 'ping' is the default for backward compatibility, but it
	* should be changed to 'member' at a compatibility break
	*/
	if (!pcmk__str_eq(type, "ping", pcmk__str_casei)) {
	pcmk__config_warn("Node %s has unrecognized type '%s', "
	"assuming 'ping'", pcmk__s(uname, "without name"),
	type);
	}
	pcmk__warn_once(pcmk__wo_ping_node,
	"Support for nodes of type 'ping' (such as %s) is "
	"deprecated and will be removed in a future release",
	pcmk__s(uname, "unnamed node"));
	new_node->details->type = node_ping;
	}

	new_node->details->attrs = pcmk__strkey_table(free, free);

	if (pe__is_guest_or_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("cluster"));
	}

	new_node->details->utilization = pcmk__strkey_table(free, free);
	new_node->details->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	scheduler->nodes = g_list_insert_sorted(scheduler->nodes, new_node,
	pe__cmp_node_name);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pcmk_scheduler_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
	attr = pcmk__xe_next(attr)) {
	const char *value = crm_element_value(attr, PCMK_XA_VALUE);
	const char *name = crm_element_value(attr, PCMK_XA_NAME);

	if (name == NULL) { // Sanity
	continue;
	}

	if (strcmp(name, PCMK_META_REMOTE_NODE) == 0) {
	remote_name = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ADDR) == 0) {
	remote_server = value;

	} else if (strcmp(name, PCMK_META_REMOTE_PORT) == 0) {
	remote_port = value;

	} else if (strcmp(name, PCMK_META_REMOTE_CONNECT_TIMEOUT) == 0) {
	connect_timeout = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ALLOW_MIGRATE) == 0) {
	remote_allow_migrate = value;

	} else if (strcmp(name, PCMK_META_IS_MANAGED) == 0) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pcmk_scheduler_t scheduler, pcmk_node_t new_node)
	{
	if ((new_node->details->type == pcmk_node_variant_remote)
	&& (new_node->details->remote_rsc == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode xml_nodes, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	pcmk_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	uname = crm_element_value(xml_obj, PCMK_XA_UNAME);
	type = crm_element_value(xml_obj, PCMK_XA_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, scheduler);

	if (new_node == NULL) {
	return FALSE;
	}

	handle_startup_fencing(scheduler, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, scheduler);

	crm_trace("Done with node %s",
	crm_element_value(xml_obj, PCMK_XA_UNAME));
	}
	}

	if (scheduler->localhost
	&& (pe_find_node(scheduler->nodes, scheduler->localhost) == NULL)) {
	crm_info("Creating a fake local node");
	pe_create_node(scheduler->localhost, scheduler->localhost, NULL, 0,
	scheduler);
	}

	return TRUE;
	}

	static void
	setup_container(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) setup_container, scheduler);
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, PCMK__META_CONTAINER);
	if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
	pcmk_resource_t *container = pe_find_resource(scheduler->resources,
	container_id);

	if (container) {
	rsc->container = container;
	pcmk__set_rsc_flags(container, pcmk_rsc_has_filler);
	container->fillers = g_list_append(container->fillers, rsc);
	pcmk__rsc_trace(rsc, "Resource %s's container is %s",
	rsc->id, container_id);
	} else {
	pcmk__config_err("Resource %s: Unknown resource container (%s)",
	rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources,
	scheduler);
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
	xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources,
	scheduler);

	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, ID(xml_obj2), ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pcmk_scheduler_t scheduler, pcmk_resource_t new_rsc)
	{
	pcmk_node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pe_find_node(scheduler->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pcmk__rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
	new_rsc->id, pe__node_name(remote_node));
	remote_node->details->remote_rsc = new_rsc;

	if (new_rsc->container == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(scheduler, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pcmk_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] scheduler Scheduler data
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when pe__unpack_resource() calls resource_location()
	*/
	gboolean
	unpack_resources(const xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	scheduler->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	pcmk_resource_t *new_rsc = NULL;
	const char *id = ID(xml_obj);

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring <%s> resource without ID",
	xml_obj->name);
	continue;
	}

	if (pcmk__str_eq((const char *) xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE,
	pcmk__str_none)) {
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(scheduler->template_rsc_sets, strdup(id),
	NULL);
	}
	continue;
	}

	crm_trace("Unpacking <%s " PCMK_XA_ID "='%s'>", xml_obj->name, id);
	if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
	scheduler) == pcmk_rc_ok) {
	scheduler->resources = g_list_append(scheduler->resources, new_rsc);
	pcmk__rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	xml_obj->name, id);
	}
	}

	for (gIter = scheduler->resources; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	setup_container(rsc, scheduler);
	link_rsc2remotenode(scheduler, rsc);
	}

	scheduler->resources = g_list_sort(scheduler->resources,
	pe__cmp_rsc_priority);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* Ignore */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the "
	PCMK_OPT_STONITH_ENABLED " option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode xml_tags, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_tag = NULL;

	scheduler->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
	xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " PCMK_XA_ID,
	(const char *) xml_tag->name);
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
	xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = ID(xml_obj_ref);

	if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " PCMK_XA_ID,
	xml_obj_ref->name, tag_id);
	continue;
	}

	if (add_tag_ref(scheduler->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode xml_ticket, pcmk_scheduler_t scheduler)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pcmk_ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, scheduler);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = pcmk__xml_attr_value(xIter);

	if (pcmk__str_eq(prop_name, PCMK_XA_ID, pcmk__str_none)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode xml_tickets, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
	continue;
	}
	unpack_ticket_state(xml_obj, scheduler);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pcmk_node_t this_node, const xmlNode state,
	pcmk_scheduler_t *scheduler)
	{
	const char *resource_discovery_enabled = NULL;
	const xmlNode *attrs = NULL;
	pcmk_resource_t *rsc = NULL;

	if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	return;
	}

	if ((this_node == NULL) \|\| !pe__is_guest_or_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing Pacemaker Remote node %s", pe__node_name(this_node));

	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
	&(this_node->details->remote_maintenance), 0);

	rsc = this_node->details->remote_rsc;
	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, scheduler);

	if (pe__shutdown_requested(this_node)) {
	crm_info("%s is shutting down", pe__node_name(this_node));
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) \|\|
	((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed))) {
	crm_info("%s is in maintenance mode", pe__node_name(this_node));
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (pe__is_remote_node(this_node)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	" attribute on Pacemaker Remote node %s"
	" because fencing is disabled",
	pe__node_name(this_node));
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("%s has resource discovery disabled",
	pe__node_name(this_node));
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in,out] node Cluster node whose attributes are being unpacked
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_transient_attributes(const xmlNode state, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS,
	FALSE);

	add_node_attrs(attrs, node, TRUE, scheduler);

	if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
	crm_info("%s is in standby mode", pe__node_name(node));
	node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
	crm_info("%s is in maintenance mode", pe__node_name(node));
	node->details->maintenance = TRUE;
	}

	discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY " attribute "
	"for %s because disabling resource discovery is "
	"not allowed for cluster nodes", pe__node_name(node));
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	* node_state element itself and node attributes inside it, but not the
	* resource history inside it. Multiple passes through the status are needed to
	* fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_state(const xmlNode state, pcmk_scheduler_t scheduler)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pcmk_node_t *this_node = NULL;

	id = crm_element_value(state, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring invalid " XML_CIB_TAG_STATE " entry without "
	PCMK_XA_ID);
	crm_log_xml_info(state, "missing-id");
	return;
	}

	uname = crm_element_value(state, PCMK_XA_UNAME);
	if (uname == NULL) {
	/* If a joining peer makes the cluster acquire the quorum from corosync
	* meanwhile it has not joined CPG membership of pacemaker-controld yet,
	* it's possible that the created node_state entry doesn't have an uname
	* yet. We should recognize the node as `pending` and wait for it to
	* join CPG.
	*/
	crm_trace("Handling " XML_CIB_TAG_STATE " entry with id=\"%s\" without "
	PCMK_XA_UNAME, id);
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	pcmk__config_warn("Ignoring recorded node state for id=\"%s\" (%s) "
	"because it is no longer in the configuration",
	id, pcmk__s(uname, "uname unknown"));
	return;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
	&(this_node->details->remote_was_fenced), 0);
	return;
	}

	unpack_transient_attributes(state, this_node, scheduler);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;

	crm_trace("Determining online status of cluster node %s (id %s)",
	pe__node_name(this_node), id);
	determine_online_status(state, this_node, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& this_node->details->online
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_fence)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(scheduler, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(const xmlNode *status, bool fence,
	pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;

	// Loop through all node_state entries in CIB status
	for (const xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
	state != NULL; state = crm_next_same_xml(state)) {

	const char *id = ID(state);
	const char *uname = crm_element_value(state, PCMK_XA_UNAME);
	pcmk_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	XML_CIB_TAG_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (this_node->details->unpacked) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pe__is_guest_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL) \|\| (rsc->role != pcmk_role_started)
	\|\| (rsc->container->role != pcmk_role_started)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because container and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pe__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)
	&& (rsc->role != pcmk_role_started))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_fencing_enabled
	\|pcmk_sched_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);
	unpack_handle_remote_attrs(this_node, state, scheduler);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	this_node->details->unpacked = TRUE;
	unpack_node_lrm(this_node, state, scheduler);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode status, pcmk_scheduler_t scheduler)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	unpack_tickets_state((xmlNode *) state, scheduler);

	} else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	unpack_node_state(state, scheduler);
	}
	}

	while (unpack_node_history(status, FALSE, scheduler) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_fencing_enabled),
	scheduler);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (scheduler->stop_needed != NULL) {
	for (GList *item = scheduler->stop_needed; item; item = item->next) {
	pcmk_resource_t *container = item->data;
	pcmk_node_t *node = pe__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(scheduler->stop_needed);
	scheduler->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *this_node = gIter->data;

	if (!pe__is_guest_or_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->details->remote_rsc != NULL)) {
	pe__set_next_role(this_node->details->remote_rsc, pcmk_role_stopped,
	"remote shutdown");
	}
	if (!this_node->details->unpacked) {
	determine_remote_online_status(scheduler, this_node);
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became a member at the cluster layer
	*
	* \param[in] node_state Node's node_state entry
	* \param[in,out] scheduler Scheduler data
	*
	* \return Epoch time when node became a cluster member
	* (or scheduler effective time for legacy entries) if a member,
	* 0 if not a member, or -1 if no valid information available
	*/
	static long long
	unpack_node_member(const xmlNode node_state, pcmk_scheduler_t scheduler)
	{
	const char *member_time = crm_element_value(node_state, PCMK__XA_IN_CCM);
	int member = 0;

	if (member_time == NULL) {
	return -1LL;

	} else if (crm_str_to_boolean(member_time, &member) == 1) {
	/* If in_ccm=0, we'll return 0 here. If in_ccm=1, either the entry was
	* recorded as a boolean for a DC < 2.1.7, or the node is pending
	* shutdown and has left the CPG, in which case it was set to 1 to avoid
	* fencing for PCMK_OPT_NODE_PENDING_TIMEOUT.
	*
	* We return the effective time for in_ccm=1 because what's important to
	* avoid fencing is that effective time minus this value is less than
	* the pending node timeout.
	*/
	return member? (long long) get_effective_time(scheduler) : 0LL;

	} else {
	long long when_member = 0LL;

	if ((pcmk__scan_ll(member_time, &when_member,
	0LL) != pcmk_rc_ok) \|\| (when_member < 0LL)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_IN_CCM
	" in " XML_CIB_TAG_STATE " entry", member_time);
	return -1LL;
	}
	return when_member;
	}
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became online in process group
	*
	* \param[in] node_state Node's node_state entry
	*
	* \return Epoch time when node became online in process group (or 0 if not
	* online, or 1 for legacy online entries)
	*/
	static long long
	unpack_node_online(const xmlNode *node_state)
	{
	const char *peer_time = crm_element_value(node_state, PCMK__XA_CRMD);

	// @COMPAT Entries recorded for DCs < 2.1.7 have "online" or "offline"
	if (pcmk__str_eq(peer_time, OFFLINESTATUS,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	return 0LL;

	} else if (pcmk__str_eq(peer_time, ONLINESTATUS, pcmk__str_casei)) {
	return 1LL;

	} else {
	long long when_online = 0LL;

	if ((pcmk__scan_ll(peer_time, &when_online, 0LL) != pcmk_rc_ok)
	\|\| (when_online < 0)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_CRMD " in "
	XML_CIB_TAG_STATE " entry, assuming offline", peer_time);
	return 0LL;
	}
	return when_online;
	}
	}

	/*!
	* \internal
	* \brief Unpack node attribute for user-requested fencing
	*
	* \param[in] node Node to check
	* \param[in] node_state Node's node_state entry in CIB status
	*
	* \return \c true if fencing has been requested for \p node, otherwise \c false
	*/
	static bool
	unpack_node_terminate(const pcmk_node_t node, const xmlNode node_state)
	{
	long long value = 0LL;
	int value_i = 0;
	const char *value_s = pe_node_attribute_raw(node, PCMK_NODE_ATTR_TERMINATE);

	// Value may be boolean or an epoch time
	if (crm_str_to_boolean(value_s, &value_i) == 1) {
	return (value_i != 0);
	}
	if (pcmk__scan_ll(value_s, &value, 0LL) == pcmk_rc_ok) {
	return (value > 0);
	}
	crm_warn("Ignoring unrecognized value '%s' for " PCMK_NODE_ATTR_TERMINATE
	"node attribute for %s", value_s, pe__node_name(node));
	return false;
	}

	static gboolean
	determine_online_status_no_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	if (when_member <= 0) {
	crm_trace("Node %s is %sdown", pe__node_name(this_node),
	((when_member < 0)? "presumed " : ""));

	} else if (when_online > 0) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node %s is not ready to run resources: %s",
	pe__node_name(this_node), join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("Node %s controller is down: "
	"member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));

	} else {
	/* mark it unclean */
	pe_fence_node(scheduler, this_node, "peer is unexpectedly down", FALSE);
	crm_info("Node %s member@%lld online@%lld join=%s expected=%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
	}
	return online;
	}

	/*!
	* \internal
	* \brief Check whether a node has taken too long to join controller group
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node to check
	* \param[in] when_member Epoch time when node became a cluster member
	* \param[in] when_online Epoch time when node joined controller group
	*
	* \return true if node has been pending (on the way up) longer than
	* \c PCMK_OPT_NODE_PENDING_TIMEOUT, otherwise false
	* \note This will also update the cluster's recheck time if appropriate.
	*/
	static inline bool
	pending_too_long(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	long long when_member, long long when_online)
	{
	if ((scheduler->node_pending_timeout > 0)
	&& (when_member > 0) && (when_online <= 0)) {
	// There is a timeout on pending nodes, and node is pending

	time_t timeout = when_member + scheduler->node_pending_timeout;

	if (get_effective_time(node->details->data_set) >= timeout) {
	return true; // Node has timed out
	}

	// Node is pending, but still has time
	pe__update_recheck_time(timeout, scheduler, "pending node timeout");
	}
	return false;
	}

	static bool
	determine_online_status_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	bool termination_requested = unpack_node_terminate(this_node, node_state);
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	/*
	- PCMK__XA_JOIN ::= member\|down\|pending\|banned
	- PCMK__XA_EXPECTED ::= member\|down

	@COMPAT with entries recorded for DCs < 2.1.7
	- PCMK__XA_IN_CCM ::= true\|false
	- PCMK__XA_CRMD ::= online\|offline

	Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
	- PCMK__XA_IN_CCM ::= <timestamp>\|0
	Since when node has been a cluster member. A value 0 of means the node is not
	a cluster member.

	- PCMK__XA_CRMD ::= <timestamp>\|0
	Since when peer has been online in CPG. A value 0 means the peer is offline
	in CPG.
	*/

	crm_trace("Node %s member@%lld online@%lld join=%s expected=%s%s",
	pe__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"),
	(termination_requested? " (termination requested)" : ""));

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", pe__node_name(this_node));

	/* Slightly different criteria since we can't shut down a dead peer */
	return (when_online > 0);
	}

	if (when_member < 0) {
	pe_fence_node(scheduler, this_node,
	"peer has not been seen by the cluster", FALSE);
	return false;
	}

	if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_none)) {
	pe_fence_node(scheduler, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (termination_requested) {
	if ((when_member <= 0) && (when_online <= 0)
	&& pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_none)) {
	crm_info("%s was fenced as requested", pe__node_name(this_node));
	return false;
	}
	pe_fence_node(scheduler, this_node, "fencing was requested", false);

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN,
	pcmk__str_null_matches)) {

	if (pending_too_long(scheduler, this_node, when_member, when_online)) {
	pe_fence_node(scheduler, this_node,
	"peer pending timed out on joining the process group",
	FALSE);

	} else if ((when_member > 0) \|\| (when_online > 0)) {
	crm_info("- %s is not ready to run resources",
	pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up",
	pe__node_name(this_node));
	}

	} else if (when_member <= 0) {
	// Consider PCMK_OPT_PRIORITY_FENCING_DELAY for lost nodes
	pe_fence_node(scheduler, this_node,
	"peer is no longer part of the cluster", TRUE);

	} else if (when_online <= 0) {
	pe_fence_node(scheduler, this_node,
	"peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_none)) {
	crm_info("%s is active", pe__node_name(this_node));

	} else if (pcmk__str_any_of(join, CRMD_JOINSTATE_PENDING,
	CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("%s is not ready to run resources", pe__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(scheduler, this_node, "peer was in an unknown state",
	FALSE);
	}

	return (when_member > 0);
	}

	static void
	determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node)
	{
	pcmk_resource_t *rsc = this_node->details->remote_rsc;
	pcmk_resource_t *container = NULL;
	pcmk_node_t *host = NULL;

	/* If there is a node state entry for a (former) Pacemaker Remote node
	* but no resource creating that node, the node's connection resource will
	* be NULL. Consider it an offline remote node in that case.
	*/
	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	if (container && pcmk__list_of_1(rsc->running_on)) {
	host = rsc->running_on->data;
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->role == pcmk_role_started) {
	crm_trace("%s node %s presumed ONLINE because connection resource is started",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if ((rsc->role == pcmk_role_started)
	&& (rsc->next_role == pcmk_role_stopped)) {

	crm_trace("%s node %s shutting down because connection resource is stopping",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;

	} else if ((rsc->role == pcmk_role_stopped)
	\|\| ((container != NULL)
	&& (container->role == pcmk_role_stopped))) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	}

	static void
	determine_online_status(const xmlNode node_state, pcmk_node_t this_node,
	pcmk_scheduler_t *scheduler)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	online = determine_online_status_no_fencing(scheduler, node_state,
	this_node);

	} else {
	online = determine_online_status_fencing(scheduler, node_state,
	this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("%s is not a Pacemaker node", pe__node_name(this_node));

	} else if (this_node->details->unclean) {
	pcmk__sched_warn("%s is unclean", pe__node_name(this_node));

	} else if (this_node->details->online) {
	crm_info("%s is %s", pe__node_name(this_node),
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("%s is offline", pe__node_name(this_node));
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = calloc(base_name_len + 3, sizeof(char));
	CRM_ASSERT(zero);
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pcmk_resource_t *
	create_fake_resource(const char rsc_id, const xmlNode rsc_entry,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (pe__unpack_resource(xml_rsc, &rsc, NULL, scheduler) != pcmk_rc_ok) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pcmk_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pe_find_node(scheduler->nodes, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, "remote", NULL, scheduler);
	}
	link_rsc2remotenode(scheduler, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, PCMK__META_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_removed_filler);
	}
	pcmk__set_rsc_flags(rsc, pcmk_rsc_removed);
	scheduler->resources = g_list_append(scheduler->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*
	* \param[in,out] parent Clone resource that orphan will be added to
	* \param[in] rsc_id Orphan's resource ID
	* \param[in] node Where orphan is active (for logging only)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly added orphaned instance of \p parent
	*/
	static pcmk_resource_t *
	create_anonymous_orphan(pcmk_resource_t parent, const char rsc_id,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *top = pe__create_clone_child(parent, scheduler);

	// find_rsc() because we might be a cloned group
	pcmk_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	pcmk__rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, pe__node_name(node));
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of \c PCMK_META_CLONE_MAX
	* instances); (3) a newly created orphan (that is, \c PCMK_META_CLONE_MAX
	* instances are already active).
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node on which to check for instance
	* \param[in,out] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (no instance)
	*/
	static pcmk_resource_t *
	find_anonymous_clone(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	pcmk_resource_t parent, const char rsc_id)
	{
	GList *rIter = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(pe_rsc_is_clone(parent));
	CRM_ASSERT(!pcmk_is_set(parent->flags, pcmk_rsc_unique));

	// Check for active (or partially active, for cloned groups) instance
	pcmk__rsc_trace(parent, "Looking for %s on %s in %s",
	rsc_id, pe__node_name(node), parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GList *locations = NULL;
	pcmk_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if
	* PCMK_META_GLOBALLY_UNIQUE was flipped from true to false); and
	* (3) when we re-run calculations on the same scheduler data as part of
	* a simulation.
	*/
	child->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (((pcmk_node_t *) locations->data)->details == node->details) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if PCMK_META_GLOBALLY_UNIQUE is switched from true
	* to false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->running_on) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, pe__node_name(node));
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pcmk__rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pcmk__rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pcmk_rsc_blocked)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if (inactive_instance && inactive_instance->pending_node
	&& (inactive_instance->pending_node->details != node->details)) {
	inactive_instance = NULL;
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pcmk__rsc_trace(parent, "Resource %s, empty slot",
	inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has "requires" set to "quorum" or "nothing", and we don't
	* have a clone instance for every node, we don't want to consume a valid
	* instance number for unclean nodes. Such instances may appear to be active
	* according to the history, but should be considered inactive, so we can
	* start an instance elsewhere. Treat such instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pe__is_guest_node(node)
	&& !pe__is_universal_clone(parent, scheduler)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, scheduler);
	pcmk__rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pcmk_resource_t *
	unpack_find_resource(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	const char *rsc_id)
	{
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when PCMK_META_CLONE_MAX=0,
	* we create a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pcmk_resource_t *clone0 = pe_find_resource(scheduler->resources,
	clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pcmk_rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->variant > pcmk_rsc_variant_primitive) {
	crm_trace("Resource history for %s is orphaned because it is no longer primitive",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pe_rsc_is_anon_clone(parent)) {

	if (pe_rsc_is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(scheduler, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
	&& !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {

	pcmk__str_update(&rsc->clone_name, rsc_id);
	pcmk__rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, pe__node_name(node), rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_removed)? " (ORPHAN)" : "");
	}
	return rsc;
	}

	static pcmk_resource_t *
	process_orphan_resource(const xmlNode rsc_entry, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, pe__node_name(node));
	rsc = create_fake_resource(rsc_id, rsc_entry, scheduler);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pcmk__rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__",
	scheduler);
	}
	return rsc;
	}

	static void
	process_rsc_state(pcmk_resource_t rsc, pcmk_node_t node,
	enum action_fail_response on_fail)
	{
	pcmk_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = pcmk_on_fail_ignore;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), pe__node_name(node),
	fail2text(on_fail));

	/* process current state */
	if (rsc->role != pcmk_role_unknown) {
	pcmk_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	pcmk_node_t *n = pe__copy_node(node);

	pcmk__rsc_trace(rsc, "%s%s%s known on %s",
	rsc->id,
	((rsc->clone_name == NULL)? "" : " also known as "),
	((rsc->clone_name == NULL)? "" : rsc->clone_name),
	pe__node_name(n));
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (pcmk_is_set(iter->flags, pcmk_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if ((rsc->role > pcmk_role_stopped)
	&& node->details->online == FALSE
	&& node->details->maintenance == FALSE
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pe__is_guest_node(node)) {
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	should_fence = TRUE;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& !pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = pcmk_on_fail_ignore;
	}

	switch (on_fail) {
	case pcmk_on_fail_ignore:
	/* nothing to do */
	break;

	case pcmk_on_fail_demote:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case pcmk_on_fail_fence_node:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	free(reason);
	break;

	case pcmk_on_fail_standby_node:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case pcmk_on_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_blocked);
	break;

	case pcmk_on_fail_ban:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__",
	rsc->cluster);
	break;

	case pcmk_on_fail_stop:
	pe__set_next_role(rsc, pcmk_role_stopped, "on-fail=stop");
	break;

	case pcmk_on_fail_restart:
	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	pcmk__set_rsc_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_restart_container:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (rsc->container && pe_rsc_is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	rsc->cluster->stop_needed =
	g_list_prepend(rsc->cluster->stop_needed, rsc->container);
	} else if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_reset_remote:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	}
	if (tmpnode &&
	pe__is_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(rsc->cluster, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (rsc->role > pcmk_role_stopped) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_ms) {
	pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__config_warn("Detected active orphan %s running on %s",
	rsc->id, pe__node_name(node));
	} else {
	pcmk__config_warn("Resource '%s' must be stopped manually on "
	"%s because cluster is configured not to "
	"stop active orphans",
	rsc->id, pe__node_name(node));
	}
	}

	native_add_running(rsc, node, rsc->cluster,
	(save_on_fail != pcmk_on_fail_ignore));
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	break;
	case pcmk_on_fail_demote:
	case pcmk_on_fail_block:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed);
	break;
	default:
	pcmk__set_rsc_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	break;
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pcmk__rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)",
	rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node,
	PCMK_ACTION_STOP, FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t stop = (pcmk_action_t ) gIter->data;

	pcmk__set_action_flags(stop, pcmk_action_optional);
	}

	g_list_free(possible_matches);
	}

	/* A successful stop after migrate_to on the migration source doesn't make
	* the partially migrated resource stopped on the migration target.
	*/
	if ((rsc->role == pcmk_role_stopped)
	&& rsc->partial_migration_source
	&& rsc->partial_migration_source->details == node->details
	&& rsc->partial_migration_target
	&& rsc->running_on) {

	rsc->role = pcmk_role_started;
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pcmk_node_t node, pcmk_resource_t rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pcmk_scheduler_t scheduler)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: node is offline",
	rsc->id, pe__node_name(node));
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: resource is not active",
	id, pe__node_name(node));
	continue;

	} else if (counter < start_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: old %d",
	id, pe__node_name(node), counter);
	continue;
	}

	crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	if (interval_ms == 0) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: non-recurring",
	id, pe__node_name(node));
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: status",
	id, pe__node_name(node));
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk__rsc_trace(rsc, "Creating %s on %s", key, pe__node_name(node));
	custom_action(rsc, key, task, node, TRUE, scheduler);
	}
	}

	void
	calculate_active_ops(const GList sorted_op_list, int start_index,
	int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	const xmlNode rsc_op = (const xmlNode ) iter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(const xmlNode rsc_entry, pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, PCMK_OPT_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((scheduler->shutdown_lock > 0)
	&& (get_effective_time(scheduler)
	> (lock_time + scheduler->shutdown_lock))) {
	pcmk__rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, pe__node_name(node));
	pe__clear_resource_history(rsc, node);
	} else {
	/* @COMPAT I don't like breaking const signatures, but
	* rsc->lock_node should really be const -- we just can't change it
	* until the next API compatibility break.
	*/
	rsc->lock_node = (pcmk_node_t *) node;
	rsc->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one lrm_resource entry from a node's CIB status
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] rsc_entry lrm_resource XML being unpacked
	* \param[in,out] scheduler Scheduler data
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pcmk_resource_t *
	unpack_lrm_resource(pcmk_node_t node, const xmlNode lrm_resource,
	pcmk_scheduler_t *scheduler)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = pcmk_role_unknown;

	const char *rsc_id = ID(lrm_resource);

	pcmk_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = pcmk_on_fail_ignore;
	enum rsc_role_e saved_role = pcmk_role_unknown;

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring invalid " XML_LRM_TAG_RESOURCE
	" entry: No " PCMK_XA_ID);
	crm_log_xml_info(lrm_resource, "missing-id");
	return NULL;
	}
	crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
	rsc_id, pe__node_name(node));

	// Build a list of individual lrm_rsc_op entries, so we can sort them
	for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(scheduler, node, rsc_id);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, scheduler);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, scheduler);
	}

	/* process operations */
	saved_role = rsc->role;
	rsc->role = pcmk_role_unknown;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list,
	scheduler);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail);

	if (get_target_role(rsc, &req_role)) {
	if ((rsc->next_role == pcmk_role_unknown)
	\|\| (req_role < rsc->next_role)) {

	pe__set_next_role(rsc, req_role, PCMK_META_TARGET_ROLE);

	} else if (req_role > rsc->next_role) {
	pcmk__rsc_info(rsc,
	"%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(const xmlNode *lrm_rsc_list,
	pcmk_scheduler_t *scheduler)
	{
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {

	pcmk_resource_t *rsc;
	pcmk_resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, PCMK__META_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(scheduler->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources, rsc_id);
	if ((rsc == NULL) \|\| (rsc->container != NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	continue;
	}

	pcmk__rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
	rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler)
	{
	bool found_orphaned_container_filler = false;

	// Drill down to lrm_resources section
	xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
	if (xml == NULL) {
	return;
	}
	xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
	if (xml == NULL) {
	return;
	}

	// Unpack each lrm_resource entry
	for (const xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	pcmk_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, scheduler);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	found_orphaned_container_filler = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* orphaned container fillers to their container resource.
	*/
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(xml, scheduler);
	}
	}

	static void
	set_active(pcmk_resource_t *rsc)
	{
	const pcmk_resource_t *top = pe__const_top_resource(rsc, false);

	if (top && pcmk_is_set(top->flags, pcmk_rsc_promotable)) {
	rsc->role = pcmk_role_unpromoted;
	} else {
	rsc->role = pcmk_role_started;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk_node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define XPATH_NODE_STATE "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE
	#define SUB_XPATH_LRM_RESOURCE "/" XML_CIB_TAG_LRM \
	"/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE
	#define SUB_XPATH_LRM_RSC_OP "/" XML_LRM_TAG_RSC_OP

	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	int target_rc, pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
	return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", resource, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_TASK "='", op, "'",
	NULL);

	/* Need to check against transition_magic too? */
	if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_TARGET "='", source, "']",
	NULL);

	} else if ((source != NULL)
	&& (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" XML_LRM_ATTR_MIGRATE_SOURCE "='", source, "']",
	NULL);
	} else {
	g_string_append_c(xpath, ']');
	}

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);
	g_string_free(xpath, TRUE);

	if (xml && target_rc >= 0) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
	if ((rc != target_rc) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static xmlNode *
	find_lrm_resource(const char rsc_id, const char node_name,
	pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc_id, "']",
	NULL);

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);

	g_string_free(xpath, TRUE);
	return xml;
	}

	/*!
	* \internal
	* \brief Check whether a resource has no completed action history on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node_name Node to check
	*
	* \return true if \p rsc_id is unknown on \p node_name, otherwise false
	*/
	static bool
	unknown_on_node(pcmk_resource_t rsc, const char node_name)
	{
	bool result = false;
	xmlXPathObjectPtr search;
	GString *xpath = g_string_sized_new(256);

	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc->id, "']"
	SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_RC "!='193']",
	NULL);
	search = xpath_search(rsc->cluster->input, (const char *) xpath->str);
	result = (numXpathResults(search) == 0);
	freeXpathObject(search);
	g_string_free(xpath, TRUE);
	return result;
	}

	/*!
	* \brief Check whether a probe/monitor indicating the resource was not running
	* on a node happened after some event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a monitor happened after event, false otherwise
	*/
	static bool
	monitor_not_running_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	/* Any probe/monitor operation on the node indicating it was not running
	* there
	*/
	xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
	NULL, PCMK_OCF_NOT_RUNNING, scheduler);

	return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
	}

	/*!
	* \brief Check whether any non-monitor operation on a node happened after some
	* event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that non-monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	non_monitor_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *lrm_resource = NULL;

	lrm_resource = find_lrm_resource(rsc_id, node_name, scheduler);
	if (lrm_resource == NULL) {
	return false;
	}

	for (xmlNode *op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	op != NULL; op = crm_next_same_xml(op)) {
	const char * task = NULL;

	if (op == xml_op) {
	continue;
	}

	task = crm_element_value(op, XML_LRM_ATTR_TASK);

	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL)
	&& pe__is_newer_op(op, xml_op, same_node) > 0) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \brief Check whether the resource has newer state on a node after a migration
	* attempt
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] migrate_to Any migrate_to event that is being compared to
	* \param[in] migrate_from Any migrate_from event that is being compared to
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	newer_state_after_migrate(const char rsc_id, const char node_name,
	const xmlNode *migrate_to,
	const xmlNode *migrate_from,
	pcmk_scheduler_t *scheduler)
	{
	const xmlNode *xml_op = migrate_to;
	const char *source = NULL;
	const char *target = NULL;
	bool same_node = false;

	if (migrate_from) {
	xml_op = migrate_from;
	}

	source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	/* It's preferred to compare to the migrate event on the same node if
	* existing, since call ids are more reliable.
	*/
	if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
	if (migrate_from) {
	xml_op = migrate_from;
	same_node = true;

	} else {
	xml_op = migrate_to;
	}

	} else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
	if (migrate_to) {
	xml_op = migrate_to;
	same_node = true;

	} else {
	xml_op = migrate_from;
	}
	}

	/* If there's any newer non-monitor operation on the node, or any newer
	* probe/monitor operation on the node indicating it was not running there,
	* the migration events potentially no longer matter for the node.
	*/
	return non_monitor_after(rsc_id, node_name, xml_op, same_node, scheduler)
	\|\| monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
	scheduler);
	}

	/*!
	* \internal
	* \brief Parse migration source and target node names from history entry
	*
	* \param[in] entry Resource history entry for a migration action
	* \param[in] source_node If not NULL, source must match this node
	* \param[in] target_node If not NULL, target must match this node
	* \param[out] source_name Where to store migration source node name
	* \param[out] target_name Where to store migration target node name
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_migration_node_names(const xmlNode entry, const pcmk_node_t source_node,
	const pcmk_node_t *target_node,
	const char source_name, const char target_name)
	{
	*source_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_SOURCE);
	*target_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_TARGET);
	if ((source_name == NULL) \|\| (target_name == NULL)) {
	pcmk__config_err("Ignoring resource history entry %s without "
	XML_LRM_ATTR_MIGRATE_SOURCE " and "
	XML_LRM_ATTR_MIGRATE_TARGET, ID(entry));
	return pcmk_rc_unpack_error;
	}

	if ((source_node != NULL)
	&& !pcmk__str_eq(*source_name, source_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_SOURCE "='%s' does not match %s",
	ID(entry), *source_name, pe__node_name(source_node));
	return pcmk_rc_unpack_error;
	}

	if ((target_node != NULL)
	&& !pcmk__str_eq(*target_name, target_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	XML_LRM_ATTR_MIGRATE_TARGET "='%s' does not match %s",
	ID(entry), *target_name, pe__node_name(target_node));
	return pcmk_rc_unpack_error;
	}

	return pcmk_rc_ok;
	}

	/*
	* \internal
	* \brief Add a migration source to a resource's list of dangling migrations
	*
	* If the migrate_to and migrate_from actions in a live migration both
	* succeeded, but there is no stop on the source, the migration is considered
	* "dangling." Add the source to the resource's dangling migration list, which
	* will be used to schedule a stop on the source without affecting the target.
	*
	* \param[in,out] rsc Resource involved in migration
	* \param[in] node Migration source
	*/
	static void
	add_dangling_migration(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk__rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
	rsc->id, pe__node_name(node));
	rsc->role = pcmk_role_stopped;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations,
	(gpointer) node);
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a successful migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_success(struct action_history *history)
	{
	/* A complete migration sequence is:
	* 1. migrate_to on source node (which succeeded if we get to this function)
	* 2. migrate_from on target node
	* 3. stop on source node
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from succeeded but no stop has happened, the
	* migration is considered to be "dangling".
	*
	* If a successful migrate_to and stop have happened on the source node, we
	* still need to check for a partial migration, due to scenarios (easier to
	* produce with batch-limit=1) like:
	*
	* - A resource is migrating from node1 to node2, and a migrate_to is
	* initiated for it on node1.
	*
	* - node2 goes into standby mode while the migrate_to is pending, which
	* aborts the transition.
	*
	* - Upon completion of the migrate_to, a new transition schedules a stop
	* on both nodes and a start on node1.
	*
	* - If the new transition is aborted for any reason while the resource is
	* stopping on node1, the transition after that stop completes will see
	* the migrate_to and stop on the source, but it's still a partial
	* migration, and the resource must be stopped on node2 because it is
	* potentially active there due to the migrate_to.
	*
	* We also need to take into account that either node's history may be
	* cleared at any point in the migration process.
	*/
	int from_rc = PCMK_OCF_OK;
	int from_status = PCMK_EXEC_PENDING;
	pcmk_node_t *target_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	bool source_newer_op = false;
	bool target_newer_state = false;
	bool active_on_target = false;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	// Check for newer state on the source
	source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
	true, history->rsc->cluster);

	// Check for a migrate_from action from this source on the target
	migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
	target, source, -1, history->rsc->cluster);
	if (migrate_from != NULL) {
	if (source_newer_op) {
	/* There's a newer non-monitor operation on the source and a
	* migrate_from on the target, so this migrate_to is irrelevant to
	* the resource's state.
	*/
	return;
	}
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS,
	&from_status);
	}

	/* If the resource has newer state on both the source and target after the
	* migration events, this migrate_to is irrelevant to the resource's state.
	*/
	target_newer_state = newer_state_after_migrate(history->rsc->id, target,
	history->xml, migrate_from,
	history->rsc->cluster);
	if (source_newer_op && target_newer_state) {
	return;
	}

	/* Check for dangling migration (migrate_from succeeded but stop not done).
	* We know there's no stop because we already returned if the target has a
	* migrate_from and the source has any newer non-monitor operation.
	*/
	if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
	add_dangling_migration(history->rsc, history->node);
	return;
	}

	/* Without newer state, this migrate_to implies the resource is active.
	* (Clones are not allowed to migrate, so role can't be promoted.)
	*/
	history->rsc->role = pcmk_role_started;

	target_node = pe_find_node(history->rsc->cluster->nodes, target);
	active_on_target = !target_newer_state && (target_node != NULL)
	&& target_node->details->online;

	if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
	if (active_on_target) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	TRUE);
	} else {
	// Mark resource as failed, require recovery, and prevent migration
	pcmk__set_rsc_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pcmk__clear_rsc_flags(history->rsc, pcmk_rsc_migratable);
	}
	return;
	}

	// The migrate_from is pending, complete but erased, or to be scheduled

	/* If there is no history at all for the resource on an online target, then
	* it was likely cleaned. Just return, and we'll schedule a probe. Once we
	* have the probe result, it will be reflected in target_newer_state.
	*/
	if ((target_node != NULL) && target_node->details->online
	&& unknown_on_node(history->rsc, target)) {
	return;
	}

	if (active_on_target) {
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	if ((source_node != NULL) && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	history->rsc->partial_migration_target = target_node;
	history->rsc->partial_migration_source = source_node;
	}

	} else if (!source_newer_op) {
	// Mark resource as failed, require recovery, and prevent migration
	pcmk__set_rsc_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pcmk__clear_rsc_flags(history->rsc, pcmk_rsc_migratable);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_failure(struct action_history *history)
	{
	xmlNode *target_migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(history->rsc->id,
	PCMK_ACTION_MIGRATE_FROM, target, source,
	PCMK_OCF_OK, history->rsc->cluster);

	if (/* If the resource state is unknown on the target, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, target)
	/* If the resource has newer state on the target after the migration
	* events, this migrate_to no longer matters for the target.
	*/
	&& !newer_state_after_migrate(history->rsc->id, target, history->xml,
	target_migrate_from,
	history->rsc->cluster)) {
	/* The resource has no newer state on the target, so assume it's still
	* active there.
	* (if it is up).
	*/
	pcmk_node_t *target_node = pe_find_node(history->rsc->cluster->nodes,
	target);

	if (target_node && target_node->details->online) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	}

	} else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
	history->rsc->cluster)) {
	/* We know the resource has newer state on the target, but this
	* migrate_to still matters for the source as long as there's no newer
	* non-monitor operation there.
	*/

	// Mark node as having dangling migration so we can force a stop later
	history->rsc->dangling_migrations =
	g_list_prepend(history->rsc->dangling_migrations,
	(gpointer) history->node);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_from action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_from_failure(struct action_history *history)
	{
	xmlNode *source_migrate_to = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, NULL, history->node, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
	source, target, PCMK_OCF_OK,
	history->rsc->cluster);

	if (/* If the resource state is unknown on the source, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, source)
	/* If the resource has newer state on the source after the migration
	* events, this migrate_from no longer matters for the source.
	*/
	&& !newer_state_after_migrate(history->rsc->id, source,
	source_migrate_to, history->xml,
	history->rsc->cluster)) {
	/* The resource has no newer state on the source, so assume it's still
	* active there (if it is up).
	*/
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	if (source_node && source_node->details->online) {
	native_add_running(history->rsc, source_node, history->rsc->cluster,
	TRUE);
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to cluster's list of failed actions
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	record_failed_op(struct action_history *history)
	{
	if (!(history->node->details->online)) {
	return;
	}

	for (const xmlNode *xIter = history->rsc->cluster->failed->children;
	xIter != NULL; xIter = xIter->next) {

	const char *key = pe__xe_history_key(xIter);
	const char *uname = crm_element_value(xIter, PCMK_XA_UNAME);

	if (pcmk__str_eq(history->key, key, pcmk__str_none)
	&& pcmk__str_eq(uname, history->node->details->uname,
	pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s",
	history->key, pe__node_name(history->node));
	return;
	}
	}

	crm_trace("Adding entry for %s on %s to failed action list",
	history->key, pe__node_name(history->node));
	crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->details->uname);
	crm_xml_add(history->xml, XML_LRM_ATTR_RSCID, history->rsc->id);
	add_node_copy(history->rsc->cluster->failed, history->xml);
	}

	static char *
	last_change_str(const xmlNode *xml_op)
	{
	time_t when;
	char *result = NULL;

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&when) == pcmk_ok) {
	char *when_s = pcmk__epoch2str(&when, 0);
	const char *p = strchr(when_s, ' ');

	// Skip day of week to make message shorter
	if ((p != NULL) && (*(++p) != '\0')) {
	result = strdup(p);
	CRM_ASSERT(result != NULL);
	}
	free(when_s);
	}

	if (result == NULL) {
	result = strdup("unknown time");
	CRM_ASSERT(result != NULL);
	}

	return result;
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case pcmk_on_fail_demote:
	switch (second) {
	case pcmk_on_fail_ignore:
	return 1;
	case pcmk_on_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_reset_remote:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return 1;
	case pcmk_on_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return 1;
	case pcmk_on_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case pcmk_on_fail_demote:
	return (first == pcmk_on_fail_ignore)? -1 : 1;

	case pcmk_on_fail_reset_remote:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return -1;
	default:
	return 1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	/*!
	* \internal
	* \brief Ban a resource (or its clone if an anonymous instance) from all nodes
	*
	* \param[in,out] rsc Resource to ban
	*/
	static void
	ban_from_all_nodes(pcmk_resource_t *rsc)
	{
	int score = -INFINITY;
	pcmk_resource_t *fail_rsc = rsc;

	if (fail_rsc->parent != NULL) {
	pcmk_resource_t *parent = uber_parent(fail_rsc);

	if (pe_rsc_is_anon_clone(parent)) {
	/* For anonymous clones, if an operation with on-fail=stop fails for
	* any instance, the entire clone must stop.
	*/
	fail_rsc = parent;
	}
	}

	// Ban the resource from all nodes
	crm_notice("%s will not be started under current conditions", fail_rsc->id);
	if (fail_rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	}
	fail_rsc->allowed_nodes = pe__node_list2table(rsc->cluster->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	/*!
	* \internal
	* \brief Get configured failure handling and role after failure for an action
	*
	* \param[in,out] history Unpacked action history entry
	* \param[out] on_fail Where to set configured failure handling
	* \param[out] fail_role Where to set to role after failure
	*/
	static void
	unpack_failure_handling(struct action_history *history,
	enum action_fail_response *on_fail,
	enum rsc_role_e *fail_role)
	{
	xmlNode *config = pcmk__find_action_config(history->rsc, history->task,
	history->interval_ms, true);

	GHashTable *meta = pcmk__unpack_action_meta(history->rsc, history->node,
	history->task,
	history->interval_ms, config);

	const char *on_fail_str = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);

	*on_fail = pcmk__parse_on_fail(history->rsc, history->task,
	history->interval_ms, on_fail_str);
	fail_role = pcmk__role_after_failure(history->rsc, history->task, on_fail,
	meta);
	g_hash_table_destroy(meta);
	}

	/*!
	* \internal
	* \brief Update resource role, failure handling, etc., after a failed action
	*
	* \param[in,out] history Parsed action result history
	* \param[in] config_on_fail Action failure handling from configuration
	* \param[in] fail_role Resource's role after failure of this action
	* \param[out] last_failure This will be set to the history XML
	* \param[in,out] on_fail Actual handling of action result
	*/
	static void
	unpack_rsc_op_failure(struct action_history *history,
	enum action_fail_response config_on_fail,
	enum rsc_role_e fail_role, xmlNode **last_failure,
	enum action_fail_response *on_fail)
	{
	bool is_probe = false;
	char *last_change_s = NULL;

	*last_failure = history->xml;

	is_probe = pcmk_xe_is_probe(history->xml);
	last_change_s = last_change_str(history->xml);

	if (!pcmk_is_set(history->rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);
	} else {
	pcmk__sched_warn("Unexpected result (%s%s%s) was recorded for %s of "
	"%s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pe__node_name(history->node), last_change_s,
	history->exit_status, history->id);

	if (is_probe && (history->exit_status != PCMK_OCF_OK)
	&& (history->exit_status != PCMK_OCF_NOT_RUNNING)
	&& (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the resource-discovery option for location constraints",
	history->rsc->id, pe__node_name(history->node));
	}

	record_failed_op(history);
	}

	free(last_change_s);

	if (cmp_on_fail(*on_fail, config_on_fail) < 0) {
	pcmk__rsc_trace(history->rsc, "on-fail %s -> %s for %s",
	fail2text(*on_fail), fail2text(config_on_fail),
	history->key);
	*on_fail = config_on_fail;
	}

	if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
	resource_location(history->rsc, history->node, -INFINITY,
	"__stop_fail__", history->rsc->cluster);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
	unpack_migrate_to_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
	unpack_migrate_from_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
	if (config_on_fail == pcmk_on_fail_block) {
	history->rsc->role = pcmk_role_promoted;
	pe__set_next_role(history->rsc, pcmk_role_stopped,
	"demote with on-fail=block");

	} else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
	history->rsc->role = pcmk_role_stopped;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	history->rsc->role = pcmk_role_unpromoted;
	}
	}

	if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	/* leave stopped */
	pcmk__rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
	history->rsc->role = pcmk_role_stopped;

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
	set_active(history->rsc);
	}

	pcmk__rsc_trace(history->rsc,
	"Resource %s: role=%s unclean=%s on_fail=%s fail_role=%s",
	history->rsc->id, role2text(history->rsc->role),
	pcmk__btoa(history->node->details->unclean),
	fail2text(config_on_fail), role2text(fail_role));

	if ((fail_role != pcmk_role_started)
	&& (history->rsc->next_role < fail_role)) {
	pe__set_next_role(history->rsc, fail_role, "failure");
	}

	if (fail_role == pcmk_role_stopped) {
	ban_from_all_nodes(history->rsc);
	}
	}

	/*!
	* \internal
	* \brief Block a resource with a failed action if it cannot be recovered
	*
	* If resource action is a failed stop and fencing is not possible, mark the
	* resource as unmanaged and blocked, since recovery cannot be done.
	*
	* \param[in,out] history Parsed action history entry
	*/
	static void
	block_if_unrecoverable(struct action_history *history)
	{
	char *last_change_s = NULL;

	if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
	return; // All actions besides stop are always recoverable
	}
	if (pe_can_fence(history->node->details->data_set, history->node)) {
	return; // Failed stops are recoverable via fencing
	}

	last_change_s = last_change_str(history->xml);
	pcmk__sched_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	CRM_XS " rc=%d id=%s",
	history->rsc->id, history->task,
	pe__node_name(history->node),
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	last_change_s, history->exit_status, history->id);

	free(last_change_s);

	pcmk__clear_rsc_flags(history->rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(history->rsc, pcmk_rsc_blocked);
	}

	/*!
	* \internal
	* \brief Update action history's execution status and why
	*
	* \param[in,out] history Parsed action history entry
	* \param[out] why Where to store reason for update
	* \param[in] value New value
	* \param[in] reason Description of why value was changed
	*/
	static inline void
	remap_because(struct action_history history, const char *why, int value,
	const char *reason)
	{
	if (history->execution_status != value) {
	history->execution_status = value;
	*why = reason;
	}
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in,out] history Parsed action history entry
	* \param[in,out] on_fail What should be done about the result
	* \param[in] expired Whether result is expired
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the scheduler data's list of failed
	* operations to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(struct action_history *history,
	enum action_fail_response *on_fail, bool expired)
	{
	bool is_probe = false;
	int orig_exit_status = history->exit_status;
	int orig_exec_status = history->execution_status;
	const char *why = NULL;
	const char *task = history->task;

	// Remap degraded results to their successful counterparts
	history->exit_status = pcmk__effective_rc(history->exit_status);
	if (history->exit_status != orig_exit_status) {
	why = "degraded result";
	if (!expired && (!history->node->details->shutdown
	\|\| history->node->details->online)) {
	record_failed_op(history);
	}
	}

	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& ((history->execution_status != PCMK_EXEC_DONE)
	\|\| (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
	history->execution_status = PCMK_EXEC_DONE;
	history->exit_status = PCMK_OCF_NOT_RUNNING;
	why = "equivalent probe result";
	}

	/* If the executor reported an execution status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	switch (history->execution_status) {
	case PCMK_EXEC_DONE:
	case PCMK_EXEC_ERROR:
	break;

	// These should be treated as node-fatal
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"node-fatal error");
	goto remap_done;

	default:
	goto remap_done;
	}

	is_probe = pcmk_xe_is_probe(history->xml);
	if (is_probe) {
	task = "probe";
	}

	if (history->expected_exit_status < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* expected exit status in the transition key, which (along with the
	* similar case of a corrupted transition key in the CIB) will be
	* reported to this function as -1. Pacemaker 2.0+ does not support
	* rolling upgrades from those versions or processing of saved CIB files
	* from those versions, so we do not need to care much about this case.
	*/
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"obsolete history format");
	pcmk__config_warn("Expected result not found for %s on %s "
	"(corrupt or obsolete CIB?)",
	history->key, pe__node_name(history->node));

	} else if (history->exit_status == history->expected_exit_status) {
	remap_because(history, &why, PCMK_EXEC_DONE, "expected result");

	} else {
	remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
	pcmk__rsc_debug(history->rsc,
	"%s on %s: expected %d (%s), got %d (%s%s%s)",
	history->key, pe__node_name(history->node),
	history->expected_exit_status,
	services_ocf_exitcode_str(history->expected_exit_status),
	history->exit_status,
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""));
	}

	switch (history->exit_status) {
	case PCMK_OCF_OK:
	if (is_probe
	&& (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe
	\|\| (history->expected_exit_status == history->exit_status)
	\|\| !pcmk_is_set(history->rsc->flags, pcmk_rsc_managed)) {

	/* For probes, recurring monitors for the Stopped role, and
	* unmanaged resources, "not running" is not considered a
	* failure.
	*/
	remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
	history->rsc->role = pcmk_role_stopped;
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe
	&& (history->exit_status != history->expected_exit_status)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active and promoted on %s at %s",
	history->rsc->id, pe__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	if (!expired
	\|\| (history->exit_status == history->expected_exit_status)) {
	history->rsc->role = pcmk_role_promoted;
	}
	break;

	case PCMK_OCF_FAILED_PROMOTED:
	if (!expired) {
	history->rsc->role = pcmk_role_promoted;
	}
	remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	{
	guint interval_ms = 0;
	crm_element_value_ms(history->xml, XML_LRM_ATTR_INTERVAL_MS,
	&interval_ms);

	if (interval_ms == 0) {
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"exit status");
	} else {
	remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
	"exit status");
	}
	}
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
	break;

	default:
	if (history->execution_status == PCMK_EXEC_DONE) {
	char *last_change_s = last_change_str(history->xml);

	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	history->exit_status, task, history->rsc->id,
	pe__node_name(history->node), last_change_s);
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"unknown exit status");
	free(last_change_s);
	}
	break;
	}

	remap_done:
	if (why != NULL) {
	pcmk__rsc_trace(history->rsc,
	"Remapped %s result from [%s: %s] to [%s: %s] "
	"because of %s",
	history->key, pcmk_exec_status_str(orig_exec_status),
	crm_exit_str(orig_exit_status),
	pcmk_exec_status_str(history->execution_status),
	crm_exit_str(history->exit_status), why);
	}
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(const xmlNode xml_op, const char task,
	pcmk_resource_t rsc, pcmk_node_t node)
	{
	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
	rsc->cluster);

	} else {
	pcmk__op_digest_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, pe__xe_history_key(xml_op),
	node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pcmk_action_t action, pcmk_resource_t remote_conn,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_node_t *remote_node = pe_find_node(scheduler->nodes, remote_conn->id);

	if (remote_node) {
	pcmk_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, scheduler);

	order_actions(fence, action, pcmk__ar_first_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(const pcmk_resource_t rsc, const char task,
	guint interval_ms, bool is_last_failure)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if (rsc->remote_reconnect_ms
	&& pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& (interval_ms != 0)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {

	pcmk_node_t *remote_node = pe_find_node(rsc->cluster->nodes, rsc->id);

	if (remote_node && !remote_node->details->remote_was_fenced) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in,out] history Parsed action result history
	*
	* \return true if operation history entry is expired, otherwise false
	*/
	static bool
	check_operation_expiry(struct action_history *history)
	{
	bool expired = false;
	bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
	time_t last_run = 0;
	int unexpired_fail_count = 0;
	const char *clear_reason = NULL;

	if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Not Installed does not expire",
	history->id, pe__node_name(history->node));
	return false; // "Not installed" must always be cleared manually
	}

	if ((history->rsc->failure_timeout > 0)
	&& (crm_element_value_epoch(history->xml, XML_RSC_OP_LAST_CHANGE,
	&last_run) == 0)) {

	/* Resource has a PCMK_META_FAILURE_TIMEOUT and history entry has a
	* timestamp
	*/

	time_t now = get_effective_time(history->rsc->cluster);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + history->rsc->failure_timeout))
	&& !should_ignore_failure_timeout(history->rsc, history->task,
	history->interval_ms,
	is_last_failure)) {
	expired = true;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
	&last_failure,
	pcmk__fc_effective,
	history->xml);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
	" last-failure@%lld",
	history->id, (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count,
	history->rsc->failure_timeout, (long long) last_failure);
	last_failure += history->rsc->failure_timeout + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, history->rsc->cluster,
	"fail count expiration");
	}
	}

	if (expired) {
	if (pe_get_failcount(history->node, history->rsc, NULL,
	pcmk__fc_default, history->xml)) {
	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Unexpired fail count",
	history->id, pe__node_name(history->node));
	expired = false;
	}

	} else if (is_last_failure
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(history->xml, history->task,
	history->rsc, history->node)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	pcmk_action_t *clear_op = NULL;

	// Schedule clearing of the fail count
	clear_op = pe__clear_failcount(history->rsc, history->node,
	clear_reason, history->rsc->cluster);

	if (pcmk_is_set(history->rsc->cluster->flags,
	pcmk_sched_fencing_enabled)
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes",
	history->task, history->rsc->id);
	order_after_remote_fencing(clear_op, history->rsc,
	history->rsc->cluster);
	}
	}

	if (expired && (history->interval_ms == 0)
	&& pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	switch (history->exit_status) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Probe result",
	history->id, pe__node_name(history->node));
	expired = false;
	break;
	}
	}

	return expired;
	}

	int
	pe__target_rc_from_xml(const xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, PCMK__XA_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	/*!
	* \internal
	* \brief Update a resource's state for an action result
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] exit_status Exit status to base new state on
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	update_resource_state(struct action_history *history, int exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool clear_past_failure = false;

	if ((exit_status == PCMK_OCF_NOT_INSTALLED)
	\|\| (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml))) {
	history->rsc->role = pcmk_role_stopped;

	} else if (exit_status == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	if ((last_failure != NULL)
	&& pcmk__str_eq(history->key, pe__xe_history_key(last_failure),
	pcmk__str_none)) {
	clear_past_failure = true;
	}
	if (history->rsc->role < pcmk_role_started) {
	set_active(history->rsc);
	}

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	history->rsc->role = pcmk_role_stopped;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_promoted;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	if (*on_fail == pcmk_on_fail_demote) {
	// Demote clears an error only if on-fail=demote
	clear_past_failure = true;
	}
	history->rsc->role = pcmk_role_unpromoted;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	unpack_migrate_to_success(history);

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "%s active on %s",
	history->rsc->id, pe__node_name(history->node));
	set_active(history->rsc);
	}

	if (!clear_past_failure) {
	return;
	}

	switch (*on_fail) {
	case pcmk_on_fail_stop:
	case pcmk_on_fail_ban:
	case pcmk_on_fail_standby_node:
	case pcmk_on_fail_fence_node:
	pcmk__rsc_trace(history->rsc,
	"%s (%s) is not cleared by a completed %s",
	history->rsc->id, fail2text(*on_fail),
	history->task);
	break;

	case pcmk_on_fail_block:
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_restart_container:
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures");
	break;

	case pcmk_on_fail_reset_remote:
	if (history->rsc->remote_reconnect_ms == 0) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures and reset remote");
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Check whether a given history entry matters for resource state
	*
	* \param[in] history Parsed action history entry
	*
	* \return true if action can affect resource state, otherwise false
	*/
	static inline bool
	can_affect_state(struct action_history *history)
	{
	#if 0
	/* @COMPAT It might be better to parse only actions we know we're interested
	* in, rather than exclude a couple we don't. However that would be a
	* behavioral change that should be done at a major or minor series release.
	* Currently, unknown operations can affect whether a resource is considered
	* active and/or failed.
	*/
	return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	"asyncmon", NULL);
	#else
	return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_META_DATA, NULL);
	#endif
	}

	/*!
	* \internal
	* \brief Unpack execution/exit status and exit reason from a history entry
	*
	* \param[in,out] history Action history entry to unpack
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_action_result(struct action_history *history)
	{
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_OPSTATUS,
	&(history->execution_status)) < 0)
	\|\| (history->execution_status < PCMK_EXEC_PENDING)
	\|\| (history->execution_status > PCMK_EXEC_MAX)
	\|\| (history->execution_status == PCMK_EXEC_CANCELLED)) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_OPSTATUS " '%s'",
	history->id, history->rsc->id,
	pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	XML_LRM_ATTR_OPSTATUS),
	""));
	return pcmk_rc_unpack_error;
	}
	if ((crm_element_value_int(history->xml, XML_LRM_ATTR_RC,
	&(history->exit_status)) < 0)
	\|\| (history->exit_status < 0) \|\| (history->exit_status > CRM_EX_MAX)) {
	#if 0
	/* @COMPAT We should ignore malformed entries, but since that would
	* change behavior, it should be done at a major or minor series
	* release.
	*/
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " XML_LRM_ATTR_RC " '%s'",
	history->id, history->rsc->id,
	pe__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	XML_LRM_ATTR_RC),
	""));
	return pcmk_rc_unpack_error;
	#else
	history->exit_status = CRM_EX_ERROR;
	#endif
	}
	history->exit_reason = crm_element_value(history->xml,
	XML_LRM_ATTR_EXIT_REASON);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Process an action history entry whose result expired
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	*
	* \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
	* entry needs no further processing)
	*/
	static int
	process_expired_result(struct action_history *history, int orig_exit_status)
	{
	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& (orig_exit_status != history->expected_exit_status)) {

	if (history->rsc->role <= pcmk_role_stopped) {
	history->rsc->role = pcmk_role_unknown;
	}
	crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
	"Masked failure expired",
	history->id, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->exit_status == history->expected_exit_status) {
	return pcmk_rc_undetermined; // Only failures expire
	}

	if (history->interval_ms == 0) {
	crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
	"Expired failure",
	history->id, history->task, history->rsc->id,
	pe__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->node->details->online && !history->node->details->unclean) {
	/* Reschedule the recurring action. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has an op-digest (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s-interval %s of %s on %s "
	"after failure expired",
	pcmk__readable_interval(history->interval_ms), history->task,
	history->rsc->id, pe__node_name(history->node));
	crm_xml_add(history->xml, XML_LRM_ATTR_RESTART_DIGEST,
	"calculated-failure-timeout");
	return pcmk_rc_ok;
	}

	return pcmk_rc_undetermined;
	}

	/*!
	* \internal
	* \brief Process a masked probe failure
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	mask_probe_failure(struct action_history *history, int orig_exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	pcmk_resource_t *ban_rsc = history->rsc;

	if (!pcmk_is_set(history->rsc->flags, pcmk_rsc_unique)) {
	ban_rsc = uber_parent(history->rsc);
	}

	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
	pe__node_name(history->node));
	update_resource_state(history, history->expected_exit_status, last_failure,
	on_fail);
	crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->details->uname);

	record_failed_op(history);
	resource_location(ban_rsc, history->node, -INFINITY, "masked-probe-failure",
	history->rsc->cluster);
	}

	/*!
	* \internal Check whether a given failure is for a given pending action
	*
	* \param[in] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*
	* \return true if \p last_failure is failure of pending action in \p history,
	* otherwise false
	* \note Both \p history and \p last_failure must come from the same
	* lrm_resource block, as node and resource are assumed to be the same.
	*/
	static bool
	failure_is_newer(const struct action_history *history,
	const xmlNode *last_failure)
	{
	guint failure_interval_ms = 0U;
	long long failure_change = 0LL;
	long long this_change = 0LL;

	if (last_failure == NULL) {
	return false; // Resource has no last_failure entry
	}

	if (!pcmk__str_eq(history->task,
	crm_element_value(last_failure, XML_LRM_ATTR_TASK),
	pcmk__str_none)) {
	return false; // last_failure is for different action
	}

	if ((crm_element_value_ms(last_failure, XML_LRM_ATTR_INTERVAL_MS,
	&failure_interval_ms) != pcmk_ok)
	\|\| (history->interval_ms != failure_interval_ms)) {
	return false; // last_failure is for action with different interval
	}

	if ((pcmk__scan_ll(crm_element_value(history->xml, XML_RSC_OP_LAST_CHANGE),
	&this_change, 0LL) != pcmk_rc_ok)
	\|\| (pcmk__scan_ll(crm_element_value(last_failure,
	XML_RSC_OP_LAST_CHANGE),
	&failure_change, 0LL) != pcmk_rc_ok)
	\|\| (failure_change < this_change)) {
	return false; // Failure is not known to be newer
	}

	return true;
	}

	/*!
	* \internal
	* \brief Update a resource's role etc. for a pending action
	*
	* \param[in,out] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*/
	static void
	process_pending_action(struct action_history *history,
	const xmlNode *last_failure)
	{
	/* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
	* and there might be a RSC_monitor_INTERVAL entry with the last successful
	* or pending result.
	*
	* If last_failure contains the failure of the pending recurring monitor
	* we're processing here, and is newer, the action is no longer pending.
	* (Pending results have call ID -1, which sorts last, so the last failure
	* if any should be known.)
	*/
	if (failure_is_newer(history, last_failure)) {
	return;
	}

	if (strcmp(history->task, PCMK_ACTION_START) == 0) {
	pcmk__set_rsc_flags(history->rsc, pcmk_rsc_start_pending);
	set_active(history->rsc);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
	&& history->node->details->unclean) {
	/* A migrate_to action is pending on a unclean source, so force a stop
	* on the target.
	*/
	const char *migrate_target = NULL;
	pcmk_node_t *target = NULL;

	migrate_target = crm_element_value(history->xml,
	XML_LRM_ATTR_MIGRATE_TARGET);
	target = pe_find_node(history->rsc->cluster->nodes, migrate_target);
	if (target != NULL) {
	stop_action(history->rsc, target, FALSE);
	}
	}

	if (history->rsc->pending_task != NULL) {
	/* There should never be multiple pending actions, but as a failsafe,
	* just remember the first one processed for display purposes.
	*/
	return;
	}

	if (pcmk_is_probe(history->task, history->interval_ms)) {
	/* Pending probes are currently never displayed, even if pending
	* operations are requested. If we ever want to change that,
	* enable the below and the corresponding part of
	* native.c:native_pending_task().
	*/
	#if 0
	history->rsc->pending_task = strdup("probe");
	history->rsc->pending_node = history->node;
	#endif
	} else {
	history->rsc->pending_task = strdup(history->task);
	history->rsc->pending_node = history->node;
	}
	}

	static void
	unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail)
	{
	int old_rc = 0;
	bool expired = false;
	pcmk_resource_t *parent = rsc;
	enum rsc_role_e fail_role = pcmk_role_unknown;
	enum action_fail_response failure_strategy = pcmk_on_fail_restart;

	struct action_history history = {
	.rsc = rsc,
	.node = node,
	.xml = xml_op,
	.execution_status = PCMK_EXEC_UNKNOWN,
	};

	CRM_CHECK(rsc && node && xml_op, return);

	history.id = ID(xml_op);
	if (history.id == NULL) {
	pcmk__config_err("Ignoring resource history entry for %s on %s "
	"without ID", rsc->id, pe__node_name(node));
	return;
	}

	// Task and interval
	history.task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (history.task == NULL) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"without " XML_LRM_ATTR_TASK,
	history.id, rsc->id, pe__node_name(node));
	return;
	}
	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS,
	&(history.interval_ms));
	if (!can_affect_state(&history)) {
	pcmk__rsc_trace(rsc,
	"Ignoring resource history entry %s for %s on %s "
	"with irrelevant action '%s'",
	history.id, rsc->id, pe__node_name(node), history.task);
	return;
	}

	if (unpack_action_result(&history) != pcmk_rc_ok) {
	return; // Error already logged
	}

	history.expected_exit_status = pe__target_rc_from_xml(xml_op);
	history.key = pe__xe_history_key(xml_op);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &(history.call_id));

	pcmk__rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
	history.id, history.task, history.call_id,
	pe__node_name(node),
	pcmk_exec_status_str(history.execution_status),
	crm_exit_str(history.exit_status));

	if (node->details->unclean) {
	pcmk__rsc_trace(rsc,
	"%s is running on %s, which is unclean (further action "
	"depends on value of stop's on-fail attribute)",
	rsc->id, pe__node_name(node));
	}

	expired = check_operation_expiry(&history);
	old_rc = history.exit_status;

	remap_operation(&history, on_fail, expired);

	if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
	goto done;
	}

	if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	mask_probe_failure(&history, old_rc, *last_failure, on_fail);
	goto done;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	switch (history.execution_status) {
	case PCMK_EXEC_PENDING:
	process_pending_action(&history, *last_failure);
	goto done;

	case PCMK_EXEC_DONE:
	update_resource_state(&history, history.exit_status, *last_failure,
	on_fail);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if (failure_strategy == pcmk_on_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	CRM_XS " status=%d rc=%d id=%s",
	history.task, rsc->id, pe__node_name(node),
	history.execution_status, history.exit_status,
	history.id);
	/* Also for printing it as "FAILED" by marking it as
	* pcmk_rsc_failed later
	*/
	*on_fail = pcmk_on_fail_ban;
	}
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pcmk__set_rsc_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	default: // No other value should be possible at this point
	break;
	}

	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if ((failure_strategy == pcmk_on_fail_ignore)
	\|\| ((failure_strategy == pcmk_on_fail_restart_container)
	&& (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {

	char *last_change_s = last_change_str(xml_op);

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
	CRM_XS " %s",
	history.task, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), rsc->id, pe__node_name(node),
	last_change_s, history.id);
	free(last_change_s);

	update_resource_state(&history, history.expected_exit_status,
	*last_failure, on_fail);
	crm_xml_add(xml_op, PCMK_XA_UNAME, node->details->uname);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_ignore_failure);

	record_failed_op(&history);

	if ((failure_strategy == pcmk_on_fail_restart_container)
	&& cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);

	if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
	uint8_t log_level = LOG_ERR;

	if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
	log_level = LOG_NOTICE;
	}
	do_crm_log(log_level,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s) " CRM_XS " %s",
	parent->id, pe__node_name(node),
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);

	} else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
	pcmk__sched_err("Preventing %s from restarting anywhere because "
	"of fatal failure (%s%s%s) " CRM_XS " %s",
	parent->id,
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, NULL, -INFINITY, "fatal-error",
	rsc->cluster);
	}
	}

	done:
	pcmk__rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
	rsc->id, pe__node_name(node), history.id,
	role2text(rsc->role), role2text(rsc->next_role));
	}

	static void
	add_node_attrs(const xmlNode xml_obj, pcmk_node_t node, bool overwrite,
	pcmk_scheduler_t *scheduler)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_UNAME), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
	strdup(node->details->id));
	if (pcmk__str_eq(node->details->id, scheduler->dc_uuid, pcmk__str_casei)) {
	scheduler->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(PCMK_VALUE_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	- strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
	+ strdup(CRM_ATTR_IS_DC), strdup(PCMK_VALUE_FALSE));
	}

	cluster_name = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_CLUSTER_NAME);
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
	strdup(cluster_name));
	}

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
	node->details->attrs, NULL, overwrite,
	scheduler);

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
	node->details->utilization, NULL,
	FALSE, scheduler);

	if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
	const char *site_name = pe_node_attribute_raw(node, "site-name");

	if (site_name) {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(cluster_name));
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
	pcmk__str_none)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, PCMK_XA_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pcmk_scheduler_t *scheduler)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(scheduler->input, XML_CIB_TAG_STATUS, TRUE);

	pcmk_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	const char *uname = crm_element_value(node_state, PCMK_XA_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pe_find_node(scheduler->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);

	} else {
	determine_online_status(node_state, this_node, scheduler);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__str_eq((const char *)lrm_rsc->name,
	XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	const char *rsc_id = crm_element_value(lrm_rsc,
	PCMK_XA_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
	index 3bcc51aeef..b0da158703 100644
	--- a/tools/crm_resource_print.c
	+++ b/tools/crm_resource_print.c
	@@ -1,822 +1,822 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm_resource.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>

	#define cons_string(x) x?x:"NA"
	static int
	print_constraint(xmlNode xml_obj, void userdata)
	{
	pcmk_scheduler_t scheduler = (pcmk_scheduler_t ) userdata;
	pcmk__output_t *out = scheduler->priv;
	xmlNode *lifetime = NULL;
	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);

	if (id == NULL) {
	return pcmk_rc_ok;
	}

	// @COMPAT lifetime is deprecated
	lifetime = first_named_child(xml_obj, "lifetime");
	if (pe_evaluate_rules(lifetime, NULL, scheduler->now, NULL) == FALSE) {
	return pcmk_rc_ok;
	}

	if (!pcmk__xe_is(xml_obj, XML_CONS_TAG_RSC_DEPEND)) {
	return pcmk_rc_ok;
	}

	out->info(out, "Constraint %s %s %s %s %s %s %s",
	xml_obj->name,
	cons_string(crm_element_value(xml_obj, PCMK_XA_ID)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
	cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));

	return pcmk_rc_ok;
	}

	void
	cli_resource_print_cts_constraints(pcmk_scheduler_t *scheduler)
	{
	pcmk__xe_foreach_child(pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_CONSTRAINTS),
	NULL, print_constraint, scheduler);
	}

	void
	cli_resource_print_cts(pcmk_resource_t rsc, pcmk__output_t out)
	{
	const char *host = NULL;
	bool needs_quorum = TRUE;
	const char *rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	const char *rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	pcmk_node_t *node = pe__current_node(rsc);

	if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	needs_quorum = FALSE;
	} else {
	// @TODO check requires in resource meta-data and rsc_defaults
	}

	if (node != NULL) {
	host = node->details->uname;
	}

	out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld %#.16llx",
	rsc->xml->name, rsc->id,
	rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
	rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
	rsc->flags);

	g_list_foreach(rsc->children, (GFunc) cli_resource_print_cts, out);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print_operations(const char rsc_id, const char host_uname,
	bool active, pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_no_output;
	GList *ops = find_operations(rsc_id, host_uname, active, scheduler);

	if (!ops) {
	return rc;
	}

	out->begin_list(out, NULL, NULL, "Resource Operations");
	rc = pcmk_rc_ok;

	for (GList *lpc = ops; lpc != NULL; lpc = lpc->next) {
	xmlNode xml_op = (xmlNode ) lpc->data;
	out->message(out, "node-and-op", scheduler, xml_op);
	}

	out->end_list(out);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	bool expanded)
	{
	pcmk__output_t *out = scheduler->priv;
	uint32_t show_opts = pcmk_show_pending;
	GList *all = NULL;

	all = g_list_prepend(all, (gpointer) "*");

	out->begin_list(out, NULL, NULL, "Resource Config");
	out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc, all, all);
	out->message(out, "resource-config", rsc, !expanded);
	out->end_list(out);

	g_list_free(all);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Attributes");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_default(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	if (status == PCMK_EXEC_DONE) {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* returned <exit-code> (<exit-description>[: <exit-reason>])
	*/
	out->info(out, "Operation %s%s%s (%s%s%s:%s) returned %d (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, (int) rc, services_ocf_exitcode_str((int) rc),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	} else {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* could not be executed (<execution-status>[: <exit-reason>])
	*/
	out->err(out,
	"Operation %s%s%s (%s%s%s:%s) could not be executed (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, pcmk_exec_status_str(status),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_xml(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action G_GNUC_UNUSED = va_arg(args, const char );
	const char name G_GNUC_UNUSED = va_arg(args, const char );
	const char class G_GNUC_UNUSED = va_arg(args, const char );
	const char provider G_GNUC_UNUSED = va_arg(args, const char );
	const char type G_GNUC_UNUSED = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	char *exit_str = pcmk__itoa(rc);
	char *status_str = pcmk__itoa(status);

	pcmk__output_create_xml_node(out, "agent-status",
	"code", exit_str,
	"message", services_ocf_exitcode_str((int) rc),
	"execution_code", status_str,
	"execution_message", pcmk_exec_status_str(status),
	"reason", exit_reason,
	NULL);

	free(exit_str);
	free(status_str);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}
	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_default(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	if (rsc_name == NULL) {
	out->list_item(out, NULL, "Overriding the cluster configuration with '%s' = '%s'",
	name, value);
	} else {
	out->list_item(out, NULL, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
	rsc_name, name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_xml(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "override",
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (rsc_name != NULL) {
	crm_xml_add(node, "rsc", rsc_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Properties");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, const char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_default(pcmk__output_t *out, va_list args) {
	int verbose = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, "overrides");

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	/* hide output for validate-all if not in verbose */
	if ((verbose == 0)
	&& pcmk__str_eq(action, PCMK_ACTION_VALIDATE_ALL, pcmk__str_casei)) {
	return pcmk_rc_ok;
	}

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	doc = string2xml(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, "command", stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_xml(pcmk__output_t *out, va_list args) {
	int verbose G_GNUC_UNUSED = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource-agent-action",
	"action", action,
	PCMK_XA_CLASS, class,
	PCMK_XA_TYPE, type,
	NULL);

	if (rsc_name) {
	crm_xml_add(node, "rsc", rsc_name);
	}

	crm_xml_add(node, PCMK_XA_PROVIDER, provider);

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, "overrides");

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	doc = string2xml(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, "command", stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_default(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	if (checks->flags == 0) {
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Checks");

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
	parent->id, checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	out->list_item(out, "check",
	"'%s' cannot run on unhealthy nodes due to "
	PCMK_OPT_NODE_HEALTH_STRATEGY "='%s'",
	parent->id,
	pe_pref(checks->rsc->cluster->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY));
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_xml(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "check",
	PCMK_XA_ID, parent->id,
	NULL);

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	pcmk__xe_set_bool_attr(node, "remain_stopped", true);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	pcmk__xe_set_bool_attr(node, "promotable", false);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	pcmk__xe_set_bool_attr(node, "unmanaged", true);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	crm_xml_add(node, "locked-to", checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	pcmk__xe_set_bool_attr(node, "unhealthy", true);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const char ")
	static int
	resource_search_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const char requested_name = va_arg(args, const char );

	bool printed = false;
	int rc = pcmk_rc_no_output;

	if (!out->is_quiet(out) && nodes == NULL) {
	out->err(out, "resource %s is NOT running", requested_name);
	return rc;
	}

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;

	if (!printed) {
	out->begin_list(out, NULL, NULL, "Nodes");
	printed = true;
	rc = pcmk_rc_ok;
	}

	if (out->is_quiet(out)) {
	out->list_item(out, "node", "%s", ni->node_name);
	} else {
	const char *role_text = "";

	if (ni->promoted) {
	#ifdef PCMK__COMPAT_2_0
	role_text = " " PCMK__ROLE_PROMOTED_LEGACY;
	#else
	role_text = " " PCMK__ROLE_PROMOTED;
	#endif
	}
	out->list_item(out, "node", "resource %s is running on: %s%s",
	requested_name, ni->node_name, role_text);
	}
	}

	if (printed) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const char ")
	static int
	resource_search_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const char requested_name = va_arg(args, const char );

	pcmk__output_xml_create_parent(out, "nodes",
	"resource", requested_name,
	NULL);

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;
	xmlNodePtr sub_node = pcmk__output_create_xml_text_node(out, "node", ni->node_name);

	if (ni->promoted) {
	crm_xml_add(sub_node, "state", "promoted");
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_default(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	out->begin_list(out, NULL, NULL, "Resource Reasons");

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	rsc->fns->location(rsc, &hosts, TRUE);

	if (hosts == NULL) {
	out->list_item(out, "reason", "Resource %s is not running", rsc->id);
	} else {
	out->list_item(out, "reason", "Resource %s is running", rsc->id);
	}

	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	hosts = NULL;
	}

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	} else {
	out->list_item(out, "reason", "Resource %s is not running on host %s",
	rsc->id, host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	out->list_item(out, "reason", "Resource %s is %srunning",
	rsc->id, (hosts? "" : "not "));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_xml(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	xmlNodePtr xml_node = pcmk__output_xml_create_parent(out, "reason", NULL);

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	pcmk__output_xml_create_parent(out, "resources", NULL);

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	rsc->fns->location(rsc, &hosts, TRUE);

	pcmk__output_xml_create_parent(out, "resource",
	PCMK_XA_ID, rsc->id,
	"running", pcmk__btoa(hosts != NULL),
	NULL);

	cli_resource_check(out, rsc, NULL);
	pcmk__output_xml_pop_parent(out);
	g_list_free(hosts);
	hosts = NULL;
	}

	pcmk__output_xml_pop_parent(out);

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	crm_xml_add(xml_node, "running_on", host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	pcmk__output_xml_create_parent(out, "resources", NULL);

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, "resource",
	PCMK_XA_ID, rsc->id,
	"running", PCMK_VALUE_TRUE,
	"host", host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, "resource",
	PCMK_XA_ID, rsc->id,
	- "running", "false",
	+ "running", PCMK_VALUE_FALSE,
	"host", host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	pcmk__output_xml_pop_parent(out);
	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	crm_xml_add(xml_node, "running", pcmk__btoa(hosts != NULL));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	return pcmk_rc_ok;
	}

	static void
	add_resource_name(pcmk_resource_t rsc, pcmk__output_t out)
	{
	if (rsc->children == NULL) {
	out->list_item(out, "resource", "%s", rsc->id);
	} else {
	g_list_foreach(rsc->children, (GFunc) add_resource_name, out);
	}
	}

	PCMK__OUTPUT_ARGS("resource-names-list", "GList *")
	static int
	resource_names(pcmk__output_t *out, va_list args) {
	GList resources = va_arg(args, GList );

	if (resources == NULL) {
	out->err(out, "NO resources configured\n");
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Names");
	g_list_foreach(resources, (GFunc) add_resource_name, out);
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "agent-status", "default", agent_status_default },
	{ "agent-status", "xml", agent_status_xml },
	{ "attribute-list", "default", attribute_list_default },
	{ "attribute-list", "text", attribute_list_text },
	{ "override", "default", override_default },
	{ "override", "xml", override_xml },
	{ "property-list", "default", property_list_default },
	{ "property-list", "text", property_list_text },
	{ "resource-agent-action", "default", resource_agent_action_default },
	{ "resource-agent-action", "xml", resource_agent_action_xml },
	{ "resource-check-list", "default", resource_check_list_default },
	{ "resource-check-list", "xml", resource_check_list_xml },
	{ "resource-search-list", "default", resource_search_list_default },
	{ "resource-search-list", "xml", resource_search_list_xml },
	{ "resource-reasons-list", "default", resource_reasons_list_default },
	{ "resource-reasons-list", "xml", resource_reasons_list_xml },
	{ "resource-names-list", "default", resource_names },

	{ NULL, NULL, NULL }
	};

	void
	crm_resource_register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/tools/crm_ticket.c b/tools/crm_ticket.c
	index 59bf2236b5..9deebee4c8 100644
	--- a/tools/crm_ticket.c
	+++ b/tools/crm_ticket.c
	@@ -1,1013 +1,1015 @@
	/*
	* Copyright 2012-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/cmdline_internal.h>

	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>

	#include <pacemaker-internal.h>

	GError *error = NULL;

	#define SUMMARY "Perform tasks related to cluster tickets\n\n" \
	"Allows ticket attributes to be queried, modified and deleted."

	struct {
	gchar *attr_default;
	gchar *attr_id;
	char *attr_name;
	char *attr_value;
	gboolean force;
	char *get_attr_name;
	gboolean quiet;
	gchar *set_name;
	char ticket_cmd;
	gchar *ticket_id;
	gchar *xml_file;
	} options = {
	.ticket_cmd = 'S'
	};

	GList *attr_delete;
	GHashTable *attr_set;
	bool modified = false;
	int cib_options = cib_sync_call;

	#define INDENT " "

	static gboolean
	attr_value_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&options.attr_value, optarg);

	if (!options.attr_name \|\| !options.attr_value) {
	return TRUE;
	}

	g_hash_table_insert(attr_set, strdup(options.attr_name), strdup(options.attr_value));
	pcmk__str_update(&options.attr_name, NULL);
	pcmk__str_update(&options.attr_value, NULL);

	modified = true;

	return TRUE;
	}

	static gboolean
	command_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	if (pcmk__str_any_of(option_name, "--info", "-l", NULL)) {
	options.ticket_cmd = 'l';
	} else if (pcmk__str_any_of(option_name, "--details", "-L", NULL)) {
	options.ticket_cmd = 'L';
	} else if (pcmk__str_any_of(option_name, "--raw", "-w", NULL)) {
	options.ticket_cmd = 'w';
	} else if (pcmk__str_any_of(option_name, "--query-xml", "-q", NULL)) {
	options.ticket_cmd = 'q';
	} else if (pcmk__str_any_of(option_name, "--constraints", "-c", NULL)) {
	options.ticket_cmd = 'c';
	} else if (pcmk__str_any_of(option_name, "--cleanup", "-C", NULL)) {
	options.ticket_cmd = 'C';
	}

	return TRUE;
	}

	static gboolean
	delete_attr_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	attr_delete = g_list_append(attr_delete, strdup(optarg));
	modified = true;
	return TRUE;
	}

	static gboolean
	get_attr_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&options.get_attr_name, optarg);
	options.ticket_cmd = 'G';
	return TRUE;
	}

	static gboolean
	grant_standby_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	if (pcmk__str_any_of(option_name, "--grant", "-g", NULL)) {
	g_hash_table_insert(attr_set, strdup("granted"),
	strdup(PCMK_VALUE_TRUE));
	modified = true;
	} else if (pcmk__str_any_of(option_name, "--revoke", "-r", NULL)) {
	- g_hash_table_insert(attr_set, strdup("granted"), strdup("false"));
	+ g_hash_table_insert(attr_set, strdup("granted"),
	+ strdup(PCMK_VALUE_FALSE));
	modified = true;
	} else if (pcmk__str_any_of(option_name, "--standby", "-s", NULL)) {
	g_hash_table_insert(attr_set, strdup("standby"),
	strdup(PCMK_VALUE_TRUE));
	modified = true;
	} else if (pcmk__str_any_of(option_name, "--activate", "-a", NULL)) {
	- g_hash_table_insert(attr_set, strdup("standby"), strdup("false"));
	+ g_hash_table_insert(attr_set, strdup("standby"),
	+ strdup(PCMK_VALUE_FALSE));
	modified = true;
	}

	return TRUE;
	}

	static gboolean
	set_attr_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&options.attr_name, optarg);

	if (!options.attr_name \|\| !options.attr_value) {
	return TRUE;
	}

	g_hash_table_insert(attr_set, strdup(options.attr_name), strdup(options.attr_value));
	pcmk__str_update(&options.attr_name, NULL);
	pcmk__str_update(&options.attr_value, NULL);

	modified = true;

	return TRUE;
	}

	static GOptionEntry query_entries[] = {
	{ "info", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the information of ticket(s)",
	NULL },

	{ "details", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the details of ticket(s)",
	NULL },

	{ "raw", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the IDs of ticket(s)",
	NULL },

	{ "query-xml", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Query the XML of ticket(s)",
	NULL },

	{ "constraints", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the rsc_ticket constraints that apply to ticket(s)",
	NULL },

	{ NULL }
	};

	static GOptionEntry command_entries[] = {
	{ "grant", 'g', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Grant a ticket to this cluster site",
	NULL },

	{ "revoke", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Revoke a ticket from this cluster site",
	NULL },

	{ "standby", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Tell this cluster site this ticket is standby",
	NULL },

	{ "activate", 'a', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, grant_standby_cb,
	"Tell this cluster site this ticket is active",
	NULL },

	{ NULL }
	};

	static GOptionEntry advanced_entries[] = {
	{ "get-attr", 'G', 0, G_OPTION_ARG_CALLBACK, get_attr_cb,
	"Display the named attribute for a ticket",
	"ATTRIBUTE" },

	{ "set-attr", 'S', 0, G_OPTION_ARG_CALLBACK, set_attr_cb,
	"Set the named attribute for a ticket",
	"ATTRIBUTE" },

	{ "delete-attr", 'D', 0, G_OPTION_ARG_CALLBACK, delete_attr_cb,
	"Delete the named attribute for a ticket",
	"ATTRIBUTE" },

	{ "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Delete all state of a ticket at this cluster site",
	NULL },

	{ NULL}
	};

	static GOptionEntry addl_entries[] = {
	{ "attr-value", 'v', 0, G_OPTION_ARG_CALLBACK, attr_value_cb,
	"Attribute value to use with -S",
	"VALUE" },

	{ "default", 'd', 0, G_OPTION_ARG_STRING, &options.attr_default,
	"(Advanced) Default attribute value to display if none is found\n"
	INDENT "(for use with -G)",
	"VALUE" },

	{ "force", 'f', 0, G_OPTION_ARG_NONE, &options.force,
	"(Advanced) Force the action to be performed",
	NULL },

	{ "ticket", 't', 0, G_OPTION_ARG_STRING, &options.ticket_id,
	"Ticket ID",
	"ID" },

	{ "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING, &options.xml_file,
	NULL,
	NULL },

	{ NULL }
	};

	static GOptionEntry deprecated_entries[] = {
	{ "set-name", 'n', 0, G_OPTION_ARG_STRING, &options.set_name,
	"(Advanced) ID of the instance_attributes object to change",
	"ID" },

	{ "nvpair", 'i', 0, G_OPTION_ARG_STRING, &options.attr_id,
	"(Advanced) ID of the nvpair object to change/delete",
	"ID" },

	{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &options.quiet,
	"Print only the value on stdout",
	NULL },

	{ NULL }
	};

	static pcmk_ticket_t *
	find_ticket(gchar ticket_id, pcmk_scheduler_t scheduler)
	{
	return g_hash_table_lookup(scheduler->tickets, ticket_id);
	}

	static void
	print_date(time_t time)
	{
	int lpc = 0;
	char date_str[26];

	asctime_r(localtime(&time), date_str);
	for (; lpc < 26; lpc++) {
	if (date_str[lpc] == '\n') {
	date_str[lpc] = 0;
	}
	}
	fprintf(stdout, "'%s'", date_str);
	}

	static void
	print_ticket(pcmk_ticket_t *ticket, bool raw, bool details)
	{
	if (raw) {
	fprintf(stdout, "%s\n", ticket->id);
	return;
	}

	fprintf(stdout, "%s\t%s %s",
	ticket->id, ticket->granted ? "granted" : "revoked",
	ticket->standby ? "[standby]" : " ");

	if (details && g_hash_table_size(ticket->state) > 0) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;
	int lpc = 0;

	fprintf(stdout, " (");

	g_hash_table_iter_init(&iter, ticket->state);
	while (g_hash_table_iter_next(&iter, (void )&name, (void )&value)) {
	if (lpc > 0) {
	fprintf(stdout, ", ");
	}
	fprintf(stdout, "%s=", name);
	if (pcmk__str_any_of(name, "last-granted", "expires", NULL)) {
	long long time_ll;

	pcmk__scan_ll(value, &time_ll, 0);
	print_date((time_t) time_ll);
	} else {
	fprintf(stdout, "%s", value);
	}
	lpc++;
	}

	fprintf(stdout, ")\n");

	} else {
	if (ticket->last_granted > -1) {
	fprintf(stdout, " last-granted=");
	print_date(ticket->last_granted);
	}
	fprintf(stdout, "\n");
	}

	return;
	}

	static void
	print_ticket_list(pcmk_scheduler_t *scheduler, bool raw, bool details)
	{
	GHashTableIter iter;
	pcmk_ticket_t *ticket = NULL;

	g_hash_table_iter_init(&iter, scheduler->tickets);

	while (g_hash_table_iter_next(&iter, NULL, (void **)&ticket)) {
	print_ticket(ticket, raw, details);
	}
	}

	static int
	find_ticket_state(cib_t * the_cib, gchar ticket_id, xmlNode * ticket_state_xml)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;

	GString *xpath = NULL;

	CRM_ASSERT(ticket_state_xml != NULL);
	*ticket_state_xml = NULL;

	xpath = g_string_sized_new(1024);
	g_string_append(xpath,
	"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS
	"/" XML_CIB_TAG_TICKETS);

	if (ticket_id != NULL) {
	pcmk__g_strcat(xpath,
	"/" XML_CIB_TAG_TICKET_STATE
	"[@" PCMK_XA_ID "=\"", ticket_id, "\"]", NULL);
	}

	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);
	g_string_free(xpath, TRUE);

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_search->children != NULL) {
	if (ticket_id) {
	fprintf(stdout, "Multiple ticket_states match ticket_id=%s\n", ticket_id);
	}
	*ticket_state_xml = xml_search;
	} else {
	*ticket_state_xml = xml_search;
	}
	return rc;
	}

	static int
	find_ticket_constraints(cib_t * the_cib, gchar ticket_id, xmlNode * ticket_cons_xml)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;

	GString *xpath = NULL;
	const char *xpath_base = NULL;

	CRM_ASSERT(ticket_cons_xml != NULL);
	*ticket_cons_xml = NULL;

	xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_CONSTRAINTS);
	if (xpath_base == NULL) {
	crm_err(XML_CIB_TAG_CONSTRAINTS " CIB element not known (bug?)");
	return -ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	pcmk__g_strcat(xpath, xpath_base, "/" XML_CONS_TAG_RSC_TICKET, NULL);

	if (ticket_id != NULL) {
	pcmk__g_strcat(xpath,
	"[@" XML_TICKET_ATTR_TICKET "=\"", ticket_id, "\"]",
	NULL);
	}

	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);
	g_string_free(xpath, TRUE);

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_log_xml_debug(xml_search, "Match");
	*ticket_cons_xml = xml_search;

	return rc;
	}

	static int
	dump_ticket_xml(cib_t * the_cib, gchar *ticket_id)
	{
	int rc = pcmk_rc_ok;
	xmlNode *state_xml = NULL;

	rc = find_ticket_state(the_cib, ticket_id, &state_xml);

	if (state_xml == NULL) {
	return rc;
	}

	fprintf(stdout, "State XML:\n");
	if (state_xml) {
	char *state_xml_str = NULL;

	state_xml_str = dump_xml_formatted(state_xml);
	fprintf(stdout, "\n%s", state_xml_str);
	free_xml(state_xml);
	free(state_xml_str);
	}

	return rc;
	}

	static int
	dump_constraints(cib_t * the_cib, gchar *ticket_id)
	{
	int rc = pcmk_rc_ok;
	xmlNode *cons_xml = NULL;
	char *cons_xml_str = NULL;

	rc = find_ticket_constraints(the_cib, ticket_id, &cons_xml);

	if (cons_xml == NULL) {
	return rc;
	}

	cons_xml_str = dump_xml_formatted(cons_xml);
	fprintf(stdout, "Constraints XML:\n\n%s", cons_xml_str);
	free_xml(cons_xml);
	free(cons_xml_str);

	return rc;
	}

	static int
	get_ticket_state_attr(gchar ticket_id, const char attr_name, const char **attr_value,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_ticket_t *ticket = NULL;

	CRM_ASSERT(attr_value != NULL);
	*attr_value = NULL;

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	return ENXIO;
	}

	*attr_value = g_hash_table_lookup(ticket->state, attr_name);
	if (*attr_value == NULL) {
	return ENXIO;
	}

	return pcmk_rc_ok;
	}

	static void
	ticket_warning(gchar ticket_id, const char action)
	{
	GString *warning = g_string_sized_new(1024);
	const char *word = NULL;

	CRM_ASSERT(action != NULL);

	if (strcmp(action, "grant") == 0) {
	pcmk__g_strcat(warning,
	"This command cannot help you verify whether '",
	ticket_id,
	"' has been already granted elsewhere.\n", NULL);
	word = "to";

	} else {
	pcmk__g_strcat(warning,
	"Revoking '", ticket_id, "' can trigger the specified "
	"'loss-policy'(s) relating to '", ticket_id, "'.\n\n"
	"You can check that with:\n"
	"crm_ticket --ticket ", ticket_id, " --constraints\n\n"
	"Otherwise before revoking '", ticket_id, "', "
	"you may want to make '", ticket_id, "' "
	"standby with:\n"
	"crm_ticket --ticket ", ticket_id, " --standby\n\n",
	NULL);
	word = "from";
	}

	pcmk__g_strcat(warning,
	"If you really want to ", action, " '", ticket_id, "' ",
	word, " this site now, and you know what you are doing,\n"
	"please specify --force.", NULL);

	fprintf(stdout, "%s\n", (const char *) warning->str);

	g_string_free(warning, TRUE);
	}

	static bool
	allow_modification(gchar *ticket_id)
	{
	const char *value = NULL;
	GList *list_iter = NULL;

	if (options.force) {
	return true;
	}

	if (g_hash_table_lookup_extended(attr_set, "granted", NULL, (gpointer *) & value)) {
	if (crm_is_true(value)) {
	ticket_warning(ticket_id, "grant");
	return false;

	} else {
	ticket_warning(ticket_id, "revoke");
	return false;
	}
	}

	for(list_iter = attr_delete; list_iter; list_iter = list_iter->next) {
	const char key = (const char )list_iter->data;

	if (pcmk__str_eq(key, "granted", pcmk__str_casei)) {
	ticket_warning(ticket_id, "revoke");
	return false;
	}
	}

	return true;
	}

	static int
	modify_ticket_state(gchar ticket_id, cib_t cib, pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_top = NULL;
	xmlNode *ticket_state_xml = NULL;
	bool found = false;

	GList *list_iter = NULL;
	GHashTableIter hash_iter;

	char *key = NULL;
	char *value = NULL;

	pcmk_ticket_t *ticket = NULL;

	rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
	if (rc == pcmk_rc_ok) {
	crm_debug("Found a match state for ticket: id=%s", ticket_id);
	xml_top = ticket_state_xml;
	found = true;

	} else if (rc != ENXIO) {
	return rc;

	} else if (g_hash_table_size(attr_set) == 0){
	return pcmk_rc_ok;

	} else {
	xmlNode *xml_obj = NULL;

	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
	ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
	crm_xml_add(ticket_state_xml, PCMK_XA_ID, ticket_id);
	}

	for(list_iter = attr_delete; list_iter; list_iter = list_iter->next) {
	const char key = (const char )list_iter->data;
	xml_remove_prop(ticket_state_xml, key);
	}

	ticket = find_ticket(ticket_id, scheduler);

	g_hash_table_iter_init(&hash_iter, attr_set);
	while (g_hash_table_iter_next(&hash_iter, (gpointer ) & key, (gpointer ) & value)) {
	crm_xml_add(ticket_state_xml, key, value);

	if (pcmk__str_eq(key, "granted", pcmk__str_casei)
	&& (ticket == NULL \|\| ticket->granted == FALSE)
	&& crm_is_true(value)) {

	char *now = pcmk__ttoa(time(NULL));

	crm_xml_add(ticket_state_xml, "last-granted", now);
	free(now);
	}
	}

	if (found && (attr_delete != NULL)) {
	crm_log_xml_debug(xml_top, "Replace");
	rc = cib->cmds->replace(cib, XML_CIB_TAG_STATUS, ticket_state_xml, cib_options);
	rc = pcmk_legacy2rc(rc);

	} else {
	crm_log_xml_debug(xml_top, "Update");
	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
	rc = pcmk_legacy2rc(rc);
	}

	free_xml(xml_top);
	return rc;
	}

	static int
	delete_ticket_state(gchar ticket_id, cib_t cib)
	{
	xmlNode *ticket_state_xml = NULL;

	int rc = pcmk_rc_ok;

	rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);

	if (rc == ENXIO) {
	return pcmk_rc_ok;

	} else if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_log_xml_debug(ticket_state_xml, "Delete");

	rc = cib->cmds->remove(cib, XML_CIB_TAG_STATUS, ticket_state_xml, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	fprintf(stdout, "Cleaned up %s\n", ticket_id);
	}

	free_xml(ticket_state_xml);
	return rc;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t *args) {
	GOptionContext *context = NULL;

	const char *description = "Examples:\n\n"
	"Display the info of tickets:\n\n"
	"\tcrm_ticket --info\n\n"
	"Display the detailed info of tickets:\n\n"
	"\tcrm_ticket --details\n\n"
	"Display the XML of 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --query-xml\n\n"
	"Display the rsc_ticket constraints that apply to 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --constraints\n\n"
	"Grant 'ticketA' to this cluster site:\n\n"
	"\tcrm_ticket --ticket ticketA --grant\n\n"
	"Revoke 'ticketA' from this cluster site:\n\n"
	"\tcrm_ticket --ticket ticketA --revoke\n\n"
	"Make 'ticketA' standby (the cluster site will treat a granted\n"
	"'ticketA' as 'standby', and the dependent resources will be\n"
	"stopped or demoted gracefully without triggering loss-policies):\n\n"
	"\tcrm_ticket --ticket ticketA --standby\n\n"
	"Activate 'ticketA' from being standby:\n\n"
	"\tcrm_ticket --ticket ticketA --activate\n\n"
	"Get the value of the 'granted' attribute for 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --get-attr granted\n\n"
	"Set the value of the 'standby' attribute for 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --set-attr standby --attr-value true\n\n"
	"Delete the 'granted' attribute for 'ticketA':\n\n"
	"\tcrm_ticket --ticket ticketA --delete-attr granted\n\n"
	"Erase the operation history of 'ticketA' at this cluster site,\n"
	"causing the cluster site to 'forget' the existing ticket state:\n\n"
	"\tcrm_ticket --ticket ticketA --cleanup\n\n";

	context = pcmk__build_arg_context(args, NULL, NULL, NULL);
	g_option_context_set_description(context, description);

	pcmk__add_arg_group(context, "queries", "Queries:",
	"Show queries", query_entries);
	pcmk__add_arg_group(context, "commands", "Commands:",
	"Show command options", command_entries);
	pcmk__add_arg_group(context, "advanced", "Advanced Options:",
	"Show advanced options", advanced_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
	"Show deprecated options", deprecated_entries);

	return context;
	}

	int
	main(int argc, char **argv)
	{
	pcmk_scheduler_t *scheduler = NULL;
	xmlNode *cib_xml_copy = NULL;

	cib_t *cib_conn = NULL;
	crm_exit_t exit_code = CRM_EX_OK;
	int rc = pcmk_rc_ok;

	pcmk__common_args_t *args = NULL;
	GOptionContext *context = NULL;
	gchar **processed_args = NULL;

	attr_set = pcmk__strkey_table(free, free);
	attr_delete = NULL;

	args = pcmk__new_common_args(SUMMARY);
	context = build_arg_context(args);
	processed_args = pcmk__cmdline_preproc(argv, "dintvxCDGS");

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_ticket", args->verbosity);

	if (args->version) {
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);
	/* FIXME: When crm_ticket is converted to use formatted output, this can go. */
	pcmk__cli_help('v');
	}

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = errno;
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not allocate scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}
	pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);

	cib_conn = cib_new();
	if (cib_conn == NULL) {
	exit_code = CRM_EX_DISCONNECT;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not connect to the CIB manager");
	goto done;
	}

	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not connect to the CIB: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	if (options.xml_file != NULL) {
	cib_xml_copy = filename2xml(options.xml_file);

	} else {
	rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Could not get local CIB: %s",
	pcmk_rc_str(rc));
	goto done;
	}
	}

	if (!cli_config_update(&cib_xml_copy, NULL, FALSE)) {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not update local CIB to latest schema version");
	goto done;
	}

	scheduler->input = cib_xml_copy;
	scheduler->now = crm_time_new(NULL);

	cluster_status(scheduler);

	/* For recording the tickets that are referenced in rsc_ticket constraints
	* but have never been granted yet. */
	pcmk__unpack_constraints(scheduler);

	if (options.ticket_cmd == 'l' \|\| options.ticket_cmd == 'L' \|\| options.ticket_cmd == 'w') {
	bool raw = false;
	bool details = false;

	if (options.ticket_cmd == 'L') {
	details = true;
	} else if (options.ticket_cmd == 'w') {
	raw = true;
	}

	if (options.ticket_id) {
	pcmk_ticket_t *ticket = find_ticket(options.ticket_id, scheduler);

	if (ticket == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"No such ticket '%s'", options.ticket_id);
	goto done;
	}
	print_ticket(ticket, raw, details);

	} else {
	print_ticket_list(scheduler, raw, details);
	}

	} else if (options.ticket_cmd == 'q') {
	rc = dump_ticket_xml(cib_conn, options.ticket_id);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not query ticket XML: %s", pcmk_rc_str(rc));
	}

	} else if (options.ticket_cmd == 'c') {
	rc = dump_constraints(cib_conn, options.ticket_id);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not show ticket constraints: %s", pcmk_rc_str(rc));
	}

	} else if (options.ticket_cmd == 'G') {
	const char *value = NULL;

	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	rc = get_ticket_state_attr(options.ticket_id, options.get_attr_name,
	&value, scheduler);
	if (rc == pcmk_rc_ok) {
	fprintf(stdout, "%s\n", value);
	} else if (rc == ENXIO && options.attr_default) {
	fprintf(stdout, "%s\n", options.attr_default);
	rc = pcmk_rc_ok;
	}
	exit_code = pcmk_rc2exitc(rc);

	} else if (options.ticket_cmd == 'C') {
	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	if (options.force == FALSE) {
	pcmk_ticket_t *ticket = NULL;

	ticket = find_ticket(options.ticket_id, scheduler);
	if (ticket == NULL) {
	exit_code = CRM_EX_NOSUCH;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"No such ticket '%s'", options.ticket_id);
	goto done;
	}

	if (ticket->granted) {
	ticket_warning(options.ticket_id, "revoke");
	exit_code = CRM_EX_INSUFFICIENT_PRIV;
	goto done;
	}
	}

	rc = delete_ticket_state(options.ticket_id, cib_conn);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not clean up ticket: %s", pcmk_rc_str(rc));
	}

	} else if (modified) {
	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	if (options.attr_value
	&& (pcmk__str_empty(options.attr_name))) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply attribute name with -S for -v %s", options.attr_value);
	goto done;
	}

	if (options.attr_name
	&& (pcmk__str_empty(options.attr_value))) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply attribute value with -v for -S %s", options.attr_value);
	goto done;
	}

	if (!allow_modification(options.ticket_id)) {
	exit_code = CRM_EX_INSUFFICIENT_PRIV;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Ticket modification not allowed");
	goto done;
	}

	rc = modify_ticket_state(options.ticket_id, cib_conn, scheduler);
	exit_code = pcmk_rc2exitc(rc);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not modify ticket: %s", pcmk_rc_str(rc));
	}

	} else if (options.ticket_cmd == 'S') {
	/* Correct usage was handled in the "if (modified)" block above, so
	* this is just for reporting usage errors
	*/

	if (pcmk__str_empty(options.attr_name)) {
	// We only get here if ticket_cmd was left as default
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Must supply a command");
	goto done;
	}

	if (options.ticket_id == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply ticket ID with -t");
	goto done;
	}

	if (pcmk__str_empty(options.attr_value)) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must supply value with -v for -S %s", options.attr_name);
	goto done;
	}

	} else {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Unknown command: %c", options.ticket_cmd);
	}

	done:
	if (attr_set) {
	g_hash_table_destroy(attr_set);
	}
	attr_set = NULL;

	if (attr_delete) {
	g_list_free_full(attr_delete, free);
	}
	attr_delete = NULL;

	pe_free_working_set(scheduler);
	scheduler = NULL;

	cib__clean_up_connection(&cib_conn);

	g_strfreev(processed_args);
	pcmk__free_arg_context(context);
	g_free(options.attr_default);
	g_free(options.attr_id);
	free(options.attr_name);
	free(options.attr_value);
	free(options.get_attr_name);
	g_free(options.set_name);
	g_free(options.ticket_id);
	g_free(options.xml_file);

	pcmk__output_and_clear_error(&error, NULL);

	crm_exit(exit_code);
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:28 PM (7 h, 12 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1986752
Default Alt Text: (741 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions