No OneTemporary
Actions

Size

1 MB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c
	index 948548f0ab..5aab5f0d47 100644
	--- a/daemons/controld/controld_cib.c
	+++ b/daemons/controld/controld_cib.c
	@@ -1,1067 +1,1067 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h> /* sleep */

	#include <crm/common/alerts_internal.h>
	#include <crm/common/xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/lrmd_internal.h>

	#include <pacemaker-controld.h>

	// Call ID of the most recent in-progress CIB resource update (or 0 if none)
	static int pending_rsc_update = 0;

	/*!
	* \internal
	* \brief Respond to a dropped CIB connection
	*
	* \param[in] user_data CIB connection that dropped
	*/
	static void
	handle_cib_disconnect(gpointer user_data)
	{
	CRM_LOG_ASSERT(user_data == controld_globals.cib_conn);

	controld_trigger_fsa();
	controld_globals.cib_conn->state = cib_disconnected;

	if (pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	// @TODO This should trigger a reconnect, not a shutdown
	crm_crit("Lost connection to the CIB manager, shutting down");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	} else { // Expected
	crm_info("Disconnected from the CIB manager");
	}
	}

	static void
	do_cib_updated(const char event, xmlNode msg)
	{
	const xmlNode *patchset = NULL;
	const char *client_name = NULL;

	crm_debug("Received CIB diff notification: DC=%s", pcmk__btoa(AM_I_DC));

	if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
	return;
	}

	if (cib__element_in_patchset(patchset, XML_CIB_TAG_ALERTS)
	\|\| cib__element_in_patchset(patchset, XML_CIB_TAG_CRMCONFIG)) {

	controld_trigger_config();
	}

	if (!AM_I_DC) {
	// We're not in control of the join sequence
	return;
	}

	client_name = crm_element_value(msg, F_CIB_CLIENTNAME);
	if (!cib__client_triggers_refresh(client_name)) {
	// The CIB is still accurate
	return;
	}

	if (cib__element_in_patchset(patchset, XML_CIB_TAG_NODES)
	\|\| cib__element_in_patchset(patchset, XML_CIB_TAG_STATUS)) {

	/* An unsafe client modified the nodes or status section. Ensure the
	* node list is up-to-date, and start the join process again so we get
	* everyone's current resource history.
	*/
	if (client_name == NULL) {
	client_name = crm_element_value(msg, F_CIB_CLIENTID);
	}
	crm_notice("Populating nodes and starting an election after %s event "
	"triggered by %s",
	event, pcmk__s(client_name, "(unidentified client)"));

	populate_cib_nodes(node_update_quick\|node_update_all, __func__);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	void
	controld_disconnect_cib_manager(void)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	CRM_ASSERT(cib_conn != NULL);

	crm_debug("Disconnecting from the CIB manager");

	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
	do_cib_updated);
	cib_free_callbacks(cib_conn);

	if (cib_conn->state != cib_disconnected) {
	cib_conn->cmds->set_secondary(cib_conn,
	cib_scope_local\|cib_discard_reply);
	cib_conn->cmds->signoff(cib_conn);
	}
	}

	/* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
	void
	do_cib_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static int cib_retries = 0;

	cib_t *cib_conn = controld_globals.cib_conn;

	void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect;
	void (update_cb) (const char event, xmlNodePtr msg) = do_cib_updated;

	int rc = pcmk_ok;

	CRM_ASSERT(cib_conn != NULL);

	if (pcmk_is_set(action, A_CIB_STOP)) {
	if ((cib_conn->state != cib_disconnected)
	&& (pending_rsc_update != 0)) {

	crm_info("Waiting for resource update %d to complete",
	pending_rsc_update);
	crmd_fsa_stall(FALSE);
	return;
	}
	controld_disconnect_cib_manager();
	}

	if (!pcmk_is_set(action, A_CIB_START)) {
	return;
	}

	if (cur_state == S_STOPPING) {
	crm_err("Ignoring request to connect to the CIB manager after "
	"shutdown");
	return;
	}

	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);

	if (rc != pcmk_ok) {
	// A short wait that usually avoids stalling the FSA
	sleep(1);
	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);
	}

	if (rc != pcmk_ok) {
	crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));

	} else if (cib_conn->cmds->set_connection_dnotify(cib_conn,
	dnotify_fn) != pcmk_ok) {
	crm_err("Could not set dnotify callback");

	} else if (cib_conn->cmds->add_notify_callback(cib_conn,
	T_CIB_DIFF_NOTIFY,
	update_cb) != pcmk_ok) {
	crm_err("Could not set CIB notification callback (update)");

	} else {
	controld_set_fsa_input_flags(R_CIB_CONNECTED);
	cib_retries = 0;
	}

	if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	cib_retries++;

	if (cib_retries < 30) {
	crm_warn("Couldn't complete CIB registration %d times... "
	"pause and retry", cib_retries);
	controld_start_wait_timer();
	crmd_fsa_stall(FALSE);

	} else {
	crm_err("Could not complete CIB registration %d times... "
	"hard error", cib_retries);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}
	}

	#define MIN_CIB_OP_TIMEOUT (30)

	/*!
	* \internal
	* \brief Get the timeout (in seconds) that should be used with CIB operations
	*
	* \return The maximum of 30 seconds, the value of the PCMK_cib_timeout
	* environment variable, or 10 seconds times one more than the number of
	* nodes in the cluster.
	*/
	unsigned int
	cib_op_timeout(void)
	{
	// @COMPAT: Drop env_timeout at 3.0.0
	static int env_timeout = -1;
	unsigned int calculated_timeout = 0;

	if (env_timeout == -1) {
	const char *env = pcmk__env_option(PCMK__ENV_CIB_TIMEOUT);

	pcmk__scan_min_int(env, &env_timeout, MIN_CIB_OP_TIMEOUT);
	crm_trace("Minimum CIB op timeout: %ds (environment: %s)",
	env_timeout, (env? env : "none"));
	}

	calculated_timeout = 1 + crm_active_peers();
	if (crm_remote_peer_cache) {
	calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
	}
	calculated_timeout *= 10;

	calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
	crm_trace("Calculated timeout: %us", calculated_timeout);

	if (controld_globals.cib_conn) {
	controld_globals.cib_conn->call_timeout = calculated_timeout;
	}
	return calculated_timeout;
	}

	/*!
	* \internal
	* \brief Get CIB call options to use local scope if primary is unavailable
	*
	* \return CIB call options
	*/
	int
	crmd_cib_smart_opt(void)
	{
	int call_opt = cib_none;

	if ((controld_globals.fsa_state == S_ELECTION)
	\|\| (controld_globals.fsa_state == S_PENDING)) {
	crm_info("Sending update to local CIB in state: %s",
	fsa_state2string(controld_globals.fsa_state));
	cib__set_call_options(call_opt, "update", cib_scope_local);
	}
	return call_opt;
	}

	static void
	cib_delete_callback(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	char *desc = user_data;

	if (rc == 0) {
	crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
	} else {
	crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
	desc, call_id, pcmk_strerror(rc), rc);
	}
	}

	// Searches for various portions of node_state to delete

	// Match a particular node's node_state (takes node name 1x)
	#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"

	// Node's lrm section (name 1x)
	#define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM

	/* Node's lrm_rsc_op entries and lrm_resource entries without unexpired lock
	* (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x)
	*/
	#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \
	"\|" XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE \
	"[not(@" PCMK_OPT_SHUTDOWN_LOCK ") " \
	"or " PCMK_OPT_SHUTDOWN_LOCK "<%lld]"

	// Node's transient_attributes section (name 1x)
	#define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS

	// Everything under node_state (name 1x)
	#define XPATH_NODE_ALL XPATH_NODE_STATE "/*"

	/* Unlocked history + transient attributes
	* (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x, name 1x)
	*/
	#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "\|" XPATH_NODE_ATTRS

	/*!
	* \internal
	* \brief Get the XPath and description of a node state section to be deleted
	*
	* \param[in] uname Desired node
	* \param[in] section Subsection of node_state to be deleted
	* \param[out] xpath Where to store XPath of \p section
	* \param[out] desc If not \c NULL, where to store description of \p section
	*/
	void
	controld_node_state_deletion_strings(const char *uname,
	enum controld_section_e section,
	char xpath, char desc)
	{
	const char *desc_pre = NULL;

	// Shutdown locks that started before this time are expired
	long long expire = (long long) time(NULL)
	- controld_globals.shutdown_lock_limit;

	switch (section) {
	case controld_section_lrm:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
	desc_pre = "resource history";
	break;
	case controld_section_lrm_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED,
	uname, uname, expire);
	desc_pre = "resource history (other than shutdown locks)";
	break;
	case controld_section_attrs:
	*xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
	desc_pre = "transient attributes";
	break;
	case controld_section_all:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
	desc_pre = "all state";
	break;
	case controld_section_all_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
	uname, uname, expire, uname);
	desc_pre = "all state (other than shutdown locks)";
	break;
	default:
	// We called this function incorrectly
	CRM_ASSERT(false);
	break;
	}

	if (desc != NULL) {
	*desc = crm_strdup_printf("%s for node %s", desc_pre, uname);
	}
	}

	/*!
	* \internal
	* \brief Delete subsection of a node's CIB node_state
	*
	* \param[in] uname Desired node
	* \param[in] section Subsection of node_state to delete
	* \param[in] options CIB call options to use
	*/
	void
	controld_delete_node_state(const char *uname, enum controld_section_e section,
	int options)
	{
	cib_t *cib = controld_globals.cib_conn;
	char *xpath = NULL;
	char *desc = NULL;
	int cib_rc = pcmk_ok;

	CRM_ASSERT((uname != NULL) && (cib != NULL));

	controld_node_state_deletion_strings(uname, section, &xpath, &desc);

	cib__set_call_options(options, "node state deletion",
	cib_xpath\|cib_multiple);
	cib_rc = cib->cmds->remove(cib, xpath, NULL, options);
	fsa_register_cib_callback(cib_rc, desc, cib_delete_callback);
	crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, cib_rc, xpath);

	// CIB library handles freeing desc
	free(xpath);
	}

	// Takes node name and resource ID
	#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \
	"[@" XML_ATTR_UNAME "='%s']/" \
	XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE \
	"[@" PCMK_XA_ID "='%s']"
	// @TODO could add "and @PCMK_OPT_SHUTDOWN_LOCK" to limit to locks

	/*!
	* \internal
	* \brief Clear resource history from CIB for a given resource and node
	*
	* \param[in] rsc_id ID of resource to be cleared
	* \param[in] node Node whose resource history should be cleared
	* \param[in] user_name ACL user name to use
	* \param[in] call_options CIB call options
	*
	* \return Standard Pacemaker return code
	*/
	int
	controld_delete_resource_history(const char rsc_id, const char node,
	const char *user_name, int call_options)
	{
	char *desc = NULL;
	char *xpath = NULL;
	int rc = pcmk_rc_ok;
	cib_t *cib = controld_globals.cib_conn;

	CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);

	desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
	if (cib == NULL) {
	crm_err("Unable to clear %s: no CIB connection", desc);
	free(desc);
	return ENOTCONN;
	}

	// Ask CIB to delete the entry
	xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);

	cib->cmds->set_user(cib, user_name);
	rc = cib->cmds->remove(cib, xpath, NULL, call_options\|cib_xpath);
	cib->cmds->set_user(cib, NULL);

	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_err("Could not delete resource status of %s on %s%s%s: %s "
	CRM_XS " rc=%d", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""),
	pcmk_rc_str(rc), rc);
	free(desc);
	free(xpath);
	return rc;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	if (pcmk_is_set(call_options, cib_dryrun)) {
	crm_debug("Deletion of %s would succeed", desc);
	} else {
	crm_debug("Deletion of %s succeeded", desc);
	}
	free(desc);

	} else {
	crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, rc, xpath);
	fsa_register_cib_callback(rc, desc, cib_delete_callback);
	// CIB library handles freeing desc
	}

	free(xpath);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Build XML and string of parameters meeting some criteria, for digest
	*
	* \param[in] op Executor event with parameter table to use
	* \param[in] metadata Parsed meta-data for executed resource agent
	* \param[in] param_type Flag used for selection criteria
	* \param[out] result Will be set to newly created XML with selected
	* parameters as attributes
	*
	* \return Newly allocated space-separated string of parameter names
	* \note Selection criteria varies by param_type: for the restart digest, we
	* want parameters that are not marked reloadable (OCF 1.1) or that
	* are marked unique (pre-1.1), for both string and XML results; for the
	* secure digest, we want parameters that are marked private for the
	* string, but parameters that are not marked private for the XML.
	* \note It is the caller's responsibility to free the string return value with
	* \p g_string_free() and the XML result with \p free_xml().
	*/
	static GString *
	build_parameter_list(const lrmd_event_data_t *op,
	const struct ra_metadata_s *metadata,
	enum ra_param_flags_e param_type, xmlNode **result)
	{
	GString *list = NULL;

	*result = create_xml_node(NULL, XML_TAG_PARAMS);

	/* Consider all parameters only except private ones to be consistent with
	* what scheduler does with calculate_secure_digest().
	*/
	if (param_type == ra_param_private
	&& compare_version(controld_globals.dc_version, "3.16.0") >= 0) {
	g_hash_table_foreach(op->params, hash2field, *result);
	pcmk__filter_op_for_digest(*result);
	}

	for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
	struct ra_param_s param = (struct ra_param_s ) iter->data;

	bool accept_for_list = false;
	bool accept_for_xml = false;

	switch (param_type) {
	case ra_param_reloadable:
	accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_unique:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_private:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = !accept_for_list;
	break;
	}

	if (accept_for_list) {
	crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));

	if (list == NULL) {
	// We will later search for " WORD ", so start list with a space
	pcmk__add_word(&list, 256, " ");
	}
	pcmk__add_word(&list, 0, param->rap_name);

	} else {
	crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
	}

	if (accept_for_xml) {
	const char *v = g_hash_table_lookup(op->params, param->rap_name);

	if (v != NULL) {
	crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
	crm_xml_add(*result, param->rap_name, v);
	}

	} else {
	crm_trace("Removing attr %s from the xml result", param->rap_name);
	xml_remove_prop(*result, param->rap_name);
	}
	}

	if (list != NULL) {
	// We will later search for " WORD ", so end list with a space
	pcmk__add_word(&list, 0, " ");
	}
	return list;
	}

	static void
	append_restart_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *restart = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	if (op->interval_ms > 0) {
	/* monitors are not reloadable */
	return;
	}

	if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
	// Add parameters not marked reloadable to the "op-force-restart" list
	list = build_parameter_list(op, metadata, ra_param_reloadable,
	&restart);

	} else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
	/* @COMPAT pre-OCF-1.1 resource agents
	*
	* Before OCF 1.1, Pacemaker abused "unique=0" to indicate
	* reloadability. Add any parameters with unique="1" to the
	* "op-force-restart" list.
	*/
	list = build_parameter_list(op, metadata, ra_param_unique, &restart);

	} else {
	// Resource does not support agent reloads
	return;
	}

	digest = calculate_operation_digest(restart, version);
	/* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
	* no matter if it actually supports any parameters with unique="1"). */
	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART,
	(list == NULL)? "" : (const char *) list->str);
	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);

	if ((list != NULL) && (list->len > 0)) {
	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	} else {
	crm_trace("%s: %s", op->rsc_id, digest);
	}

	if (list != NULL) {
	g_string_free(list, TRUE);
	}
	free_xml(restart);
	free(digest);
	}

	static void
	append_secure_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *secure = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	/*
	* To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
	* secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
	* the insecure ones
	*/
	list = build_parameter_list(op, metadata, ra_param_private, &secure);

	if (list != NULL) {
	digest = calculate_operation_digest(secure, version);
	crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, (const char *) list->str);
	crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);

	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	g_string_free(list, TRUE);
	} else {
	crm_trace("%s: no secure parameters", op->rsc_id);
	}

	free_xml(secure);
	free(digest);
	}

	/*!
	* \internal
	* \brief Create XML for a resource history entry
	*
	* \param[in] func Function name of caller
	* \param[in,out] parent XML to add entry to
	* \param[in] rsc Affected resource
	* \param[in,out] op Action to add an entry for (or NULL to do nothing)
	* \param[in] node_name Node where action occurred
	*/
	void
	controld_add_resource_history_xml_as(const char func, xmlNode parent,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op,
	const char *node_name)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	struct ra_metadata_s *metadata = NULL;
	const char *caller_version = NULL;
	lrm_state_t *lrm_state = NULL;

	if (op == NULL) {
	return;
	}

	target_rc = rsc_op_expected_rc(op);

	caller_version = g_hash_table_lookup(op->params, PCMK_XA_CRM_FEATURE_SET);
	CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);

	xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
	controld_globals.our_nodename, func);
	if (xml_op == NULL) {
	return;
	}

	if ((rsc == NULL) \|\| (op->params == NULL)
	\|\| !crm_op_needs_metadata(rsc->standard, op->op_type)) {

	crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
	op->op_type, op->rsc_id, op->params, rsc);
	return;
	}

	lrm_state = lrm_state_find(node_name);
	if (lrm_state == NULL) {
	crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
	" because we have no connection to executor for %s",
	op->rsc_id, op->op_type, op->interval_ms, node_name);
	return;
	}

	/* Ideally the metadata is cached, and the agent is just a fallback.
	*
	* @TODO Go through all callers and ensure they get metadata asynchronously
	* first.
	*/
	metadata = controld_get_rsc_metadata(lrm_state, rsc,
	controld_metadata_from_agent
	\|controld_metadata_from_cache);
	if (metadata == NULL) {
	return;
	}

	crm_trace("Including additional digests for %s:%s:%s",
	rsc->standard, rsc->provider, rsc->type);
	append_restart_list(op, metadata, xml_op, caller_version);
	append_secure_list(op, metadata, xml_op, caller_version);

	return;
	}

	/*!
	* \internal
	* \brief Record an action as pending in the CIB, if appropriate
	*
	* \param[in] node_name Node where the action is pending
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Pending action
	*
	* \return true if action was recorded in CIB, otherwise false
	*/
	bool
	controld_record_pending_op(const char node_name, const lrmd_rsc_info_t rsc,
	lrmd_event_data_t *op)
	{
	const char *record_pending = NULL;

	CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL),
	return false);

	// Never record certain operation types as pending
	if ((op->op_type == NULL) \|\| (op->params == NULL)
	\|\| !controld_action_is_recordable(op->op_type)) {
	return false;
	}

	// Check action's record-pending meta-attribute (defaults to true)
	record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
	if ((record_pending != NULL) && !crm_is_true(record_pending)) {
	return false;
	}

	op->call_id = -1;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);

	crm_debug("Recording pending %s-interval %s for %s on %s in the CIB",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	node_name);
	controld_update_resource_history(node_name, rsc, op, 0);
	return true;
	}

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource history update completed (call=%d rc=%d)",
	call_id, rc);
	break;
	default:
	if (call_id > 0) {
	crm_warn("Resource history update %d failed: %s "
	CRM_XS " rc=%d", call_id, pcmk_strerror(rc), rc);
	} else {
	crm_warn("Resource history update failed: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}
	}

	if (call_id == pending_rsc_update) {
	pending_rsc_update = 0;
	controld_trigger_fsa();
	}
	}

	/* Only successful stops, and probes that found the resource inactive, get locks
	* recorded in the history. This ensures the resource stays locked to the node
	* until it is active there again after the node comes back up.
	*/
	static bool
	should_preserve_lock(lrmd_event_data_t *op)
	{
	if (!pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
	return false;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_STOP) && (op->rc == PCMK_OCF_OK)) {
	return true;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_MONITOR)
	&& (op->rc == PCMK_OCF_NOT_RUNNING)) {
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Request a CIB update
	*
	* \param[in] section Section of CIB to update
	* \param[in] data New XML of CIB section to update
	* \param[in] options CIB call options
	* \param[in] callback If not \c NULL, set this as the operation callback
	*
	* \return Standard Pacemaker return code
	*
	* \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is
	* stored in \p pending_rsc_update on success.
	*/
	int
	controld_update_cib(const char section, xmlNode data, int options,
	void (callback)(xmlNode , int, int, xmlNode , void ))
	{
	cib_t *cib = controld_globals.cib_conn;
	int cib_rc = -ENOTCONN;

	CRM_ASSERT(data != NULL);

	if (cib != NULL) {
	cib_rc = cib->cmds->modify(cib, section, data, options);
	if (cib_rc >= 0) {
	crm_debug("Submitted CIB update %d for %s section",
	cib_rc, section);
	}
	}

	if (callback == NULL) {
	if (cib_rc < 0) {
	crm_err("Failed to update CIB %s section: %s",
	section, pcmk_rc_str(pcmk_legacy2rc(cib_rc)));
	}

	} else {
	if ((cib_rc >= 0) && (callback == cib_rsc_callback)) {
	/* Checking for a particular callback is a little hacky, but it
	* didn't seem worth adding an output argument for cib_rc for just
	* one use case.
	*/
	pending_rsc_update = cib_rc;
	}
	fsa_register_cib_callback(cib_rc, NULL, callback);
	}

	return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc);
	}

	/*!
	* \internal
	* \brief Update resource history entry in CIB
	*
	* \param[in] node_name Node where action occurred
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Action to record
	* \param[in] lock_time If nonzero, when resource was locked to node
	*
	* \note On success, the CIB update's call ID will be stored in
	* pending_rsc_update.
	*/
	void
	controld_update_resource_history(const char *node_name,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op, time_t lock_time)
	{
	xmlNode *update = NULL;
	xmlNode *xml = NULL;
	int call_opt = crmd_cib_smart_opt();
	const char *node_id = NULL;
	const char *container = NULL;

	CRM_CHECK((node_name != NULL) && (op != NULL), return);

	if (rsc == NULL) {
	crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
	controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
	return;
	}

	// <status>
	update = create_xml_node(NULL, XML_CIB_TAG_STATUS);

	// <node_state ...>
	xml = create_xml_node(update, XML_CIB_TAG_STATE);
	if (pcmk__str_eq(node_name, controld_globals.our_nodename,
	pcmk__str_casei)) {
	node_id = controld_globals.our_uuid;
	} else {
	node_id = node_name;
	pcmk__xe_set_bool_attr(xml, XML_NODE_IS_REMOTE, true);
	}
	crm_xml_add(xml, PCMK_XA_ID, node_id);
	crm_xml_add(xml, XML_ATTR_UNAME, node_name);
	crm_xml_add(xml, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);

	// <lrm ...>
	xml = create_xml_node(xml, XML_CIB_TAG_LRM);
	crm_xml_add(xml, PCMK_XA_ID, node_id);

	// <lrm_resources>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCES);

	// <lrm_resource ...>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCE);
	crm_xml_add(xml, PCMK_XA_ID, op->rsc_id);
	- crm_xml_add(xml, XML_AGENT_ATTR_CLASS, rsc->standard);
	+ crm_xml_add(xml, PCMK_XA_CLASS, rsc->standard);
	crm_xml_add(xml, PCMK_XA_PROVIDER, rsc->provider);
	crm_xml_add(xml, PCMK_XA_TYPE, rsc->type);
	if (lock_time != 0) {
	/* Actions on a locked resource should either preserve the lock by
	* recording it with the action result, or clear it.
	*/
	if (!should_preserve_lock(op)) {
	lock_time = 0;
	}
	crm_xml_add_ll(xml, PCMK_OPT_SHUTDOWN_LOCK, (long long) lock_time);
	}
	if (op->params != NULL) {
	container = g_hash_table_lookup(op->params,
	CRM_META "_" PCMK__META_CONTAINER);
	if (container != NULL) {
	crm_trace("Resource %s is a part of container resource %s",
	op->rsc_id, container);
	crm_xml_add(xml, PCMK__META_CONTAINER, container);
	}
	}

	// <lrm_resource_op ...> (possibly more than one)
	controld_add_resource_history_xml(xml, rsc, op, node_name);

	/* Update CIB asynchronously. Even if it fails, the resource state should be
	* discovered during the next election. Worst case, the node is wrongly
	* fenced for running a resource it isn't.
	*/
	crm_log_xml_trace(update, __func__);
	controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, cib_rsc_callback);
	free_xml(update);
	}

	/*!
	* \internal
	* \brief Erase an LRM history entry from the CIB, given the operation data
	*
	* \param[in] op Operation whose history should be deleted
	*/
	void
	controld_delete_action_history(const lrmd_event_data_t *op)
	{
	xmlNode *xml_top = NULL;

	CRM_CHECK(op != NULL, return);

	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);

	if (op->interval_ms > 0) {
	char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);

	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, PCMK_XA_ID, op_id);
	free(op_id);
	}

	crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
	op->rsc_id, op->op_type, op->interval_ms, op->call_id);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn,
	XML_CIB_TAG_STATUS, xml_top,
	cib_none);
	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	/* Define xpath to find LRM resource history entry by node and resource */
	#define XPATH_HISTORY \
	"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS \
	"/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']" \
	"/" XML_LRM_TAG_RSC_OP

	/* ... and also by operation key */
	#define XPATH_HISTORY_ID XPATH_HISTORY "[@" PCMK_XA_ID "='%s']"

	/* ... and also by operation key and operation call ID */
	#define XPATH_HISTORY_CALL XPATH_HISTORY \
	"[@" PCMK_XA_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"

	/* ... and also by operation key and original operation key */
	#define XPATH_HISTORY_ORIG XPATH_HISTORY \
	"[@" PCMK_XA_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"

	/*!
	* \internal
	* \brief Delete a last_failure resource history entry from the CIB
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] action If specified, delete only if this was failed action
	* \param[in] interval_ms If \p action is specified, it has this interval
	*/
	void
	controld_cib_delete_last_failure(const char rsc_id, const char node,
	const char *action, guint interval_ms)
	{
	char *xpath = NULL;
	char *last_failure_key = NULL;
	CRM_CHECK((rsc_id != NULL) && (node != NULL), return);

	// Generate XPath to match desired entry
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
	if (action == NULL) {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id,
	last_failure_key);
	} else {
	char *action_key = pcmk__op_key(rsc_id, action, interval_ms);

	xpath = crm_strdup_printf(XPATH_HISTORY_ORIG, node, rsc_id,
	last_failure_key, action_key);
	free(action_key);
	}
	free(last_failure_key);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}

	/*!
	* \internal
	* \brief Delete resource history entry from the CIB, given operation key
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] key Operation key of operation to clear history for
	* \param[in] call_id If specified, delete entry only if it has this call ID
	*/
	void
	controld_delete_action_history_by_key(const char rsc_id, const char node,
	const char *key, int call_id)
	{
	char *xpath = NULL;

	CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return);

	if (call_id > 0) {
	xpath = crm_strdup_printf(XPATH_HISTORY_CALL, node, rsc_id, key,
	call_id);
	} else {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id, key);
	}
	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}
	diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c
	index 77c3c31808..e0dd959004 100644
	--- a/daemons/controld/controld_execd.c
	+++ b/daemons/controld/controld_execd.c
	@@ -1,2446 +1,2446 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <regex.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h> // lrmd_event_data_t, lrmd_rsc_info_t, etc.
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/rules.h>
	#include <crm/lrmd_internal.h>

	#include <pacemaker-internal.h>
	#include <pacemaker-controld.h>

	#define START_DELAY_THRESHOLD 5 * 60 * 1000
	#define MAX_LRM_REG_FAILS 30

	struct delete_event_s {
	int rc;
	const char *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
	static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
	static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);

	static lrmd_event_data_t construct_op(const lrm_state_t lrm_state,
	const xmlNode *rsc_op,
	const char *rsc_id,
	const char *operation);
	static void do_lrm_rsc_op(lrm_state_t lrm_state, lrmd_rsc_info_t rsc,
	xmlNode msg, struct ra_metadata_s md);

	static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
	int log_level);

	static void
	lrm_connection_destroy(void)
	{
	if (pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) {
	crm_crit("Lost connection to local executor");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	controld_clear_fsa_input_flags(R_LRM_CONNECTED);
	}
	}

	static char *
	make_stop_id(const char *rsc, int call_id)
	{
	return crm_strdup_printf("%s:%d", rsc, call_id);
	}

	static void
	copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") == NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	static void
	copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
	{
	if (strstr(key, CRM_META "_") != NULL) {
	g_hash_table_replace(user_data, strdup((const char )key), strdup((const char )value));
	}
	}

	/*!
	* \internal
	* \brief Remove a recurring operation from a resource's history
	*
	* \param[in,out] history Resource history to modify
	* \param[in] op Operation to remove
	*
	* \return TRUE if the operation was found and removed, FALSE otherwise
	*/
	static gboolean
	history_remove_recurring_op(rsc_history_t history, const lrmd_event_data_t op)
	{
	GList *iter;

	for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
	lrmd_event_data_t *existing = iter->data;

	if ((op->interval_ms == existing->interval_ms)
	&& pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
	&& pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {

	history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
	lrmd_free_event(existing);
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Free all recurring operations in resource history
	*
	* \param[in,out] history Resource history to modify
	*/
	static void
	history_free_recurring_ops(rsc_history_t *history)
	{
	GList *iter;

	for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
	lrmd_free_event(iter->data);
	}
	g_list_free(history->recurring_op_list);
	history->recurring_op_list = NULL;
	}

	/*!
	* \internal
	* \brief Free resource history
	*
	* \param[in,out] history Resource history to free
	*/
	void
	history_free(gpointer data)
	{
	rsc_history_t history = (rsc_history_t)data;

	if (history->stop_params) {
	g_hash_table_destroy(history->stop_params);
	}

	/* Don't need to free history->rsc.id because it's set to history->id */
	free(history->rsc.type);
	free(history->rsc.standard);
	free(history->rsc.provider);

	lrmd_free_event(history->failed);
	lrmd_free_event(history->last);
	free(history->id);
	history_free_recurring_ops(history);
	free(history);
	}

	static void
	update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
	{
	int target_rc = 0;
	rsc_history_t *entry = NULL;

	if (op->rsc_deleted) {
	crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
	controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
	NULL, crmd_cib_smart_opt());
	return;
	}

	if (pcmk__str_eq(op->op_type, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
	return;
	}

	crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);

	entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
	if (entry == NULL && rsc) {
	entry = calloc(1, sizeof(rsc_history_t));
	entry->id = strdup(op->rsc_id);
	g_hash_table_insert(lrm_state->resource_history, entry->id, entry);

	entry->rsc.id = entry->id;
	entry->rsc.type = strdup(rsc->type);
	entry->rsc.standard = strdup(rsc->standard);
	pcmk__str_update(&entry->rsc.provider, rsc->provider);

	} else if (entry == NULL) {
	crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
	return;
	}

	entry->last_callid = op->call_id;
	target_rc = rsc_op_expected_rc(op);
	if (op->op_status == PCMK_EXEC_CANCELLED) {
	if (op->interval_ms > 0) {
	crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
	op->rsc_id, op->op_type, op->interval_ms);
	history_remove_recurring_op(entry, op);
	return;
	} else {
	crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
	op->rsc_id, op->op_type, op->interval_ms, op->rc,
	op->op_status);
	}

	} else if (did_rsc_op_fail(op, target_rc)) {
	/* Store failed monitors here, otherwise the block below will cause them
	* to be forgotten when a stop happens.
	*/
	if (entry->failed) {
	lrmd_free_event(entry->failed);
	}
	entry->failed = lrmd_copy_event(op);

	} else if (op->interval_ms == 0) {
	if (entry->last) {
	lrmd_free_event(entry->last);
	}
	entry->last = lrmd_copy_event(op);

	if (op->params && pcmk__strcase_any_of(op->op_type, PCMK_ACTION_START,
	PCMK_ACTION_RELOAD,
	PCMK_ACTION_RELOAD_AGENT,
	PCMK_ACTION_MONITOR, NULL)) {
	if (entry->stop_params) {
	g_hash_table_destroy(entry->stop_params);
	}
	entry->stop_params = pcmk__strkey_table(free, free);

	g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
	}
	}

	if (op->interval_ms > 0) {
	/* Ensure there are no duplicates */
	history_remove_recurring_op(entry, op);

	crm_trace("Adding recurring op: " PCMK__OP_FMT,
	op->rsc_id, op->op_type, op->interval_ms);
	entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));

	} else if ((entry->recurring_op_list != NULL)
	&& !pcmk__str_eq(op->op_type, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
	g_list_length(entry->recurring_op_list), op->rsc_id,
	op->op_type, op->interval_ms);
	history_free_recurring_ops(entry);
	}
	}

	/*!
	* \internal
	* \brief Send a direct OK ack for a resource task
	*
	* \param[in] lrm_state LRM connection
	* \param[in] input Input message being ack'ed
	* \param[in] rsc_id ID of affected resource
	* \param[in] rsc Affected resource (if available)
	* \param[in] task Operation task being ack'ed
	* \param[in] ack_host Name of host to send ack to
	* \param[in] ack_sys IPC system name to ack
	*/
	static void
	send_task_ok_ack(const lrm_state_t lrm_state, const ha_msg_input_t input,
	const char rsc_id, const lrmd_rsc_info_t rsc,
	const char task, const char ack_host, const char *ack_sys)
	{
	lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);

	lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
	lrmd_free_event(op);
	}

	static inline const char *
	op_node_name(lrmd_event_data_t *op)
	{
	return pcmk__s(op->remote_nodename, controld_globals.our_nodename);
	}

	void
	lrm_op_callback(lrmd_event_data_t * op)
	{
	CRM_CHECK(op != NULL, return);
	switch (op->type) {
	case lrmd_event_disconnect:
	if (op->remote_nodename == NULL) {
	/* If this is the local executor IPC connection, set the right
	* bits in the controller when the connection goes down.
	*/
	lrm_connection_destroy();
	}
	break;

	case lrmd_event_exec_complete:
	{
	lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));

	CRM_ASSERT(lrm_state != NULL);
	process_lrm_event(lrm_state, op, NULL, NULL);
	}
	break;

	default:
	break;
	}
	}

	static void
	try_local_executor_connect(long long action, fsa_data_t *msg_data,
	lrm_state_t *lrm_state)
	{
	int rc = pcmk_rc_ok;

	crm_debug("Connecting to the local executor");

	// If we can connect, great
	rc = controld_connect_local_executor(lrm_state);
	if (rc == pcmk_rc_ok) {
	controld_set_fsa_input_flags(R_LRM_CONNECTED);
	crm_info("Connection to the local executor established");
	return;
	}

	// Otherwise, if we can try again, set a timer to do so
	if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
	crm_warn("Failed to connect to the local executor %d time%s "
	"(%d max): %s", lrm_state->num_lrm_register_fails,
	pcmk__plural_s(lrm_state->num_lrm_register_fails),
	MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
	controld_start_wait_timer();
	crmd_fsa_stall(FALSE);
	return;
	}

	// Otherwise give up
	crm_err("Failed to connect to the executor the max allowed "
	"%d time%s: %s", lrm_state->num_lrm_register_fails,
	pcmk__plural_s(lrm_state->num_lrm_register_fails),
	pcmk_rc_str(rc));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}

	/* A_LRM_CONNECT */
	void
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* This only pertains to local executor connections. Remote connections are
	* handled as resources within the scheduler. Connecting and disconnecting
	* from remote executor instances is handled differently.
	*/

	lrm_state_t *lrm_state = NULL;

	if (controld_globals.our_nodename == NULL) {
	return; /* Nothing to do */
	}
	lrm_state = lrm_state_find_or_create(controld_globals.our_nodename);
	if (lrm_state == NULL) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	if (action & A_LRM_DISCONNECT) {
	if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
	if (action == A_LRM_DISCONNECT) {
	crmd_fsa_stall(FALSE);
	return;
	}
	}

	controld_clear_fsa_input_flags(R_LRM_CONNECTED);
	lrm_state_disconnect(lrm_state);
	lrm_state_reset_tables(lrm_state, FALSE);
	}

	if (action & A_LRM_CONNECT) {
	try_local_executor_connect(action, msg_data, lrm_state);
	}

	if (action & ~(A_LRM_CONNECT \| A_LRM_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action),
	__func__);
	}
	}

	static gboolean
	lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
	{
	int counter = 0;
	gboolean rc = TRUE;
	const char *when = "lrm disconnect";

	GHashTableIter gIter;
	const char *key = NULL;
	rsc_history_t *entry = NULL;
	active_op_t *pending = NULL;

	crm_debug("Checking for active resources before exit");

	if (cur_state == S_TERMINATE) {
	log_level = LOG_ERR;
	when = "shutdown";

	} else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	when = "shutdown... waiting";
	}

	if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) {
	guint removed = g_hash_table_foreach_remove(lrm_state->active_ops,
	stop_recurring_actions,
	lrm_state);
	guint nremaining = g_hash_table_size(lrm_state->active_ops);

	if (removed \|\| nremaining) {
	crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
	removed, pcmk__plural_s(removed), when, nremaining);
	}
	}

	if (lrm_state->active_ops != NULL) {
	g_hash_table_iter_init(&gIter, lrm_state->active_ops);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
	/* Ignore recurring actions in the shutdown calculations */
	if (pending->interval_ms == 0) {
	counter++;
	}
	}
	}

	if (counter > 0) {
	do_crm_log(log_level, "%d pending executor operation%s at %s",
	counter, pcmk__plural_s(counter), when);

	if ((cur_state == S_TERMINATE)
	\|\| !pcmk_is_set(controld_globals.fsa_input_register,
	R_SENT_RSC_STOP)) {
	g_hash_table_iter_init(&gIter, lrm_state->active_ops);
	while (g_hash_table_iter_next(&gIter, (gpointer)&key, (gpointer)&pending)) {
	do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
	}

	} else {
	rc = FALSE;
	}
	return rc;
	}

	if (lrm_state->resource_history == NULL) {
	return rc;
	}

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	/* At this point we're not waiting, we're just shutting down */
	when = "shutdown";
	}

	counter = 0;
	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
	if (is_rsc_active(lrm_state, entry->id) == FALSE) {
	continue;
	}

	counter++;
	if (log_level == LOG_ERR) {
	crm_info("Found %s active at %s", entry->id, when);
	} else {
	crm_trace("Found %s active at %s", entry->id, when);
	}
	if (lrm_state->active_ops != NULL) {
	GHashTableIter hIter;

	g_hash_table_iter_init(&hIter, lrm_state->active_ops);
	while (g_hash_table_iter_next(&hIter, (gpointer)&key, (gpointer)&pending)) {
	if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
	crm_notice("%sction %s (%s) incomplete at %s",
	pending->interval_ms == 0 ? "A" : "Recurring a",
	key, pending->op_key, when);
	}
	}
	}
	}

	if (counter) {
	crm_err("%d resource%s active at %s",
	counter, (counter == 1)? " was" : "s were", when);
	}

	return rc;
	}

	static gboolean
	is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
	{
	rsc_history_t *entry = NULL;

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	if (entry == NULL \|\| entry->last == NULL) {
	return FALSE;
	}

	crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
	entry->last->interval_ms, entry->last->rc);
	if ((entry->last->rc == PCMK_OCF_OK)
	&& pcmk__str_eq(entry->last->op_type, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_OK
	&& pcmk__str_eq(entry->last->op_type, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_casei)) {
	// A stricter check is too complex ... leave that to the scheduler
	return FALSE;

	} else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
	return FALSE;

	} else if ((entry->last->interval_ms == 0)
	&& (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
	/* Badly configured resources can't be reliably stopped */
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
	{
	GHashTableIter iter;
	rsc_history_t *entry = NULL;

	g_hash_table_iter_init(&iter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {

	GList *gIter = NULL;
	xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);

	crm_xml_add(xml_rsc, PCMK_XA_ID, entry->id);
	crm_xml_add(xml_rsc, PCMK_XA_TYPE, entry->rsc.type);
	- crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
	+ crm_xml_add(xml_rsc, PCMK_XA_CLASS, entry->rsc.standard);
	crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, entry->rsc.provider);

	if (entry->last && entry->last->params) {
	static const char *name = CRM_META "_" PCMK__META_CONTAINER;
	const char *container = g_hash_table_lookup(entry->last->params,
	name);

	if (container) {
	crm_trace("Resource %s is a part of container resource %s", entry->id, container);
	crm_xml_add(xml_rsc, PCMK__META_CONTAINER, container);
	}
	}
	controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed,
	lrm_state->node_name);
	controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last,
	lrm_state->node_name);
	for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
	controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data,
	lrm_state->node_name);
	}
	}

	return FALSE;
	}

	xmlNode *
	controld_query_executor_state(void)
	{
	xmlNode *xml_state = NULL;
	xmlNode *xml_data = NULL;
	xmlNode *rsc_list = NULL;
	crm_node_t *peer = NULL;
	lrm_state_t *lrm_state = lrm_state_find(controld_globals.our_nodename);

	if (!lrm_state) {
	crm_err("Could not find executor state for node %s",
	controld_globals.our_nodename);
	return NULL;
	}

	peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
	CRM_CHECK(peer != NULL, return NULL);

	xml_state = create_node_state_update(peer,
	node_update_cluster\|node_update_peer,
	NULL, __func__);
	if (xml_state == NULL) {
	return NULL;
	}

	xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
	crm_xml_add(xml_data, PCMK_XA_ID, peer->uuid);
	rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);

	/* Build a list of active (not always running) resources */
	build_active_RAs(lrm_state, rsc_list);

	crm_log_xml_trace(xml_state, "Current executor state");

	return xml_state;
	}

	/*!
	* \internal
	* \brief Map standard Pacemaker return code to operation status and OCF code
	*
	* \param[out] event Executor event whose status and return code should be set
	* \param[in] rc Standard Pacemaker return code
	*/
	void
	controld_rc2event(lrmd_event_data_t *event, int rc)
	{
	/* This is called for cleanup requests from controller peers/clients, not
	* for resource actions, so no exit reason is needed.
	*/
	switch (rc) {
	case pcmk_rc_ok:
	lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	break;
	case EACCES:
	lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
	PCMK_EXEC_ERROR, NULL);
	break;
	default:
	lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	NULL);
	break;
	}
	}

	/*!
	* \internal
	* \brief Trigger a new transition after CIB status was deleted
	*
	* If a CIB status delete was not expected (as part of the transition graph),
	* trigger a new transition by updating the (arbitrary) "last-lrm-refresh"
	* cluster property.
	*
	* \param[in] from_sys IPC name that requested the delete
	* \param[in] rsc_id Resource whose status was deleted (for logging only)
	*/
	void
	controld_trigger_delete_refresh(const char from_sys, const char rsc_id)
	{
	if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
	char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));

	crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
	cib__update_node_attr(controld_globals.logger_out,
	controld_globals.cib_conn, cib_none,
	XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"last-lrm-refresh", now_s, NULL, NULL);
	free(now_s);
	}
	}

	static void
	notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, (from_host? from_host : "localhost"), rsc_id,
	((rc == pcmk_ok)? "" : " not"));
	op = construct_op(lrm_state, input->xml, rsc_id, PCMK_ACTION_DELETE);
	controld_rc2event(op, pcmk_legacy2rc(rc));
	controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);
	controld_trigger_delete_refresh(from_sys, rsc_id);
	}

	static gboolean
	lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	struct delete_event_s *event = user_data;
	struct pending_deletion_op_s *op = value;

	if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
	notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
	return TRUE;
	}
	return FALSE;
	}

	static gboolean
	lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
	{
	const char *rsc = user_data;
	active_op_t *pending = value;

	if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
	crm_info("Removing op %s:%d for deleted resource %s",
	pending->op_key, pending->call_id, rsc);
	return TRUE;
	}
	return FALSE;
	}

	static void
	delete_rsc_entry(lrm_state_t lrm_state, ha_msg_input_t input,
	const char rsc_id, GHashTableIter rsc_iter, int rc,
	const char *user_name, bool from_cib)
	{
	struct delete_event_s event;

	CRM_CHECK(rsc_id != NULL, return);

	if (rc == pcmk_ok) {
	char *rsc_id_copy = strdup(rsc_id);

	if (rsc_iter) {
	g_hash_table_iter_remove(rsc_iter);
	} else {
	g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
	}

	if (from_cib) {
	controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
	user_name, crmd_cib_smart_opt());
	}
	g_hash_table_foreach_remove(lrm_state->active_ops,
	lrm_remove_deleted_op, rsc_id_copy);
	free(rsc_id_copy);
	}

	if (input) {
	notify_deleted(lrm_state, input, rsc_id, rc);
	}

	event.rc = rc;
	event.rsc = rsc_id;
	event.lrm_state = lrm_state;
	g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
	}

	static inline gboolean
	last_failed_matches_op(rsc_history_t entry, const char op, guint interval_ms)
	{
	if (entry == NULL) {
	return FALSE;
	}
	if (op == NULL) {
	return TRUE;
	}
	return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
	&& (interval_ms == entry->failed->interval_ms));
	}

	/*!
	* \internal
	* \brief Clear a resource's last failure
	*
	* Erase a resource's last failure on a particular node from both the
	* LRM resource history in the CIB, and the resource history remembered
	* for the LRM state.
	*
	* \param[in] rsc_id Resource name
	* \param[in] node_name Node name
	* \param[in] operation If specified, only clear if matching this operation
	* \param[in] interval_ms If operation is specified, it has this interval
	*/
	void
	lrm_clear_last_failure(const char rsc_id, const char node_name,
	const char *operation, guint interval_ms)
	{
	lrm_state_t *lrm_state = lrm_state_find(node_name);

	if (lrm_state == NULL) {
	return;
	}
	if (lrm_state->resource_history != NULL) {
	rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
	rsc_id);

	if (last_failed_matches_op(entry, operation, interval_ms)) {
	lrmd_free_event(entry->failed);
	entry->failed = NULL;
	}
	}
	}

	/* Returns: gboolean - cancellation is in progress */
	static gboolean
	cancel_op(lrm_state_t * lrm_state, const char rsc_id, const char key, int op, gboolean remove)
	{
	int rc = pcmk_ok;
	char *local_key = NULL;
	active_op_t *pending = NULL;

	CRM_CHECK(op != 0, return FALSE);
	CRM_CHECK(rsc_id != NULL, return FALSE);
	if (key == NULL) {
	local_key = make_stop_id(rsc_id, op);
	key = local_key;
	}
	pending = g_hash_table_lookup(lrm_state->active_ops, key);

	if (pending) {
	if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
	controld_set_active_op_flags(pending, active_op_remove);
	crm_debug("Scheduling %s for removal", key);
	}

	if (pcmk_is_set(pending->flags, active_op_cancelled)) {
	crm_debug("Operation %s already cancelled", key);
	free(local_key);
	return FALSE;
	}
	controld_set_active_op_flags(pending, active_op_cancelled);

	} else {
	crm_info("No pending op found for %s", key);
	free(local_key);
	return FALSE;
	}

	crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
	rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
	pending->interval_ms);
	if (rc == pcmk_ok) {
	crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
	free(local_key);
	return TRUE;
	}

	crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
	/* The caller needs to make sure the entry is
	* removed from the active operations list
	*
	* Usually by returning TRUE inside the worker function
	* supplied to g_hash_table_foreach_remove()
	*
	* Not removing the entry from active operations will block
	* the node from shutting down
	*/
	free(local_key);
	return FALSE;
	}

	struct cancel_data {
	gboolean done;
	gboolean remove;
	const char *key;
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct cancel_data *data = user_data;
	active_op_t *op = value;

	if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
	data->done = TRUE;
	remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
	}
	return remove;
	}

	static gboolean
	cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
	{
	guint removed = 0;
	struct cancel_data data;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(key != NULL, return FALSE);

	data.key = key;
	data.rsc = rsc;
	data.done = FALSE;
	data.remove = remove;
	data.lrm_state = lrm_state;

	removed = g_hash_table_foreach_remove(lrm_state->active_ops,
	cancel_action_by_key, &data);
	crm_trace("Removed %u op cache entries, new size: %u",
	removed, g_hash_table_size(lrm_state->active_ops));
	return data.done;
	}

	/*!
	* \internal
	* \brief Retrieve resource information from LRM
	*
	* \param[in,out] lrm_state Executor connection state to use
	* \param[in] rsc_xml XML containing resource configuration
	* \param[in] do_create If true, register resource if not already
	* \param[out] rsc_info Where to store information obtained from executor
	*
	* \retval pcmk_ok Success (and rsc_info holds newly allocated result)
	* \retval -EINVAL Required information is missing from arguments
	* \retval -ENOTCONN No active connection to LRM
	* \retval -ENODEV Resource not found
	* \retval -errno Error communicating with executor when registering resource
	*
	* \note Caller is responsible for freeing result on success.
	*/
	static int
	get_lrm_resource(lrm_state_t lrm_state, const xmlNode rsc_xml,
	gboolean do_create, lrmd_rsc_info_t **rsc_info)
	{
	const char *id = ID(rsc_xml);

	CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
	CRM_CHECK(id, return -EINVAL);

	if (lrm_state_is_connected(lrm_state) == FALSE) {
	return -ENOTCONN;
	}

	crm_trace("Retrieving resource information for %s from the executor", id);
	*rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);

	// If resource isn't known by ID, try clone name, if provided
	if (!*rsc_info) {
	const char *long_id = crm_element_value(rsc_xml, PCMK__XA_LONG_ID);

	if (long_id) {
	*rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
	}
	}

	if ((*rsc_info == NULL) && do_create) {
	- const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc_xml, PCMK_XA_CLASS);
	const char *provider = crm_element_value(rsc_xml, PCMK_XA_PROVIDER);
	const char *type = crm_element_value(rsc_xml, PCMK_XA_TYPE);
	int rc;

	crm_trace("Registering resource %s with the executor", id);
	rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
	lrmd_opt_drop_recurring);
	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL;

	crm_err("Could not register resource %s with the executor on %s: %s "
	CRM_XS " rc=%d",
	id, lrm_state->node_name, pcmk_strerror(rc), rc);

	/* Register this as an internal error if this involves the local
	* executor. Otherwise, we're likely dealing with an unresponsive
	* remote node, which is not an FSA failure.
	*/
	if (lrm_state_is_local(lrm_state) == TRUE) {
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	return rc;
	}

	*rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
	}
	return *rsc_info? pcmk_ok : -ENODEV;
	}

	static void
	delete_resource(lrm_state_t lrm_state, const char id, lrmd_rsc_info_t *rsc,
	GHashTableIter iter, const char sys, const char *user,
	ha_msg_input_t *request, bool unregister, bool from_cib)
	{
	int rc = pcmk_ok;

	crm_info("Removing resource %s from executor for %s%s%s",
	id, sys, (user? " as " : ""), (user? user : ""));

	if (rsc && unregister) {
	rc = lrm_state_unregister_rsc(lrm_state, id, 0);
	}

	if (rc == pcmk_ok) {
	crm_trace("Resource %s deleted from executor", id);
	} else if (rc == -EINPROGRESS) {
	crm_info("Deletion of resource '%s' from executor is pending", id);
	if (request) {
	struct pending_deletion_op_s *op = NULL;
	char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);

	op = calloc(1, sizeof(struct pending_deletion_op_s));
	op->rsc = strdup(rsc->id);
	op->input = copy_ha_msg_input(request);
	g_hash_table_insert(lrm_state->deletion_ops, ref, op);
	}
	return;
	} else {
	crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
	CRM_XS " rc=%d", id, sys, (user? " as " : ""),
	(user? user : ""), pcmk_strerror(rc), rc);
	}

	delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib);
	}

	static int
	get_fake_call_id(lrm_state_t lrm_state, const char rsc_id)
	{
	int call_id = 999999999;
	rsc_history_t *entry = NULL;

	if(lrm_state) {
	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	}

	/* Make sure the call id is greater than the last successful operation,
	* otherwise the failure will not result in a possible recovery of the resource
	* as it could appear the failure occurred before the successful start */
	if (entry) {
	call_id = entry->last_callid + 1;
	}

	if (call_id < 0) {
	call_id = 1;
	}
	return call_id;
	}

	static void
	fake_op_status(lrm_state_t lrm_state, lrmd_event_data_t op, int op_status,
	enum ocf_exitcode op_exitcode, const char *exit_reason)
	{
	op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;
	lrmd__set_result(op, op_exitcode, op_status, exit_reason);
	}

	static void
	force_reprobe(lrm_state_t lrm_state, const char from_sys,
	const char from_host, const char user_name,
	gboolean is_remote_node, bool reprobe_all_nodes)
	{
	GHashTableIter gIter;
	rsc_history_t *entry = NULL;

	crm_info("Clearing resource history on node %s", lrm_state->node_name);
	g_hash_table_iter_init(&gIter, lrm_state->resource_history);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	/* only unregister the resource during a reprobe if it is not a remote connection
	* resource. otherwise unregistering the connection will terminate remote-node
	* membership */
	bool unregister = true;

	if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
	unregister = false;

	if (reprobe_all_nodes) {
	lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);

	if (remote_lrm_state != NULL) {
	/* If reprobing all nodes, be sure to reprobe the remote
	* node before clearing its connection resource
	*/
	force_reprobe(remote_lrm_state, from_sys, from_host,
	user_name, TRUE, reprobe_all_nodes);
	}
	}
	}

	/* Don't delete from the CIB, since we'll delete the whole node's LRM
	* state from the CIB soon
	*/
	delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
	user_name, NULL, unregister, false);
	}

	/* Now delete the copy in the CIB */
	controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
	cib_scope_local);

	// @COMPAT DCs < 1.1.14 need this deleted (in case it was explicitly false)
	update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
	}

	/*!
	* \internal
	* \brief Fail a requested action without actually executing it
	*
	* For an action that can't be executed, process it similarly to an actual
	* execution result, with specified error status (except for notify actions,
	* which will always be treated as successful).
	*
	* \param[in,out] lrm_state Executor connection that action is for
	* \param[in] action Action XML from request
	* \param[in] rc Desired return code to use
	* \param[in] op_status Desired operation status to use
	* \param[in] exit_reason Human-friendly detail, if error
	*/
	static void
	synthesize_lrmd_failure(lrm_state_t lrm_state, const xmlNode action,
	int op_status, enum ocf_exitcode rc,
	const char *exit_reason)
	{
	lrmd_event_data_t *op = NULL;
	const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
	const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
	xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);

	if ((xml_rsc == NULL) \|\| (ID(xml_rsc) == NULL)) {
	/* @TODO Should we do something else, like direct ack? */
	crm_info("Can't fake %s failure (%d) on %s without resource configuration",
	crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
	target_node);
	return;

	} else if(operation == NULL) {
	/* This probably came from crm_resource -C, nothing to do */
	crm_info("Can't fake %s failure (%d) on %s without operation",
	ID(xml_rsc), rc, target_node);
	return;
	}

	op = construct_op(lrm_state, action, ID(xml_rsc), operation);

	if (pcmk__str_eq(operation, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
	// Notifications can't fail
	fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL);
	} else {
	fake_op_status(lrm_state, op, op_status, rc, exit_reason);
	}

	crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
	op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);

	// Process the result as if it came from the LRM
	process_lrm_event(lrm_state, op, NULL, action);
	lrmd_free_event(op);
	}

	/*!
	* \internal
	* \brief Get target of an LRM operation (replacing \p NULL with local node
	* name)
	*
	* \param[in] xml LRM operation data XML
	*
	* \return LRM operation target node name (local node or Pacemaker Remote node)
	*/
	static const char *
	lrm_op_target(const xmlNode *xml)
	{
	const char *target = NULL;

	if (xml) {
	target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
	}
	if (target == NULL) {
	target = controld_globals.our_nodename;
	}
	return target;
	}

	static void
	fail_lrm_resource(xmlNode xml, lrm_state_t lrm_state, const char *user_name,
	const char from_host, const char from_sys)
	{
	lrmd_event_data_t *op = NULL;
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);

	CRM_CHECK(xml_rsc != NULL, return);

	/* The executor simply executes operations and reports the results, without
	* any concept of success or failure, so to fail a resource, we must fake
	* what a failure looks like.
	*
	* To do this, we create a fake executor operation event for the resource,
	* and pass that event to the executor client callback so it will be
	* processed as if it came from the executor.
	*/
	op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");

	free((char*) op->user_data);
	op->user_data = NULL;
	op->interval_ms = 0;

	if (user_name && !pcmk__is_privileged(user_name)) {
	crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
	fake_op_status(lrm_state, op, PCMK_EXEC_ERROR,
	PCMK_OCF_INSUFFICIENT_PRIV,
	"Unprivileged user cannot fail resources");
	controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	return;
	}


	if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
	crm_info("Failing resource %s...", rsc->id);
	fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR,
	"Simulated failure");
	process_lrm_event(lrm_state, op, NULL, xml);
	op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded
	lrmd_free_rsc_info(rsc);

	} else {
	crm_info("Cannot find/create resource in order to fail it...");
	crm_log_xml_warn(xml, "bad input");
	fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR,
	"Cannot fail unknown resource");
	}

	controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
	lrmd_free_event(op);
	}

	static void
	handle_reprobe_op(lrm_state_t lrm_state, const char from_sys,
	const char from_host, const char user_name,
	gboolean is_remote_node, bool reprobe_all_nodes)
	{
	crm_notice("Forcing the status of all resources to be redetected");
	force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node,
	reprobe_all_nodes);

	if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {

	xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
	from_sys, CRM_SYSTEM_LRMD,
	controld_globals.our_uuid);

	crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}
	free_xml(reply);
	}
	}

	static bool do_lrm_cancel(ha_msg_input_t input, lrm_state_t lrm_state,
	lrmd_rsc_info_t rsc, const char from_host, const char *from_sys)
	{
	char *op_key = NULL;
	char *meta_key = NULL;
	int call = 0;
	const char *call_id = NULL;
	const char *op_task = NULL;
	guint interval_ms = 0;
	gboolean in_progress = FALSE;
	xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);

	CRM_CHECK(params != NULL, return FALSE);

	meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
	op_task = crm_element_value(params, meta_key);
	free(meta_key);
	CRM_CHECK(op_task != NULL, return FALSE);

	meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
	if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
	free(meta_key);
	return FALSE;
	}
	free(meta_key);

	op_key = pcmk__op_key(rsc->id, op_task, interval_ms);

	meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
	call_id = crm_element_value(params, meta_key);
	free(meta_key);

	crm_debug("Scheduler requested op %s (call=%s) be cancelled",
	op_key, (call_id? call_id : "NA"));
	pcmk__scan_min_int(call_id, &call, 0);
	if (call == 0) {
	// Normal case when the scheduler cancels a recurring op
	in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);

	} else {
	// Normal case when the scheduler cancels an orphan op
	in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
	}

	// Acknowledge cancellation operation if for a remote connection resource
	if (!in_progress \|\| is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
	char *op_id = make_stop_id(rsc->id, call);

	if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
	crm_info("Nothing known about operation %d for %s", call, op_key);
	}
	controld_delete_action_history_by_key(rsc->id, lrm_state->node_name,
	op_key, call);
	send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
	from_host, from_sys);

	/* needed at least for cancellation of a remote operation */
	if (lrm_state->active_ops != NULL) {
	g_hash_table_remove(lrm_state->active_ops, op_id);
	}
	free(op_id);

	} else {
	/* No ack is needed since abcdaa8, but peers with older versions
	* in a rolling upgrade need one. We didn't bump the feature set
	* at that commit, so we can only compare against the previous
	* CRM version (3.0.8). If any peers have feature set 3.0.9 but
	* not abcdaa8, they will time out waiting for the ack (no
	* released versions of Pacemaker are affected).
	*/
	const char *peer_version = crm_element_value(params,
	PCMK_XA_CRM_FEATURE_SET);

	if (compare_version(peer_version, "3.0.8") <= 0) {
	crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
	op_key, from_host, peer_version);
	send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
	from_host, from_sys);
	}
	}

	free(op_key);
	return TRUE;
	}

	static void
	do_lrm_delete(ha_msg_input_t input, lrm_state_t lrm_state,
	lrmd_rsc_info_t rsc, const char from_sys, const char *from_host,
	bool crm_rsc_delete, const char *user_name)
	{
	bool unregister = true;
	int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
	user_name,
	cib_dryrun\|cib_sync_call);

	if (cib_rc != pcmk_rc_ok) {
	lrmd_event_data_t *op = NULL;

	op = construct_op(lrm_state, input->xml, rsc->id, PCMK_ACTION_DELETE);

	/* These are resource clean-ups, not actions, so no exit reason is
	* needed.
	*/
	lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL);
	controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
	lrmd_free_event(op);
	return;
	}

	if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
	unregister = false;
	}

	delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
	user_name, input, unregister, true);
	}

	// User data for asynchronous metadata execution
	struct metadata_cb_data {
	lrmd_rsc_info_t *rsc; // Copy of resource information
	xmlNode *input_xml; // Copy of FSA input XML
	};

	static struct metadata_cb_data *
	new_metadata_cb_data(lrmd_rsc_info_t rsc, xmlNode input_xml)
	{
	struct metadata_cb_data *data = NULL;

	data = calloc(1, sizeof(struct metadata_cb_data));
	CRM_ASSERT(data != NULL);
	data->input_xml = copy_xml(input_xml);
	data->rsc = lrmd_copy_rsc_info(rsc);
	return data;
	}

	static void
	free_metadata_cb_data(struct metadata_cb_data *data)
	{
	lrmd_free_rsc_info(data->rsc);
	free_xml(data->input_xml);
	free(data);
	}

	/*!
	* \internal
	* \brief Execute an action after metadata has been retrieved
	*
	* \param[in] pid Ignored
	* \param[in] result Result of metadata action
	* \param[in] user_data Metadata callback data
	*/
	static void
	metadata_complete(int pid, const pcmk__action_result_t result, void user_data)
	{
	struct metadata_cb_data data = (struct metadata_cb_data ) user_data;

	struct ra_metadata_s *md = NULL;
	lrm_state_t *lrm_state = lrm_state_find(lrm_op_target(data->input_xml));

	if ((lrm_state != NULL) && pcmk__result_ok(result)) {
	md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc,
	result->action_stdout);
	}
	if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
	do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md);
	}
	free_metadata_cb_data(data);
	}

	/* A_LRM_INVOKE */
	void
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	lrm_state_t *lrm_state = NULL;
	const char *crm_op = NULL;
	const char *from_sys = NULL;
	const char *from_host = NULL;
	const char *operation = NULL;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *user_name = NULL;
	const char *target_node = lrm_op_target(input->xml);
	gboolean is_remote_node = FALSE;
	bool crm_rsc_delete = FALSE;

	// Message routed to the local node is targeting a specific, non-local node
	is_remote_node = !pcmk__str_eq(target_node, controld_globals.our_nodename,
	pcmk__str_casei);

	lrm_state = lrm_state_find(target_node);
	if ((lrm_state == NULL) && is_remote_node) {
	crm_err("Failing action because local node has never had connection to remote node %s",
	target_node);
	synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
	PCMK_OCF_UNKNOWN_ERROR,
	"Local node has no connection to remote");
	return;
	}
	CRM_ASSERT(lrm_state != NULL);

	user_name = pcmk__update_acl_user(input->msg, F_CRM_USER, NULL);
	crm_op = crm_element_value(input->msg, F_CRM_TASK);
	from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
	if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
	from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
	}

	if (pcmk__str_eq(crm_op, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
	if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
	crm_rsc_delete = TRUE; // from crm_resource
	}
	operation = PCMK_ACTION_DELETE;

	} else if (input->xml != NULL) {
	operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
	}

	CRM_CHECK(!pcmk__str_empty(crm_op) \|\| !pcmk__str_empty(operation), return);

	crm_trace("'%s' execution request from %s as %s user",
	pcmk__s(crm_op, operation),
	pcmk__s(from_sys, "unknown subsystem"),
	pcmk__s(user_name, "current"));

	if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) {
	fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
	from_sys);

	} else if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_none)) {
	/* @COMPAT This can only be sent by crm_resource --refresh on a
	* Pacemaker Remote node running Pacemaker 1.1.9, which is extremely
	* unlikely. It previously would cause the controller to re-write its
	* resource history to the CIB. Just ignore it.
	*/
	crm_notice("Ignoring refresh request from Pacemaker Remote 1.1.9 node");

	// @COMPAT DCs <1.1.14 in a rolling upgrade might schedule this op
	} else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_none)) {
	update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
	user_name, is_remote_node);

	} else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none)
	\|\| pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
	const char *raw_target = NULL;

	if (input->xml != NULL) {
	// For CRM_OP_REPROBE, a NULL target means we're targeting all nodes
	raw_target = crm_element_value(input->xml, XML_LRM_ATTR_TARGET);
	}
	handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
	is_remote_node, (raw_target == NULL));

	} else if (operation != NULL) {
	lrmd_rsc_info_t *rsc = NULL;
	xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
	gboolean create_rsc = !pcmk__str_eq(operation, PCMK_ACTION_DELETE,
	pcmk__str_none);
	int rc;

	// We can't return anything meaningful without a resource ID
	CRM_CHECK(xml_rsc && ID(xml_rsc), return);

	rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
	if (rc == -ENOTCONN) {
	synthesize_lrmd_failure(lrm_state, input->xml,
	PCMK_EXEC_NOT_CONNECTED,
	PCMK_OCF_UNKNOWN_ERROR,
	"Not connected to remote executor");
	return;

	} else if ((rc < 0) && !create_rsc) {
	/* Delete of malformed or nonexistent resource
	* (deleting something that does not exist is a success)
	*/
	crm_notice("Not registering resource '%s' for a %s event "
	CRM_XS " get-rc=%d (%s) transition-key=%s",
	ID(xml_rsc), operation,
	rc, pcmk_strerror(rc), ID(input->xml));
	delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
	user_name, true);
	return;

	} else if (rc == -EINVAL) {
	// Resource operation on malformed resource
	crm_err("Invalid resource definition for %s", ID(xml_rsc));
	crm_log_xml_warn(input->msg, "invalid resource");
	synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
	PCMK_OCF_NOT_CONFIGURED, // fatal error
	"Invalid resource definition");
	return;

	} else if (rc < 0) {
	// Error communicating with the executor
	crm_err("Could not register resource '%s' with executor: %s "
	CRM_XS " rc=%d",
	ID(xml_rsc), pcmk_strerror(rc), rc);
	crm_log_xml_warn(input->msg, "failed registration");
	synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
	PCMK_OCF_INVALID_PARAM, // hard error
	"Could not register resource with executor");
	return;
	}

	if (pcmk__str_eq(operation, PCMK_ACTION_CANCEL, pcmk__str_none)) {
	if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
	crm_log_xml_warn(input->xml, "Bad command");
	}

	} else if (pcmk__str_eq(operation, PCMK_ACTION_DELETE,
	pcmk__str_none)) {
	do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
	crm_rsc_delete, user_name);

	} else {
	struct ra_metadata_s *md = NULL;

	/* Getting metadata from cache is OK except for start actions --
	* always refresh from the agent for those, in case the resource
	* agent was updated.
	*
	* @TODO Only refresh metadata for starts if the agent actually
	* changed (using something like inotify, or a hash or modification
	* time of the agent executable).
	*/
	if (strcmp(operation, PCMK_ACTION_START) != 0) {
	md = controld_get_rsc_metadata(lrm_state, rsc,
	controld_metadata_from_cache);
	}

	if ((md == NULL) && crm_op_needs_metadata(rsc->standard,
	operation)) {
	/* Most likely, we'll need the agent metadata to record the
	* pending operation and the operation result. Get it now rather
	* than wait until then, so the metadata action doesn't eat into
	* the real action's timeout.
	*
	* @TODO Metadata is retrieved via direct execution of the
	* agent, which has a couple of related issues: the executor
	* should execute agents, not the controller; and metadata for
	* Pacemaker Remote nodes should be collected on those nodes,
	* not locally.
	*/
	struct metadata_cb_data *data = NULL;

	data = new_metadata_cb_data(rsc, input->xml);
	crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously",
	rsc->id, rsc->standard,
	((rsc->provider == NULL)? "" : ":"),
	((rsc->provider == NULL)? "" : rsc->provider),
	rsc->type);
	(void) lrmd__metadata_async(rsc, metadata_complete,
	(void *) data);
	} else {
	do_lrm_rsc_op(lrm_state, rsc, input->xml, md);
	}
	}

	lrmd_free_rsc_info(rsc);

	} else {
	crm_err("Invalid execution request: unknown command '%s' (bug?)",
	crm_op);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	static lrmd_event_data_t *
	construct_op(const lrm_state_t lrm_state, const xmlNode rsc_op,
	const char rsc_id, const char operation)
	{
	lrmd_event_data_t *op = NULL;
	const char *op_delay = NULL;
	const char *op_timeout = NULL;
	GHashTable *params = NULL;

	xmlNode *primitive = NULL;
	const char *class = NULL;

	const char *transition = NULL;

	CRM_ASSERT(rsc_id && operation);

	op = lrmd_new_event(rsc_id, operation, 0);
	op->type = lrmd_event_exec_complete;
	op->timeout = 0;
	op->start_delay = 0;
	lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);

	if (rsc_op == NULL) {
	CRM_LOG_ASSERT(pcmk__str_eq(operation, PCMK_ACTION_STOP,
	pcmk__str_casei));
	op->user_data = NULL;
	/* the stop_all_resources() case
	* by definition there is no DC (or they'd be shutting
	* us down).
	* So we should put our version here.
	*/
	op->params = pcmk__strkey_table(free, free);

	g_hash_table_insert(op->params, strdup(PCMK_XA_CRM_FEATURE_SET),
	strdup(CRM_FEATURE_SET));

	crm_trace("Constructed %s op for %s", operation, rsc_id);
	return op;
	}

	params = xml2list(rsc_op);
	g_hash_table_remove(params, CRM_META "_op_target_rc");

	op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
	pcmk__scan_min_int(op_delay, &op->start_delay, 0);

	op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
	pcmk__scan_min_int(op_timeout, &op->timeout, 0);

	if (pcmk__guint_from_hash(params, CRM_META "_" XML_LRM_ATTR_INTERVAL_MS, 0,
	&(op->interval_ms)) != pcmk_rc_ok) {
	op->interval_ms = 0;
	}

	/* Use pcmk_monitor_timeout instead of meta timeout for stonith
	recurring monitor, if set */
	primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE);
	- class = crm_element_value(primitive, XML_AGENT_ATTR_CLASS);
	+ class = crm_element_value(primitive, PCMK_XA_CLASS);

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
	&& pcmk__str_eq(operation, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& (op->interval_ms > 0)) {

	op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
	if (op_timeout != NULL) {
	op->timeout = crm_get_msec(op_timeout);
	}
	}

	if (!pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)) {
	op->params = params;

	} else {
	rsc_history_t *entry = NULL;

	if (lrm_state) {
	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	}

	/* If we do not have stop parameters cached, use
	* whatever we are given */
	if (!entry \|\| !entry->stop_params) {
	op->params = params;
	} else {
	/* Copy the cached parameter list so that we stop the resource
	* with the old attributes, not the new ones */
	op->params = pcmk__strkey_table(free, free);

	g_hash_table_foreach(params, copy_meta_keys, op->params);
	g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
	g_hash_table_destroy(params);
	params = NULL;
	}
	}

	/* sanity */
	if (op->timeout <= 0) {
	op->timeout = op->interval_ms;
	}
	if (op->start_delay < 0) {
	op->start_delay = 0;
	}

	transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
	CRM_CHECK(transition != NULL, return op);

	op->user_data = strdup(transition);

	if (op->interval_ms != 0) {
	if (pcmk__strcase_any_of(operation, PCMK_ACTION_START, PCMK_ACTION_STOP,
	NULL)) {
	crm_err("Start and Stop actions cannot have an interval: %u",
	op->interval_ms);
	op->interval_ms = 0;
	}
	}

	crm_trace("Constructed %s op for %s: interval=%u",
	operation, rsc_id, op->interval_ms);

	return op;
	}

	/*!
	* \internal
	* \brief Send a (synthesized) event result
	*
	* Reply with a synthesized event result directly, as opposed to going through
	* the executor.
	*
	* \param[in] to_host Host to send result to
	* \param[in] to_sys IPC name to send result (NULL for transition engine)
	* \param[in] rsc Type information about resource the result is for
	* \param[in,out] op Event with result to send
	* \param[in] rsc_id ID of resource the result is for
	*/
	void
	controld_ack_event_directly(const char to_host, const char to_sys,
	const lrmd_rsc_info_t rsc, lrmd_event_data_t op,
	const char *rsc_id)
	{
	xmlNode *reply = NULL;
	xmlNode update, iter;
	crm_node_t *peer = NULL;

	CRM_CHECK(op != NULL, return);
	if (op->rsc_id == NULL) {
	CRM_ASSERT(rsc_id != NULL);
	op->rsc_id = strdup(rsc_id);
	}
	if (to_sys == NULL) {
	to_sys = CRM_SYSTEM_TENGINE;
	}

	peer = crm_get_peer(0, controld_globals.our_nodename);
	update = create_node_state_update(peer, node_update_none, NULL,
	__func__);

	iter = create_xml_node(update, XML_CIB_TAG_LRM);
	crm_xml_add(iter, PCMK_XA_ID, controld_globals.our_uuid);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);

	crm_xml_add(iter, PCMK_XA_ID, op->rsc_id);

	controld_add_resource_history_xml(iter, rsc, op,
	controld_globals.our_nodename);
	reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);

	crm_log_xml_trace(update, "[direct ACK]");

	crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
	op->rsc_id, op->op_type, op->interval_ms, op->user_data,
	crm_element_value(reply, XML_ATTR_REFERENCE));

	if (relay_message(reply, TRUE) == FALSE) {
	crm_log_xml_err(reply, "Unable to route reply");
	}

	free_xml(update);
	free_xml(reply);
	}

	gboolean
	verify_stopped(enum crmd_fsa_state cur_state, int log_level)
	{
	gboolean res = TRUE;
	GList *lrm_state_list = lrm_state_get_list();
	GList *state_entry;

	for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
	lrm_state_t *lrm_state = state_entry->data;

	if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
	/* keep iterating through all even when false is returned */
	res = FALSE;
	}
	}

	controld_set_fsa_input_flags(R_SENT_RSC_STOP);
	g_list_free(lrm_state_list); lrm_state_list = NULL;
	return res;
	}

	struct stop_recurring_action_s {
	lrmd_rsc_info_t *rsc;
	lrm_state_t *lrm_state;
	};

	static gboolean
	stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	struct stop_recurring_action_s *event = user_data;
	active_op_t *op = value;

	if ((op->interval_ms != 0)
	&& pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {

	crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
	remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
	}

	return remove;
	}

	static gboolean
	stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
	{
	gboolean remove = FALSE;
	lrm_state_t *lrm_state = user_data;
	active_op_t *op = value;

	if (op->interval_ms != 0) {
	crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
	(const char *) key);
	remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
	}

	return remove;
	}

	/*!
	* \internal
	* \brief Check whether recurring actions should be cancelled before an action
	*
	* \param[in] rsc_id Resource that action is for
	* \param[in] action Action being performed
	* \param[in] interval_ms Operation interval of \p action (in milliseconds)
	*
	* \return true if recurring actions should be cancelled, otherwise false
	*/
	static bool
	should_cancel_recurring(const char rsc_id, const char action, guint interval_ms)
	{
	if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0)
	&& (strcmp(action, PCMK_ACTION_MIGRATE_TO) == 0)) {
	/* Don't stop monitoring a migrating Pacemaker Remote connection
	* resource until the entire migration has completed. We must detect if
	* the connection is unexpectedly severed, even during a migration.
	*/
	return false;
	}

	// Cancel recurring actions before changing resource state
	return (interval_ms == 0)
	&& !pcmk__str_any_of(action, PCMK_ACTION_MONITOR,
	PCMK_ACTION_NOTIFY, NULL);
	}

	/*!
	* \internal
	* \brief Check whether an action should not be performed at this time
	*
	* \param[in] operation Action to be performed
	*
	* \return Readable description of why action should not be performed,
	* or NULL if it should be performed
	*/
	static const char *
	should_nack_action(const char *action)
	{
	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)
	&& pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {

	register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
	return "Not attempting start due to shutdown in progress";
	}

	switch (controld_globals.fsa_state) {
	case S_NOT_DC:
	case S_POLICY_ENGINE: // Recalculating
	case S_TRANSITION_ENGINE:
	break;
	default:
	if (!pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
	return "Controller cannot attempt actions at this time";
	}
	break;
	}
	return NULL;
	}

	static void
	do_lrm_rsc_op(lrm_state_t lrm_state, lrmd_rsc_info_t rsc, xmlNode *msg,
	struct ra_metadata_s *md)
	{
	int rc;
	int call_id = 0;
	char *op_id = NULL;
	lrmd_event_data_t *op = NULL;
	fsa_data_t *msg_data = NULL;
	const char *transition = NULL;
	const char *operation = NULL;
	const char *nack_reason = NULL;

	CRM_CHECK((rsc != NULL) && (msg != NULL), return);

	operation = crm_element_value(msg, XML_LRM_ATTR_TASK);
	CRM_CHECK(!pcmk__str_empty(operation), return);

	transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
	if (pcmk__str_empty(transition)) {
	crm_log_xml_err(msg, "Missing transition number");
	}

	if (lrm_state == NULL) {
	// This shouldn't be possible, but provide a failsafe just in case
	crm_err("Cannot execute %s of %s: No executor connection "
	CRM_XS " transition_key=%s",
	operation, rsc->id, pcmk__s(transition, ""));
	synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID,
	PCMK_OCF_UNKNOWN_ERROR,
	"No executor connection");
	return;
	}

	if (pcmk__str_any_of(operation, PCMK_ACTION_RELOAD,
	PCMK_ACTION_RELOAD_AGENT, NULL)) {
	/* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs
	* will schedule reload-agent actions only. In either case, we need
	* to map that to whatever the resource agent actually supports.
	* Default to the OCF 1.1 name.
	*/
	if ((md != NULL)
	&& pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
	operation = PCMK_ACTION_RELOAD;
	} else {
	operation = PCMK_ACTION_RELOAD_AGENT;
	}
	}

	op = construct_op(lrm_state, msg, rsc->id, operation);
	CRM_CHECK(op != NULL, return);

	if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) {
	guint removed = 0;
	struct stop_recurring_action_s data;

	data.rsc = rsc;
	data.lrm_state = lrm_state;
	removed = g_hash_table_foreach_remove(lrm_state->active_ops,
	stop_recurring_action_by_rsc,
	&data);

	if (removed) {
	crm_debug("Stopped %u recurring operation%s in preparation for "
	PCMK__OP_FMT, removed, pcmk__plural_s(removed),
	rsc->id, operation, op->interval_ms);
	}
	}

	/* now do the op */
	crm_notice("Requesting local execution of %s operation for %s on %s "
	CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
	pcmk__readable_action(op->op_type, op->interval_ms), rsc->id,
	lrm_state->node_name, pcmk__s(transition, ""), rsc->id,
	operation, op->interval_ms);

	nack_reason = should_nack_action(operation);
	if (nack_reason != NULL) {
	crm_notice("Discarding attempt to perform action %s on %s in state %s "
	"(shutdown=%s)", operation, rsc->id,
	fsa_state2string(controld_globals.fsa_state),
	pcmk__btoa(pcmk_is_set(controld_globals.fsa_input_register,
	R_SHUTDOWN)));

	lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
	nack_reason);
	controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
	lrmd_free_event(op);
	free(op_id);
	return;
	}

	controld_record_pending_op(lrm_state->node_name, rsc, op);

	op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);

	if (op->interval_ms > 0) {
	/* cancel it so we can then restart it without conflict */
	cancel_op_key(lrm_state, rsc, op_id, FALSE);
	}

	rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
	op->user_data, op->interval_ms,
	op->timeout, op->start_delay,
	op->params, &call_id);
	if (rc == pcmk_rc_ok) {
	/* record all operations so we can wait
	* for them to complete during shutdown
	*/
	char *call_id_s = make_stop_id(rsc->id, call_id);
	active_op_t *pending = NULL;

	pending = calloc(1, sizeof(active_op_t));
	crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);

	pending->call_id = call_id;
	pending->interval_ms = op->interval_ms;
	pending->op_type = strdup(operation);
	pending->op_key = strdup(op_id);
	pending->rsc_id = strdup(rsc->id);
	pending->start_time = time(NULL);
	pcmk__str_update(&pending->user_data, op->user_data);
	if (crm_element_value_epoch(msg, PCMK_OPT_SHUTDOWN_LOCK,
	&(pending->lock_time)) != pcmk_ok) {
	pending->lock_time = 0;
	}
	g_hash_table_replace(lrm_state->active_ops, call_id_s, pending);

	if ((op->interval_ms > 0)
	&& (op->start_delay > START_DELAY_THRESHOLD)) {
	int target_rc = PCMK_OCF_OK;

	crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
	decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
	lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
	controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
	}

	pending->params = op->params;
	op->params = NULL;

	} else if (lrm_state_is_local(lrm_state)) {
	crm_err("Could not initiate %s action for resource %s locally: %s "
	CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
	fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
	PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
	process_lrm_event(lrm_state, op, NULL, NULL);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);

	} else {
	crm_err("Could not initiate %s action for resource %s remotely on %s: "
	"%s " CRM_XS " rc=%d",
	operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
	fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
	PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
	process_lrm_event(lrm_state, op, NULL, NULL);
	}

	free(op_id);
	lrmd_free_event(op);
	}

	void
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
	{
	CRM_CHECK(FALSE, return);
	}

	static char *
	unescape_newlines(const char *string)
	{
	char *pch = NULL;
	char *ret = NULL;
	static const char *escaped_newline = "\\n";

	if (!string) {
	return NULL;
	}

	ret = strdup(string);
	pch = strstr(ret, escaped_newline);
	while (pch != NULL) {
	/* Replace newline escape pattern with actual newline (and a space so we
	* don't have to shuffle the rest of the buffer)
	*/
	pch[0] = '\n';
	pch[1] = ' ';
	pch = strstr(pch, escaped_newline);
	}

	return ret;
	}

	static bool
	did_lrm_rsc_op_fail(lrm_state_t lrm_state, const char rsc_id,
	const char * op_type, guint interval_ms)
	{
	rsc_history_t *entry = NULL;

	CRM_CHECK(lrm_state != NULL, return FALSE);
	CRM_CHECK(rsc_id != NULL, return FALSE);
	CRM_CHECK(op_type != NULL, return FALSE);

	entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
	if (entry == NULL \|\| entry->failed == NULL) {
	return FALSE;
	}

	if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
	&& pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
	&& entry->failed->interval_ms == interval_ms) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \internal
	* \brief Log the result of an executor action (actual or synthesized)
	*
	* \param[in] op Executor action to log result for
	* \param[in] op_key Operation key for action
	* \param[in] node_name Name of node action was performed on, if known
	* \param[in] confirmed Whether to log that graph action was confirmed
	*/
	static void
	log_executor_event(const lrmd_event_data_t op, const char op_key,
	const char *node_name, gboolean confirmed)
	{
	int log_level = LOG_ERR;
	GString *str = g_string_sized_new(100); // reasonable starting size

	pcmk__g_strcat(str,
	"Result of ",
	pcmk__readable_action(op->op_type, op->interval_ms),
	" operation for ", op->rsc_id, NULL);

	if (node_name != NULL) {
	pcmk__g_strcat(str, " on ", node_name, NULL);
	}

	switch (op->op_status) {
	case PCMK_EXEC_DONE:
	log_level = LOG_NOTICE;
	pcmk__g_strcat(str, ": ", services_ocf_exitcode_str(op->rc), NULL);
	break;

	case PCMK_EXEC_TIMEOUT:
	pcmk__g_strcat(str,
	": ", pcmk_exec_status_str(op->op_status), " after ",
	pcmk__readable_interval(op->timeout), NULL);
	break;

	case PCMK_EXEC_CANCELLED:
	log_level = LOG_INFO;
	/* order of __attribute__ and Fall through comment is IMPORTANT!
	* do not change it without proper testing with both clang and gcc
	* in multiple versions.
	* the clang check allows to build with all versions of clang.
	* the has_c_attribute check is to workaround a bug in clang version
	* in rhel7. has_attribute would happily return "YES SIR WE GOT IT"
	* and fail the build the next line.
	*/
	#ifdef __clang__
	#ifdef __has_c_attribute
	#if __has_attribute(fallthrough)
	__attribute__((fallthrough));
	#endif
	#endif
	#endif
	// Fall through
	default:
	pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status),
	NULL);
	}

	if ((op->exit_reason != NULL)
	&& ((op->op_status != PCMK_EXEC_DONE) \|\| (op->rc != PCMK_OCF_OK))) {

	pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL);
	}

	g_string_append(str, " " CRM_XS);
	g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s",
	(confirmed? "" : "un"), op->call_id, op_key);
	if (op->op_status == PCMK_EXEC_DONE) {
	g_string_append_printf(str, " rc=%d", op->rc);
	}

	do_crm_log(log_level, "%s", str->str);
	g_string_free(str, TRUE);

	/* The services library has already logged the output at info or debug
	* level, so just raise to notice if it looks like a failure.
	*/
	if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) {
	char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output",
	op->rsc_id, op->op_type,
	op->interval_ms, node_name);

	crm_log_output(LOG_NOTICE, prefix, op->output);
	free(prefix);
	}
	}

	void
	process_lrm_event(lrm_state_t lrm_state, lrmd_event_data_t op,
	active_op_t pending, const xmlNode action_xml)
	{
	char *op_id = NULL;
	char *op_key = NULL;

	gboolean remove = FALSE;
	gboolean removed = FALSE;
	bool need_direct_ack = FALSE;
	lrmd_rsc_info_t *rsc = NULL;
	const char *node_name = NULL;

	CRM_CHECK(op != NULL, return);
	CRM_CHECK(op->rsc_id != NULL, return);

	// Remap new status codes for older DCs
	if (compare_version(controld_globals.dc_version, "3.2.0") < 0) {
	switch (op->op_status) {
	case PCMK_EXEC_NOT_CONNECTED:
	lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
	PCMK_EXEC_ERROR, op->exit_reason);
	break;
	case PCMK_EXEC_INVALID:
	lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
	op->exit_reason);
	break;
	default:
	break;
	}
	}

	op_id = make_stop_id(op->rsc_id, op->call_id);
	op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);

	// Get resource info if available (from executor state or action XML)
	if (lrm_state) {
	rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
	}
	if ((rsc == NULL) && action_xml) {
	xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);

	- const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
	+ const char *standard = crm_element_value(xml, PCMK_XA_CLASS);
	const char *provider = crm_element_value(xml, PCMK_XA_PROVIDER);
	const char *type = crm_element_value(xml, PCMK_XA_TYPE);

	if (standard && type) {
	crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
	op->rsc_id, standard,
	(provider? ":" : ""), (provider? provider : ""), type);
	rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
	} else {
	crm_err("Can't process %s result because %s agent information not cached or in XML",
	op_key, op->rsc_id);
	}
	}

	// Get node name if available (from executor state or action XML)
	if (lrm_state) {
	node_name = lrm_state->node_name;
	} else if (action_xml) {
	node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
	}

	if(pending == NULL) {
	remove = TRUE;
	if (lrm_state) {
	pending = g_hash_table_lookup(lrm_state->active_ops, op_id);
	}
	}

	if (op->op_status == PCMK_EXEC_ERROR) {
	switch(op->rc) {
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Leave it to the TE/scheduler to decide if this is an error
	op->op_status = PCMK_EXEC_DONE;
	break;
	default:
	/* Nothing to do */
	break;
	}
	}

	if (op->op_status != PCMK_EXEC_CANCELLED) {
	/* We might not record the result, so directly acknowledge it to the
	* originator instead, so it doesn't time out waiting for the result
	* (especially important if part of a transition).
	*/
	need_direct_ack = TRUE;

	if (controld_action_is_recordable(op->op_type)) {
	if (node_name && rsc) {
	// We should record the result, and happily, we can
	time_t lock_time = (pending == NULL)? 0 : pending->lock_time;

	controld_update_resource_history(node_name, rsc, op, lock_time);
	need_direct_ack = FALSE;

	} else if (op->rsc_deleted) {
	/* We shouldn't record the result (likely the resource was
	* refreshed, cleaned, or removed while this operation was
	* in flight).
	*/
	crm_notice("Not recording %s result in CIB because "
	"resource information was removed since it was initiated",
	op_key);
	} else {
	/* This shouldn't be possible; the executor didn't consider the
	* resource deleted, but we couldn't find resource or node
	* information.
	*/
	crm_err("Unable to record %s result in CIB: %s", op_key,
	(node_name? "No resource information" : "No node name"));
	}
	}

	} else if (op->interval_ms == 0) {
	/* A non-recurring operation was cancelled. Most likely, the
	* never-initiated action was removed from the executor's pending
	* operations list upon resource removal.
	*/
	need_direct_ack = TRUE;

	} else if (pending == NULL) {
	/* This recurring operation was cancelled, but was not pending. No
	* transition actions are waiting on it, nothing needs to be done.
	*/

	} else if (op->user_data == NULL) {
	/* This recurring operation was cancelled and pending, but we don't
	* have a transition key. This should never happen.
	*/
	crm_err("Recurring operation %s was cancelled without transition information",
	op_key);

	} else if (pcmk_is_set(pending->flags, active_op_remove)) {
	/* This recurring operation was cancelled (by us) and pending, and we
	* have been waiting for it to finish.
	*/
	if (lrm_state) {
	controld_delete_action_history(op);
	}

	/* Directly acknowledge failed recurring actions here. The above call to
	* controld_delete_action_history() will not erase any corresponding
	* last_failure entry, which means that the DC won't confirm the
	* cancellation via process_op_deletion(), and the transition would
	* otherwise wait for the action timer to pop.
	*/
	if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
	pending->op_type, pending->interval_ms)) {
	need_direct_ack = TRUE;
	}

	} else if (op->rsc_deleted) {
	/* This recurring operation was cancelled (but not by us, and the
	* executor does not have resource information, likely due to resource
	* cleanup, refresh, or removal) and pending.
	*/
	crm_debug("Recurring op %s was cancelled due to resource deletion",
	op_key);
	need_direct_ack = TRUE;

	} else {
	/* This recurring operation was cancelled (but not by us, likely by the
	* executor before stopping the resource) and pending. We don't need to
	* do anything special.
	*/
	}

	if (need_direct_ack) {
	controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
	}

	if(remove == FALSE) {
	/* The caller will do this afterwards, but keep the logging consistent */
	removed = TRUE;

	} else if (lrm_state && ((op->interval_ms == 0)
	\|\| (op->op_status == PCMK_EXEC_CANCELLED))) {

	gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id);

	if (op->interval_ms != 0) {
	removed = TRUE;
	} else if (found) {
	removed = TRUE;
	crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
	op_key, op->call_id, op_id,
	g_hash_table_size(lrm_state->active_ops));
	}
	}

	log_executor_event(op, op_key, node_name, removed);

	if (lrm_state) {
	if (!pcmk__str_eq(op->op_type, PCMK_ACTION_META_DATA,
	pcmk__str_casei)) {
	crmd_alert_resource_op(lrm_state->node_name, op);
	} else if (rsc && (op->rc == PCMK_OCF_OK)) {
	char *metadata = unescape_newlines(op->output);

	controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata);
	free(metadata);
	}
	}

	if (op->rsc_deleted) {
	crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
	if (lrm_state) {
	delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL,
	true);
	}
	}

	/* If a shutdown was escalated while operations were pending,
	* then the FSA will be stalled right now... allow it to continue
	*/
	controld_trigger_fsa();
	if (lrm_state && rsc) {
	update_history_cache(lrm_state, rsc, op);
	}

	lrmd_free_rsc_info(rsc);
	free(op_key);
	free(op_id);
	}
	diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c
	index 7b9bfe91b8..82f4cc5d39 100644
	--- a/daemons/controld/controld_te_actions.c
	+++ b/daemons/controld/controld_te_actions.c
	@@ -1,749 +1,749 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/lrmd.h> // lrmd_event_data_t, lrmd_free_event()
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>

	#include <pacemaker-internal.h>
	#include <pacemaker-controld.h>

	static GHashTable *te_targets = NULL;
	void send_rsc_command(pcmk__graph_action_t *action);
	static void te_update_job_count(pcmk__graph_action_t *action, int offset);

	static void
	te_start_action_timer(const pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	action->timer = g_timeout_add(action->timeout + graph->network_delay,
	action_timer_callback, (void *) action);
	CRM_ASSERT(action->timer != 0);
	}

	/*!
	* \internal
	* \brief Execute a graph pseudo-action
	*
	* \param[in,out] graph Transition graph being executed
	* \param[in,out] pseudo Pseudo-action to execute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	execute_pseudo_action(pcmk__graph_t graph, pcmk__graph_action_t pseudo)
	{
	const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);

	/* send to peers as well? */
	if (pcmk__str_eq(task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_casei)) {
	GHashTableIter iter;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *cmd = NULL;

	if (pcmk__str_eq(controld_globals.our_nodename, node->uname,
	pcmk__str_casei)) {
	continue;
	}

	cmd = create_request(task, pseudo->xml, node->uname,
	CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
	send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
	free_xml(cmd);
	}

	remote_ra_process_maintenance_nodes(pseudo->xml);
	} else {
	/* Check action for Pacemaker Remote node side effects */
	remote_ra_process_pseudo(pseudo->xml);
	}

	crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
	crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
	te_action_confirmed(pseudo, graph);
	return pcmk_rc_ok;
	}

	static int
	get_target_rc(pcmk__graph_action_t *action)
	{
	int exit_status;

	pcmk__scan_min_int(crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC),
	&exit_status, 0);
	return exit_status;
	}

	/*!
	* \internal
	* \brief Execute a cluster action from a transition graph
	*
	* \param[in,out] graph Transition graph being executed
	* \param[in,out] action Cluster action to execute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	execute_cluster_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	char *counter = NULL;
	xmlNode *cmd = NULL;
	gboolean is_local = FALSE;

	const char *id = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;

	id = ID(action->xml);
	CRM_CHECK(!pcmk__str_empty(id), return EPROTO);

	task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	CRM_CHECK(!pcmk__str_empty(task), return EPROTO);

	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown);

	router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
	if (router_node == NULL) {
	router_node = on_node;
	if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
	const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);

	if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_none)) {
	router_node = controld_globals.our_nodename;
	}
	}
	}

	if (pcmk__str_eq(router_node, controld_globals.our_nodename,
	pcmk__str_casei)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_info("Handling controller request '%s' (%s on %s)%s%s",
	id, task, on_node, (is_local? " locally" : ""),
	(no_wait? " without waiting" : ""));

	if (is_local
	&& pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
	/* defer until everything else completes */
	crm_info("Controller request '%s' is a local shutdown", id);
	graph->completion_action = pcmk__graph_shutdown;
	graph->abort_reason = "local shutdown";
	te_action_confirmed(action, graph);
	return pcmk_rc_ok;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
	crm_node_t *peer = crm_get_peer(0, router_node);

	pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
	}

	cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);

	counter = pcmk__transition_key(controld_globals.transition_graph->id,
	action->id, get_target_rc(action),
	controld_globals.te_uuid);
	crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);

	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
	free(counter);
	free_xml(cmd);

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return ECOMM;

	} else if (no_wait) {
	te_action_confirmed(action, graph);

	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %ums instead",
	action->id, task, on_node, action->timeout, graph->network_delay);
	action->timeout = (int) graph->network_delay;
	}
	te_start_action_timer(graph, action);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Synthesize an executor event for a resource action timeout
	*
	* \param[in] action Resource action that timed out
	* \param[in] target_rc Expected result of action that timed out
	*
	* Synthesize an executor event for a resource action timeout. (If the executor
	* gets a timeout while waiting for a resource action to complete, that will be
	* reported via the usual callback. This timeout means we didn't hear from the
	* executor itself or the controller that relayed the action to the executor.)
	*
	* \return Newly created executor event for result of \p action
	* \note The caller is responsible for freeing the return value using
	* lrmd_free_event().
	*/
	static lrmd_event_data_t *
	synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *reason = NULL;
	char *dynamic_reason = NULL;

	if (pcmk__str_eq(target, get_local_node_name(), pcmk__str_casei)) {
	reason = "Local executor did not return result in time";
	} else {
	const char *router_node = NULL;

	router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
	if (router_node == NULL) {
	router_node = target;
	}
	dynamic_reason = crm_strdup_printf("Controller on %s did not return "
	"result in time", router_node);
	reason = dynamic_reason;
	}

	op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT,
	PCMK_OCF_UNKNOWN_ERROR, reason);
	op->call_id = -1;
	op->user_data = pcmk__transition_key(controld_globals.transition_graph->id,
	action->id, target_rc,
	controld_globals.te_uuid);
	free(dynamic_reason);
	return op;
	}

	static void
	controld_record_action_event(pcmk__graph_action_t *action,
	lrmd_event_data_t *op)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	xmlNode *state = NULL;
	xmlNode *rsc = NULL;
	xmlNode *action_rsc = NULL;

	int rc = pcmk_ok;

	const char *rsc_id = NULL;
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);

	int target_rc = get_target_rc(action);

	action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
	if (action_rsc == NULL) {
	return;
	}

	rsc_id = ID(action_rsc);
	CRM_CHECK(rsc_id != NULL,
	crm_log_xml_err(action->xml, "Bad:action"); return);

	/*
	update the CIB

	<node_state id="hadev">
	<lrm>
	<lrm_resources>
	<lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
	*/

	state = create_xml_node(NULL, XML_CIB_TAG_STATE);

	crm_xml_add(state, PCMK_XA_ID, target_uuid);
	crm_xml_add(state, XML_ATTR_UNAME, target);

	rsc = create_xml_node(state, XML_CIB_TAG_LRM);
	crm_xml_add(rsc, PCMK_XA_ID, target_uuid);

	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
	crm_xml_add(rsc, PCMK_XA_ID, rsc_id);


	crm_copy_xml_element(action_rsc, rsc, PCMK_XA_TYPE);
	- crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_CLASS);
	+ crm_copy_xml_element(action_rsc, rsc, PCMK_XA_CLASS);
	crm_copy_xml_element(action_rsc, rsc, PCMK_XA_PROVIDER);

	pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target,
	__func__);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS, state,
	cib_scope_local);
	fsa_register_cib_callback(rc, NULL, cib_action_updated);
	free_xml(state);

	crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)",
	rc, action->id, task_uuid, target);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update);
	}

	void
	controld_record_action_timeout(pcmk__graph_action_t *action)
	{
	lrmd_event_data_t *op = NULL;

	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	int target_rc = get_target_rc(action);

	crm_warn("%s %d: %s on %s timed out",
	action->xml->name, action->id, task_uuid, target);

	op = synthesize_timeout_event(action, target_rc);
	controld_record_action_event(action, op);
	lrmd_free_event(op);
	}

	/*!
	* \internal
	* \brief Execute a resource action from a transition graph
	*
	* \param[in,out] graph Transition graph being executed
	* \param[in,out] action Resource action to execute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	execute_rsc_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	/* never overwrite stop actions in the CIB with
	* anything other than completed results
	*
	* Writing pending stops makes it look like the
	* resource is running again
	*/
	xmlNode *cmd = NULL;
	xmlNode *rsc_op = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;
	gboolean is_local = FALSE;

	char *counter = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;
	const char *task_uuid = NULL;

	CRM_ASSERT(action != NULL);
	CRM_ASSERT(action->xml != NULL);

	pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed);
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	CRM_CHECK(!pcmk__str_empty(on_node),
	crm_err("Corrupted command(id=%s) %s: no node",
	ID(action->xml), pcmk__s(task, "without task"));
	return pcmk_rc_node_unknown);

	rsc_op = action->xml;
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	}

	counter = pcmk__transition_key(controld_globals.transition_graph->id,
	action->id, get_target_rc(action),
	controld_globals.te_uuid);
	crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);

	if (pcmk__str_eq(router_node, controld_globals.our_nodename,
	pcmk__str_casei)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
	task, task_uuid, (is_local? " locally" : ""), on_node,
	(no_wait? " without waiting" : ""), action->id);

	cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
	CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);

	if (is_local) {
	/* shortcut local resource commands */
	ha_msg_input_t data = {
	.msg = cmd,
	.xml = rsc_op,
	};

	fsa_data_t msg = {
	.id = 0,
	.data = &data,
	.data_type = fsa_dt_ha_msg,
	.fsa_input = I_NULL,
	.fsa_cause = C_FSA_INTERNAL,
	.actions = A_LRM_INVOKE,
	.origin = __func__,
	};

	do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state,
	I_NULL, &msg);

	} else {
	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
	}

	free(counter);
	free_xml(cmd);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return ECOMM;

	} else if (no_wait) {
	/* Just mark confirmed. Don't bump the job count only to immediately
	* decrement it.
	*/
	crm_info("Action %d confirmed - no wait", action->id);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(controld_globals.transition_graph, action);
	trigger_graph();

	} else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
	crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
	action->id, task, task_uuid, on_node, action->timeout);
	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %ums instead",
	action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
	action->timeout = (int) graph->network_delay;
	}
	te_update_job_count(action, 1);
	te_start_action_timer(graph, action);
	}

	return pcmk_rc_ok;
	}

	struct te_peer_s
	{
	char *name;
	int jobs;
	int migrate_jobs;
	};

	static void te_peer_free(gpointer p)
	{
	struct te_peer_s *peer = p;

	free(peer->name);
	free(peer);
	}

	void te_reset_job_counts(void)
	{
	GHashTableIter iter;
	struct te_peer_s *peer = NULL;

	if(te_targets == NULL) {
	te_targets = pcmk__strkey_table(NULL, te_peer_free);
	}

	g_hash_table_iter_init(&iter, te_targets);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
	peer->jobs = 0;
	peer->migrate_jobs = 0;
	}
	}

	static void
	te_update_job_count_on(const char *target, int offset, bool migrate)
	{
	struct te_peer_s *r = NULL;

	if(target == NULL \|\| te_targets == NULL) {
	return;
	}

	r = g_hash_table_lookup(te_targets, target);
	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	r->jobs += offset;
	if(migrate) {
	r->migrate_jobs += offset;
	}
	crm_trace("jobs[%s] = %d", target, r->jobs);
	}

	static void
	te_update_job_count(pcmk__graph_action_t *action, int offset)
	{
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	if ((action->type != pcmk__rsc_graph_action) \|\| (target == NULL)) {
	/* No limit on these */
	return;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all actions occurring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL)
	&& pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	te_update_job_count_on(t1, offset, TRUE);
	te_update_job_count_on(t2, offset, TRUE);
	return;
	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	te_update_job_count_on(target, offset, FALSE);
	}

	/*!
	* \internal
	* \brief Check whether a graph action is allowed to be executed on a node
	*
	* \param[in] graph Transition graph being executed
	* \param[in] action Graph action being executed
	* \param[in] target Name of node where action should be executed
	*
	* \return true if action is allowed, otherwise false
	*/
	static bool
	allowed_on_node(const pcmk__graph_t graph, const pcmk__graph_action_t action,
	const char *target)
	{
	int limit = 0;
	struct te_peer_s *r = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	if(target == NULL) {
	/* No limit on these */
	return true;

	} else if(te_targets == NULL) {
	return false;
	}

	r = g_hash_table_lookup(te_targets, target);
	limit = throttle_get_job_limit(target);

	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	if(limit <= r->jobs) {
	crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
	target, limit, r->jobs, id);
	return false;

	} else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
	if (pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
	target, graph->migration_limit, r->migrate_jobs, id);
	return false;
	}
	}

	crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);

	return true;
	}

	/*!
	* \internal
	* \brief Check whether a graph action is allowed to be executed
	*
	* \param[in] graph Transition graph being executed
	* \param[in] action Graph action being executed
	*
	* \return true if action is allowed, otherwise false
	*/
	static bool
	graph_action_allowed(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *target = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);

	if (action->type != pcmk__rsc_graph_action) {
	/* No limit on these */
	return true;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all actions occurring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL)
	&& pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	if (!allowed_on_node(graph, action, target)) {
	return false;
	}

	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	return allowed_on_node(graph, action, target);
	}

	/*!
	* \brief Confirm a graph action (and optionally update graph)
	*
	* \param[in,out] action Action to confirm
	* \param[in,out] graph Update and trigger this graph (if non-NULL)
	*/
	void
	te_action_confirmed(pcmk__graph_action_t action, pcmk__graph_t graph)
	{
	if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
	if ((action->type == pcmk__rsc_graph_action)
	&& (crm_element_value(action->xml, XML_LRM_ATTR_TARGET) != NULL)) {
	te_update_job_count(action, -1);
	}
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	}
	if (graph) {
	pcmk__update_graph(graph, action);
	trigger_graph();
	}
	}

	static pcmk__graph_functions_t te_graph_fns = {
	execute_pseudo_action,
	execute_rsc_action,
	execute_cluster_action,
	controld_execute_fence_action,
	graph_action_allowed,
	};

	/*
	* \internal
	* \brief Register the transitioner's graph functions with \p libpacemaker
	*/
	void
	controld_register_graph_functions(void)
	{
	pcmk__set_graph_functions(&te_graph_fns);
	}

	void
	notify_crmd(pcmk__graph_t *graph)
	{
	const char *type = "unknown";
	enum crmd_fsa_input event = I_NULL;

	crm_debug("Processing transition completion in state %s",
	fsa_state2string(controld_globals.fsa_state));

	CRM_CHECK(graph->complete, graph->complete = true);

	switch (graph->completion_action) {
	case pcmk__graph_wait:
	type = "stop";
	if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;
	case pcmk__graph_done:
	type = "done";
	if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;

	case pcmk__graph_restart:
	type = "restart";
	if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
	if (controld_get_period_transition_timer() > 0) {
	controld_stop_transition_timer();
	controld_start_transition_timer();
	} else {
	event = I_PE_CALC;
	}

	} else if (controld_globals.fsa_state == S_POLICY_ENGINE) {
	controld_set_fsa_action_flags(A_PE_INVOKE);
	controld_trigger_fsa();
	}
	break;

	case pcmk__graph_shutdown:
	type = "shutdown";
	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	event = I_STOP;

	} else {
	crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
	event = I_TERMINATE;
	}
	}

	crm_debug("Transition %d status: %s - %s", graph->id, type,
	pcmk__s(graph->abort_reason, "unspecified reason"));

	graph->abort_reason = NULL;
	graph->completion_action = pcmk__graph_done;

	if (event != I_NULL) {
	register_fsa_input(C_FSA_INTERNAL, event, NULL);
	} else {
	controld_trigger_fsa();
	}
	}
	diff --git a/daemons/fenced/fenced_cib.c b/daemons/fenced/fenced_cib.c
	index 66165f1c8b..087621656a 100644
	--- a/daemons/fenced/fenced_cib.c
	+++ b/daemons/fenced/fenced_cib.c
	@@ -1,738 +1,738 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <stdio.h>
	#include <libxml/tree.h>
	#include <libxml/xpath.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/cluster/internal.h>

	#include <crm/cib.h>
	#include <crm/cib/internal.h>

	#include <pacemaker-fenced.h>

	static xmlNode *local_cib = NULL;
	static cib_t *cib_api = NULL;
	static bool have_cib_devices = FALSE;

	/*!
	* \internal
	* \brief Check whether a node has a specific attribute name/value
	*
	* \param[in] node Name of node to check
	* \param[in] name Name of an attribute to look for
	* \param[in] value The value the named attribute needs to be set to in order to be considered a match
	*
	* \return TRUE if the locally cached CIB has the specified node attribute
	*/
	gboolean
	node_has_attr(const char node, const char name, const char *value)
	{
	GString *xpath = NULL;
	xmlNode *match;

	CRM_CHECK((local_cib != NULL) && (node != NULL) && (name != NULL)
	&& (value != NULL), return FALSE);

	/* Search for the node's attributes in the CIB. While the schema allows
	* multiple sets of instance attributes, and allows instance attributes to
	* use id-ref to reference values elsewhere, that is intended for resources,
	* so we ignore that here.
	*/
	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	"//" XML_CIB_TAG_NODES "/" XML_CIB_TAG_NODE
	"[@" XML_ATTR_UNAME "='", node, "']/" XML_TAG_ATTR_SETS
	"/" XML_CIB_TAG_NVPAIR
	"[@" PCMK_XA_NAME "='", name, "' "
	"and @" PCMK_XA_VALUE "='", value, "']", NULL);

	match = get_xpath_object((const char *) xpath->str, local_cib, LOG_NEVER);

	g_string_free(xpath, TRUE);
	return (match != NULL);
	}

	static void
	add_topology_level(xmlNode *match)
	{
	char *desc = NULL;
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	CRM_CHECK(match != NULL, return);

	fenced_register_level(match, &desc, &result);
	fenced_send_level_notification(STONITH_OP_LEVEL_ADD, &result, desc);
	pcmk__reset_result(&result);
	free(desc);
	}

	static void
	topology_remove_helper(const char *node, int level)
	{
	char *desc = NULL;
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
	xmlNode *data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);

	crm_xml_add(data, F_STONITH_ORIGIN, __func__);
	crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
	crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);

	fenced_unregister_level(data, &desc, &result);
	fenced_send_level_notification(STONITH_OP_LEVEL_DEL, &result, desc);
	pcmk__reset_result(&result);
	free_xml(data);
	free(desc);
	}

	static void
	remove_topology_level(xmlNode *match)
	{
	int index = 0;
	char *key = NULL;

	CRM_CHECK(match != NULL, return);

	key = stonith_level_key(match, fenced_target_by_unknown);
	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	topology_remove_helper(key, index);
	free(key);
	}

	static void
	register_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	remove_topology_level(match);
	add_topology_level(match);
	}
	}

	/* Fencing
	<diff crm_feature_set="3.0.6">
	<diff-removed>
	<fencing-topology>
	<fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
	<fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
	<fencing-level devices="disk,network" id="f-p2.1"/>
	</fencing-topology>
	</diff-removed>
	<diff-added>
	<fencing-topology>
	<fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
	<fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
	<fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
	</fencing-topology>
	</diff-added>
	</diff>
	*/

	void
	fencing_topology_init(void)
	{
	xmlXPathObjectPtr xpathObj = NULL;
	const char *xpath = "//" XML_TAG_FENCING_LEVEL;

	crm_trace("Full topology refresh");
	free_topology_list();
	init_topology_list();

	/* Grab everything */
	xpathObj = xpath_search(local_cib, xpath);
	register_fencing_topology(xpathObj);

	freeXpathObject(xpathObj);
	}

	static void
	remove_cib_device(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match != NULL) {
	- standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
	+ standard = crm_element_value(match, PCMK_XA_CLASS);
	}

	if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	continue;
	}

	rsc_id = crm_element_value(match, PCMK_XA_ID);

	stonith_device_remove(rsc_id, true);
	}
	}

	#define XPATH_WATCHDOG_TIMEOUT "//" XML_CIB_TAG_NVPAIR \
	"[@" PCMK_XA_NAME "='" \
	PCMK_OPT_STONITH_WATCHDOG_TIMEOUT "']"

	static void
	update_stonith_watchdog_timeout_ms(xmlNode *cib)
	{
	long timeout_ms = 0;
	xmlNode *stonith_watchdog_xml = NULL;
	const char *value = NULL;

	stonith_watchdog_xml = get_xpath_object(XPATH_WATCHDOG_TIMEOUT, cib,
	LOG_NEVER);
	if (stonith_watchdog_xml) {
	value = crm_element_value(stonith_watchdog_xml, PCMK_XA_VALUE);
	}
	if (value) {
	timeout_ms = crm_get_msec(value);
	}

	if (timeout_ms < 0) {
	timeout_ms = pcmk__auto_watchdog_timeout();
	}

	stonith_watchdog_timeout_ms = timeout_ms;
	}

	/*!
	* \internal
	* \brief Update all STONITH device definitions based on current CIB
	*/
	static void
	cib_devices_update(void)
	{
	GHashTableIter iter;
	stonith_device_t *device = NULL;

	crm_info("Updating devices to version %s.%s.%s",
	crm_element_value(local_cib, PCMK_XA_ADMIN_EPOCH),
	crm_element_value(local_cib, PCMK_XA_EPOCH),
	crm_element_value(local_cib, PCMK_XA_NUM_UPDATES));

	g_hash_table_iter_init(&iter, device_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) {
	if (device->cib_registered) {
	device->dirty = TRUE;
	}
	}

	/* have list repopulated if cib has a watchdog-fencing-resource
	TODO: keep a cached list for queries happening while we are refreshing
	*/
	g_list_free_full(stonith_watchdog_targets, free);
	stonith_watchdog_targets = NULL;

	fenced_scheduler_run(local_cib);

	g_hash_table_iter_init(&iter, device_list);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&device)) {
	if (device->dirty) {
	g_hash_table_iter_remove(&iter);
	}
	}
	}

	static void
	update_cib_stonith_devices_v1(const char event, xmlNode msg)
	{
	const char *reason = "none";
	gboolean needs_update = FALSE;
	xmlXPathObjectPtr xpath_obj = NULL;

	/* process new constraints */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_CONS_TAG_RSC_LOCATION);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	/* Safest and simplest to always recompute */
	needs_update = TRUE;
	reason = "new location constraint";

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	crm_log_xml_trace(match, "new constraint");
	}
	}
	freeXpathObject(xpath_obj);

	/* process deletions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	remove_cib_device(xpath_obj);
	}
	freeXpathObject(xpath_obj);

	/* process additions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	rsc_id = crm_element_value(match, PCMK_XA_ID);
	- standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
	+ standard = crm_element_value(match, PCMK_XA_CLASS);

	if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	continue;
	}

	crm_trace("Fencing resource %s was added or modified", rsc_id);
	reason = "new resource";
	needs_update = TRUE;
	}
	}
	freeXpathObject(xpath_obj);

	if(needs_update) {
	crm_info("Updating device list from CIB: %s", reason);
	cib_devices_update();
	}
	}

	static void
	update_cib_stonith_devices_v2(const char event, xmlNode msg)
	{
	xmlNode *change = NULL;
	char *reason = NULL;
	bool needs_update = FALSE;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	for (change = pcmk__xml_first_child(patchset); change != NULL;
	change = pcmk__xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);
	const char *shortpath = NULL;

	if ((op == NULL) \|\|
	(strcmp(op, "move") == 0) \|\|
	strstr(xpath, "/"XML_CIB_TAG_STATUS)) {
	continue;
	} else if (pcmk__str_eq(op, "delete", pcmk__str_casei) && strstr(xpath, "/"XML_CIB_TAG_RESOURCE)) {
	const char *rsc_id = NULL;
	char *search = NULL;
	char *mutable = NULL;

	if (strstr(xpath, XML_TAG_ATTR_SETS) \|\|
	strstr(xpath, XML_TAG_META_SETS)) {
	needs_update = TRUE;
	pcmk__str_update(&reason,
	"(meta) attribute deleted from resource");
	break;
	}
	pcmk__str_update(&mutable, xpath);
	rsc_id = strstr(mutable, "primitive[@" PCMK_XA_ID "=\'");
	if (rsc_id != NULL) {
	rsc_id += strlen("primitive[@" PCMK_XA_ID "=\'");
	search = strchr(rsc_id, '\'');
	}
	if (search != NULL) {
	*search = 0;
	stonith_device_remove(rsc_id, true);
	/* watchdog_device_update called afterwards
	to fall back to implicit definition if needed */
	} else {
	crm_warn("Ignoring malformed CIB update (resource deletion)");
	}
	free(mutable);

	} else if (strstr(xpath, "/"XML_CIB_TAG_RESOURCES) \|\|
	strstr(xpath, "/"XML_CIB_TAG_CONSTRAINTS) \|\|
	strstr(xpath, "/"XML_CIB_TAG_RSCCONFIG)) {
	shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
	reason = crm_strdup_printf("%s %s", op, shortpath+1);
	needs_update = TRUE;
	break;
	}
	}

	if(needs_update) {
	crm_info("Updating device list from CIB: %s", reason);
	cib_devices_update();
	} else {
	crm_trace("No updates for device list found in CIB");
	}
	free(reason);
	}

	static void
	update_cib_stonith_devices(const char event, xmlNode msg)
	{
	int format = 1;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, PCMK_XA_FORMAT, &format);
	switch(format) {
	case 1:
	update_cib_stonith_devices_v1(event, msg);
	break;
	case 2:
	update_cib_stonith_devices_v2(event, msg);
	break;
	default:
	crm_warn("Unknown patch format: %d", format);
	}
	}

	static void
	watchdog_device_update(void)
	{
	if (stonith_watchdog_timeout_ms > 0) {
	if (!g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID) &&
	!stonith_watchdog_targets) {
	/* getting here watchdog-fencing enabled, no device there yet
	and reason isn't stonith_watchdog_targets preventing that
	*/
	int rc;
	xmlNode *xml;

	xml = create_device_registration_xml(
	STONITH_WATCHDOG_ID,
	st_namespace_internal,
	STONITH_WATCHDOG_AGENT,
	NULL, /* stonith_device_register will add our
	own name as PCMK_STONITH_HOST_LIST param
	so we can skip that here
	*/
	NULL);
	rc = stonith_device_register(xml, TRUE);
	free_xml(xml);
	if (rc != pcmk_ok) {
	rc = pcmk_legacy2rc(rc);
	exit_code = CRM_EX_FATAL;
	crm_crit("Cannot register watchdog pseudo fence agent: %s",
	pcmk_rc_str(rc));
	stonith_shutdown(0);
	}
	}

	} else if (g_hash_table_lookup(device_list, STONITH_WATCHDOG_ID) != NULL) {
	/* be silent if no device - todo parameter to stonith_device_remove */
	stonith_device_remove(STONITH_WATCHDOG_ID, true);
	}
	}

	/*!
	* \internal
	* \brief Query the full CIB
	*
	* \return Standard Pacemaker return code
	*/
	static int
	fenced_query_cib(void)
	{
	int rc = pcmk_ok;

	crm_trace("Re-requesting full CIB");
	rc = cib_api->cmds->query(cib_api, NULL, &local_cib,
	cib_scope_local\|cib_sync_call);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	CRM_ASSERT(local_cib != NULL);
	} else {
	crm_err("Couldn't retrieve the CIB: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	}
	return rc;
	}

	static void
	remove_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if (match && crm_element_value(match, XML_DIFF_MARKER)) {
	/* Deletion */
	int index = 0;
	char *target = stonith_level_key(match, fenced_target_by_unknown);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	if (target == NULL) {
	crm_err("Invalid fencing target in element %s", ID(match));

	} else if (index <= 0) {
	crm_err("Invalid level for %s in element %s", target, ID(match));

	} else {
	topology_remove_helper(target, index);
	}
	/* } else { Deal with modifications during the 'addition' stage */
	}
	}
	}

	static void
	update_fencing_topology(const char event, xmlNode msg)
	{
	int format = 1;
	const char *xpath;
	xmlXPathObjectPtr xpathObj = NULL;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, PCMK_XA_FORMAT, &format);

	if(format == 1) {
	/* Process deletions (only) */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	remove_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	/* Process additions and changes */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	register_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	} else if(format == 2) {
	xmlNode *change = NULL;
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(patchset, add, del);

	for (change = pcmk__xml_first_child(patchset); change != NULL;
	change = pcmk__xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);

	if(op == NULL) {
	continue;

	} else if(strstr(xpath, "/" XML_TAG_FENCING_LEVEL) != NULL) {
	/* Change to a specific entry */

	crm_trace("Handling %s operation %d.%d.%d for %s", op, add[0], add[1], add[2], xpath);
	if(strcmp(op, "move") == 0) {
	continue;

	} else if(strcmp(op, "create") == 0) {
	add_topology_level(change->children);

	} else if(strcmp(op, "modify") == 0) {
	xmlNode *match = first_named_child(change, XML_DIFF_RESULT);

	if(match) {
	remove_topology_level(match->children);
	add_topology_level(match->children);
	}

	} else if(strcmp(op, "delete") == 0) {
	/* Nuclear option, all we have is the path and an id... not enough to remove a specific entry */
	crm_info("Re-initializing fencing topology after %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else if (strstr(xpath, "/" XML_TAG_FENCING_TOPOLOGY) != NULL) {
	/* Change to the topology in general */
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;

	} else if (strstr(xpath, "/" XML_CIB_TAG_CONFIGURATION)) {
	/* Changes to the whole config section, possibly including the topology as a whild */
	if(first_named_child(change, XML_TAG_FENCING_TOPOLOGY) == NULL) {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);

	} else if(strcmp(op, "delete") == 0 \|\| strcmp(op, "create") == 0) {
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	}
	}

	} else {
	crm_warn("Unknown patch format: %d", format);
	}
	}

	static void
	update_cib_cache_cb(const char event, xmlNode msg)
	{
	long timeout_ms_saved = stonith_watchdog_timeout_ms;
	bool need_full_refresh = false;

	if(!have_cib_devices) {
	crm_trace("Skipping updates until we get a full dump");
	return;

	} else if(msg == NULL) {
	crm_trace("Missing %s update", event);
	return;
	}

	/* Maintain a local copy of the CIB so that we have full access
	* to device definitions, location constraints, and node attributes
	*/
	if (local_cib != NULL) {
	int rc = pcmk_ok;
	xmlNode *patchset = NULL;

	crm_element_value_int(msg, F_CIB_RC, &rc);
	if (rc != pcmk_ok) {
	return;
	}

	patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
	rc = xml_apply_patchset(local_cib, patchset, TRUE);
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_old_data:
	break;
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	break;
	default:
	crm_warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	}
	}

	if (local_cib == NULL) {
	if (fenced_query_cib() != pcmk_rc_ok) {
	return;
	}
	need_full_refresh = true;
	}

	pcmk__refresh_node_caches_from_cib(local_cib);
	update_stonith_watchdog_timeout_ms(local_cib);

	if (timeout_ms_saved != stonith_watchdog_timeout_ms) {
	need_full_refresh = true;
	}

	if (need_full_refresh) {
	fencing_topology_init();
	cib_devices_update();
	} else {
	// Partial refresh
	update_fencing_topology(event, msg);
	update_cib_stonith_devices(event, msg);
	}

	watchdog_device_update();
	}

	static void
	init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	crm_info("Updating device list from CIB");
	have_cib_devices = TRUE;
	local_cib = copy_xml(output);

	pcmk__refresh_node_caches_from_cib(local_cib);
	update_stonith_watchdog_timeout_ms(local_cib);

	fencing_topology_init();
	cib_devices_update();
	watchdog_device_update();
	}

	static void
	cib_connection_destroy(gpointer user_data)
	{
	if (stonith_shutdown_flag) {
	crm_info("Connection to the CIB manager closed");
	return;
	} else {
	crm_crit("Lost connection to the CIB manager, shutting down");
	}
	if (cib_api) {
	cib_api->cmds->signoff(cib_api);
	}
	stonith_shutdown(0);
	}

	/*!
	* \internal
	* \brief Disconnect from CIB manager
	*/
	void
	fenced_cib_cleanup(void)
	{
	if (cib_api != NULL) {
	cib_api->cmds->del_notify_callback(cib_api, T_CIB_DIFF_NOTIFY,
	update_cib_cache_cb);
	cib__clean_up_connection(&cib_api);
	}
	free_xml(local_cib);
	local_cib = NULL;
	}

	void
	setup_cib(void)
	{
	int rc, retries = 0;

	cib_api = cib_new();
	if (cib_api == NULL) {
	crm_err("No connection to the CIB manager");
	return;
	}

	do {
	sleep(retries);
	rc = cib_api->cmds->signon(cib_api, CRM_SYSTEM_STONITHD, cib_command);
	} while (rc == -ENOTCONN && ++retries < 5);

	if (rc != pcmk_ok) {
	crm_err("Could not connect to the CIB manager: %s (%d)", pcmk_strerror(rc), rc);

	} else if (pcmk_ok !=
	cib_api->cmds->add_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb)) {
	crm_err("Could not set CIB notification callback");

	} else {
	rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_scope_local);
	cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL, "init_cib_cache_cb",
	init_cib_cache_cb);
	cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy);
	crm_info("Watching for fencing topology changes");
	}
	}
	diff --git a/daemons/fenced/fenced_scheduler.c b/daemons/fenced/fenced_scheduler.c
	index b2c7cdbd48..1e3b524185 100644
	--- a/daemons/fenced/fenced_scheduler.c
	+++ b/daemons/fenced/fenced_scheduler.c
	@@ -1,225 +1,225 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <errno.h>
	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>

	#include <pacemaker-internal.h>
	#include <pacemaker-fenced.h>

	static pcmk_scheduler_t *scheduler = NULL;

	/*!
	* \internal
	* \brief Initialize scheduler data for fencer purposes
	*
	* \return Standard Pacemaker return code
	*/
	int
	fenced_scheduler_init(void)
	{
	pcmk__output_t *logger = NULL;
	int rc = pcmk__log_output_new(&logger);

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	pcmk__output_free(logger);
	return ENOMEM;
	}

	pe__register_messages(logger);
	pcmk__register_lib_messages(logger);
	pcmk__output_set_log_level(logger, LOG_TRACE);
	scheduler->priv = logger;

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Free all scheduler-related resources
	*/
	void
	fenced_scheduler_cleanup(void)
	{
	if (scheduler != NULL) {
	pcmk__output_t *logger = scheduler->priv;

	if (logger != NULL) {
	logger->finish(logger, CRM_EX_OK, true, NULL);
	pcmk__output_free(logger);
	scheduler->priv = NULL;
	}
	pe_free_working_set(scheduler);
	scheduler = NULL;
	}
	}

	/*!
	* \internal
	* \brief Check whether the local node is in a resource's allowed node list
	*
	* \param[in] rsc Resource to check
	*
	* \return Pointer to node if found, otherwise NULL
	*/
	static pcmk_node_t *
	local_node_allowed_for(const pcmk_resource_t *rsc)
	{
	if ((rsc != NULL) && (stonith_our_uname != NULL)) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (pcmk__str_eq(node->details->uname, stonith_our_uname,
	pcmk__str_casei)) {
	return node;
	}
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief If a given resource or any of its children are fencing devices,
	* register the devices
	*
	* \param[in,out] data Resource to check
	* \param[in,out] user_data Ignored
	*/
	static void
	register_if_fencing_device(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	xmlNode *xml = NULL;
	GHashTableIter hash_iter;
	pcmk_node_t *node = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *rclass = NULL;
	const char *agent = NULL;
	const char *rsc_provides = NULL;
	stonith_key_value_t *params = NULL;

	// If this is a collective resource, check children instead
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	register_if_fencing_device(iter->data, NULL);
	if (pe_rsc_is_clone(rsc)) {
	return; // Only one instance needs to be checked for clones
	}
	}
	return;
	}

	- rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	return; // Not a fencing device
	}

	if (pe__resource_is_disabled(rsc)) {
	crm_info("Ignoring fencing device %s because it is disabled", rsc->id);
	return;
	}

	if ((stonith_watchdog_timeout_ms <= 0) &&
	pcmk__str_eq(rsc->id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
	crm_info("Ignoring fencing device %s "
	"because watchdog fencing is disabled", rsc->id);
	return;
	}

	// Check whether local node is allowed to run resource
	node = local_node_allowed_for(rsc);
	if (node == NULL) {
	crm_info("Ignoring fencing device %s "
	"because local node is not allowed to run it", rsc->id);
	return;
	}
	if (node->weight < 0) {
	crm_info("Ignoring fencing device %s "
	"because local node has preference %s for it",
	rsc->id, pcmk_readable_score(node->weight));
	return;
	}

	// If device is in a group, check whether local node is allowed for group
	if ((rsc->parent != NULL)
	&& (rsc->parent->variant == pcmk_rsc_variant_group)) {
	pcmk_node_t *group_node = local_node_allowed_for(rsc->parent);

	if ((group_node != NULL) && (group_node->weight < 0)) {
	crm_info("Ignoring fencing device %s "
	"because local node has preference %s for its group",
	rsc->id, pcmk_readable_score(group_node->weight));
	return;
	}
	}

	crm_debug("Reloading configuration of fencing device %s", rsc->id);

	agent = crm_element_value(rsc->xml, PCMK_XA_TYPE);

	get_meta_attributes(rsc->meta, rsc, node, scheduler);
	rsc_provides = g_hash_table_lookup(rsc->meta, PCMK_STONITH_PROVIDES);

	g_hash_table_iter_init(&hash_iter, pe_rsc_params(rsc, node, scheduler));
	while (g_hash_table_iter_next(&hash_iter, (gpointer *) &name,
	(gpointer *) &value)) {
	if ((name == NULL) \|\| (value == NULL)) {
	continue;
	}
	params = stonith_key_value_add(params, name, value);
	}

	xml = create_device_registration_xml(pcmk__s(rsc->clone_name, rsc->id),
	st_namespace_any, agent, params,
	rsc_provides);
	stonith_key_value_freeall(params, 1, 1);
	CRM_ASSERT(stonith_device_register(xml, TRUE) == pcmk_ok);
	free_xml(xml);
	}

	/*!
	* \internal
	* \brief Run the scheduler for fencer purposes
	*
	* \param[in] cib Cluster's current CIB
	*/
	void
	fenced_scheduler_run(xmlNode *cib)
	{
	CRM_CHECK((cib != NULL) && (scheduler != NULL), return);

	if (scheduler->now != NULL) {
	crm_time_free(scheduler->now);
	scheduler->now = NULL;
	}
	scheduler->localhost = stonith_our_uname;
	pcmk__schedule_actions(cib, pcmk_sched_location_only
	\|pcmk_sched_no_compat
	\|pcmk_sched_no_counts, scheduler);
	g_list_foreach(scheduler->resources, register_if_fencing_device, NULL);

	scheduler->input = NULL; // Wasn't a copy, so don't let API free it
	pe_reset_working_set(scheduler);
	}
	diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
	index f8668645e3..7a8343f4db 100644
	--- a/include/crm/msg_xml.h
	+++ b/include/crm/msg_xml.h
	@@ -1,418 +1,419 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_MSG_XML__H
	# define PCMK__CRM_MSG_XML__H

	# include <crm/common/xml.h>

	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	#include <crm/msg_xml_compat.h>
	#endif

	#ifdef __cplusplus
	extern "C" {
	#endif

	/* This file defines constants for various XML syntax (mainly element and
	* attribute names).
	*
	* For consistency, new constants should start with "PCMK_", followed by "XE"
	* for XML element names, "XA" for XML attribute names, and "META" for meta
	* attribute names. Old names that don't follow this policy should eventually be
	* deprecated and replaced with names that do.
	*
	* Symbols should be public if the user may specify them somewhere (especially
	* the CIB). They should be internal if they're used only internally to
	* Pacemaker (such as daemon IPC/CPG message XML).
	*
	* For meta-attributes that can be specified as either XML attributes or nvpair
	* names, use "META" unless using both "XA" and "META" constants adds clarity.
	*/

	/*
	* XML elements
	*/

	#define PCMK_XE_DATE_EXPRESSION "date_expression"
	#define PCMK_XE_OP_EXPRESSION "op_expression"

	/* This has been deprecated as a CIB element (an alias for <clone> with
	* PCMK_META_PROMOTABLE set to "true") since 2.0.0.
	*/
	#define PCMK_XE_PROMOTABLE_LEGACY "master"

	#define PCMK_XE_RSC_EXPRESSION "rsc_expression"


	/*
	* XML attributes
	*/

	#define PCMK_XA_ADMIN_EPOCH "admin_epoch"
	#define PCMK_XA_CIB_LAST_WRITTEN "cib-last-written"
	+#define PCMK_XA_CLASS "class"
	#define PCMK_XA_CRM_DEBUG_ORIGIN "crm-debug-origin"
	#define PCMK_XA_CRM_FEATURE_SET "crm_feature_set"
	#define PCMK_XA_CRM_TIMESTAMP "crm-timestamp"
	#define PCMK_XA_DESCRIPTION "description"
	#define PCMK_XA_EPOCH "epoch"
	#define PCMK_XA_FORMAT "format"
	#define PCMK_XA_HAVE_QUORUM "have-quorum"
	#define PCMK_XA_ID "id"
	#define PCMK_XA_ID_REF "id-ref"
	#define PCMK_XA_NAME "name"
	#define PCMK_XA_NO_QUORUM_PANIC "no-quorum-panic"
	#define PCMK_XA_NUM_UPDATES "num_updates"
	#define PCMK_XA_PROVIDER "provider"
	#define PCMK_XA_TYPE "type"
	#define PCMK_XA_VALIDATE_WITH "validate-with"
	#define PCMK_XA_VALUE "value"
	#define PCMK_XA_VERSION "version"


	/*
	* Older constants that don't follow current naming
	*/

	# ifndef F_ORIG
	# define F_ORIG "src"
	# endif

	# ifndef F_SEQ
	# define F_SEQ "seq"
	# endif

	# ifndef F_SUBTYPE
	# define F_SUBTYPE "subt"
	# endif

	# ifndef F_TYPE
	# define F_TYPE "t"
	# endif

	# ifndef F_CLIENTNAME
	# define F_CLIENTNAME "cn"
	# endif

	# ifndef F_XML_TAGNAME
	# define F_XML_TAGNAME "__name__"
	# endif

	# ifndef T_CRM
	# define T_CRM "crmd"
	# endif

	# ifndef T_ATTRD
	# define T_ATTRD "attrd"
	# endif

	# define CIB_OPTIONS_FIRST "cib-bootstrap-options"

	# define F_CRM_DATA "crm_xml"
	# define F_CRM_TASK "crm_task"
	# define F_CRM_HOST_TO "crm_host_to"
	# define F_CRM_MSG_TYPE F_SUBTYPE
	# define F_CRM_SYS_TO "crm_sys_to"
	# define F_CRM_SYS_FROM "crm_sys_from"
	# define F_CRM_HOST_FROM F_ORIG
	# define F_CRM_REFERENCE XML_ATTR_REFERENCE
	# define F_CRM_VERSION PCMK_XA_VERSION
	# define F_CRM_ORIGIN "origin"
	# define F_CRM_USER "crm_user"
	# define F_CRM_JOIN_ID "join_id"
	# define F_CRM_DC_LEAVING "dc-leaving"
	# define F_CRM_ELECTION_ID "election-id"
	# define F_CRM_ELECTION_AGE_S "election-age-sec"
	# define F_CRM_ELECTION_AGE_US "election-age-nano-sec"
	# define F_CRM_ELECTION_OWNER "election-owner"
	# define F_CRM_TGRAPH "crm-tgraph-file"
	# define F_CRM_TGRAPH_INPUT "crm-tgraph-in"

	# define F_CRM_THROTTLE_MODE "crm-limit-mode"
	# define F_CRM_THROTTLE_MAX "crm-limit-max"

	/---- Common tags/attrs /
	# define XML_DIFF_MARKER "__crm_diff_marker__"
	# define XML_TAG_CIB "cib"
	# define XML_TAG_FAILED "failed"

	# define XML_ATTR_TIMEOUT "timeout"
	# define XML_ATTR_OP "op"
	# define XML_ATTR_DC_UUID "dc-uuid"
	# define XML_ATTR_UPDATE_ORIG "update-origin"
	# define XML_ATTR_UPDATE_CLIENT "update-client"
	# define XML_ATTR_UPDATE_USER "update-user"

	# define XML_BOOLEAN_TRUE "true"
	# define XML_BOOLEAN_FALSE "false"

	# define XML_TAG_OPTIONS "options"

	/---- top level tags/attrs /
	# define XML_ATTR_REQUEST "request"
	# define XML_ATTR_RESPONSE "response"

	# define XML_ATTR_UNAME "uname"
	# define XML_ATTR_REFERENCE "reference"

	# define XML_CRM_TAG_PING "ping_response"
	# define XML_PING_ATTR_STATUS "result"
	# define XML_PING_ATTR_SYSFROM "crm_subsystem"
	# define XML_PING_ATTR_CRMDSTATE "crmd_state"
	# define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state"
	# define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init"
	# define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons"
	# define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping"
	# define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running"
	# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down"
	# define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete"
	# define XML_PING_ATTR_PACEMAKERDSTATE_REMOTE "remote"

	# define XML_FAIL_TAG_CIB "failed_update"

	# define XML_FAILCIB_ATTR_OBJTYPE "object_type"
	# define XML_FAILCIB_ATTR_OP "operation"
	# define XML_FAILCIB_ATTR_REASON "reason"

	/---- CIB specific tags/attrs /
	# define XML_CIB_TAG_SECTION_ALL "all"
	# define XML_CIB_TAG_CONFIGURATION "configuration"
	# define XML_CIB_TAG_STATUS "status"
	# define XML_CIB_TAG_RESOURCES "resources"
	# define XML_CIB_TAG_NODES "nodes"
	# define XML_CIB_TAG_CONSTRAINTS "constraints"
	# define XML_CIB_TAG_CRMCONFIG "crm_config"
	# define XML_CIB_TAG_OPCONFIG "op_defaults"
	# define XML_CIB_TAG_RSCCONFIG "rsc_defaults"
	# define XML_CIB_TAG_ACLS "acls"
	# define XML_CIB_TAG_ALERTS "alerts"
	# define XML_CIB_TAG_ALERT "alert"
	# define XML_CIB_TAG_ALERT_RECIPIENT "recipient"
	# define XML_CIB_TAG_ALERT_SELECT "select"
	# define XML_CIB_TAG_ALERT_ATTRIBUTES "select_attributes"
	# define XML_CIB_TAG_ALERT_FENCING "select_fencing"
	# define XML_CIB_TAG_ALERT_NODES "select_nodes"
	# define XML_CIB_TAG_ALERT_RESOURCES "select_resources"
	# define XML_CIB_TAG_ALERT_ATTR "attribute"

	# define XML_CIB_TAG_STATE "node_state"
	# define XML_CIB_TAG_NODE "node"
	# define XML_CIB_TAG_NVPAIR "nvpair"

	# define XML_CIB_TAG_PROPSET "cluster_property_set"
	# define XML_TAG_ATTR_SETS "instance_attributes"
	# define XML_TAG_META_SETS "meta_attributes"
	# define XML_TAG_ATTRS "attributes"
	# define XML_TAG_PARAMS "parameters"
	# define XML_TAG_PARAM "param"
	# define XML_TAG_UTILIZATION "utilization"

	# define XML_TAG_RESOURCE_REF "resource_ref"
	# define XML_CIB_TAG_RESOURCE "primitive"
	# define XML_CIB_TAG_GROUP "group"
	# define XML_CIB_TAG_INCARNATION "clone"
	# define XML_CIB_TAG_CONTAINER "bundle"

	# define XML_CIB_TAG_RSC_TEMPLATE "template"

	# define XML_OP_ATTR_ON_FAIL "on-fail"
	# define XML_OP_ATTR_START_DELAY "start-delay"
	# define XML_OP_ATTR_ORIGIN "interval-origin"
	# define XML_OP_ATTR_PENDING "record-pending"
	# define XML_OP_ATTR_DIGESTS_ALL "digests-all"
	# define XML_OP_ATTR_DIGESTS_SECURE "digests-secure"

	# define XML_CIB_TAG_LRM "lrm"
	# define XML_LRM_TAG_RESOURCES "lrm_resources"
	# define XML_LRM_TAG_RESOURCE "lrm_resource"
	# define XML_LRM_TAG_RSC_OP "lrm_rsc_op"
	# define XML_AGENT_ATTR_CLASS "class"

	//! \deprecated Do not use (will be removed in a future release)
	# define XML_CIB_ATTR_REPLACE "replace"

	# define XML_CIB_ATTR_PRIORITY "priority"

	# define XML_NODE_IS_REMOTE "remote_node"
	# define XML_NODE_IS_FENCED "node_fenced"
	# define XML_NODE_IS_MAINTENANCE "node_in_maintenance"

	# define XML_CIB_ATTR_SHUTDOWN "shutdown"

	/* Aside from being an old name for the executor, LRM is a misnomer here because
	* the controller and scheduler use these to track actions, which are not always
	* executor operations.
	*/

	// XML attribute that takes interval specification (user-facing configuration)
	# define XML_LRM_ATTR_INTERVAL "interval"

	// XML attribute that takes interval in milliseconds (daemon APIs)
	// (identical value as above, but different constant allows clearer code intent)
	# define XML_LRM_ATTR_INTERVAL_MS XML_LRM_ATTR_INTERVAL

	# define XML_LRM_ATTR_TASK "operation"
	# define XML_LRM_ATTR_TASK_KEY "operation_key"
	# define XML_LRM_ATTR_TARGET "on_node"
	# define XML_LRM_ATTR_TARGET_UUID "on_node_uuid"
	/*! Actions to be executed on Pacemaker Remote nodes are routed through the
	* controller on the cluster node hosting the remote connection. That cluster
	* node is considered the router node for the action.
	*/
	# define XML_LRM_ATTR_ROUTER_NODE "router_node"
	# define XML_LRM_ATTR_RSCID "rsc-id"
	# define XML_LRM_ATTR_OPSTATUS "op-status"
	# define XML_LRM_ATTR_RC "rc-code"
	# define XML_LRM_ATTR_CALLID "call-id"
	# define XML_LRM_ATTR_OP_DIGEST "op-digest"
	# define XML_LRM_ATTR_OP_RESTART "op-force-restart"
	# define XML_LRM_ATTR_OP_SECURE "op-secure-params"
	# define XML_LRM_ATTR_RESTART_DIGEST "op-restart-digest"
	# define XML_LRM_ATTR_SECURE_DIGEST "op-secure-digest"
	# define XML_LRM_ATTR_EXIT_REASON "exit-reason"

	# define XML_RSC_OP_LAST_CHANGE "last-rc-change"
	# define XML_RSC_OP_T_EXEC "exec-time"
	# define XML_RSC_OP_T_QUEUE "queue-time"

	# define XML_LRM_ATTR_MIGRATE_SOURCE "migrate_source"
	# define XML_LRM_ATTR_MIGRATE_TARGET "migrate_target"

	# define XML_TAG_GRAPH "transition_graph"
	# define XML_GRAPH_TAG_RSC_OP "rsc_op"
	# define XML_GRAPH_TAG_PSEUDO_EVENT "pseudo_event"
	# define XML_GRAPH_TAG_CRM_EVENT "crm_event"
	# define XML_GRAPH_TAG_DOWNED "downed"
	# define XML_GRAPH_TAG_MAINTENANCE "maintenance"

	# define XML_TAG_RULE "rule"
	# define XML_RULE_ATTR_SCORE "score"
	# define XML_RULE_ATTR_SCORE_ATTRIBUTE "score-attribute"
	# define XML_RULE_ATTR_ROLE "role"
	# define XML_RULE_ATTR_BOOLEAN_OP "boolean-op"

	# define XML_TAG_EXPRESSION "expression"
	# define XML_EXPR_ATTR_ATTRIBUTE "attribute"
	# define XML_EXPR_ATTR_OPERATION "operation"
	# define XML_EXPR_ATTR_VALUE_SOURCE "value-source"

	# define XML_CONS_TAG_RSC_DEPEND "rsc_colocation"
	# define XML_CONS_TAG_RSC_ORDER "rsc_order"
	# define XML_CONS_TAG_RSC_LOCATION "rsc_location"
	# define XML_CONS_TAG_RSC_TICKET "rsc_ticket"
	# define XML_CONS_TAG_RSC_SET "resource_set"
	# define XML_CONS_ATTR_SYMMETRICAL "symmetrical"

	# define XML_LOCATION_ATTR_DISCOVERY "resource-discovery"

	# define XML_COLOC_ATTR_SOURCE "rsc"
	# define XML_COLOC_ATTR_SOURCE_ROLE "rsc-role"
	# define XML_COLOC_ATTR_TARGET "with-rsc"
	# define XML_COLOC_ATTR_TARGET_ROLE "with-rsc-role"
	# define XML_COLOC_ATTR_NODE_ATTR "node-attribute"
	# define XML_COLOC_ATTR_INFLUENCE "influence"

	//! \deprecated Deprecated since 2.1.5
	# define XML_COLOC_ATTR_SOURCE_INSTANCE "rsc-instance"

	//! \deprecated Deprecated since 2.1.5
	# define XML_COLOC_ATTR_TARGET_INSTANCE "with-rsc-instance"

	# define XML_LOC_ATTR_SOURCE "rsc"
	# define XML_LOC_ATTR_SOURCE_PATTERN "rsc-pattern"

	# define XML_ORDER_ATTR_FIRST "first"
	# define XML_ORDER_ATTR_THEN "then"
	# define XML_ORDER_ATTR_FIRST_ACTION "first-action"
	# define XML_ORDER_ATTR_THEN_ACTION "then-action"
	# define XML_ORDER_ATTR_KIND "kind"

	//! \deprecated Deprecated since 2.1.5
	# define XML_ORDER_ATTR_FIRST_INSTANCE "first-instance"

	//! \deprecated Deprecated since 2.1.5
	# define XML_ORDER_ATTR_THEN_INSTANCE "then-instance"

	# define XML_TICKET_ATTR_TICKET "ticket"
	# define XML_TICKET_ATTR_LOSS_POLICY "loss-policy"

	# define XML_NODE_ATTR_RSC_DISCOVERY "resource-discovery-enabled"

	# define XML_ALERT_ATTR_PATH "path"
	# define XML_ALERT_ATTR_TIMEOUT "timeout"
	# define XML_ALERT_ATTR_TSTAMP_FORMAT "timestamp-format"

	# define XML_CIB_TAG_GENERATION_TUPPLE "generation_tuple"

	# define XML_ATTR_TRANSITION_MAGIC "transition-magic"
	# define XML_ATTR_TRANSITION_KEY "transition-key"

	# define XML_ATTR_TE_NOWAIT "op_no_wait"
	# define XML_ATTR_TE_TARGET_RC "op_target_rc"
	# define XML_TAG_TRANSIENT_NODEATTRS "transient_attributes"

	//! \deprecated Do not use (will be removed in a future release)
	# define XML_TAG_DIFF_ADDED "diff-added"

	//! \deprecated Do not use (will be removed in a future release)
	# define XML_TAG_DIFF_REMOVED "diff-removed"

	# define XML_ACL_TAG_USER "acl_target"
	# define XML_ACL_TAG_USERv1 "acl_user"
	# define XML_ACL_TAG_GROUP "acl_group"
	# define XML_ACL_TAG_ROLE "acl_role"
	# define XML_ACL_TAG_PERMISSION "acl_permission"
	# define XML_ACL_TAG_ROLE_REF "role"
	# define XML_ACL_TAG_ROLE_REFv1 "role_ref"
	# define XML_ACL_ATTR_KIND "kind"
	# define XML_ACL_TAG_READ "read"
	# define XML_ACL_TAG_WRITE "write"
	# define XML_ACL_TAG_DENY "deny"
	# define XML_ACL_ATTR_REF "reference"
	# define XML_ACL_ATTR_REFv1 "ref"
	# define XML_ACL_ATTR_TAG "object-type"
	# define XML_ACL_ATTR_TAGv1 "tag"
	# define XML_ACL_ATTR_XPATH "xpath"
	# define XML_ACL_ATTR_ATTRIBUTE "attribute"

	# define XML_CIB_TAG_TICKETS "tickets"
	# define XML_CIB_TAG_TICKET_STATE "ticket_state"

	# define XML_CIB_TAG_TAGS "tags"
	# define XML_CIB_TAG_TAG "tag"
	# define XML_CIB_TAG_OBJ_REF "obj_ref"

	# define XML_TAG_FENCING_TOPOLOGY "fencing-topology"
	# define XML_TAG_FENCING_LEVEL "fencing-level"
	# define XML_ATTR_STONITH_INDEX "index"
	# define XML_ATTR_STONITH_TARGET "target"
	# define XML_ATTR_STONITH_TARGET_VALUE "target-value"
	# define XML_ATTR_STONITH_TARGET_PATTERN "target-pattern"
	# define XML_ATTR_STONITH_TARGET_ATTRIBUTE "target-attribute"
	# define XML_ATTR_STONITH_DEVICES "devices"

	# define XML_TAG_DIFF "diff"
	# define XML_DIFF_VERSION "version"
	# define XML_DIFF_VSOURCE "source"
	# define XML_DIFF_VTARGET "target"
	# define XML_DIFF_CHANGE "change"
	# define XML_DIFF_LIST "change-list"
	# define XML_DIFF_ATTR "change-attr"
	# define XML_DIFF_RESULT "change-result"
	# define XML_DIFF_OP "operation"
	# define XML_DIFF_PATH "path"
	# define XML_DIFF_POSITION "position"

	# define ID(x) crm_element_value(x, PCMK_XA_ID)

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
	index 408e10afec..f9638ff9f8 100644
	--- a/lib/common/ipc_controld.c
	+++ b/lib/common/ipc_controld.c
	@@ -1,657 +1,657 @@
	/*
	* Copyright 2020-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <stdbool.h>
	#include <errno.h>
	#include <libxml/tree.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/ipc_controld.h>
	#include "crmcommon_private.h"

	struct controld_api_private_s {
	char *client_uuid;
	unsigned int replies_expected;
	};

	/*!
	* \internal
	* \brief Get a string representation of a controller API reply type
	*
	* \param[in] reply Controller API reply type
	*
	* \return String representation of a controller API reply type
	*/
	const char *
	pcmk__controld_api_reply2str(enum pcmk_controld_api_reply reply)
	{
	switch (reply) {
	case pcmk_controld_reply_reprobe:
	return "reprobe";
	case pcmk_controld_reply_info:
	return "info";
	case pcmk_controld_reply_resource:
	return "resource";
	case pcmk_controld_reply_ping:
	return "ping";
	case pcmk_controld_reply_nodes:
	return "nodes";
	default:
	return "unknown";
	}
	}

	// \return Standard Pacemaker return code
	static int
	new_data(pcmk_ipc_api_t *api)
	{
	struct controld_api_private_s *private = NULL;

	api->api_data = calloc(1, sizeof(struct controld_api_private_s));

	if (api->api_data == NULL) {
	return errno;
	}

	private = api->api_data;

	/* This is set to the PID because that's how it was always done, but PIDs
	* are not unique because clients can be remote. The value appears to be
	* unused other than as part of F_CRM_SYS_FROM in IPC requests, which is
	* only compared against the internal system names (CRM_SYSTEM_TENGINE,
	* etc.), so it shouldn't be a problem.
	*/
	private->client_uuid = pcmk__getpid_s();

	/* @TODO Implement a call ID model similar to the CIB, executor, and fencer
	* IPC APIs, so that requests and replies can be matched, and
	* duplicate replies can be discarded.
	*/
	return pcmk_rc_ok;
	}

	static void
	free_data(void *data)
	{
	free(((struct controld_api_private_s *) data)->client_uuid);
	free(data);
	}

	// \return Standard Pacemaker return code
	static int
	post_connect(pcmk_ipc_api_t *api)
	{
	/* The controller currently requires clients to register via a hello
	* request, but does not reply back.
	*/
	struct controld_api_private_s *private = api->api_data;
	const char *client_name = crm_system_name? crm_system_name : "client";
	xmlNode *hello;
	int rc;

	hello = create_hello_message(private->client_uuid, client_name,
	PCMK__CONTROLD_API_MAJOR,
	PCMK__CONTROLD_API_MINOR);
	rc = pcmk__send_ipc_request(api, hello);
	free_xml(hello);
	if (rc != pcmk_rc_ok) {
	crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
	} else {
	crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
	}
	return rc;
	}

	static void
	set_node_info_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	data->reply_type = pcmk_controld_reply_info;
	if (msg_data == NULL) {
	return;
	}
	data->data.node_info.have_quorum =
	pcmk__xe_attr_is_true(msg_data, PCMK_XA_HAVE_QUORUM);
	data->data.node_info.is_remote = pcmk__xe_attr_is_true(msg_data, XML_NODE_IS_REMOTE);

	/* Integer node_info.id is currently valid only for Corosync nodes.
	*
	* @TODO: Improve handling after crm_node_t is refactored to handle layer-
	* specific data better.
	*/
	crm_element_value_int(msg_data, PCMK_XA_ID, &(data->data.node_info.id));

	data->data.node_info.uuid = crm_element_value(msg_data, PCMK_XA_ID);
	data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME);
	data->data.node_info.state = crm_element_value(msg_data, PCMK__XA_CRMD);
	}

	static void
	set_ping_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	data->reply_type = pcmk_controld_reply_ping;
	if (msg_data == NULL) {
	return;
	}
	data->data.ping.sys_from = crm_element_value(msg_data,
	XML_PING_ATTR_SYSFROM);
	data->data.ping.fsa_state = crm_element_value(msg_data,
	XML_PING_ATTR_CRMDSTATE);
	data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS);
	}

	static void
	set_nodes_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	pcmk_controld_api_node_t *node_info;

	data->reply_type = pcmk_controld_reply_nodes;
	for (xmlNode *node = first_named_child(msg_data, XML_CIB_TAG_NODE);
	node != NULL; node = crm_next_same_xml(node)) {

	long long id_ll = 0;

	node_info = calloc(1, sizeof(pcmk_controld_api_node_t));
	crm_element_value_ll(node, PCMK_XA_ID, &id_ll);
	if (id_ll > 0) {
	node_info->id = id_ll;
	}
	node_info->uname = crm_element_value(node, XML_ATTR_UNAME);
	node_info->state = crm_element_value(node, PCMK__XA_IN_CCM);
	data->data.nodes = g_list_prepend(data->data.nodes, node_info);
	}
	}

	static bool
	reply_expected(pcmk_ipc_api_t api, const xmlNode request)
	{
	// We only need to handle commands that API functions can send
	return pcmk__str_any_of(crm_element_value(request, F_CRM_TASK),
	PCMK__CONTROLD_CMD_NODES,
	CRM_OP_LRM_DELETE,
	CRM_OP_LRM_FAIL,
	CRM_OP_NODE_INFO,
	CRM_OP_PING,
	CRM_OP_REPROBE,
	CRM_OP_RM_NODE_CACHE,
	NULL);
	}

	static bool
	dispatch(pcmk_ipc_api_t api, xmlNode reply)
	{
	struct controld_api_private_s *private = api->api_data;
	crm_exit_t status = CRM_EX_OK;
	xmlNode *msg_data = NULL;
	const char *value = NULL;
	pcmk_controld_api_reply_t reply_data = {
	pcmk_controld_reply_unknown, NULL, NULL,
	};

	if (pcmk__xe_is(reply, "ack")) {
	/* ACKs are trivial responses that do not count toward expected replies,
	* and do not have all the fields that validation requires, so skip that
	* processing.
	*/
	return private->replies_expected > 0;
	}

	if (private->replies_expected > 0) {
	private->replies_expected--;
	}

	// Do some basic validation of the reply

	/* @TODO We should be able to verify that value is always a response, but
	* currently the controller doesn't always properly set the type. Even
	* if we fix the controller, we'll still need to handle replies from
	* old versions (feature set could be used to differentiate).
	*/
	value = crm_element_value(reply, F_CRM_MSG_TYPE);
	if (pcmk__str_empty(value)
	\|\| !pcmk__str_any_of(value, XML_ATTR_REQUEST, XML_ATTR_RESPONSE, NULL)) {
	crm_info("Unrecognizable message from controller: "
	"invalid message type '%s'", pcmk__s(value, ""));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	if (pcmk__str_empty(crm_element_value(reply, XML_ATTR_REFERENCE))) {
	crm_info("Unrecognizable message from controller: no reference");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	value = crm_element_value(reply, F_CRM_TASK);
	if (pcmk__str_empty(value)) {
	crm_info("Unrecognizable message from controller: no command name");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	// Parse useful info from reply

	reply_data.feature_set = crm_element_value(reply, PCMK_XA_VERSION);
	reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM);
	msg_data = get_message_xml(reply, F_CRM_DATA);

	if (!strcmp(value, CRM_OP_REPROBE)) {
	reply_data.reply_type = pcmk_controld_reply_reprobe;

	} else if (!strcmp(value, CRM_OP_NODE_INFO)) {
	set_node_info_data(&reply_data, msg_data);

	} else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
	reply_data.reply_type = pcmk_controld_reply_resource;
	reply_data.data.resource.node_state = msg_data;

	} else if (!strcmp(value, CRM_OP_PING)) {
	set_ping_data(&reply_data, msg_data);

	} else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
	set_nodes_data(&reply_data, msg_data);

	} else {
	crm_info("Unrecognizable message from controller: unknown command '%s'",
	value);
	status = CRM_EX_PROTOCOL;
	}

	done:
	pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);

	// Free any reply data that was allocated
	if (pcmk__str_eq(value, PCMK__CONTROLD_CMD_NODES, pcmk__str_casei)) {
	g_list_free_full(reply_data.data.nodes, free);
	}

	return false; // No further replies needed
	}

	pcmk__ipc_methods_t *
	pcmk__controld_api_methods(void)
	{
	pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));

	if (cmds != NULL) {
	cmds->new_data = new_data;
	cmds->free_data = free_data;
	cmds->post_connect = post_connect;
	cmds->reply_expected = reply_expected;
	cmds->dispatch = dispatch;
	}
	return cmds;
	}

	/*!
	* \internal
	* \brief Create XML for a controller IPC request
	*
	* \param[in] api Controller connection
	* \param[in] op Controller IPC command name
	* \param[in] node Node name to set as destination host
	* \param[in] msg_data XML to attach to request as message data
	*
	* \return Newly allocated XML for request
	*/
	static xmlNode *
	create_controller_request(const pcmk_ipc_api_t api, const char op,
	const char node, xmlNode msg_data)
	{
	struct controld_api_private_s *private = NULL;
	const char *sys_to = NULL;

	if (api == NULL) {
	return NULL;
	}
	private = api->api_data;
	if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
	sys_to = CRM_SYSTEM_DC;
	} else {
	sys_to = CRM_SYSTEM_CRMD;
	}
	return create_request(op, msg_data, node, sys_to,
	(crm_system_name? crm_system_name : "client"),
	private->client_uuid);
	}

	// \return Standard Pacemaker return code
	static int
	send_controller_request(pcmk_ipc_api_t api, const xmlNode request,
	bool reply_is_expected)
	{
	if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) {
	return EINVAL;
	}
	if (reply_is_expected) {
	struct controld_api_private_s *private = api->api_data;

	private->replies_expected++;
	}
	return pcmk__send_ipc_request(api, request);
	}

	static xmlNode *
	create_reprobe_message_data(const char target_node, const char router_node)
	{
	xmlNode *msg_data;

	msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
	crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
	if ((router_node != NULL) && !pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
	crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	}
	return msg_data;
	}

	/*!
	* \brief Send a reprobe controller operation
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node to reprobe
	* \param[in] router_node Router node for host
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
	*/
	int
	pcmk_controld_api_reprobe(pcmk_ipc_api_t api, const char target_node,
	const char *router_node)
	{
	xmlNode *request;
	xmlNode *msg_data;
	int rc = pcmk_rc_ok;

	if (api == NULL) {
	return EINVAL;
	}
	if (router_node == NULL) {
	router_node = target_node;
	}
	crm_debug("Sending %s IPC request to reprobe %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(target_node, "local node"),
	pcmk__s(router_node, "local node"));
	msg_data = create_reprobe_message_data(target_node, router_node);
	request = create_controller_request(api, CRM_OP_REPROBE, router_node,
	msg_data);
	rc = send_controller_request(api, request, true);
	free_xml(msg_data);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Send a "node info" controller operation
	*
	* \param[in,out] api Controller connection
	* \param[in] nodeid ID of node to get info for (or 0 for local node)
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_info.
	*/
	int
	pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
	{
	xmlNode *request;
	int rc = pcmk_rc_ok;

	request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
	if (request == NULL) {
	return EINVAL;
	}
	if (nodeid > 0) {
	crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
	}

	rc = send_controller_request(api, request, true);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Ask the controller for status
	*
	* \param[in,out] api Controller connection
	* \param[in] node_name Name of node whose status is desired (NULL for DC)
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_ping.
	*/
	int
	pcmk_controld_api_ping(pcmk_ipc_api_t api, const char node_name)
	{
	xmlNode *request;
	int rc = pcmk_rc_ok;

	request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
	if (request == NULL) {
	return EINVAL;
	}
	rc = send_controller_request(api, request, true);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Ask the controller for cluster information
	*
	* \param[in,out] api Controller connection
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_nodes.
	*/
	int
	pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api)
	{
	xmlNode *request;
	int rc = EINVAL;

	request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL,
	NULL);
	if (request != NULL) {
	rc = send_controller_request(api, request, true);
	free_xml(request);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	controller_resource_op(pcmk_ipc_api_t api, const char op,
	const char target_node, const char router_node,
	bool cib_only, const char *rsc_id,
	const char rsc_long_id, const char standard,
	const char provider, const char type)
	{
	int rc = pcmk_rc_ok;
	char *key;
	xmlNode request, msg_data, xml_rsc, params;

	if (api == NULL) {
	return EINVAL;
	}
	if (router_node == NULL) {
	router_node = target_node;
	}

	msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);

	/* The controller logs the transition key from resource op requests, so we
	* need to have something for it.
	* @TODO don't use "crm-resource"
	*/
	key = pcmk__transition_key(0, getpid(), 0,
	"xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
	crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
	free(key);

	crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
	if (!pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
	crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
	}

	if (cib_only) {
	// Indicate that only the CIB needs to be cleaned
	crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
	}

	xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
	crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
	crm_xml_add(xml_rsc, PCMK__XA_LONG_ID, rsc_long_id);
	- crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard);
	+ crm_xml_add(xml_rsc, PCMK_XA_CLASS, standard);
	crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(xml_rsc, PCMK_XA_TYPE, type);

	params = create_xml_node(msg_data, XML_TAG_ATTRS);
	crm_xml_add(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);

	// The controller parses the timeout from the request
	key = crm_meta_name(XML_ATTR_TIMEOUT);
	crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg
	free(key);

	request = create_controller_request(api, op, router_node, msg_data);
	rc = send_controller_request(api, request, true);
	free_xml(msg_data);
	free_xml(request);
	return rc;
	}

	/*!
	* \brief Ask the controller to fail a resource
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node resource is on
	* \param[in] router_node Router node for target
	* \param[in] rsc_id ID of resource to fail
	* \param[in] rsc_long_id Long ID of resource (if any)
	* \param[in] standard Standard of resource
	* \param[in] provider Provider of resource (if any)
	* \param[in] type Type of resource to fail
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_resource.
	*/
	int
	pcmk_controld_api_fail(pcmk_ipc_api_t *api,
	const char target_node, const char router_node,
	const char rsc_id, const char rsc_long_id,
	const char standard, const char provider,
	const char *type)
	{
	crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
	pcmk__s(rsc_long_id, "no other names"),
	pcmk__s(target_node, "unspecified node"),
	pcmk__s(router_node, "unspecified node"));
	return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
	router_node, false, rsc_id, rsc_long_id,
	standard, provider, type);
	}

	/*!
	* \brief Ask the controller to refresh a resource
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node resource is on
	* \param[in] router_node Router node for target
	* \param[in] rsc_id ID of resource to refresh
	* \param[in] rsc_long_id Long ID of resource (if any)
	* \param[in] standard Standard of resource
	* \param[in] provider Provider of resource (if any)
	* \param[in] type Type of resource
	* \param[in] cib_only If true, clean resource from CIB only
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_resource.
	*/
	int
	pcmk_controld_api_refresh(pcmk_ipc_api_t api, const char target_node,
	const char *router_node,
	const char rsc_id, const char rsc_long_id,
	const char standard, const char provider,
	const char *type, bool cib_only)
	{
	crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
	pcmk__s(rsc_long_id, "no other names"),
	pcmk__s(target_node, "unspecified node"),
	pcmk__s(router_node, "unspecified node"));
	return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
	router_node, cib_only, rsc_id, rsc_long_id,
	standard, provider, type);
	}

	/*!
	* \brief Get the number of IPC replies currently expected from the controller
	*
	* \param[in] api Controller IPC API connection
	*
	* \return Number of replies expected
	*/
	unsigned int
	pcmk_controld_api_replies_expected(const pcmk_ipc_api_t *api)
	{
	struct controld_api_private_s *private = api->api_data;

	return private->replies_expected;
	}

	/*!
	* \brief Create XML for a controller IPC "hello" message
	*
	* \deprecated This function is deprecated as part of the public C API.
	*/
	// \todo make this static to this file when breaking API backward compatibility
	xmlNode *
	create_hello_message(const char uuid, const char client_name,
	const char major_version, const char minor_version)
	{
	xmlNode *hello_node = NULL;
	xmlNode *hello = NULL;

	if (pcmk__str_empty(uuid) \|\| pcmk__str_empty(client_name)
	\|\| pcmk__str_empty(major_version) \|\| pcmk__str_empty(minor_version)) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"missing information",
	client_name? client_name : "unknown client",
	uuid? uuid : "unknown");
	return NULL;
	}

	hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
	if (hello_node == NULL) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"Message data creation failed", client_name, uuid);
	return NULL;
	}

	crm_xml_add(hello_node, "major_version", major_version);
	crm_xml_add(hello_node, "minor_version", minor_version);
	crm_xml_add(hello_node, "client_name", client_name);
	crm_xml_add(hello_node, "client_uuid", uuid);

	hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
	if (hello == NULL) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"Request creation failed", client_name, uuid);
	return NULL;
	}
	free_xml(hello_node);

	crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
	return hello;
	}
	diff --git a/lib/common/output_html.c b/lib/common/output_html.c
	index 975888b9a6..791b1eb4a3 100644
	--- a/lib/common/output_html.c
	+++ b/lib/common/output_html.c
	@@ -1,478 +1,478 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <libxml/HTMLtree.h>
	#include <stdarg.h>
	#include <stdlib.h>
	#include <stdio.h>

	#include <crm/msg_xml.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/xml.h>

	static const char *stylesheet_default =
	".bold { font-weight: bold }\n"

	".online { color: green }\n"
	".offline { color: red }\n"
	".maint { color: blue }\n"
	".standby { color: blue }\n"
	".health_red { color: red }\n"
	".health_yellow { color: GoldenRod }\n"

	".rsc-failed { color: red }\n"
	".rsc-failure-ignored { color: DarkGreen }\n"
	".rsc-managed { color: blue }\n"
	".rsc-multiple { color: orange }\n"
	".rsc-ok { color: green }\n"

	".warning { color: red; font-weight: bold }";

	static gboolean cgi_output = FALSE;
	static char *stylesheet_link = NULL;
	static char *title = NULL;
	static GSList *extra_headers = NULL;

	GOptionEntry pcmk__html_output_entries[] = {
	{ "html-cgi", 0, 0, G_OPTION_ARG_NONE, &cgi_output,
	"Add CGI headers (requires --output-as=html)",
	NULL },

	{ "html-stylesheet", 0, 0, G_OPTION_ARG_STRING, &stylesheet_link,
	"Link to an external stylesheet (requires --output-as=html)",
	"URI" },

	{ "html-title", 0, 0, G_OPTION_ARG_STRING, &title,
	"Specify a page title (requires --output-as=html)",
	"TITLE" },

	{ NULL }
	};

	/* The first several elements of this struct must be the same as the first
	* several elements of private_data_s in lib/common/output_xml.c. This
	* struct gets passed to a bunch of the pcmk__output_xml_* functions which
	* assume an XML private_data_s. Keeping them laid out the same means this
	* still works.
	*/
	typedef struct private_data_s {
	/* Begin members that must match the XML version */
	xmlNode *root;
	GQueue *parent_q;
	GSList *errors;
	/* End members that must match the XML version */
	} private_data_t;

	static void
	html_free_priv(pcmk__output_t *out) {
	private_data_t *priv = NULL;

	if (out == NULL \|\| out->priv == NULL) {
	return;
	}

	priv = out->priv;

	xmlFreeNode(priv->root);
	g_queue_free(priv->parent_q);
	g_slist_free(priv->errors);
	free(priv);
	out->priv = NULL;
	}

	static bool
	html_init(pcmk__output_t *out) {
	private_data_t *priv = NULL;

	CRM_ASSERT(out != NULL);

	/* If html_init was previously called on this output struct, just return. */
	if (out->priv != NULL) {
	return true;
	} else {
	out->priv = calloc(1, sizeof(private_data_t));
	if (out->priv == NULL) {
	return false;
	}

	priv = out->priv;
	}

	priv->parent_q = g_queue_new();

	priv->root = create_xml_node(NULL, "html");
	xmlCreateIntSubset(priv->root->doc, (pcmkXmlStr) "html", NULL, NULL);

	crm_xml_add(priv->root, "lang", "en");
	g_queue_push_tail(priv->parent_q, priv->root);
	priv->errors = NULL;

	pcmk__output_xml_create_parent(out, "body", NULL);

	return true;
	}

	static void
	add_error_node(gpointer data, gpointer user_data) {
	char str = (char ) data;
	pcmk__output_t out = (pcmk__output_t ) user_data;
	out->list_item(out, NULL, "%s", str);
	}

	static void
	html_finish(pcmk__output_t out, crm_exit_t exit_status, bool print, void *copy_dest) {
	private_data_t *priv = NULL;
	htmlNodePtr head_node = NULL;
	htmlNodePtr charset_node = NULL;

	CRM_ASSERT(out != NULL);

	priv = out->priv;

	/* If root is NULL, html_init failed and we are being called from pcmk__output_free
	* in the pcmk__output_new path.
	*/
	if (priv == NULL \|\| priv->root == NULL) {
	return;
	}

	if (cgi_output && print) {
	fprintf(out->dest, "Content-Type: text/html\n\n");
	}

	/* Add the head node last - it's not needed earlier because it doesn't contain
	* anything else that the user could add, and we want it done last to pick up
	* any options that may have been given.
	*/
	head_node = xmlNewDocRawNode(NULL, NULL, (pcmkXmlStr) "head", NULL);

	if (title != NULL ) {
	pcmk_create_xml_text_node(head_node, "title", title);
	} else if (out->request != NULL) {
	pcmk_create_xml_text_node(head_node, "title", out->request);
	}

	charset_node = create_xml_node(head_node, "meta");
	crm_xml_add(charset_node, "charset", "utf-8");

	/* Add any extra header nodes the caller might have created. */
	for (int i = 0; i < g_slist_length(extra_headers); i++) {
	xmlAddChild(head_node, xmlCopyNode(g_slist_nth_data(extra_headers, i), 1));
	}

	/* Stylesheets are included two different ways. The first is via a built-in
	* default (see the stylesheet_default const above). The second is via the
	* html-stylesheet option, and this should obviously be a link to a
	* stylesheet. The second can override the first. At least one should be
	* given.
	*/
	pcmk_create_xml_text_node(head_node, "style", stylesheet_default);

	if (stylesheet_link != NULL) {
	htmlNodePtr link_node = create_xml_node(head_node, "link");
	pcmk__xe_set_props(link_node, "rel", "stylesheet",
	"href", stylesheet_link,
	NULL);
	}

	xmlAddPrevSibling(priv->root->children, head_node);

	if (g_slist_length(priv->errors) > 0) {
	out->begin_list(out, "Errors", NULL, NULL);
	g_slist_foreach(priv->errors, add_error_node, (gpointer) out);
	out->end_list(out);
	}

	if (print) {
	htmlDocDump(out->dest, priv->root->doc);
	}

	if (copy_dest != NULL) {
	*copy_dest = copy_xml(priv->root);
	}

	g_slist_free_full(extra_headers, (GDestroyNotify) xmlFreeNode);
	extra_headers = NULL;
	}

	static void
	html_reset(pcmk__output_t *out) {
	CRM_ASSERT(out != NULL);

	out->dest = freopen(NULL, "w", out->dest);
	CRM_ASSERT(out->dest != NULL);

	html_free_priv(out);
	html_init(out);
	}

	static void
	html_subprocess_output(pcmk__output_t *out, int exit_status,
	const char proc_stdout, const char proc_stderr) {
	char *rc_buf = NULL;

	CRM_ASSERT(out != NULL);

	rc_buf = crm_strdup_printf("Return code: %d", exit_status);

	pcmk__output_create_xml_text_node(out, "h2", "Command Output");
	pcmk__output_create_html_node(out, "div", NULL, NULL, rc_buf);

	if (proc_stdout != NULL) {
	pcmk__output_create_html_node(out, "div", NULL, NULL, "Stdout");
	pcmk__output_create_html_node(out, "div", NULL, "output", proc_stdout);
	}
	if (proc_stderr != NULL) {
	pcmk__output_create_html_node(out, "div", NULL, NULL, "Stderr");
	pcmk__output_create_html_node(out, "div", NULL, "output", proc_stderr);
	}

	free(rc_buf);
	}

	static void
	html_version(pcmk__output_t *out, bool extended) {
	CRM_ASSERT(out != NULL);

	pcmk__output_create_xml_text_node(out, "h2", "Version Information");
	pcmk__output_create_html_node(out, "div", NULL, NULL, "Program: Pacemaker");
	pcmk__output_create_html_node(out, "div", NULL, NULL, crm_strdup_printf("Version: %s", PACEMAKER_VERSION));
	pcmk__output_create_html_node(out, "div", NULL, NULL,
	"Author: Andrew Beekhof and "
	"the Pacemaker project contributors");
	pcmk__output_create_html_node(out, "div", NULL, NULL, crm_strdup_printf("Build: %s", BUILD_VERSION));
	pcmk__output_create_html_node(out, "div", NULL, NULL, crm_strdup_printf("Features: %s", CRM_FEATURES));
	}

	G_GNUC_PRINTF(2, 3)
	static void
	html_err(pcmk__output_t out, const char format, ...) {
	private_data_t *priv = NULL;
	int len = 0;
	char *buf = NULL;
	va_list ap;

	CRM_ASSERT(out != NULL && out->priv != NULL);
	priv = out->priv;

	va_start(ap, format);
	len = vasprintf(&buf, format, ap);
	CRM_ASSERT(len >= 0);
	va_end(ap);

	priv->errors = g_slist_append(priv->errors, buf);
	}

	G_GNUC_PRINTF(2, 3)
	static int
	html_info(pcmk__output_t out, const char format, ...) {
	return pcmk_rc_no_output;
	}

	static void
	html_output_xml(pcmk__output_t out, const char name, const char *buf) {
	htmlNodePtr node = NULL;

	CRM_ASSERT(out != NULL);

	node = pcmk__output_create_html_node(out, "pre", NULL, NULL, buf);
	crm_xml_add(node, "lang", "xml");
	}

	G_GNUC_PRINTF(4, 5)
	static void
	html_begin_list(pcmk__output_t out, const char singular_noun,
	const char plural_noun, const char format, ...) {
	int q_len = 0;
	private_data_t *priv = NULL;
	xmlNodePtr node = NULL;

	CRM_ASSERT(out != NULL && out->priv != NULL);
	priv = out->priv;

	/* If we are already in a list (the queue depth is always at least
	* one because of the <html> element), first create a <li> element
	* to hold the <h2> and the new list.
	*/
	q_len = g_queue_get_length(priv->parent_q);
	if (q_len > 2) {
	pcmk__output_xml_create_parent(out, "li", NULL);
	}

	if (format != NULL) {
	va_list ap;
	char *buf = NULL;
	int len;

	va_start(ap, format);
	len = vasprintf(&buf, format, ap);
	va_end(ap);
	CRM_ASSERT(len >= 0);

	if (q_len > 2) {
	pcmk__output_create_xml_text_node(out, "h3", buf);
	} else {
	pcmk__output_create_xml_text_node(out, "h2", buf);
	}

	free(buf);
	}

	node = pcmk__output_xml_create_parent(out, "ul", NULL);
	g_queue_push_tail(priv->parent_q, node);
	}

	G_GNUC_PRINTF(3, 4)
	static void
	html_list_item(pcmk__output_t out, const char name, const char *format, ...) {
	htmlNodePtr item_node = NULL;
	va_list ap;
	char *buf = NULL;
	int len;

	CRM_ASSERT(out != NULL);

	va_start(ap, format);
	len = vasprintf(&buf, format, ap);
	CRM_ASSERT(len >= 0);
	va_end(ap);

	item_node = pcmk__output_create_xml_text_node(out, "li", buf);
	free(buf);

	if (name != NULL) {
	- crm_xml_add(item_node, "class", name);
	+ crm_xml_add(item_node, PCMK_XA_CLASS, name);
	}
	}

	static void
	html_increment_list(pcmk__output_t *out) {
	/* This function intentially left blank */
	}

	static void
	html_end_list(pcmk__output_t *out) {
	private_data_t *priv = NULL;

	CRM_ASSERT(out != NULL && out->priv != NULL);
	priv = out->priv;

	/* Remove the <ul> tag. */
	g_queue_pop_tail(priv->parent_q);
	pcmk__output_xml_pop_parent(out);

	/* Remove the <li> created for nested lists. */
	if (g_queue_get_length(priv->parent_q) > 2) {
	pcmk__output_xml_pop_parent(out);
	}
	}

	static bool
	html_is_quiet(pcmk__output_t *out) {
	return false;
	}

	static void
	html_spacer(pcmk__output_t *out) {
	CRM_ASSERT(out != NULL);
	pcmk__output_create_xml_node(out, "br", NULL);
	}

	static void
	html_progress(pcmk__output_t *out, bool end) {
	/* This function intentially left blank */
	}

	pcmk__output_t *
	pcmk__mk_html_output(char **argv) {
	pcmk__output_t *retval = calloc(1, sizeof(pcmk__output_t));

	if (retval == NULL) {
	return NULL;
	}

	retval->fmt_name = "html";
	retval->request = pcmk__quote_cmdline(argv);

	retval->init = html_init;
	retval->free_priv = html_free_priv;
	retval->finish = html_finish;
	retval->reset = html_reset;

	retval->register_message = pcmk__register_message;
	retval->message = pcmk__call_message;

	retval->subprocess_output = html_subprocess_output;
	retval->version = html_version;
	retval->info = html_info;
	retval->transient = html_info;
	retval->err = html_err;
	retval->output_xml = html_output_xml;

	retval->begin_list = html_begin_list;
	retval->list_item = html_list_item;
	retval->increment_list = html_increment_list;
	retval->end_list = html_end_list;

	retval->is_quiet = html_is_quiet;
	retval->spacer = html_spacer;
	retval->progress = html_progress;
	retval->prompt = pcmk__text_prompt;

	return retval;
	}

	xmlNodePtr
	pcmk__output_create_html_node(pcmk__output_t out, const char element_name, const char *id,
	const char class_name, const char text) {
	htmlNodePtr node = NULL;

	CRM_ASSERT(out != NULL);
	CRM_CHECK(pcmk__str_eq(out->fmt_name, "html", pcmk__str_none), return NULL);

	node = pcmk__output_create_xml_text_node(out, element_name, text);

	if (class_name != NULL) {
	- crm_xml_add(node, "class", class_name);
	+ crm_xml_add(node, PCMK_XA_CLASS, class_name);
	}

	if (id != NULL) {
	crm_xml_add(node, PCMK_XA_ID, id);
	}

	return node;
	}

	void
	pcmk__html_add_header(const char *name, ...) {
	htmlNodePtr header_node;
	va_list ap;

	va_start(ap, name);

	header_node = xmlNewDocRawNode(NULL, NULL, (pcmkXmlStr) name, NULL);
	while (1) {
	char key = va_arg(ap, char );
	char *value;

	if (key == NULL) {
	break;
	}

	value = va_arg(ap, char *);
	crm_xml_add(header_node, key, value);
	}

	extra_headers = g_slist_append(extra_headers, header_node);

	va_end(ap);
	}
	diff --git a/lib/common/xml.c b/lib/common/xml.c
	index 8982b36da9..d27dbd7f20 100644
	--- a/lib/common/xml.c
	+++ b/lib/common/xml.c
	@@ -1,2763 +1,2763 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <time.h>
	#include <string.h>
	#include <stdlib.h>
	#include <stdarg.h>
	#include <bzlib.h>

	#include <libxml/parser.h>
	#include <libxml/tree.h>
	#include <libxml/xmlIO.h> /* xmlAllocOutputBuffer */

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h> // PCMK__XML_LOG_BASE, etc.
	#include "crmcommon_private.h"

	// Define this as 1 in development to get insanely verbose trace messages
	#ifndef XML_PARSER_DEBUG
	#define XML_PARSER_DEBUG 0
	#endif

	/* @TODO XML_PARSE_RECOVER allows some XML errors to be silently worked around
	* by libxml2, which is potentially ambiguous and dangerous. We should drop it
	* when we can break backward compatibility with configurations that might be
	* relying on it (i.e. pacemaker 3.0.0).
	*
	* It might be a good idea to have a transitional period where we first try
	* parsing without XML_PARSE_RECOVER, and if that fails, try parsing again with
	* it, logging a warning if it succeeds.
	*/
	#define PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER (XML_PARSE_NOBLANKS)
	#define PCMK__XML_PARSE_OPTS_WITH_RECOVER (XML_PARSE_NOBLANKS \| XML_PARSE_RECOVER)

	bool
	pcmk__tracking_xml_changes(xmlNode *xml, bool lazy)
	{
	if(xml == NULL \|\| xml->doc == NULL \|\| xml->doc->_private == NULL) {
	return FALSE;
	} else if (!pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
	pcmk__xf_tracking)) {
	return FALSE;
	} else if (lazy && !pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
	pcmk__xf_lazy)) {
	return FALSE;
	}
	return TRUE;
	}

	static inline void
	set_parent_flag(xmlNode *xml, long flag)
	{
	for(; xml; xml = xml->parent) {
	xml_node_private_t *nodepriv = xml->_private;

	if (nodepriv == NULL) {
	/* During calls to xmlDocCopyNode(), _private will be unset for parent nodes */
	} else {
	pcmk__set_xml_flags(nodepriv, flag);
	}
	}
	}

	void
	pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag)
	{
	if(xml && xml->doc && xml->doc->_private){
	/* During calls to xmlDocCopyNode(), xml->doc may be unset */
	xml_doc_private_t *docpriv = xml->doc->_private;

	pcmk__set_xml_flags(docpriv, flag);
	}
	}

	// Mark document, element, and all element's parents as changed
	void
	pcmk__mark_xml_node_dirty(xmlNode *xml)
	{
	pcmk__set_xml_doc_flag(xml, pcmk__xf_dirty);
	set_parent_flag(xml, pcmk__xf_dirty);
	}

	// Clear flags on XML node and its children
	static void
	reset_xml_node_flags(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_node_private_t *nodepriv = xml->_private;

	if (nodepriv) {
	nodepriv->flags = 0;
	}

	for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	reset_xml_node_flags(cIter);
	}
	}

	// Set xpf_created flag on XML node and any children
	void
	pcmk__mark_xml_created(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_node_private_t *nodepriv = NULL;

	CRM_ASSERT(xml != NULL);
	nodepriv = xml->_private;

	if (nodepriv && pcmk__tracking_xml_changes(xml, FALSE)) {
	if (!pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
	pcmk__set_xml_flags(nodepriv, pcmk__xf_created);
	pcmk__mark_xml_node_dirty(xml);
	}
	for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	pcmk__mark_xml_created(cIter);
	}
	}
	}

	#define XML_DOC_PRIVATE_MAGIC 0x81726354UL
	#define XML_NODE_PRIVATE_MAGIC 0x54637281UL

	// Free an XML object previously marked as deleted
	static void
	free_deleted_object(void *data)
	{
	if(data) {
	pcmk__deleted_xml_t *deleted_obj = data;

	free(deleted_obj->path);
	free(deleted_obj);
	}
	}

	// Free and NULL user, ACLs, and deleted objects in an XML node's private data
	static void
	reset_xml_private_data(xml_doc_private_t *docpriv)
	{
	if (docpriv != NULL) {
	CRM_ASSERT(docpriv->check == XML_DOC_PRIVATE_MAGIC);

	free(docpriv->user);
	docpriv->user = NULL;

	if (docpriv->acls != NULL) {
	pcmk__free_acls(docpriv->acls);
	docpriv->acls = NULL;
	}

	if(docpriv->deleted_objs) {
	g_list_free_full(docpriv->deleted_objs, free_deleted_object);
	docpriv->deleted_objs = NULL;
	}
	}
	}

	// Free all private data associated with an XML node
	static void
	free_private_data(xmlNode *node)
	{
	/* Note:

	This function frees private data assosciated with an XML node,
	unless the function is being called as a result of internal
	XSLT cleanup.

	That could happen through, for example, the following chain of
	function calls:

	xsltApplyStylesheetInternal
	-> xsltFreeTransformContext
	-> xsltFreeRVTs
	-> xmlFreeDoc

	And in that case, the node would fulfill three conditions:

	1. It would be a standalone document (i.e. it wouldn't be
	part of a document)
	2. It would have a space-prefixed name (for reference, please
	see xsltInternals.h: XSLT_MARK_RES_TREE_FRAG)
	3. It would carry its own payload in the _private field.

	We do not free data in this circumstance to avoid a failed
	assertion on the XML_*_PRIVATE_MAGIC later.

	*/
	if (node->name == NULL \|\| node->name[0] != ' ') {
	if (node->_private) {
	if (node->type == XML_DOCUMENT_NODE) {
	reset_xml_private_data(node->_private);
	} else {
	CRM_ASSERT(((xml_node_private_t *) node->_private)->check
	== XML_NODE_PRIVATE_MAGIC);
	/* nothing dynamically allocated nested */
	}
	free(node->_private);
	node->_private = NULL;
	}
	}
	}

	// Allocate and initialize private data for an XML node
	static void
	new_private_data(xmlNode *node)
	{
	switch (node->type) {
	case XML_DOCUMENT_NODE: {
	xml_doc_private_t *docpriv = NULL;
	docpriv = calloc(1, sizeof(xml_doc_private_t));
	CRM_ASSERT(docpriv != NULL);
	docpriv->check = XML_DOC_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	pcmk__set_xml_flags(docpriv, pcmk__xf_dirty\|pcmk__xf_created);
	node->_private = docpriv;
	break;
	}
	case XML_ELEMENT_NODE:
	case XML_ATTRIBUTE_NODE:
	case XML_COMMENT_NODE: {
	xml_node_private_t *nodepriv = NULL;
	nodepriv = calloc(1, sizeof(xml_node_private_t));
	CRM_ASSERT(nodepriv != NULL);
	nodepriv->check = XML_NODE_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty\|pcmk__xf_created);
	node->_private = nodepriv;
	if (pcmk__tracking_xml_changes(node, FALSE)) {
	/* XML_ELEMENT_NODE doesn't get picked up here, node->doc is
	* not hooked up at the point we are called
	*/
	pcmk__mark_xml_node_dirty(node);
	}
	break;
	}
	case XML_TEXT_NODE:
	case XML_DTD_NODE:
	case XML_CDATA_SECTION_NODE:
	break;
	default:
	/* Ignore */
	crm_trace("Ignoring %p %d", node, node->type);
	CRM_LOG_ASSERT(node->type == XML_ELEMENT_NODE);
	break;
	}
	}

	void
	xml_track_changes(xmlNode * xml, const char user, xmlNode acl_source, bool enforce_acls)
	{
	xml_accept_changes(xml);
	crm_trace("Tracking changes%s to %p", enforce_acls?" with ACLs":"", xml);
	pcmk__set_xml_doc_flag(xml, pcmk__xf_tracking);
	if(enforce_acls) {
	if(acl_source == NULL) {
	acl_source = xml;
	}
	pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_enabled);
	pcmk__unpack_acl(acl_source, xml, user);
	pcmk__apply_acl(xml);
	}
	}

	bool xml_tracking_changes(xmlNode * xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_tracking);
	}

	bool xml_document_dirty(xmlNode *xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_dirty);
	}

	/*!
	* \internal
	* \brief Return ordinal position of an XML node among its siblings
	*
	* \param[in] xml XML node to check
	* \param[in] ignore_if_set Don't count siblings with this flag set
	*
	* \return Ordinal position of \p xml (starting with 0)
	*/
	int
	pcmk__xml_position(const xmlNode *xml, enum xml_private_flags ignore_if_set)
	{
	int position = 0;

	for (const xmlNode *cIter = xml; cIter->prev; cIter = cIter->prev) {
	xml_node_private_t nodepriv = ((xmlNode)cIter->prev)->_private;

	if (!pcmk_is_set(nodepriv->flags, ignore_if_set)) {
	position++;
	}
	}

	return position;
	}

	// Remove all attributes marked as deleted from an XML node
	static void
	accept_attr_deletions(xmlNode *xml)
	{
	// Clear XML node's flags
	((xml_node_private_t *) xml->_private)->flags = pcmk__xf_none;

	// Remove this XML node's attributes that were marked as deleted
	pcmk__xe_remove_matching_attrs(xml, pcmk__marked_as_deleted, NULL);

	// Recursively do the same for this XML node's children
	for (xmlNodePtr cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	accept_attr_deletions(cIter);
	}
	}

	/*!
	* \internal
	* \brief Find first child XML node matching another given XML node
	*
	* \param[in] haystack XML whose children should be checked
	* \param[in] needle XML to match (comment content or element name and ID)
	* \param[in] exact If true and needle is a comment, position must match
	*/
	xmlNode *
	pcmk__xml_match(const xmlNode haystack, const xmlNode needle, bool exact)
	{
	CRM_CHECK(needle != NULL, return NULL);

	if (needle->type == XML_COMMENT_NODE) {
	return pcmk__xc_match(haystack, needle, exact);

	} else {
	const char *id = ID(needle);
	const char *attr = (id == NULL)? NULL : PCMK_XA_ID;

	return pcmk__xe_match(haystack, (const char *) needle->name, attr, id);
	}
	}

	void
	xml_accept_changes(xmlNode * xml)
	{
	xmlNode *top = NULL;
	xml_doc_private_t *docpriv = NULL;

	if(xml == NULL) {
	return;
	}

	crm_trace("Accepting changes to %p", xml);
	docpriv = xml->doc->_private;
	top = xmlDocGetRootElement(xml->doc);

	reset_xml_private_data(xml->doc->_private);

	if (!pcmk_is_set(docpriv->flags, pcmk__xf_dirty)) {
	docpriv->flags = pcmk__xf_none;
	return;
	}

	docpriv->flags = pcmk__xf_none;
	accept_attr_deletions(top);
	}

	xmlNode *
	find_xml_node(const xmlNode root, const char search_path, gboolean must_find)
	{
	xmlNode *a_child = NULL;
	const char name = (root == NULL)? "<NULL>" : (const char ) root->name;

	if (search_path == NULL) {
	crm_warn("Will never find <NULL>");
	return NULL;
	}

	for (a_child = pcmk__xml_first_child(root); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	if (strcmp((const char *)a_child->name, search_path) == 0) {
	return a_child;
	}
	}

	if (must_find) {
	crm_warn("Could not find %s in %s.", search_path, name);
	} else if (root != NULL) {
	crm_trace("Could not find %s in %s.", search_path, name);
	} else {
	crm_trace("Could not find %s in <NULL>.", search_path);
	}

	return NULL;
	}

	#define attr_matches(c, n, v) pcmk__str_eq(crm_element_value((c), (n)), \
	(v), pcmk__str_none)

	/*!
	* \internal
	* \brief Find first XML child element matching given criteria
	*
	* \param[in] parent XML element to search
	* \param[in] node_name If not NULL, only match children of this type
	* \param[in] attr_n If not NULL, only match children with an attribute
	* of this name.
	* \param[in] attr_v If \p attr_n and this are not NULL, only match children
	* with an attribute named \p attr_n and this value
	*
	* \return Matching XML child element, or NULL if none found
	*/
	xmlNode *
	pcmk__xe_match(const xmlNode parent, const char node_name,
	const char attr_n, const char attr_v)
	{
	CRM_CHECK(parent != NULL, return NULL);
	CRM_CHECK(attr_v == NULL \|\| attr_n != NULL, return NULL);

	for (xmlNode *child = pcmk__xml_first_child(parent); child != NULL;
	child = pcmk__xml_next(child)) {
	if (pcmk__str_eq(node_name, (const char *) (child->name),
	pcmk__str_null_matches)
	&& ((attr_n == NULL) \|\|
	(attr_v == NULL && xmlHasProp(child, (pcmkXmlStr) attr_n)) \|\|
	(attr_v != NULL && attr_matches(child, attr_n, attr_v)))) {
	return child;
	}
	}
	crm_trace("XML child node <%s%s%s%s%s> not found in %s",
	(node_name? node_name : "(any)"),
	(attr_n? " " : ""),
	(attr_n? attr_n : ""),
	(attr_n? "=" : ""),
	(attr_n? attr_v : ""),
	(const char *) parent->name);
	return NULL;
	}

	void
	copy_in_properties(xmlNode target, const xmlNode src)
	{
	if (src == NULL) {
	crm_warn("No node to copy properties from");

	} else if (target == NULL) {
	crm_err("No node to copy properties into");

	} else {
	for (xmlAttrPtr a = pcmk__xe_first_attr(src); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	expand_plus_plus(target, p_name, p_value);
	if (xml_acl_denied(target)) {
	crm_trace("Cannot copy %s=%s to %s", p_name, p_value, target->name);
	return;
	}
	}
	}

	return;
	}

	/*!
	* \brief Parse integer assignment statements on this node and all its child
	* nodes
	*
	* \param[in,out] target Root XML node to be processed
	*
	* \note This function is recursive
	*/
	void
	fix_plus_plus_recursive(xmlNode *target)
	{
	/* TODO: Remove recursion and use xpath searches for value++ */
	xmlNode *child = NULL;

	for (xmlAttrPtr a = pcmk__xe_first_attr(target); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	expand_plus_plus(target, p_name, p_value);
	}
	for (child = pcmk__xml_first_child(target); child != NULL;
	child = pcmk__xml_next(child)) {
	fix_plus_plus_recursive(child);
	}
	}

	/*!
	* \brief Update current XML attribute value per parsed integer assignment
	statement
	*
	* \param[in,out] target an XML node, containing a XML attribute that is
	* initialized to some numeric value, to be processed
	* \param[in] name name of the XML attribute, e.g. X, whose value
	* should be updated
	* \param[in] value assignment statement, e.g. "X++" or
	* "X+=5", to be applied to the initialized value.
	*
	* \note The original XML attribute value is treated as 0 if non-numeric and
	* truncated to be an integer if decimal-point-containing.
	* \note The final XML attribute value is truncated to not exceed 1000000.
	* \note Undefined behavior if unexpected input.
	*/
	void
	expand_plus_plus(xmlNode * target, const char name, const char value)
	{
	int offset = 1;
	int name_len = 0;
	int int_value = 0;
	int value_len = 0;

	const char *old_value = NULL;

	if (target == NULL \|\| value == NULL \|\| name == NULL) {
	return;
	}

	old_value = crm_element_value(target, name);

	if (old_value == NULL) {
	/* if no previous value, set unexpanded */
	goto set_unexpanded;

	} else if (strstr(value, name) != value) {
	goto set_unexpanded;
	}

	name_len = strlen(name);
	value_len = strlen(value);
	if (value_len < (name_len + 2)
	\|\| value[name_len] != '+' \|\| (value[name_len + 1] != '+' && value[name_len + 1] != '=')) {
	goto set_unexpanded;
	}

	/* if we are expanding ourselves,
	* then no previous value was set and leave int_value as 0
	*/
	if (old_value != value) {
	int_value = char2score(old_value);
	}

	if (value[name_len + 1] != '+') {
	const char *offset_s = value + (name_len + 2);

	offset = char2score(offset_s);
	}
	int_value += offset;

	if (int_value > INFINITY) {
	int_value = (int)INFINITY;
	}

	crm_xml_add_int(target, name, int_value);
	return;

	set_unexpanded:
	if (old_value == value) {
	/* the old value is already set, nothing to do */
	return;
	}
	crm_xml_add(target, name, value);
	return;
	}

	/*!
	* \internal
	* \brief Remove an XML element's attributes that match some criteria
	*
	* \param[in,out] element XML element to modify
	* \param[in] match If not NULL, only remove attributes for which
	* this function returns true
	* \param[in,out] user_data Data to pass to \p match
	*/
	void
	pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data)
	{
	xmlAttrPtr next = NULL;

	for (xmlAttrPtr a = pcmk__xe_first_attr(element); a != NULL; a = next) {
	next = a->next; // Grab now because attribute might get removed
	if ((match == NULL) \|\| match(a, user_data)) {
	if (!pcmk__check_acl(element, NULL, pcmk__xf_acl_write)) {
	crm_trace("ACLs prevent removal of attributes (%s and "
	"possibly others) from %s element",
	(const char ) a->name, (const char ) element->name);
	return; // ACLs apply to element, not particular attributes
	}

	if (pcmk__tracking_xml_changes(element, false)) {
	// Leave (marked for removal) until after diff is calculated
	set_parent_flag(element, pcmk__xf_dirty);
	pcmk__set_xml_flags((xml_node_private_t *) a->_private,
	pcmk__xf_deleted);
	} else {
	xmlRemoveProp(a);
	}
	}
	}
	}

	xmlNode *
	add_node_copy(xmlNode * parent, xmlNode * src_node)
	{
	xmlNode *child = NULL;

	CRM_CHECK((parent != NULL) && (src_node != NULL), return NULL);

	child = xmlDocCopyNode(src_node, parent->doc, 1);
	if (child == NULL) {
	return NULL;
	}
	xmlAddChild(parent, child);
	pcmk__mark_xml_created(child);
	return child;
	}

	xmlNode *
	create_xml_node(xmlNode * parent, const char *name)
	{
	xmlDoc *doc = NULL;
	xmlNode *node = NULL;

	if (pcmk__str_empty(name)) {
	CRM_CHECK(name != NULL && name[0] == 0, return NULL);
	return NULL;
	}

	if (parent == NULL) {
	doc = xmlNewDoc((pcmkXmlStr) "1.0");
	if (doc == NULL) {
	return NULL;
	}

	node = xmlNewDocRawNode(doc, NULL, (pcmkXmlStr) name, NULL);
	if (node == NULL) {
	xmlFreeDoc(doc);
	return NULL;
	}
	xmlDocSetRootElement(doc, node);

	} else {
	node = xmlNewChild(parent, NULL, (pcmkXmlStr) name, NULL);
	if (node == NULL) {
	return NULL;
	}
	}
	pcmk__mark_xml_created(node);
	return node;
	}

	xmlNode *
	pcmk_create_xml_text_node(xmlNode * parent, const char name, const char content)
	{
	xmlNode *node = create_xml_node(parent, name);

	if (node != NULL) {
	xmlNodeSetContent(node, (pcmkXmlStr) content);
	}

	return node;
	}

	xmlNode *
	pcmk_create_html_node(xmlNode * parent, const char element_name, const char id,
	const char class_name, const char text)
	{
	xmlNode *node = pcmk_create_xml_text_node(parent, element_name, text);

	if (class_name != NULL) {
	- crm_xml_add(node, "class", class_name);
	+ crm_xml_add(node, PCMK_XA_CLASS, class_name);
	}

	if (id != NULL) {
	crm_xml_add(node, PCMK_XA_ID, id);
	}

	return node;
	}

	/*!
	* Free an XML element and all of its children, removing it from its parent
	*
	* \param[in,out] xml XML element to free
	*/
	void
	pcmk_free_xml_subtree(xmlNode *xml)
	{
	xmlUnlinkNode(xml); // Detaches from parent and siblings
	xmlFreeNode(xml); // Frees
	}

	static void
	free_xml_with_position(xmlNode * child, int position)
	{
	if (child != NULL) {
	xmlNode *top = NULL;
	xmlDoc *doc = child->doc;
	xml_node_private_t *nodepriv = child->_private;
	xml_doc_private_t *docpriv = NULL;

	if (doc != NULL) {
	top = xmlDocGetRootElement(doc);
	}

	if (doc != NULL && top == child) {
	/* Free everything */
	xmlFreeDoc(doc);

	} else if (pcmk__check_acl(child, NULL, pcmk__xf_acl_write) == FALSE) {
	GString *xpath = NULL;

	pcmk__if_tracing({}, return);
	xpath = pcmk__element_xpath(child);
	qb_log_from_external_source(__func__, __FILE__,
	"Cannot remove %s %x", LOG_TRACE,
	__LINE__, 0, (const char *) xpath->str,
	nodepriv->flags);
	g_string_free(xpath, TRUE);
	return;

	} else {
	if (doc && pcmk__tracking_xml_changes(child, FALSE)
	&& !pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {

	GString *xpath = pcmk__element_xpath(child);

	if (xpath != NULL) {
	pcmk__deleted_xml_t *deleted_obj = NULL;

	crm_trace("Deleting %s %p from %p",
	(const char *) xpath->str, child, doc);

	deleted_obj = calloc(1, sizeof(pcmk__deleted_xml_t));
	deleted_obj->path = strdup((const char *) xpath->str);

	CRM_ASSERT(deleted_obj->path != NULL);
	g_string_free(xpath, TRUE);

	deleted_obj->position = -1;
	/* Record the "position" only for XML comments for now */
	if (child->type == XML_COMMENT_NODE) {
	if (position >= 0) {
	deleted_obj->position = position;

	} else {
	deleted_obj->position = pcmk__xml_position(child,
	pcmk__xf_skip);
	}
	}

	docpriv = doc->_private;
	docpriv->deleted_objs = g_list_append(docpriv->deleted_objs, deleted_obj);
	pcmk__set_xml_doc_flag(child, pcmk__xf_dirty);
	}
	}
	pcmk_free_xml_subtree(child);
	}
	}
	}


	void
	free_xml(xmlNode * child)
	{
	free_xml_with_position(child, -1);
	}

	xmlNode *
	copy_xml(xmlNode * src)
	{
	xmlDoc *doc = xmlNewDoc((pcmkXmlStr) "1.0");
	xmlNode *copy = xmlDocCopyNode(src, doc, 1);

	CRM_ASSERT(copy != NULL);
	xmlDocSetRootElement(doc, copy);
	return copy;
	}

	xmlNode *
	string2xml(const char *input)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	xmlParserCtxtPtr ctxt = NULL;
	const xmlError *last_error = NULL;

	if (input == NULL) {
	crm_err("Can't parse NULL input");
	return NULL;
	}

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err);
	output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);

	if (output == NULL) {
	output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	PCMK__XML_PARSE_OPTS_WITH_RECOVER);
	if (output) {
	crm_warn("Successfully recovered from XML errors "
	"(note: a future release will treat this as a fatal failure)");
	}
	}

	if (output) {
	xml = xmlDocGetRootElement(output);
	}
	last_error = xmlCtxtGetLastError(ctxt);
	if (last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__func__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level, last_error->code, last_error->message);

	if (last_error->code == XML_ERR_DOCUMENT_EMPTY) {
	CRM_LOG_ASSERT("Cannot parse an empty string");

	} else if (last_error->code != XML_ERR_DOCUMENT_END) {
	crm_err("Couldn't%s parse %d chars: %s", xml ? " fully" : "", (int)strlen(input),
	input);
	if (xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}

	} else {
	int len = strlen(input);
	int lpc = 0;

	while(lpc < len) {
	crm_warn("Parse error[+%.3d]: %.80s", lpc, input+lpc);
	lpc += 80;
	}

	CRM_LOG_ASSERT("String parsing error");
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	xmlNode *
	stdin2xml(void)
	{
	size_t data_length = 0;
	size_t read_chars = 0;

	char *xml_buffer = NULL;
	xmlNode *xml_obj = NULL;

	do {
	xml_buffer = pcmk__realloc(xml_buffer, data_length + PCMK__BUFFER_SIZE);
	read_chars = fread(xml_buffer + data_length, 1, PCMK__BUFFER_SIZE,
	stdin);
	data_length += read_chars;
	} while (read_chars == PCMK__BUFFER_SIZE);

	if (data_length == 0) {
	crm_warn("No XML supplied on stdin");
	free(xml_buffer);
	return NULL;
	}

	xml_buffer[data_length] = '\0';
	xml_obj = string2xml(xml_buffer);
	free(xml_buffer);

	crm_log_xml_trace(xml_obj, "Created fragment");
	return xml_obj;
	}

	static char *
	decompress_file(const char *filename)
	{
	char *buffer = NULL;
	int rc = 0;
	size_t length = 0, read_len = 0;
	BZFILE *bz_file = NULL;
	FILE *input = fopen(filename, "r");

	if (input == NULL) {
	crm_perror(LOG_ERR, "Could not open %s for reading", filename);
	return NULL;
	}

	bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_err("Could not prepare to read compressed %s: %s "
	CRM_XS " rc=%d", filename, pcmk_rc_str(rc), rc);
	BZ2_bzReadClose(&rc, bz_file);
	fclose(input);
	return NULL;
	}

	rc = BZ_OK;
	// cppcheck seems not to understand the abort-logic in pcmk__realloc
	// cppcheck-suppress memleak
	while (rc == BZ_OK) {
	buffer = pcmk__realloc(buffer, PCMK__BUFFER_SIZE + length + 1);
	read_len = BZ2_bzRead(&rc, bz_file, buffer + length, PCMK__BUFFER_SIZE);

	crm_trace("Read %ld bytes from file: %d", (long)read_len, rc);

	if (rc == BZ_OK \|\| rc == BZ_STREAM_END) {
	length += read_len;
	}
	}

	buffer[length] = '\0';

	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_err("Could not read compressed %s: %s " CRM_XS " rc=%d",
	filename, pcmk_rc_str(rc), rc);
	free(buffer);
	buffer = NULL;
	}

	BZ2_bzReadClose(&rc, bz_file);
	fclose(input);
	return buffer;
	}

	/*!
	* \internal
	* \brief Remove XML text nodes from specified XML and all its children
	*
	* \param[in,out] xml XML to strip text from
	*/
	void
	pcmk__strip_xml_text(xmlNode *xml)
	{
	xmlNode *iter = xml->children;

	while (iter) {
	xmlNode *next = iter->next;

	switch (iter->type) {
	case XML_TEXT_NODE:
	/* Remove it */
	pcmk_free_xml_subtree(iter);
	break;

	case XML_ELEMENT_NODE:
	/* Search it */
	pcmk__strip_xml_text(iter);
	break;

	default:
	/* Leave it */
	break;
	}

	iter = next;
	}
	}

	xmlNode *
	filename2xml(const char *filename)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	bool uncompressed = true;
	xmlParserCtxtPtr ctxt = NULL;
	const xmlError *last_error = NULL;

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err);

	if (filename) {
	uncompressed = !pcmk__ends_with_ext(filename, ".bz2");
	}

	if (pcmk__str_eq(filename, "-", pcmk__str_null_matches)) {
	/* STDIN_FILENO == fileno(stdin) */
	output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL,
	PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);

	if (output == NULL) {
	output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL,
	PCMK__XML_PARSE_OPTS_WITH_RECOVER);
	if (output) {
	crm_warn("Successfully recovered from XML errors "
	"(note: a future release will treat this as a fatal failure)");
	}
	}

	} else if (uncompressed) {
	output = xmlCtxtReadFile(ctxt, filename, NULL,
	PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);

	if (output == NULL) {
	output = xmlCtxtReadFile(ctxt, filename, NULL,
	PCMK__XML_PARSE_OPTS_WITH_RECOVER);
	if (output) {
	crm_warn("Successfully recovered from XML errors "
	"(note: a future release will treat this as a fatal failure)");
	}
	}

	} else {
	char *input = decompress_file(filename);

	output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);

	if (output == NULL) {
	output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	PCMK__XML_PARSE_OPTS_WITH_RECOVER);
	if (output) {
	crm_warn("Successfully recovered from XML errors "
	"(note: a future release will treat this as a fatal failure)");
	}
	}

	free(input);
	}

	if (output && (xml = xmlDocGetRootElement(output))) {
	pcmk__strip_xml_text(xml);
	}

	last_error = xmlCtxtGetLastError(ctxt);
	if (last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__func__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_err("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level, last_error->code, last_error->message);

	if (last_error && last_error->code != XML_ERR_OK) {
	crm_err("Couldn't%s parse %s", xml ? " fully" : "", filename);
	if (xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	/*!
	* \internal
	* \brief Add a "last written" attribute to an XML element, set to current time
	*
	* \param[in,out] xe XML element to add attribute to
	*
	* \return Value that was set, or NULL on error
	*/
	const char *
	pcmk__xe_add_last_written(xmlNode *xe)
	{
	char *now_s = pcmk__epoch2str(NULL, 0);
	const char *result = NULL;

	result = crm_xml_add(xe, PCMK_XA_CIB_LAST_WRITTEN,
	pcmk__s(now_s, "Could not determine current time"));
	free(now_s);
	return result;
	}

	/*!
	* \brief Sanitize a string so it is usable as an XML ID
	*
	* \param[in,out] id String to sanitize
	*/
	void
	crm_xml_sanitize_id(char *id)
	{
	char *c;

	for (c = id; *c; ++c) {
	/* @TODO Sanitize more comprehensively */
	switch (*c) {
	case ':':
	case '#':
	*c = '.';
	}
	}
	}

	/*!
	* \brief Set the ID of an XML element using a format
	*
	* \param[in,out] xml XML element
	* \param[in] fmt printf-style format
	* \param[in] ... any arguments required by format
	*/
	void
	crm_xml_set_id(xmlNode xml, const char format, ...)
	{
	va_list ap;
	int len = 0;
	char *id = NULL;

	/* equivalent to crm_strdup_printf() */
	va_start(ap, format);
	len = vasprintf(&id, format, ap);
	va_end(ap);
	CRM_ASSERT(len > 0);

	crm_xml_sanitize_id(id);
	crm_xml_add(xml, PCMK_XA_ID, id);
	free(id);
	}

	/*!
	* \internal
	* \brief Write XML to a file stream
	*
	* \param[in] xml XML to write
	* \param[in] filename Name of file being written (for logging only)
	* \param[in,out] stream Open file stream corresponding to filename
	* \param[in] compress Whether to compress XML before writing
	* \param[out] nbytes Number of bytes written
	*
	* \return Standard Pacemaker return code
	*/
	static int
	write_xml_stream(const xmlNode xml, const char filename, FILE *stream,
	bool compress, unsigned int *nbytes)
	{
	int rc = pcmk_rc_ok;
	char *buffer = NULL;

	*nbytes = 0;
	crm_log_xml_trace(xml, "writing");

	buffer = dump_xml_formatted(xml);
	CRM_CHECK(buffer && strlen(buffer),
	crm_log_xml_warn(xml, "formatting failed");
	rc = pcmk_rc_error;
	goto bail);

	if (compress) {
	unsigned int in = 0;
	BZFILE *bz_file = NULL;

	rc = BZ_OK;
	bz_file = BZ2_bzWriteOpen(&rc, stream, 5, 0, 30);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_warn("Not compressing %s: could not prepare file stream: %s "
	CRM_XS " rc=%d", filename, pcmk_rc_str(rc), rc);
	} else {
	BZ2_bzWrite(&rc, bz_file, buffer, strlen(buffer));
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_warn("Not compressing %s: could not compress data: %s "
	CRM_XS " rc=%d errno=%d",
	filename, pcmk_rc_str(rc), rc, errno);
	}
	}

	if (rc == pcmk_rc_ok) {
	BZ2_bzWriteClose(&rc, bz_file, 0, &in, nbytes);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_warn("Not compressing %s: could not write compressed data: %s "
	CRM_XS " rc=%d errno=%d",
	filename, pcmk_rc_str(rc), rc, errno);
	*nbytes = 0; // retry without compression
	} else {
	crm_trace("Compressed XML for %s from %u bytes to %u",
	filename, in, *nbytes);
	}
	}
	rc = pcmk_rc_ok; // Either true, or we'll retry without compression
	}

	if (*nbytes == 0) {
	rc = fprintf(stream, "%s", buffer);
	if (rc < 0) {
	rc = errno;
	crm_perror(LOG_ERR, "writing %s", filename);
	} else {
	*nbytes = (unsigned int) rc;
	rc = pcmk_rc_ok;
	}
	}

	bail:

	if (fflush(stream) != 0) {
	rc = errno;
	crm_perror(LOG_ERR, "flushing %s", filename);
	}

	/* Don't report error if the file does not support synchronization */
	if (fsync(fileno(stream)) < 0 && errno != EROFS && errno != EINVAL) {
	rc = errno;
	crm_perror(LOG_ERR, "synchronizing %s", filename);
	}

	fclose(stream);

	crm_trace("Saved %d bytes to %s as XML", *nbytes, filename);
	free(buffer);

	return rc;
	}

	/*!
	* \brief Write XML to a file descriptor
	*
	* \param[in] xml XML to write
	* \param[in] filename Name of file being written (for logging only)
	* \param[in] fd Open file descriptor corresponding to filename
	* \param[in] compress Whether to compress XML before writing
	*
	* \return Number of bytes written on success, -errno otherwise
	*/
	int
	write_xml_fd(const xmlNode xml, const char filename, int fd,
	gboolean compress)
	{
	FILE *stream = NULL;
	unsigned int nbytes = 0;
	int rc = pcmk_rc_ok;

	CRM_CHECK((xml != NULL) && (fd > 0), return -EINVAL);
	stream = fdopen(fd, "w");
	if (stream == NULL) {
	return -errno;
	}
	rc = write_xml_stream(xml, filename, stream, compress, &nbytes);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}
	return (int) nbytes;
	}

	/*!
	* \brief Write XML to a file
	*
	* \param[in] xml XML to write
	* \param[in] filename Name of file to write
	* \param[in] compress Whether to compress XML before writing
	*
	* \return Number of bytes written on success, -errno otherwise
	*/
	int
	write_xml_file(const xmlNode xml, const char filename, gboolean compress)
	{
	FILE *stream = NULL;
	unsigned int nbytes = 0;
	int rc = pcmk_rc_ok;

	CRM_CHECK((xml != NULL) && (filename != NULL), return -EINVAL);
	stream = fopen(filename, "w");
	if (stream == NULL) {
	return -errno;
	}
	rc = write_xml_stream(xml, filename, stream, compress, &nbytes);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}
	return (int) nbytes;
	}

	// Replace a portion of a dynamically allocated string (reallocating memory)
	static char *
	replace_text(char text, int start, size_t length, const char *replace)
	{
	size_t offset = strlen(replace) - 1; // We have space for 1 char already

	*length += offset;
	text = pcmk__realloc(text, *length);

	for (size_t lpc = (*length) - 1; lpc > (start + offset); lpc--) {
	text[lpc] = text[lpc - offset];
	}

	memcpy(text + start, replace, offset + 1);
	return text;
	}

	/*!
	* \brief Replace special characters with their XML escape sequences
	*
	* \param[in] text Text to escape
	*
	* \return Newly allocated string equivalent to \p text but with special
	* characters replaced with XML escape sequences (or NULL if \p text
	* is NULL)
	*/
	char *
	crm_xml_escape(const char *text)
	{
	size_t length;
	char *copy;

	/*
	* When xmlCtxtReadDoc() parses < and friends in a
	* value, it converts them to their human readable
	* form.
	*
	* If one uses xmlNodeDump() to convert it back to a
	* string, all is well, because special characters are
	* converted back to their escape sequences.
	*
	* However xmlNodeDump() is randomly dog slow, even with the same
	* input. So we need to replicate the escaping in our custom
	* version so that the result can be re-parsed by xmlCtxtReadDoc()
	* when necessary.
	*/

	if (text == NULL) {
	return NULL;
	}

	length = 1 + strlen(text);
	copy = strdup(text);
	CRM_ASSERT(copy != NULL);
	for (size_t index = 0; index < length; index++) {
	if(copy[index] & 0x80 && copy[index+1] & 0x80){
	index++;
	break;
	}
	switch (copy[index]) {
	case 0:
	break;
	case '<':
	copy = replace_text(copy, index, &length, "<");
	break;
	case '>':
	copy = replace_text(copy, index, &length, ">");
	break;
	case '"':
	copy = replace_text(copy, index, &length, """);
	break;
	case '\'':
	copy = replace_text(copy, index, &length, "'");
	break;
	case '&':
	copy = replace_text(copy, index, &length, "&");
	break;
	case '\t':
	/* Might as well just expand to a few spaces... */
	copy = replace_text(copy, index, &length, " ");
	break;
	case '\n':
	copy = replace_text(copy, index, &length, "\\n");
	break;
	case '\r':
	copy = replace_text(copy, index, &length, "\\r");
	break;
	default:
	/* Check for and replace non-printing characters with their octal equivalent */
	if(copy[index] < ' ' \|\| copy[index] > '~') {
	char *replace = crm_strdup_printf("\\%.3o", copy[index]);

	copy = replace_text(copy, index, &length, replace);
	free(replace);
	}
	}
	}
	return copy;
	}

	/*!
	* \internal
	* \brief Append a string representation of an XML element to a buffer
	*
	* \param[in] data XML whose representation to append
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_element(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	bool filtered = pcmk_is_set(options, pcmk__xml_fmt_filtered);
	int spaces = pretty? (2 * depth) : 0;

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "<", data->name, NULL);

	for (const xmlAttr *attr = pcmk__xe_first_attr(data); attr != NULL;
	attr = attr->next) {

	if (!filtered \|\| !pcmk__xa_filterable((const char *) (attr->name))) {
	pcmk__dump_xml_attr(attr, buffer);
	}
	}

	if (data->children == NULL) {
	g_string_append(buffer, "/>");

	} else {
	g_string_append_c(buffer, '>');
	}

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}

	if (data->children) {
	for (const xmlNode *child = data->children; child != NULL;
	child = child->next) {
	pcmk__xml2text(child, options, buffer, depth + 1);
	}

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "</", data->name, ">", NULL);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	}
	}

	/*!
	* \internal
	* \brief Append XML text content to a buffer
	*
	* \param[in] data XML whose content to append
	* \param[in] options Group of \p xml_log_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_text(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	/* @COMPAT: Remove when log_data_element() is removed. There are no internal
	* code paths to this, except through the deprecated log_data_element().
	*/
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	int spaces = pretty? (2 * depth) : 0;

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	g_string_append(buffer, (const gchar *) data->content);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	}

	/*!
	* \internal
	* \brief Append XML CDATA content to a buffer
	*
	* \param[in] data XML whose content to append
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_cdata(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	int spaces = pretty? (2 * depth) : 0;

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "<![CDATA[", (const char *) data->content, "]]>",
	NULL);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	}

	/*!
	* \internal
	* \brief Append an XML comment to a buffer
	*
	* \param[in] data XML whose content to append
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_comment(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	int spaces = pretty? (2 * depth) : 0;

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "<!--", (const char *) data->content, "-->", NULL);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	}

	/*!
	* \internal
	* \brief Get a string representation of an XML element type
	*
	* \param[in] type XML element type
	*
	* \return String representation of \p type
	*/
	static const char *
	xml_element_type2str(xmlElementType type)
	{
	static const char *const element_type_names[] = {
	[XML_ELEMENT_NODE] = "element",
	[XML_ATTRIBUTE_NODE] = "attribute",
	[XML_TEXT_NODE] = "text",
	[XML_CDATA_SECTION_NODE] = "CDATA section",
	[XML_ENTITY_REF_NODE] = "entity reference",
	[XML_ENTITY_NODE] = "entity",
	[XML_PI_NODE] = "PI",
	[XML_COMMENT_NODE] = "comment",
	[XML_DOCUMENT_NODE] = "document",
	[XML_DOCUMENT_TYPE_NODE] = "document type",
	[XML_DOCUMENT_FRAG_NODE] = "document fragment",
	[XML_NOTATION_NODE] = "notation",
	[XML_HTML_DOCUMENT_NODE] = "HTML document",
	[XML_DTD_NODE] = "DTD",
	[XML_ELEMENT_DECL] = "element declaration",
	[XML_ATTRIBUTE_DECL] = "attribute declaration",
	[XML_ENTITY_DECL] = "entity declaration",
	[XML_NAMESPACE_DECL] = "namespace declaration",
	[XML_XINCLUDE_START] = "XInclude start",
	[XML_XINCLUDE_END] = "XInclude end",
	};

	if ((type < 0) \|\| (type >= PCMK__NELEM(element_type_names))) {
	return "unrecognized type";
	}
	return element_type_names[type];
	}

	/*!
	* \internal
	* \brief Create a text representation of an XML object
	*
	* \param[in] data XML to convert
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to store the text (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	void
	pcmk__xml2text(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	CRM_ASSERT(buffer != NULL);
	CRM_CHECK(depth >= 0, depth = 0);

	switch(data->type) {
	case XML_ELEMENT_NODE:
	/* Handle below */
	dump_xml_element(data, options, buffer, depth);
	break;
	case XML_TEXT_NODE:
	if (pcmk_is_set(options, pcmk__xml_fmt_text)) {
	dump_xml_text(data, options, buffer, depth);
	}
	break;
	case XML_COMMENT_NODE:
	dump_xml_comment(data, options, buffer, depth);
	break;
	case XML_CDATA_SECTION_NODE:
	dump_xml_cdata(data, options, buffer, depth);
	break;
	default:
	crm_warn("Cannot convert XML %s node to text " CRM_XS " type=%d",
	xml_element_type2str(data->type), data->type);
	break;
	}
	}

	char *
	dump_xml_formatted_with_text(const xmlNode *xml)
	{
	/* libxml's xmlNodeDumpOutput() would work here since we're not specifically
	* filtering out any nodes. However, use pcmk__xml2text() for consistency,
	* to escape attribute values, and to allow a const argument.
	*/
	char *buffer = NULL;
	GString *g_buffer = g_string_sized_new(1024);

	pcmk__xml2text(xml, pcmk__xml_fmt_pretty\|pcmk__xml_fmt_text, g_buffer, 0);

	pcmk__str_update(&buffer, g_buffer->str);
	g_string_free(g_buffer, TRUE);
	return buffer;
	}

	char *
	dump_xml_formatted(const xmlNode *xml)
	{
	char *buffer = NULL;
	GString *g_buffer = g_string_sized_new(1024);

	pcmk__xml2text(xml, pcmk__xml_fmt_pretty, g_buffer, 0);

	pcmk__str_update(&buffer, g_buffer->str);
	g_string_free(g_buffer, TRUE);
	return buffer;
	}

	char *
	dump_xml_unformatted(const xmlNode *xml)
	{
	char *buffer = NULL;
	GString *g_buffer = g_string_sized_new(1024);

	pcmk__xml2text(xml, 0, g_buffer, 0);

	pcmk__str_update(&buffer, g_buffer->str);
	g_string_free(g_buffer, TRUE);
	return buffer;
	}

	int
	pcmk__xml2fd(int fd, xmlNode *cur)
	{
	bool success;

	xmlOutputBuffer *fd_out = xmlOutputBufferCreateFd(fd, NULL);
	CRM_ASSERT(fd_out != NULL);
	xmlNodeDumpOutput(fd_out, cur->doc, cur, 0, pcmk__xml_fmt_pretty, NULL);

	success = xmlOutputBufferWrite(fd_out, sizeof("\n") - 1, "\n") != -1;

	success = xmlOutputBufferClose(fd_out) != -1 && success;

	if (!success) {
	return EIO;
	}

	fsync(fd);
	return pcmk_rc_ok;
	}

	void
	xml_remove_prop(xmlNode * obj, const char *name)
	{
	if (crm_element_value(obj, name) == NULL) {
	return;
	}

	if (pcmk__check_acl(obj, NULL, pcmk__xf_acl_write) == FALSE) {
	crm_trace("Cannot remove %s from %s", name, obj->name);

	} else if (pcmk__tracking_xml_changes(obj, FALSE)) {
	/* Leave in place (marked for removal) until after the diff is calculated */
	xmlAttr *attr = xmlHasProp(obj, (pcmkXmlStr) name);
	xml_node_private_t *nodepriv = attr->_private;

	set_parent_flag(obj, pcmk__xf_dirty);
	pcmk__set_xml_flags(nodepriv, pcmk__xf_deleted);
	} else {
	xmlUnsetProp(obj, (pcmkXmlStr) name);
	}
	}

	void
	save_xml_to_file(const xmlNode xml, const char desc, const char *filename)
	{
	char *f = NULL;

	if (filename == NULL) {
	char *uuid = crm_generate_uuid();

	f = crm_strdup_printf("%s/%s", pcmk__get_tmpdir(), uuid);
	filename = f;
	free(uuid);
	}

	crm_info("Saving %s to %s", desc, filename);
	write_xml_file(xml, filename, FALSE);
	free(f);
	}

	/*!
	* \internal
	* \brief Set a flag on all attributes of an XML element
	*
	* \param[in,out] xml XML node to set flags on
	* \param[in] flag XML private flag to set
	*/
	static void
	set_attrs_flag(xmlNode *xml, enum xml_private_flags flag)
	{
	for (xmlAttr *attr = pcmk__xe_first_attr(xml); attr; attr = attr->next) {
	pcmk__set_xml_flags((xml_node_private_t *) (attr->_private), flag);
	}
	}

	/*!
	* \internal
	* \brief Add an XML attribute to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when an attribute has been
	* deleted. Add the attribute back to the new XML, so that we can check the
	* removal against ACLs, and mark it as deleted for later removal after
	* differences have been calculated.
	*
	* \param[in,out] new_xml XML to modify
	* \param[in] element Name of XML element that changed (for logging)
	* \param[in] attr_name Name of attribute that was deleted
	* \param[in] old_value Value of attribute that was deleted
	*/
	static void
	mark_attr_deleted(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	xml_doc_private_t *docpriv = new_xml->doc->_private;
	xmlAttr *attr = NULL;
	xml_node_private_t *nodepriv;

	// Prevent the dirty flag being set recursively upwards
	pcmk__clear_xml_flags(docpriv, pcmk__xf_tracking);

	// Restore the old value (and the tracking flag)
	attr = xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);
	pcmk__set_xml_flags(docpriv, pcmk__xf_tracking);

	// Reset flags (so the attribute doesn't appear as newly created)
	nodepriv = attr->_private;
	nodepriv->flags = 0;

	// Check ACLs and mark restored value for later removal
	xml_remove_prop(new_xml, attr_name);

	crm_trace("XML attribute %s=%s was removed from %s",
	attr_name, old_value, element);
	}

	/*
	* \internal
	* \brief Check ACLs for a changed XML attribute
	*/
	static void
	mark_attr_changed(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	char *vcopy = crm_element_value_copy(new_xml, attr_name);

	crm_trace("XML attribute %s was changed from '%s' to '%s' in %s",
	attr_name, old_value, vcopy, element);

	// Restore the original value
	xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);

	// Change it back to the new value, to check ACLs
	crm_xml_add(new_xml, attr_name, vcopy);
	free(vcopy);
	}

	/*!
	* \internal
	* \brief Mark an XML attribute as having changed position
	*
	* \param[in,out] new_xml XML to modify
	* \param[in] element Name of XML element that changed (for logging)
	* \param[in,out] old_attr Attribute that moved, in original XML
	* \param[in,out] new_attr Attribute that moved, in \p new_xml
	* \param[in] p_old Ordinal position of \p old_attr in original XML
	* \param[in] p_new Ordinal position of \p new_attr in \p new_xml
	*/
	static void
	mark_attr_moved(xmlNode new_xml, const char element, xmlAttr *old_attr,
	xmlAttr *new_attr, int p_old, int p_new)
	{
	xml_node_private_t *nodepriv = new_attr->_private;

	crm_trace("XML attribute %s moved from position %d to %d in %s",
	old_attr->name, p_old, p_new, element);

	// Mark document, element, and all element's parents as changed
	pcmk__mark_xml_node_dirty(new_xml);

	// Mark attribute as changed
	pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty\|pcmk__xf_moved);

	nodepriv = (p_old > p_new)? old_attr->_private : new_attr->_private;
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	}

	/*!
	* \internal
	* \brief Calculate differences in all previously existing XML attributes
	*
	* \param[in,out] old_xml Original XML to compare
	* \param[in,out] new_xml New XML to compare
	*/
	static void
	xml_diff_old_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(old_xml);

	while (attr_iter != NULL) {
	const char name = (const char ) attr_iter->name;
	xmlAttr *old_attr = attr_iter;
	xmlAttr *new_attr = xmlHasProp(new_xml, attr_iter->name);
	const char *old_value = pcmk__xml_attr_value(attr_iter);

	attr_iter = attr_iter->next;
	if (new_attr == NULL) {
	mark_attr_deleted(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else {
	xml_node_private_t *nodepriv = new_attr->_private;
	int new_pos = pcmk__xml_position((xmlNode*) new_attr,
	pcmk__xf_skip);
	int old_pos = pcmk__xml_position((xmlNode*) old_attr,
	pcmk__xf_skip);
	const char *new_value = crm_element_value(new_xml, name);

	// This attribute isn't new
	pcmk__clear_xml_flags(nodepriv, pcmk__xf_created);

	if (strcmp(new_value, old_value) != 0) {
	mark_attr_changed(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else if ((old_pos != new_pos)
	&& !pcmk__tracking_xml_changes(new_xml, TRUE)) {
	mark_attr_moved(new_xml, (const char *) old_xml->name,
	old_attr, new_attr, old_pos, new_pos);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Check all attributes in new XML for creation
	*
	* For each of a given XML element's attributes marked as newly created, accept
	* (and mark as dirty) or reject the creation according to ACLs.
	*
	* \param[in,out] new_xml XML to check
	*/
	static void
	mark_created_attrs(xmlNode *new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(new_xml);

	while (attr_iter != NULL) {
	xmlAttr *new_attr = attr_iter;
	xml_node_private_t *nodepriv = attr_iter->_private;

	attr_iter = attr_iter->next;
	if (pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
	const char attr_name = (const char ) new_attr->name;

	crm_trace("Created new attribute %s=%s in %s",
	attr_name, pcmk__xml_attr_value(new_attr),
	new_xml->name);

	/* Check ACLs (we can't use the remove-then-create trick because it
	* would modify the attribute position).
	*/
	if (pcmk__check_acl(new_xml, attr_name, pcmk__xf_acl_write)) {
	pcmk__mark_xml_attr_dirty(new_attr);
	} else {
	// Creation was not allowed, so remove the attribute
	xmlUnsetProp(new_xml, new_attr->name);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Calculate differences in attributes between two XML nodes
	*
	* \param[in,out] old_xml Original XML to compare
	* \param[in,out] new_xml New XML to compare
	*/
	static void
	xml_diff_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	set_attrs_flag(new_xml, pcmk__xf_created); // cleared later if not really new
	xml_diff_old_attrs(old_xml, new_xml);
	mark_created_attrs(new_xml);
	}

	/*!
	* \internal
	* \brief Add an XML child element to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when a child element has been
	* deleted. Add the child back to the new XML, so that we can check the removal
	* against ACLs, and mark it as deleted for later removal after differences have
	* been calculated.
	*
	* \param[in,out] old_child Child element from original XML
	* \param[in,out] new_parent New XML to add marked copy to
	*/
	static void
	mark_child_deleted(xmlNode old_child, xmlNode new_parent)
	{
	// Re-create the child element so we can check ACLs
	xmlNode *candidate = add_node_copy(new_parent, old_child);

	// Clear flags on new child and its children
	reset_xml_node_flags(candidate);

	// Check whether ACLs allow the deletion
	pcmk__apply_acl(xmlDocGetRootElement(candidate->doc));

	// Remove the child again (which will track it in document's deleted_objs)
	free_xml_with_position(candidate,
	pcmk__xml_position(old_child, pcmk__xf_skip));

	if (pcmk__xml_match(new_parent, old_child, true) == NULL) {
	pcmk__set_xml_flags((xml_node_private_t *) (old_child->_private),
	pcmk__xf_skip);
	}
	}

	static void
	mark_child_moved(xmlNode old_child, xmlNode new_parent, xmlNode *new_child,
	int p_old, int p_new)
	{
	xml_node_private_t *nodepriv = new_child->_private;

	crm_trace("Child element %s with id='%s' moved from position %d to %d under %s",
	new_child->name, (ID(new_child)? ID(new_child) : "<no id>"),
	p_old, p_new, new_parent->name);
	pcmk__mark_xml_node_dirty(new_parent);
	pcmk__set_xml_flags(nodepriv, pcmk__xf_moved);

	if (p_old > p_new) {
	nodepriv = old_child->_private;
	} else {
	nodepriv = new_child->_private;
	}
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	}

	// Given original and new XML, mark new XML portions that have changed
	static void
	mark_xml_changes(xmlNode old_xml, xmlNode new_xml, bool check_top)
	{
	xmlNode *cIter = NULL;
	xml_node_private_t *nodepriv = NULL;

	CRM_CHECK(new_xml != NULL, return);
	if (old_xml == NULL) {
	pcmk__mark_xml_created(new_xml);
	pcmk__apply_creation_acl(new_xml, check_top);
	return;
	}

	nodepriv = new_xml->_private;
	CRM_CHECK(nodepriv != NULL, return);

	if(nodepriv->flags & pcmk__xf_processed) {
	/* Avoid re-comparing nodes */
	return;
	}
	pcmk__set_xml_flags(nodepriv, pcmk__xf_processed);

	xml_diff_attrs(old_xml, new_xml);

	// Check for differences in the original children
	for (cIter = pcmk__xml_first_child(old_xml); cIter != NULL; ) {
	xmlNode *old_child = cIter;
	xmlNode *new_child = pcmk__xml_match(new_xml, cIter, true);

	cIter = pcmk__xml_next(cIter);
	if(new_child) {
	mark_xml_changes(old_child, new_child, TRUE);

	} else {
	mark_child_deleted(old_child, new_xml);
	}
	}

	// Check for moved or created children
	for (cIter = pcmk__xml_first_child(new_xml); cIter != NULL; ) {
	xmlNode *new_child = cIter;
	xmlNode *old_child = pcmk__xml_match(old_xml, cIter, true);

	cIter = pcmk__xml_next(cIter);
	if(old_child == NULL) {
	// This is a newly created child
	nodepriv = new_child->_private;
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	mark_xml_changes(old_child, new_child, TRUE);

	} else {
	/* Check for movement, we already checked for differences */
	int p_new = pcmk__xml_position(new_child, pcmk__xf_skip);
	int p_old = pcmk__xml_position(old_child, pcmk__xf_skip);

	if(p_old != p_new) {
	mark_child_moved(old_child, new_xml, new_child, p_old, p_new);
	}
	}
	}
	}

	void
	xml_calculate_significant_changes(xmlNode old_xml, xmlNode new_xml)
	{
	pcmk__set_xml_doc_flag(new_xml, pcmk__xf_lazy);
	xml_calculate_changes(old_xml, new_xml);
	}

	// Called functions may set the \p pcmk__xf_skip flag on parts of \p old_xml
	void
	xml_calculate_changes(xmlNode old_xml, xmlNode new_xml)
	{
	CRM_CHECK((old_xml != NULL) && (new_xml != NULL)
	&& pcmk__xe_is(old_xml, (const char *) new_xml->name)
	&& pcmk__str_eq(ID(old_xml), ID(new_xml), pcmk__str_none),
	return);

	if(xml_tracking_changes(new_xml) == FALSE) {
	xml_track_changes(new_xml, NULL, NULL, FALSE);
	}

	mark_xml_changes(old_xml, new_xml, FALSE);
	}

	gboolean
	can_prune_leaf(xmlNode * xml_node)
	{
	xmlNode *cIter = NULL;
	gboolean can_prune = TRUE;

	CRM_CHECK(xml_node != NULL, return FALSE);

	if (pcmk__strcase_any_of((const char *) xml_node->name,
	XML_TAG_RESOURCE_REF, XML_CIB_TAG_OBJ_REF,
	XML_ACL_TAG_ROLE_REF, XML_ACL_TAG_ROLE_REFv1,
	NULL)) {
	return FALSE;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(xml_node); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;

	if (strcmp(p_name, PCMK_XA_ID) == 0) {
	continue;
	}
	can_prune = FALSE;
	}

	cIter = pcmk__xml_first_child(xml_node);
	while (cIter) {
	xmlNode *child = cIter;

	cIter = pcmk__xml_next(cIter);
	if (can_prune_leaf(child)) {
	free_xml(child);
	} else {
	can_prune = FALSE;
	}
	}
	return can_prune;
	}

	/*!
	* \internal
	* \brief Find a comment with matching content in specified XML
	*
	* \param[in] root XML to search
	* \param[in] search_comment Comment whose content should be searched for
	* \param[in] exact If true, comment must also be at same position
	*/
	xmlNode *
	pcmk__xc_match(const xmlNode root, const xmlNode search_comment, bool exact)
	{
	xmlNode *a_child = NULL;
	int search_offset = pcmk__xml_position(search_comment, pcmk__xf_skip);

	CRM_CHECK(search_comment->type == XML_COMMENT_NODE, return NULL);

	for (a_child = pcmk__xml_first_child(root); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	if (exact) {
	int offset = pcmk__xml_position(a_child, pcmk__xf_skip);
	xml_node_private_t *nodepriv = a_child->_private;

	if (offset < search_offset) {
	continue;

	} else if (offset > search_offset) {
	return NULL;
	}

	if (pcmk_is_set(nodepriv->flags, pcmk__xf_skip)) {
	continue;
	}
	}

	if (a_child->type == XML_COMMENT_NODE
	&& pcmk__str_eq((const char )a_child->content, (const char )search_comment->content, pcmk__str_casei)) {
	return a_child;

	} else if (exact) {
	return NULL;
	}
	}

	return NULL;
	}

	/*!
	* \internal
	* \brief Make one XML comment match another (in content)
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* comment child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML comment node
	* \param[in] update Make comment content match this (must not be NULL)
	*
	* \note At least one of \parent and \target must be non-NULL
	*/
	void
	pcmk__xc_update(xmlNode parent, xmlNode target, xmlNode *update)
	{
	CRM_CHECK(update != NULL, return);
	CRM_CHECK(update->type == XML_COMMENT_NODE, return);

	if (target == NULL) {
	target = pcmk__xc_match(parent, update, false);
	}

	if (target == NULL) {
	add_node_copy(parent, update);

	} else if (!pcmk__str_eq((const char )target->content, (const char )update->content, pcmk__str_casei)) {
	xmlFree(target->content);
	target->content = xmlStrdup(update->content);
	}
	}

	/*!
	* \internal
	* \brief Make one XML tree match another (in children and attributes)
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML
	* \param[in] update Make the desired XML match this (must not be NULL)
	* \param[in] as_diff If false, expand "++" when making attributes match
	*
	* \note At least one of \p parent and \p target must be non-NULL
	*/
	void
	pcmk__xml_update(xmlNode parent, xmlNode target, xmlNode *update,
	bool as_diff)
	{
	xmlNode *a_child = NULL;
	const char *object_name = NULL,
	*object_href = NULL,
	*object_href_val = NULL;

	#if XML_PARSER_DEBUG
	crm_log_xml_trace(update, "update:");
	crm_log_xml_trace(target, "target:");
	#endif

	CRM_CHECK(update != NULL, return);

	if (update->type == XML_COMMENT_NODE) {
	pcmk__xc_update(parent, target, update);
	return;
	}

	object_name = (const char *) update->name;
	object_href_val = ID(update);
	if (object_href_val != NULL) {
	object_href = PCMK_XA_ID;
	} else {
	object_href_val = crm_element_value(update, PCMK_XA_ID_REF);
	object_href = (object_href_val == NULL)? NULL : PCMK_XA_ID_REF;
	}

	CRM_CHECK(object_name != NULL, return);
	CRM_CHECK(target != NULL \|\| parent != NULL, return);

	if (target == NULL) {
	target = pcmk__xe_match(parent, object_name,
	object_href, object_href_val);
	}

	if (target == NULL) {
	target = create_xml_node(parent, object_name);
	CRM_CHECK(target != NULL, return);
	#if XML_PARSER_DEBUG
	crm_trace("Added <%s%s%s%s%s/>", pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");

	} else {
	crm_trace("Found node <%s%s%s%s%s/> to update",
	pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	}

	CRM_CHECK(pcmk__xe_is(target, (const char *) update->name), return);

	if (as_diff == FALSE) {
	/* So that expand_plus_plus() gets called */
	copy_in_properties(target, update);

	} else {
	/* No need for expand_plus_plus(), just raw speed */
	for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
	a = a->next) {
	const char *p_value = pcmk__xml_attr_value(a);

	/* Remove it first so the ordering of the update is preserved */
	xmlUnsetProp(target, a->name);
	xmlSetProp(target, a->name, (pcmkXmlStr) p_value);
	}
	}

	for (a_child = pcmk__xml_first_child(update); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	#if XML_PARSER_DEBUG
	crm_trace("Updating child <%s%s%s%s%s/>",
	pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	pcmk__xml_update(target, NULL, a_child, as_diff);
	}

	#if XML_PARSER_DEBUG
	crm_trace("Finished with <%s%s%s%s%s/>", pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	}

	gboolean
	update_xml_child(xmlNode * child, xmlNode * to_update)
	{
	gboolean can_update = TRUE;
	xmlNode *child_of_child = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(to_update != NULL, return FALSE);

	if (!pcmk__xe_is(to_update, (const char *) child->name)) {
	can_update = FALSE;

	} else if (!pcmk__str_eq(ID(to_update), ID(child), pcmk__str_none)) {
	can_update = FALSE;

	} else if (can_update) {
	#if XML_PARSER_DEBUG
	crm_log_xml_trace(child, "Update match found...");
	#endif
	pcmk__xml_update(NULL, child, to_update, false);
	}

	for (child_of_child = pcmk__xml_first_child(child); child_of_child != NULL;
	child_of_child = pcmk__xml_next(child_of_child)) {
	/* only update the first one */
	if (can_update) {
	break;
	}
	can_update = update_xml_child(child_of_child, to_update);
	}

	return can_update;
	}

	int
	find_xml_children(xmlNode ** children, xmlNode * root,
	const char tag, const char field, const char *value, gboolean search_matches)
	{
	int match_found = 0;

	CRM_CHECK(root != NULL, return FALSE);
	CRM_CHECK(children != NULL, return FALSE);

	if ((tag != NULL) && !pcmk__xe_is(root, tag)) {

	} else if (value != NULL && !pcmk__str_eq(value, crm_element_value(root, field), pcmk__str_casei)) {

	} else {
	if (*children == NULL) {
	*children = create_xml_node(NULL, __func__);
	}
	add_node_copy(*children, root);
	match_found = 1;
	}

	if (search_matches \|\| match_found == 0) {
	xmlNode *child = NULL;

	for (child = pcmk__xml_first_child(root); child != NULL;
	child = pcmk__xml_next(child)) {
	match_found += find_xml_children(children, child, tag, field, value, search_matches);
	}
	}

	return match_found;
	}

	gboolean
	replace_xml_child(xmlNode * parent, xmlNode * child, xmlNode * update, gboolean delete_only)
	{
	gboolean can_delete = FALSE;
	xmlNode *child_of_child = NULL;

	const char *up_id = NULL;
	const char *child_id = NULL;
	const char *right_val = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(update != NULL, return FALSE);

	up_id = ID(update);
	child_id = ID(child);

	if (up_id == NULL \|\| (child_id && strcmp(child_id, up_id) == 0)) {
	can_delete = TRUE;
	}
	if (!pcmk__xe_is(update, (const char *) child->name)) {
	can_delete = FALSE;
	}
	if (can_delete && delete_only) {
	for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
	a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	right_val = crm_element_value(child, p_name);
	if (!pcmk__str_eq(p_value, right_val, pcmk__str_casei)) {
	can_delete = FALSE;
	}
	}
	}

	if (can_delete && parent != NULL) {
	crm_log_xml_trace(child, "Delete match found...");
	if (delete_only \|\| update == NULL) {
	free_xml(child);

	} else {
	xmlNode *old = child;
	xmlNode *new = xmlCopyNode(update, 1);

	CRM_ASSERT(new != NULL);

	// May be unnecessary but avoids slight changes to some test outputs
	reset_xml_node_flags(new);

	old = xmlReplaceNode(old, new);

	if (xml_tracking_changes(new)) {
	// Replaced sections may have included relevant ACLs
	pcmk__apply_acl(new);
	}
	xml_calculate_changes(old, new);
	xmlFreeNode(old);
	}
	return TRUE;

	} else if (can_delete) {
	crm_log_xml_debug(child, "Cannot delete the search root");
	can_delete = FALSE;
	}

	child_of_child = pcmk__xml_first_child(child);
	while (child_of_child) {
	xmlNode *next = pcmk__xml_next(child_of_child);

	can_delete = replace_xml_child(child, child_of_child, update, delete_only);

	/* only delete the first one */
	if (can_delete) {
	child_of_child = NULL;
	} else {
	child_of_child = next;
	}
	}

	return can_delete;
	}

	xmlNode *
	sorted_xml(xmlNode input, xmlNode parent, gboolean recursive)
	{
	xmlNode *child = NULL;
	GSList *nvpairs = NULL;
	xmlNode *result = NULL;

	CRM_CHECK(input != NULL, return NULL);

	result = create_xml_node(parent, (const char *) input->name);
	nvpairs = pcmk_xml_attrs2nvpairs(input);
	nvpairs = pcmk_sort_nvpairs(nvpairs);
	pcmk_nvpairs2xml_attrs(nvpairs, result);
	pcmk_free_nvpairs(nvpairs);

	for (child = pcmk__xml_first_child(input); child != NULL;
	child = pcmk__xml_next(child)) {

	if (recursive) {
	sorted_xml(child, result, recursive);
	} else {
	add_node_copy(result, child);
	}
	}

	return result;
	}

	xmlNode *
	first_named_child(const xmlNode parent, const char name)
	{
	xmlNode *match = NULL;

	for (match = pcmk__xe_first_child(parent); match != NULL;
	match = pcmk__xe_next(match)) {
	/*
	* name == NULL gives first child regardless of name; this is
	* semantically incorrect in this function, but may be necessary
	* due to prior use of xml_child_iter_filter
	*/
	if (pcmk__str_eq(name, (const char *)match->name, pcmk__str_null_matches)) {
	return match;
	}
	}
	return NULL;
	}

	/*!
	* \brief Get next instance of same XML tag
	*
	* \param[in] sibling XML tag to start from
	*
	* \return Next sibling XML tag with same name
	*/
	xmlNode *
	crm_next_same_xml(const xmlNode *sibling)
	{
	xmlNode *match = pcmk__xe_next(sibling);

	while (match != NULL) {
	if (pcmk__xe_is(match, (const char *) sibling->name)) {
	return match;
	}
	match = pcmk__xe_next(match);
	}
	return NULL;
	}

	void
	crm_xml_init(void)
	{
	static bool init = true;

	if(init) {
	init = false;
	/* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
	* pcmk__realloc()s and it can take upwards of 18 seconds (yes, seconds)
	* to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
	* less than 1 second.
	*/
	xmlSetBufferAllocationScheme(XML_BUFFER_ALLOC_DOUBLEIT);

	/* Populate and free the _private field when nodes are created and destroyed */
	xmlDeregisterNodeDefault(free_private_data);
	xmlRegisterNodeDefault(new_private_data);

	crm_schema_init();
	}
	}

	void
	crm_xml_cleanup(void)
	{
	crm_schema_cleanup();
	xmlCleanupParser();
	}

	#define XPATH_MAX 512

	xmlNode *
	expand_idref(xmlNode * input, xmlNode * top)
	{
	char *xpath = NULL;
	const char *ref = NULL;
	xmlNode *result = NULL;

	if (input == NULL) {
	return NULL;
	}

	ref = crm_element_value(input, PCMK_XA_ID_REF);
	if (ref == NULL) {
	return input;
	}

	if (top == NULL) {
	top = input;
	}

	xpath = crm_strdup_printf("//%s[@" PCMK_XA_ID "='%s']", input->name, ref);
	result = get_xpath_object(xpath, top, LOG_DEBUG);
	if (result == NULL) { // Not possible with schema validation enabled
	pcmk__config_err("Ignoring invalid %s configuration: "
	PCMK_XA_ID_REF " '%s' does not reference "
	"a valid object " CRM_XS " xpath=%s",
	input->name, ref, xpath);
	}
	free(xpath);
	return result;
	}

	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns)
	{
	static const char *base = NULL;
	char *ret = NULL;

	if (base == NULL) {
	base = pcmk__env_option(PCMK__ENV_SCHEMA_DIRECTORY);
	}
	if (pcmk__str_empty(base)) {
	base = CRM_SCHEMA_DIRECTORY;
	}

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_legacy_xslt:
	ret = strdup(base);
	break;
	case pcmk__xml_artefact_ns_base_rng:
	case pcmk__xml_artefact_ns_base_xslt:
	ret = crm_strdup_printf("%s/base", base);
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}
	return ret;
	}

	static char *
	find_artefact(enum pcmk__xml_artefact_ns ns, const char path, const char filespec)
	{
	char *ret = NULL;

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_base_rng:
	if (pcmk__ends_with(filespec, ".rng")) {
	ret = crm_strdup_printf("%s/%s", path, filespec);
	} else {
	ret = crm_strdup_printf("%s/%s.rng", path, filespec);
	}
	break;
	case pcmk__xml_artefact_ns_legacy_xslt:
	case pcmk__xml_artefact_ns_base_xslt:
	if (pcmk__ends_with(filespec, ".xsl")) {
	ret = crm_strdup_printf("%s/%s", path, filespec);
	} else {
	ret = crm_strdup_printf("%s/%s.xsl", path, filespec);
	}
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}

	return ret;
	}

	char *
	pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec)
	{
	struct stat sb;
	char *base = pcmk__xml_artefact_root(ns);
	char *ret = NULL;

	ret = find_artefact(ns, base, filespec);
	free(base);

	if (stat(ret, &sb) != 0 \|\| !S_ISREG(sb.st_mode)) {
	const char *remote_schema_dir = pcmk__remote_schema_dir();
	ret = find_artefact(ns, remote_schema_dir, filespec);
	}

	return ret;
	}

	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs)
	{
	while (true) {
	const char name, value;

	name = va_arg(pairs, const char *);
	if (name == NULL) {
	return;
	}

	value = va_arg(pairs, const char *);
	if (value != NULL) {
	crm_xml_add(node, name, value);
	}
	}
	}

	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	{
	va_list pairs;
	va_start(pairs, node);
	pcmk__xe_set_propv(node, pairs);
	va_end(pairs);
	}

	int
	pcmk__xe_foreach_child(xmlNode xml, const char child_element_name,
	int (handler)(xmlNode xml, void *userdata),
	void *userdata)
	{
	xmlNode *children = (xml? xml->children : NULL);

	CRM_ASSERT(handler != NULL);

	for (xmlNode *node = children; node != NULL; node = node->next) {
	if (node->type == XML_ELEMENT_NODE &&
	pcmk__str_eq(child_element_name, (const char *) node->name, pcmk__str_null_matches)) {
	int rc = handler(node, userdata);

	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	}

	return pcmk_rc_ok;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/common/xml_compat.h>

	xmlNode *
	find_entity(xmlNode parent, const char node_name, const char *id)
	{
	return pcmk__xe_match(parent, node_name,
	((id == NULL)? id : PCMK_XA_ID), id);
	}

	void
	crm_destroy_xml(gpointer data)
	{
	free_xml(data);
	}

	xmlDoc *
	getDocPtr(xmlNode *node)
	{
	xmlDoc *doc = NULL;

	CRM_CHECK(node != NULL, return NULL);

	doc = node->doc;
	if (doc == NULL) {
	doc = xmlNewDoc((pcmkXmlStr) "1.0");
	xmlDocSetRootElement(doc, node);
	}
	return doc;
	}

	int
	add_node_nocopy(xmlNode parent, const char name, xmlNode *child)
	{
	add_node_copy(parent, child);
	free_xml(child);
	return 1;
	}

	gboolean
	xml_has_children(const xmlNode * xml_root)
	{
	if (xml_root != NULL && xml_root->children != NULL) {
	return TRUE;
	}
	return FALSE;
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
	index 4b8dbacf05..0b9873f41d 100644
	--- a/lib/pacemaker/pcmk_graph_producer.c
	+++ b/lib/pacemaker/pcmk_graph_producer.c
	@@ -1,1097 +1,1097 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Add an XML node tag for a specified ID
	*
	* \param[in] id Node UUID to add
	* \param[in,out] xml Parent XML tag to add to
	*/
	static xmlNode*
	add_node_to_xml_by_id(const char id, xmlNode xml)
	{
	xmlNode *node_xml;

	node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
	crm_xml_add(node_xml, PCMK_XA_ID, id);

	return node_xml;
	}

	/*!
	* \internal
	* \brief Add an XML node tag for a specified node
	*
	* \param[in] node Node to add
	* \param[in,out] xml XML to add node to
	*/
	static void
	add_node_to_xml(const pcmk_node_t node, void xml)
	{
	add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
	}

	/*!
	* \internal
	* \brief Count (optionally add to XML) nodes needing maintenance state update
	*
	* \param[in,out] xml Parent XML tag to add to, if any
	* \param[in] scheduler Scheduler data
	*
	* \return Count of nodes added
	* \note Only Pacemaker Remote nodes are considered currently
	*/
	static int
	add_maintenance_nodes(xmlNode xml, const pcmk_scheduler_t scheduler)
	{
	xmlNode *maintenance = NULL;
	int count = 0;

	if (xml != NULL) {
	maintenance = create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE);
	}
	for (const GList *iter = scheduler->nodes;
	iter != NULL; iter = iter->next) {
	const pcmk_node_t *node = iter->data;

	if (pe__is_guest_or_remote_node(node) &&
	(node->details->maintenance != node->details->remote_maintenance)) {

	if (maintenance != NULL) {
	crm_xml_add(add_node_to_xml_by_id(node->details->id,
	maintenance),
	XML_NODE_IS_MAINTENANCE,
	(node->details->maintenance? "1" : "0"));
	}
	count++;
	}
	}
	crm_trace("%s %d nodes in need of maintenance mode update in state",
	((maintenance == NULL)? "Counted" : "Added"), count);
	return count;
	}

	/*!
	* \internal
	* \brief Add pseudo action with nodes needing maintenance state update
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_maintenance_update(pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *action = NULL;

	if (add_maintenance_nodes(NULL, scheduler) != 0) {
	action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, scheduler);
	pcmk__set_action_flags(action, pcmk_action_always_in_graph);
	}
	}

	/*!
	* \internal
	* \brief Add XML with nodes that an action is expected to bring down
	*
	* If a specified action is expected to bring any nodes down, add an XML block
	* with their UUIDs. When a node is lost, this allows the controller to
	* determine whether it was expected.
	*
	* \param[in,out] xml Parent XML tag to add to
	* \param[in] action Action to check for downed nodes
	*/
	static void
	add_downed_nodes(xmlNode xml, const pcmk_action_t action)
	{
	CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL),
	return);

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {

	/* Shutdown makes the action's node down */
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {

	/* Fencing makes the action's node and any hosted guest nodes down */
	const char *fence = g_hash_table_lookup(action->meta, "stonith_action");

	if (pcmk__is_fencing_action(fence)) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->node->details->id, downed);
	pe_foreach_guest_node(action->node->details->data_set,
	action->node, add_node_to_xml, downed);
	}

	} else if (action->rsc && action->rsc->is_remote_node
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {

	/* Stopping a remote connection resource makes connected node down,
	* unless it's part of a migration
	*/
	GList *iter;
	pcmk_action_t *input;
	bool migrating = false;

	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	input = ((pcmk__related_action_t *) iter->data)->action;
	if ((input->rsc != NULL)
	&& pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none)
	&& pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	migrating = true;
	break;
	}
	}
	if (!migrating) {
	xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
	add_node_to_xml_by_id(action->rsc->id, downed);
	}
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph operation key for a clone action
	*
	* \param[in] action Clone action
	* \param[in] interval_ms Action interval in milliseconds
	*
	* \return Newly allocated string with transition graph operation key
	*/
	static char *
	clone_op_key(const pcmk_action_t *action, guint interval_ms)
	{
	if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_operation");

	CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL));
	return pcmk__notify_key(action->rsc->clone_name, n_type, n_task);

	} else if (action->cancel_task != NULL) {
	return pcmk__op_key(action->rsc->clone_name, action->cancel_task,
	interval_ms);
	} else {
	return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms);
	}
	}

	/*!
	* \internal
	* \brief Add node details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] xml Transition graph action XML for \p action
	*/
	static void
	add_node_details(const pcmk_action_t action, xmlNode xml)
	{
	pcmk_node_t *router_node = pcmk__connection_host_for_action(action);

	crm_xml_add(xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
	crm_xml_add(xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
	if (router_node != NULL) {
	crm_xml_add(xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
	}
	}

	/*!
	* \internal
	* \brief Add resource details to transition graph action XML
	*
	* \param[in] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_resource_details(const pcmk_action_t action, xmlNode action_xml)
	{
	xmlNode *rsc_xml = NULL;
	const char *attr_list[] = {
	- XML_AGENT_ATTR_CLASS,
	+ PCMK_XA_CLASS,
	PCMK_XA_PROVIDER,
	PCMK_XA_TYPE,
	};

	/* If a resource is locked to a node via PCMK_OPT_SHUTDOWN_LOCK, mark its
	* actions so the controller can preserve the lock when the action
	* completes.
	*/
	if (pcmk__action_locks_rsc_to_node(action)) {
	crm_xml_add_ll(action_xml, PCMK_OPT_SHUTDOWN_LOCK,
	(long long) action->rsc->lock_time);
	}

	// List affected resource

	rsc_xml = create_xml_node(action_xml,
	(const char *) action->rsc->xml->name);
	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_removed)
	&& (action->rsc->clone_name != NULL)) {
	/* Use the numbered instance name here, because if there is more
	* than one instance on a node, we need to make sure the command
	* goes to the right one.
	*
	* This is important even for anonymous clones, because the clone's
	* unique meta-attribute might have just been toggled from on to
	* off.
	*/
	crm_debug("Using orphan clone name %s instead of %s",
	action->rsc->id, action->rsc->clone_name);
	crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->clone_name);
	crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);

	} else if (!pcmk_is_set(action->rsc->flags, pcmk_rsc_unique)) {
	const char *xml_id = ID(action->rsc->xml);

	crm_debug("Using anonymous clone name %s for %s (aka %s)",
	xml_id, action->rsc->id, action->rsc->clone_name);

	/* ID is what we'd like client to use
	* LONG_ID is what they might know it as instead
	*
	* LONG_ID is only strictly needed /here/ during the
	* transition period until all nodes in the cluster
	* are running the new software /and/ have rebooted
	* once (meaning that they've only ever spoken to a DC
	* supporting this feature).
	*
	* If anyone toggles the unique flag to 'on', the
	* 'instance free' name will correspond to an orphan
	* and fall into the clause above instead
	*/
	crm_xml_add(rsc_xml, PCMK_XA_ID, xml_id);
	if ((action->rsc->clone_name != NULL)
	&& !pcmk__str_eq(xml_id, action->rsc->clone_name,
	pcmk__str_none)) {
	crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->clone_name);
	} else {
	crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);
	}

	} else {
	CRM_ASSERT(action->rsc->clone_name == NULL);
	crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->id);
	}

	for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
	crm_xml_add(rsc_xml, attr_list[lpc],
	g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
	}
	}

	/*!
	* \internal
	* \brief Add action attributes to transition graph action XML
	*
	* \param[in,out] action Scheduled action
	* \param[in,out] action_xml Transition graph action XML for \p action
	*/
	static void
	add_action_attributes(pcmk_action_t action, xmlNode action_xml)
	{
	xmlNode *args_xml = NULL;

	/* We create free-standing XML to start, so we can sort the attributes
	* before adding it to action_xml, which keeps the scheduler regression
	* test graphs comparable.
	*/
	args_xml = create_xml_node(NULL, XML_TAG_ATTRS);

	crm_xml_add(args_xml, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
	g_hash_table_foreach(action->extra, hash2field, args_xml);

	if ((action->rsc != NULL) && (action->node != NULL)) {
	// Get the resource instance attributes, evaluated properly for node
	GHashTable *params = pe_rsc_params(action->rsc, action->node,
	action->rsc->cluster);

	pcmk__substitute_remote_addr(action->rsc, params);

	g_hash_table_foreach(params, hash2smartfield, args_xml);

	} else if ((action->rsc != NULL)
	&& (action->rsc->variant <= pcmk_rsc_variant_primitive)) {
	GHashTable *params = pe_rsc_params(action->rsc, NULL,
	action->rsc->cluster);

	g_hash_table_foreach(params, hash2smartfield, args_xml);
	}

	g_hash_table_foreach(action->meta, hash2metafield, args_xml);
	if (action->rsc != NULL) {
	pcmk_resource_t *parent = action->rsc;

	while (parent != NULL) {
	parent->cmds->add_graph_meta(parent, args_xml);
	parent = parent->parent;
	}

	pcmk__add_guest_meta_to_xml(args_xml, action);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& (action->node != NULL)) {
	/* Pass the node's attributes as meta-attributes.
	*
	* @TODO: Determine whether it is still necessary to do this. It was
	* added in 33d99707, probably for the libfence-based implementation in
	* c9a90bd, which is no longer used.
	*/
	g_hash_table_foreach(action->node->details->attrs, hash2metafield,
	args_xml);
	}

	sorted_xml(args_xml, action_xml, FALSE);
	free_xml(args_xml);
	}

	/*!
	* \internal
	* \brief Create the transition graph XML for a scheduled action
	*
	* \param[in,out] parent Parent XML element to add action to
	* \param[in,out] action Scheduled action
	* \param[in] skip_details If false, add action details as sub-elements
	* \param[in] scheduler Scheduler data
	*/
	static void
	create_graph_action(xmlNode parent, pcmk_action_t action, bool skip_details,
	const pcmk_scheduler_t *scheduler)
	{
	bool needs_node_info = true;
	bool needs_maintenance_info = false;
	xmlNode *action_xml = NULL;

	if ((action == NULL) \|\| (scheduler == NULL)) {
	return;
	}

	// Create the top-level element based on task

	if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) {
	/* All fences need node info; guest node fences are pseudo-events */
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	}

	} else if (pcmk__str_any_of(action->task,
	PCMK_ACTION_DO_SHUTDOWN,
	PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE,
	pcmk__str_none)) {
	// CIB-only clean-up for shutdown locks
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
	crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);

	} else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES,
	pcmk__str_none)) {
	needs_maintenance_info = true;
	}
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
	needs_node_info = false;

	} else {
	action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP);
	}

	crm_xml_add_int(action_xml, PCMK_XA_ID, action->id);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);

	if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) {
	char *clone_key = NULL;
	guint interval_ms;

	if (pcmk__guint_from_hash(action->meta, XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}
	clone_key = clone_op_key(action, interval_ms);
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
	crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY,
	action->uuid);
	free(clone_key);
	} else {
	crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
	}

	if (needs_node_info && (action->node != NULL)) {
	add_node_details(action, action_xml);
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET),
	strdup(action->node->details->uname));
	g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID),
	strdup(action->node->details->id));
	}

	if (skip_details) {
	return;
	}

	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_pseudo)) {

	// This is a real resource action, so add resource details
	add_resource_details(action, action_xml);
	}

	/* List any attributes in effect */
	add_action_attributes(action, action_xml);

	/* List any nodes this action is expected to make down */
	if (needs_node_info && (action->node != NULL)) {
	add_downed_nodes(action_xml, action);
	}

	if (needs_maintenance_info) {
	add_maintenance_nodes(action_xml, scheduler);
	}
	}

	/*!
	* \internal
	* \brief Check whether an action should be added to the transition graph
	*
	* \param[in] action Action to check
	*
	* \return true if action should be added to graph, otherwise false
	*/
	static bool
	should_add_action_to_graph(const pcmk_action_t *action)
	{
	if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	crm_trace("Ignoring action %s (%d): unrunnable",
	action->uuid, action->id);
	return false;
	}

	if (pcmk_is_set(action->flags, pcmk_action_optional)
	&& !pcmk_is_set(action->flags, pcmk_action_always_in_graph)) {
	crm_trace("Ignoring action %s (%d): optional",
	action->uuid, action->id);
	return false;
	}

	/* Actions for unmanaged resources should be excluded from the graph,
	* with the exception of monitors and cancellation of recurring monitors.
	*/
	if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {

	const char *interval_ms_s;

	/* A cancellation of a recurring monitor will get here because the task
	* is cancel rather than monitor, but the interval can still be used to
	* recognize it. The interval has been normalized to milliseconds by
	* this point, so a string comparison is sufficient.
	*/
	interval_ms_s = g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
	crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
	action->uuid, action->id, action->rsc->id);
	return false;
	}
	}

	/* Always add pseudo-actions, fence actions, and shutdown actions (already
	* determined to be required and runnable by this point)
	*/
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)
	\|\| pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH,
	PCMK_ACTION_DO_SHUTDOWN, NULL)) {
	return true;
	}

	if (action->node == NULL) {
	pcmk__sched_err("Skipping action %s (%d) "
	"because it was not assigned to a node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Unassigned", action, false);
	return false;
	}

	if (pcmk_is_set(action->flags, pcmk_action_on_dc)) {
	crm_trace("Action %s (%d) should be dumped: "
	"can run on DC instead of %s",
	action->uuid, action->id, pe__node_name(action->node));

	} else if (pe__is_guest_node(action->node)
	&& !action->node->details->remote_requires_reset) {
	crm_trace("Action %s (%d) should be dumped: "
	"assuming will be runnable on guest %s",
	action->uuid, action->id, pe__node_name(action->node));

	} else if (!action->node->details->online) {
	pcmk__sched_err("Skipping action %s (%d) "
	"because it was scheduled for offline node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Offline node", action, false);
	return false;

	} else if (action->node->details->unclean) {
	pcmk__sched_err("Skipping action %s (%d) "
	"because it was scheduled for unclean node (bug?)",
	action->uuid, action->id);
	pcmk__log_action("Unclean node", action, false);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering's flags can change an action
	*
	* \param[in] ordering Ordering to check
	*
	* \return true if ordering has flags that can change an action, false otherwise
	*/
	static bool
	ordering_can_change_actions(const pcmk__related_action_t *ordering)
	{
	return pcmk_any_flags_set(ordering->type,
	~(pcmk__ar_then_implies_first_graphed
	\|pcmk__ar_first_implies_then_graphed
	\|pcmk__ar_ordered));
	}

	/*!
	* \internal
	* \brief Check whether an action input should be in the transition graph
	*
	* \param[in] action Action to check
	* \param[in,out] input Action input to check
	*
	* \return true if input should be in graph, false otherwise
	* \note This function may not only check an input, but disable it under certian
	* circumstances (load or anti-colocation orderings that are not needed).
	*/
	static bool
	should_add_input_to_graph(const pcmk_action_t *action,
	pcmk__related_action_t *input)
	{
	if (input->state == pe_link_dumped) {
	return true;
	}

	if ((uint32_t) input->type == pcmk__ar_none) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering disabled",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
	&& !ordering_can_change_actions(input)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional and input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
	&& pcmk_is_set(input->type, pcmk__ar_min_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"minimum number of instances required but input unrunnable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pcmk__ar_unmigratable_then_blocks)
	&& !pcmk_is_set(input->action->flags, pcmk_action_runnable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input blocked if 'then' unmigratable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (pcmk_is_set(input->type, pcmk__ar_if_first_unmigratable)
	&& pcmk_is_set(input->action->flags, pcmk_action_migratable)) {
	crm_trace("Ignoring %s (%d) input %s (%d): ordering applies "
	"only if input is unmigratable, but it is migratable",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if (((uint32_t) input->type == pcmk__ar_ordered)
	&& pcmk_is_set(input->action->flags, pcmk_action_migratable)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"optional but stop in migration",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;

	} else if ((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target) {
	pcmk_node_t *input_node = input->action->node;

	if ((action->rsc != NULL)
	&& pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {

	pcmk_node_t *assigned = action->rsc->allocated_to;

	/* For load_stopped -> migrate_to orderings, we care about where
	* the resource has been assigned, not where migrate_to will be
	* executed.
	*/
	if (!pe__same_node(input_node, assigned)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"migration target %s is not same as input node %s",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(assigned? assigned->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if (!pe__same_node(input_node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"not on same node (%s vs %s)",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	(action->node? action->node->details->uname : "<none>"),
	(input_node? input_node->details->uname : "<none>"));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"ordering optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if ((uint32_t) input->type == pcmk__ar_if_required_on_same_node) {
	if (input->action->node && action->node
	&& !pe__same_node(input->action->node, action->node)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"not on same node (%s vs %s)",
	action->uuid, action->id,
	input->action->uuid, input->action->id,
	pe__node_name(action->node),
	pe__node_name(input->action->node));
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
	crm_trace("Ignoring %s (%d) input %s (%d): optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	input->type = (enum pe_ordering) pcmk__ar_none;
	return false;
	}

	} else if (input->action->rsc
	&& input->action->rsc != action->rsc
	&& pcmk_is_set(input->action->rsc->flags, pcmk_rsc_failed)
	&& !pcmk_is_set(input->action->rsc->flags, pcmk_rsc_managed)
	&& pcmk__ends_with(input->action->uuid, "_stop_0")
	&& action->rsc && pe_rsc_is_clone(action->rsc)) {
	crm_warn("Ignoring requirement that %s complete before %s:"
	" unmanaged failed resources cannot prevent clone shutdown",
	input->action->uuid, action->uuid);
	return false;

	} else if (pcmk_is_set(input->action->flags, pcmk_action_optional)
	&& !pcmk_any_flags_set(input->action->flags,
	pcmk_action_always_in_graph
	\|pcmk_action_added_to_graph)
	&& !should_add_action_to_graph(input->action)) {
	crm_trace("Ignoring %s (%d) input %s (%d): "
	"input optional",
	action->uuid, action->id,
	input->action->uuid, input->action->id);
	return false;
	}

	crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
	action->uuid, action->id, action_type_str(input->action->flags),
	input->action->uuid, input->action->id,
	action_node_str(input->action),
	action_runnable_str(input->action->flags),
	action_optional_str(input->action->flags), input->type);
	return true;
	}

	/*!
	* \internal
	* \brief Check whether an ordering creates an ordering loop
	*
	* \param[in] init_action "First" action in ordering
	* \param[in] action Callers should always set this the same as
	* \p init_action (this function may use a different
	* value for recursive calls)
	* \param[in,out] input Action wrapper for "then" action in ordering
	*
	* \return true if the ordering creates a loop, otherwise false
	*/
	bool
	pcmk__graph_has_loop(const pcmk_action_t *init_action,
	const pcmk_action_t action, pcmk__related_action_t input)
	{
	bool has_loop = false;

	if (pcmk_is_set(input->action->flags, pcmk_action_detect_loop)) {
	crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	return false;
	}

	// Don't need to check inputs that won't be used
	if (!should_add_input_to_graph(action, input)) {
	return false;
	}

	if (input->action == init_action) {
	crm_debug("Input loop found in %s@%s ->...-> %s@%s",
	action->uuid,
	action->node? action->node->details->uname : "",
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");
	return true;
	}

	pcmk__set_action_flags(input->action, pcmk_action_detect_loop);

	crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
	"for graph loop with %s@%s ",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	input->type,
	init_action->uuid,
	init_action->node? init_action->node->details->uname : "");

	// Recursively check input itself for loops
	for (GList *iter = input->action->actions_before;
	iter != NULL; iter = iter->next) {

	if (pcmk__graph_has_loop(init_action, input->action,
	(pcmk__related_action_t *) iter->data)) {
	// Recursive call already logged a debug message
	has_loop = true;
	break;
	}
	}

	pcmk__clear_action_flags(input->action, pcmk_action_detect_loop);

	if (!has_loop) {
	crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
	input->action->uuid,
	input->action->node? input->action->node->details->uname : "",
	action->uuid,
	action->node? action->node->details->uname : "",
	input->type);
	}
	return has_loop;
	}

	/*!
	* \internal
	* \brief Create a synapse XML element for a transition graph
	*
	* \param[in] action Action that synapse is for
	* \param[in,out] scheduler Scheduler data containing graph
	*
	* \return Newly added XML element for new graph synapse
	*/
	static xmlNode *
	create_graph_synapse(const pcmk_action_t action, pcmk_scheduler_t scheduler)
	{
	int synapse_priority = 0;
	xmlNode *syn = create_xml_node(scheduler->graph, "synapse");

	crm_xml_add_int(syn, PCMK_XA_ID, scheduler->num_synapse);
	scheduler->num_synapse++;

	if (action->rsc != NULL) {
	synapse_priority = action->rsc->priority;
	}
	if (action->priority > synapse_priority) {
	synapse_priority = action->priority;
	}
	if (synapse_priority > 0) {
	crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
	}
	return syn;
	}

	/*!
	* \internal
	* \brief Add an action to the transition graph XML if appropriate
	*
	* \param[in,out] data Action to possibly add
	* \param[in,out] user_data Scheduler data
	*
	* \note This will de-duplicate the action inputs, meaning that the
	* pcmk__related_action_t:type flags can no longer be relied on to retain
	* their original settings. That means this MUST be called after
	* pcmk__apply_orderings() is complete, and nothing after this should rely
	* on those type flags. (For example, some code looks for type equal to
	* some flag rather than whether the flag is set, and some code looks for
	* particular combinations of flags -- such code must be done before
	* pcmk__create_graph().)
	*/
	static void
	add_action_to_graph(gpointer data, gpointer user_data)
	{
	pcmk_action_t action = (pcmk_action_t ) data;
	pcmk_scheduler_t scheduler = (pcmk_scheduler_t ) user_data;

	xmlNode *syn = NULL;
	xmlNode *set = NULL;
	xmlNode *in = NULL;

	/* If we haven't already, de-duplicate inputs (even if we won't be adding
	* the action to the graph, so that crm_simulate's dot graphs don't have
	* duplicates).
	*/
	if (!pcmk_is_set(action->flags, pcmk_action_inputs_deduplicated)) {
	pcmk__deduplicate_action_inputs(action);
	pcmk__set_action_flags(action, pcmk_action_inputs_deduplicated);
	}

	if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)
	\|\| !should_add_action_to_graph(action)) {
	return; // Already added, or shouldn't be
	}
	pcmk__set_action_flags(action, pcmk_action_added_to_graph);

	crm_trace("Adding action %d (%s%s%s) to graph",
	action->id, action->uuid,
	((action->node == NULL)? "" : " on "),
	((action->node == NULL)? "" : action->node->details->uname));

	syn = create_graph_synapse(action, scheduler);
	set = create_xml_node(syn, "action_set");
	in = create_xml_node(syn, "inputs");

	create_graph_action(set, action, false, scheduler);

	for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
	pcmk__related_action_t *input = lpc->data;

	if (should_add_input_to_graph(action, input)) {
	xmlNode *input_xml = create_xml_node(in, "trigger");

	input->state = pe_link_dumped;
	create_graph_action(input_xml, input->action, true, scheduler);
	}
	}
	}

	static int transition_id = -1;

	/*!
	* \internal
	* \brief Log a message after calculating a transition
	*
	* \param[in] filename Where transition input is stored
	*/
	void
	pcmk__log_transition_summary(const char *filename)
	{
	if (was_processing_error \|\| crm_config_error) {
	crm_err("Calculated transition %d (with errors)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else if (was_processing_warning \|\| crm_config_warning) {
	crm_warn("Calculated transition %d (with warnings)%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);

	} else {
	crm_notice("Calculated transition %d%s%s",
	transition_id,
	(filename == NULL)? "" : ", saving inputs in ",
	(filename == NULL)? "" : filename);
	}
	if (crm_config_error) {
	crm_notice("Configuration errors found during scheduler processing,"
	" please run \"crm_verify -L\" to identify issues");
	}
	}

	/*!
	* \internal
	* \brief Add a resource's actions to the transition graph
	*
	* \param[in,out] rsc Resource whose actions should be added
	*/
	void
	pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc)
	{
	GList *iter = NULL;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);

	// First add the resource's own actions
	g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster);

	// Then recursively add its children's actions (appropriate to variant)
	for (iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child_rsc = (pcmk_resource_t ) iter->data;

	child_rsc->cmds->add_actions_to_graph(child_rsc);
	}
	}

	/*!
	* \internal
	* \brief Create a transition graph with all cluster actions needed
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__create_graph(pcmk_scheduler_t *scheduler)
	{
	GList *iter = NULL;
	const char *value = NULL;
	long long limit = 0LL;

	transition_id++;
	crm_trace("Creating transition graph %d", transition_id);

	scheduler->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_CLUSTER_DELAY);
	crm_xml_add(scheduler->graph, PCMK_OPT_CLUSTER_DELAY, value);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_STONITH_TIMEOUT);
	crm_xml_add(scheduler->graph, PCMK_OPT_STONITH_TIMEOUT, value);

	crm_xml_add(scheduler->graph, "failed-stop-offset", "INFINITY");

	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_xml_add(scheduler->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(scheduler->graph, "failed-start-offset", "1");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_BATCH_LIMIT);
	crm_xml_add(scheduler->graph, PCMK_OPT_BATCH_LIMIT, value);

	crm_xml_add_int(scheduler->graph, "transition_id", transition_id);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_MIGRATION_LIMIT);
	if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
	crm_xml_add(scheduler->graph, PCMK_OPT_MIGRATION_LIMIT, value);
	}

	if (scheduler->recheck_by > 0) {
	char *recheck_epoch = NULL;

	recheck_epoch = crm_strdup_printf("%llu",
	(long long) scheduler->recheck_by);
	crm_xml_add(scheduler->graph, "recheck-by", recheck_epoch);
	free(recheck_epoch);
	}

	/* The following code will de-duplicate action inputs, so nothing past this
	* should rely on the action input type flags retaining their original
	* values.
	*/

	// Add resource actions to graph
	for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk__rsc_trace(rsc, "Processing actions for %s", rsc->id);
	rsc->cmds->add_actions_to_graph(rsc);
	}

	// Add pseudo-action for list of nodes with maintenance state update
	add_maintenance_update(scheduler);

	// Add non-resource (node) actions
	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	if ((action->rsc != NULL)
	&& (action->node != NULL)
	&& action->node->details->shutdown
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)
	&& !pcmk_any_flags_set(action->flags,
	pcmk_action_optional\|pcmk_action_runnable)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* Eventually we should just ignore the 'fence' case, but for now
	* it's the best way to detect (in CTS) when CIB resource updates
	* are being lost.
	*/
	if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	\|\| (scheduler->no_quorum_policy == pcmk_no_quorum_ignore)) {
	const bool managed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_managed);
	const bool failed = pcmk_is_set(action->rsc->flags,
	pcmk_rsc_failed);

	crm_crit("Cannot %s %s because of %s:%s%s (%s)",
	action->node->details->unclean? "fence" : "shut down",
	pe__node_name(action->node), action->rsc->id,
	(managed? " blocked" : " unmanaged"),
	(failed? " failed" : ""), action->uuid);
	}
	}

	add_action_to_graph((gpointer) action, (gpointer) scheduler);
	}

	crm_log_xml_trace(scheduler->graph, "graph");
	}
	diff --git a/lib/pacemaker/pcmk_injections.c b/lib/pacemaker/pcmk_injections.c
	index 93b18355a1..2c62e0f4c3 100644
	--- a/lib/pacemaker/pcmk_injections.c
	+++ b/lib/pacemaker/pcmk_injections.c
	@@ -1,788 +1,788 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>

	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/util.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/xml_internal.h>
	#include <crm/lrmd_events.h> // lrmd_event_data_t, etc.
	#include <crm/lrmd_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	bool pcmk__simulate_node_config = false;

	#define XPATH_NODE_CONFIG "//" XML_CIB_TAG_NODE "[@" XML_ATTR_UNAME "='%s']"
	#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"
	#define XPATH_NODE_STATE_BY_ID "//" XML_CIB_TAG_STATE "[@" PCMK_XA_ID "='%s']"
	#define XPATH_RSC_HISTORY XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']"


	/*!
	* \internal
	* \brief Inject a fictitious transient node attribute into scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node node_state XML to inject attribute into
	* \param[in] name Transient node attribute name to inject
	* \param[in] value Transient node attribute value to inject
	*/
	static void
	inject_transient_attr(pcmk__output_t out, xmlNode cib_node,
	const char name, const char value)
	{
	xmlNode *attrs = NULL;
	xmlNode *instance_attrs = NULL;
	const char *node_uuid = ID(cib_node);

	out->message(out, "inject-attr", name, value, cib_node);

	attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	if (attrs == NULL) {
	attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	crm_xml_add(attrs, PCMK_XA_ID, node_uuid);
	}

	instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
	if (instance_attrs == NULL) {
	instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
	crm_xml_add(instance_attrs, PCMK_XA_ID, node_uuid);
	}

	crm_create_nvpair_xml(instance_attrs, NULL, name, value);
	}

	/*!
	* \internal
	* \brief Inject a fictitious fail count into a scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node Node state XML to inject into
	* \param[in] resource ID of resource for fail count to inject
	* \param[in] task Action name for fail count to inject
	* \param[in] interval_ms Action interval (in milliseconds) for fail count
	* \param[in] rc Action result for fail count to inject (if 0, or
	* 7 when interval_ms is 0, inject nothing)
	*/
	void
	pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char task,
	guint interval_ms, int rc)
	{
	if (rc == 0) {
	return;

	} else if ((rc == 7) && (interval_ms == 0)) {
	return;

	} else {
	char *name = NULL;
	char *now = pcmk__ttoa(time(NULL));

	name = pcmk__failcount_name(resource, task, interval_ms);
	inject_transient_attr(out, cib_node, name, "value++");
	free(name);

	name = pcmk__lastfailure_name(resource, task, interval_ms);
	inject_transient_attr(out, cib_node, name, now);
	free(name);

	free(now);
	}
	}

	/*!
	* \internal
	* \brief Create a CIB configuration entry for a fictitious node
	*
	* \param[in,out] cib_conn CIB object to use
	* \param[in] node Node name to use
	*/
	static void
	create_node_entry(cib_t cib_conn, const char node)
	{
	int rc = pcmk_ok;
	char *xpath = crm_strdup_printf(XPATH_NODE_CONFIG, node);

	rc = cib_conn->cmds->query(cib_conn, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if (rc == -ENXIO) { // Only add if not already existing
	xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);

	crm_xml_add(cib_object, PCMK_XA_ID, node); // Use node name as ID
	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
	cib_sync_call\|cib_scope_local);
	/* Not bothering with subsequent query to see if it exists,
	we'll bomb out later in the call to query_node_uuid()... */

	free_xml(cib_object);
	}

	free(xpath);
	}

	/*!
	* \internal
	* \brief Synthesize a fake executor event for an action
	*
	* \param[in] cib_resource XML for any existing resource action history
	* \param[in] task Name of action to synthesize
	* \param[in] interval_ms Interval of action to synthesize
	* \param[in] outcome Result of action to synthesize
	*
	* \return Newly allocated executor event
	* \note It is the caller's responsibility to free the result with
	* lrmd_free_event().
	*/
	static lrmd_event_data_t *
	create_op(const xmlNode cib_resource, const char task, guint interval_ms,
	int outcome)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *xop = NULL;

	op = lrmd_new_event(ID(cib_resource), task, interval_ms);
	lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
	op->params = NULL; // Not needed for simulation purposes
	op->t_run = (unsigned int) time(NULL);
	op->t_rcchange = op->t_run;

	// Use a call ID higher than any existing history entries
	op->call_id = 0;
	for (xop = pcmk__xe_first_child(cib_resource); xop != NULL;
	xop = pcmk__xe_next(xop)) {

	int tmp = 0;

	crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
	if (tmp > op->call_id) {
	op->call_id = tmp;
	}
	}
	op->call_id++;

	return op;
	}

	/*!
	* \internal
	* \brief Inject a fictitious resource history entry into a scheduler input
	*
	* \param[in,out] cib_resource Resource history XML to inject entry into
	* \param[in,out] op Action result to inject
	* \param[in] target_rc Expected result for action to inject
	*
	* \return XML of injected resource history entry
	*/
	xmlNode *
	pcmk__inject_action_result(xmlNode cib_resource, lrmd_event_data_t op,
	int target_rc)
	{
	return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
	target_rc, NULL, crm_system_name);
	}

	/*!
	* \internal
	* \brief Inject a fictitious node into a scheduler input
	*
	* \param[in,out] cib_conn Scheduler input CIB to inject node into
	* \param[in] node Name of node to inject
	* \param[in] uuid UUID of node to inject
	*
	* \return XML of node_state entry for new node
	* \note If the global pcmk__simulate_node_config has been set to true, a
	* node entry in the configuration section will be added, as well as a
	* node state entry in the status section.
	*/
	xmlNode *
	pcmk__inject_node(cib_t cib_conn, const char node, const char *uuid)
	{
	int rc = pcmk_ok;
	xmlNode *cib_object = NULL;
	char *xpath = crm_strdup_printf(XPATH_NODE_STATE, node);
	bool duplicate = false;
	char *found_uuid = NULL;

	if (pcmk__simulate_node_config) {
	create_node_entry(cib_conn, node);
	}

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
	crm_err("Detected multiple node_state entries for xpath=%s, bailing",
	xpath);
	duplicate = true;
	goto done;
	}

	if (rc == -ENXIO) {
	if (uuid == NULL) {
	query_node_uuid(cib_conn, node, &found_uuid, NULL);
	} else {
	found_uuid = strdup(uuid);
	}

	if (found_uuid) {
	char *xpath_by_uuid = crm_strdup_printf(XPATH_NODE_STATE_BY_ID,
	found_uuid);

	// It's possible that a node_state entry doesn't have an uname yet.
	rc = cib_conn->cmds->query(cib_conn, xpath_by_uuid, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
	crm_err("Can't inject node state for %s because multiple "
	"state entries found for ID %s", node, found_uuid);
	duplicate = true;
	free(xpath_by_uuid);
	goto done;

	} else if (cib_object != NULL) {
	crm_xml_add(cib_object, XML_ATTR_UNAME, node);

	rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS,
	cib_object,
	cib_sync_call\|cib_scope_local);
	}

	free(xpath_by_uuid);
	}
	}

	if (rc == -ENXIO) {
	cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
	crm_xml_add(cib_object, PCMK_XA_ID, found_uuid);
	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
	cib_sync_call\|cib_scope_local);
	free_xml(cib_object);

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	crm_trace("Injecting node state for %s (rc=%d)", node, rc);
	}

	done:
	free(found_uuid);
	free(xpath);

	if (duplicate) {
	crm_log_xml_warn(cib_object, "Duplicates");
	crm_exit(CRM_EX_SOFTWARE);
	return NULL; // not reached, but makes static analysis happy
	}

	CRM_ASSERT(rc == pcmk_ok);
	return cib_object;
	}

	/*!
	* \internal
	* \brief Inject a fictitious node state change into a scheduler input
	*
	* \param[in,out] cib_conn Scheduler input CIB to inject into
	* \param[in] node Name of node to inject change for
	* \param[in] up If true, change state to online, otherwise offline
	*
	* \return XML of changed (or added) node state entry
	*/
	xmlNode *
	pcmk__inject_node_state_change(cib_t cib_conn, const char node, bool up)
	{
	xmlNode *cib_node = pcmk__inject_node(cib_conn, node, NULL);

	if (up) {
	pcmk__xe_set_props(cib_node,
	PCMK__XA_IN_CCM, XML_BOOLEAN_TRUE,
	PCMK__XA_CRMD, ONLINESTATUS,
	PCMK__XA_JOIN, CRMD_JOINSTATE_MEMBER,
	PCMK__XA_EXPECTED, CRMD_JOINSTATE_MEMBER,
	NULL);
	} else {
	pcmk__xe_set_props(cib_node,
	PCMK__XA_IN_CCM, XML_BOOLEAN_FALSE,
	PCMK__XA_CRMD, OFFLINESTATUS,
	PCMK__XA_JOIN, CRMD_JOINSTATE_DOWN,
	PCMK__XA_EXPECTED, CRMD_JOINSTATE_DOWN,
	NULL);
	}
	crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, crm_system_name);
	return cib_node;
	}

	/*!
	* \internal
	* \brief Check whether a node has history for a given resource
	*
	* \param[in,out] cib_node Node state XML to check
	* \param[in] resource Resource name to check for
	*
	* \return Resource's lrm_resource XML entry beneath \p cib_node if found,
	* otherwise NULL
	*/
	static xmlNode *
	find_resource_xml(xmlNode cib_node, const char resource)
	{
	const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
	char *xpath = crm_strdup_printf(XPATH_RSC_HISTORY, node, resource);
	xmlNode *match = get_xpath_object(xpath, cib_node, LOG_TRACE);

	free(xpath);
	return match;
	}

	/*!
	* \internal
	* \brief Inject a resource history element into a scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node Node state XML to inject resource history entry into
	* \param[in] resource ID (in configuration) of resource to inject
	* \param[in] lrm_name ID as used in history (could be clone instance)
	* \param[in] rclass Resource agent class of resource to inject
	* \param[in] rtype Resource agent type of resource to inject
	* \param[in] rprovider Resource agent provider of resource to inject
	*
	* \return XML of injected resource history element
	* \note If a history element already exists under either \p resource or
	* \p lrm_name, this will return it rather than injecting a new one.
	*/
	xmlNode *
	pcmk__inject_resource_history(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char lrm_name,
	const char rclass, const char rtype,
	const char *rprovider)
	{
	xmlNode *lrm = NULL;
	xmlNode *container = NULL;
	xmlNode *cib_resource = NULL;

	cib_resource = find_resource_xml(cib_node, resource);
	if (cib_resource != NULL) {
	/* If an existing LRM history entry uses the resource name,
	* continue using it, even if lrm_name is different.
	*/
	return cib_resource;
	}

	// Check for history entry under preferred name
	if (strcmp(resource, lrm_name) != 0) {
	cib_resource = find_resource_xml(cib_node, lrm_name);
	if (cib_resource != NULL) {
	return cib_resource;
	}
	}

	if ((rclass == NULL) \|\| (rtype == NULL)) {
	// @TODO query configuration for class, provider, type
	out->err(out,
	"Resource %s not found in the status section of %s "
	"(supply class and type to continue)",
	resource, ID(cib_node));
	return NULL;

	} else if (!pcmk__strcase_any_of(rclass,
	PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_STONITH,
	PCMK_RESOURCE_CLASS_SERVICE,
	PCMK_RESOURCE_CLASS_UPSTART,
	PCMK_RESOURCE_CLASS_SYSTEMD,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	out->err(out, "Invalid class for %s: %s", resource, rclass);
	return NULL;

	} else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
	&& (rprovider == NULL)) {
	// @TODO query configuration for provider
	out->err(out, "Please specify the provider for resource %s", resource);
	return NULL;
	}

	crm_info("Injecting new resource %s into node state '%s'",
	lrm_name, ID(cib_node));

	lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
	if (lrm == NULL) {
	const char *node_uuid = ID(cib_node);

	lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
	crm_xml_add(lrm, PCMK_XA_ID, node_uuid);
	}

	container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
	if (container == NULL) {
	container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
	}

	cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);

	// If we're creating a new entry, use the preferred name
	crm_xml_add(cib_resource, PCMK_XA_ID, lrm_name);

	- crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
	+ crm_xml_add(cib_resource, PCMK_XA_CLASS, rclass);
	crm_xml_add(cib_resource, PCMK_XA_PROVIDER, rprovider);
	crm_xml_add(cib_resource, PCMK_XA_TYPE, rtype);

	return cib_resource;
	}

	static int
	find_ticket_state(pcmk__output_t out, cib_t the_cib, const char *ticket_id,
	xmlNode **ticket_state_xml)
	{
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;

	GString *xpath = g_string_sized_new(256);

	CRM_ASSERT(ticket_state_xml != NULL);
	*ticket_state_xml = NULL;

	g_string_append(xpath,
	"/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS
	"/" XML_CIB_TAG_TICKETS);

	if (ticket_id) {
	pcmk__g_strcat(xpath,
	"/" XML_CIB_TAG_TICKET_STATE
	"[@" PCMK_XA_ID "=\"", ticket_id, "\"]", NULL);
	}
	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call\|cib_scope_local\|cib_xpath);
	g_string_free(xpath, TRUE);

	if (rc != pcmk_ok) {
	return rc;
	}

	crm_log_xml_debug(xml_search, "Match");
	if ((xml_search->children != NULL) && (ticket_id != NULL)) {
	out->err(out, "Multiple ticket_states match ticket_id=%s", ticket_id);
	}
	*ticket_state_xml = xml_search;

	return rc;
	}

	/*!
	* \internal
	* \brief Inject a ticket attribute into ticket state
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in] ticket_id Ticket whose state should be changed
	* \param[in] attr_name Ticket attribute name to inject
	* \param[in] attr_value Boolean value of ticket attribute to inject
	* \param[in,out] cib CIB object to use
	*
	* \return Standard Pacemaker return code
	*/
	static int
	set_ticket_state_attr(pcmk__output_t out, const char ticket_id,
	const char attr_name, bool attr_value, cib_t cib)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_top = NULL;
	xmlNode *ticket_state_xml = NULL;

	// Check for an existing ticket state entry
	rc = find_ticket_state(out, cib, ticket_id, &ticket_state_xml);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) { // Ticket state found, use it
	crm_debug("Injecting attribute into existing ticket state %s",
	ticket_id);
	xml_top = ticket_state_xml;

	} else if (rc == ENXIO) { // No ticket state, create it
	xmlNode *xml_obj = NULL;

	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
	ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
	crm_xml_add(ticket_state_xml, PCMK_XA_ID, ticket_id);

	} else { // Error
	return rc;
	}

	// Add the attribute to the ticket state
	pcmk__xe_set_bool_attr(ticket_state_xml, attr_name, attr_value);
	crm_log_xml_debug(xml_top, "Update");

	// Commit the change to the CIB
	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top,
	cib_sync_call\|cib_scope_local);
	rc = pcmk_legacy2rc(rc);

	free_xml(xml_top);
	return rc;
	}

	/*!
	* \internal
	* \brief Inject a fictitious action into the cluster
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in] spec Action specification to inject
	* \param[in,out] cib CIB object for scheduler input
	* \param[in] scheduler Scheduler data
	*/
	static void
	inject_action(pcmk__output_t out, const char spec, cib_t *cib,
	const pcmk_scheduler_t *scheduler)
	{
	int rc;
	int outcome = PCMK_OCF_OK;
	guint interval_ms = 0;

	char *key = NULL;
	char *node = NULL;
	char *task = NULL;
	char *resource = NULL;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *rprovider = NULL;

	xmlNode *cib_op = NULL;
	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	const pcmk_resource_t *rsc = NULL;
	lrmd_event_data_t *op = NULL;

	out->message(out, "inject-spec", spec);

	key = calloc(1, strlen(spec) + 1);
	node = calloc(1, strlen(spec) + 1);
	rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
	if (rc != 3) {
	out->err(out, "Invalid operation spec: %s. Only found %d fields",
	spec, rc);
	goto done;
	}

	parse_op_key(key, &resource, &task, &interval_ms);

	rsc = pe_find_resource(scheduler->resources, resource);
	if (rsc == NULL) {
	out->err(out, "Invalid resource name: %s", resource);
	goto done;
	}

	- rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	rprovider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);

	cib_node = pcmk__inject_node(cib, node, NULL);
	CRM_ASSERT(cib_node != NULL);

	pcmk__inject_failcount(out, cib_node, resource, task, interval_ms, outcome);

	cib_resource = pcmk__inject_resource_history(out, cib_node,
	resource, resource,
	rclass, rtype, rprovider);
	CRM_ASSERT(cib_resource != NULL);

	op = create_op(cib_resource, task, interval_ms, outcome);
	CRM_ASSERT(op != NULL);

	cib_op = pcmk__inject_action_result(cib_resource, op, 0);
	CRM_ASSERT(cib_op != NULL);
	lrmd_free_event(op);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(task);
	free(node);
	free(key);
	}

	/*!
	* \internal
	* \brief Inject fictitious scheduler inputs
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] cib CIB object for scheduler input to modify
	* \param[in] injections Injections to apply
	*/
	void
	pcmk__inject_scheduler_input(pcmk_scheduler_t scheduler, cib_t cib,
	const pcmk_injections_t *injections)
	{
	int rc = pcmk_ok;
	const GList *iter = NULL;
	xmlNode *cib_node = NULL;
	pcmk__output_t *out = scheduler->priv;

	out->message(out, "inject-modify-config", injections->quorum,
	injections->watchdog);
	if (injections->quorum != NULL) {
	xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);

	/* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid); */
	crm_xml_add(top, PCMK_XA_HAVE_QUORUM, injections->quorum);

	rc = cib->cmds->modify(cib, NULL, top, cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	}

	if (injections->watchdog != NULL) {
	rc = cib__update_node_attr(out, cib, cib_sync_call\|cib_scope_local,
	XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL,
	NULL, PCMK_OPT_HAVE_WATCHDOG,
	injections->watchdog, NULL, NULL);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->node_up; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;

	out->message(out, "inject-modify-node", "Online", node);

	cib_node = pcmk__inject_node_state_change(cib, node, true);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (iter = injections->node_down; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;
	char *xpath = NULL;

	out->message(out, "inject-modify-node", "Offline", node);

	cib_node = pcmk__inject_node_state_change(cib, node, false);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);

	xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
	node, XML_CIB_TAG_LRM);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	free(xpath);

	xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
	node, XML_TAG_TRANSIENT_NODEATTRS);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	free(xpath);
	}

	for (iter = injections->node_fail; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;

	out->message(out, "inject-modify-node", "Failing", node);

	cib_node = pcmk__inject_node_state_change(cib, node, true);
	crm_xml_add(cib_node, PCMK__XA_IN_CCM, XML_BOOLEAN_FALSE);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (iter = injections->ticket_grant; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Granting", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "granted", true, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_revoke; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Revoking", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "granted", false, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_standby; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Standby", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "standby", true, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_activate; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Activating", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "standby", false, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->op_inject; iter != NULL; iter = iter->next) {
	inject_action(out, (const char *) iter->data, cib, scheduler);
	}

	if (!out->is_quiet(out)) {
	out->end_list(out);
	}
	}

	void
	pcmk_free_injections(pcmk_injections_t *injections)
	{
	if (injections == NULL) {
	return;
	}

	g_list_free_full(injections->node_up, g_free);
	g_list_free_full(injections->node_down, g_free);
	g_list_free_full(injections->node_fail, g_free);
	g_list_free_full(injections->op_fail, g_free);
	g_list_free_full(injections->op_inject, g_free);
	g_list_free_full(injections->ticket_grant, g_free);
	g_list_free_full(injections->ticket_revoke, g_free);
	g_list_free_full(injections->ticket_standby, g_free);
	g_list_free_full(injections->ticket_activate, g_free);
	free(injections->quorum);
	free(injections->watchdog);

	free(injections);
	}
	diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c
	index d268887b20..199d98fd79 100644
	--- a/lib/pacemaker/pcmk_sched_primitive.c
	+++ b/lib/pacemaker/pcmk_sched_primitive.c
	@@ -1,1671 +1,1671 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdbool.h>
	#include <stdint.h> // uint8_t, uint32_t

	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	static void stop_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void start_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void demote_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void promote_resource(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);
	static void assert_role_error(pcmk_resource_t rsc, pcmk_node_t node,
	bool optional);

	#define RSC_ROLE_MAX (pcmk_role_promoted + 1)

	static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the immediate next role when transitioning from one role
	* to a target role. For example, when going from Stopped to Promoted, the
	* next role is Unpromoted, because the resource must be started before it
	* can be promoted. The current state then becomes Started, which is fed
	* into this array again, giving a next role of Promoted.
	*
	* Current role Immediate next role Final target role
	* ------------ ------------------- -----------------
	*/
	/* Unknown / { pcmk_role_unknown, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_stopped, /* Unpromoted */
	pcmk_role_stopped, /* Promoted */
	},
	/* Stopped / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_unpromoted, /* Promoted */
	},
	/* Started / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_started, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Unpromoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_stopped, /* Stopped */
	pcmk_role_stopped, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	/* Promoted / { pcmk_role_stopped, / Unknown */
	pcmk_role_unpromoted, /* Stopped */
	pcmk_role_unpromoted, /* Started */
	pcmk_role_unpromoted, /* Unpromoted */
	pcmk_role_promoted, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Function to schedule actions needed for a role change
	*
	* \param[in,out] rsc Resource whose role is changing
	* \param[in,out] node Node where resource will be in its next role
	* \param[in] optional Whether scheduled actions should be optional
	*/
	typedef void (rsc_transition_fn)(pcmk_resource_t rsc, pcmk_node_t *node,
	bool optional);

	static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
	/* This array lists the function needed to transition directly from one role
	* to another. NULL indicates that nothing is needed.
	*
	* Current role Transition function Next role
	* ------------ ------------------- ----------
	*/
	/* Unknown / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	assert_role_error, /* Started */
	assert_role_error, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Stopped / { assert_role_error, / Unknown */
	NULL, /* Stopped */
	start_resource, /* Started */
	start_resource, /* Unpromoted */
	assert_role_error, /* Promoted */
	},
	/* Started / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	NULL, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Unpromoted / { assert_role_error, / Unknown */
	stop_resource, /* Stopped */
	stop_resource, /* Started */
	NULL, /* Unpromoted */
	promote_resource, /* Promoted */
	},
	/* Promoted / { assert_role_error, / Unknown */
	demote_resource, /* Stopped */
	demote_resource, /* Started */
	demote_resource, /* Unpromoted */
	NULL, /* Promoted */
	},
	};

	/*!
	* \internal
	* \brief Get a list of a resource's allowed nodes sorted by node score
	*
	* \param[in] rsc Resource to check
	*
	* \return List of allowed nodes sorted by node score
	*/
	static GList *
	sorted_allowed_nodes(const pcmk_resource_t *rsc)
	{
	if (rsc->allowed_nodes != NULL) {
	GList *nodes = g_hash_table_get_values(rsc->allowed_nodes);

	if (nodes != NULL) {
	return pcmk__sort_nodes(nodes, pe__current_node(rsc));
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Assign a resource to its best allowed node, if possible
	*
	* \param[in,out] rsc Resource to choose a node for
	* \param[in] prefer If not \c NULL, prefer this node when all else
	* equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return true if \p rsc could be assigned to a node, otherwise false
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	static bool
	assign_best_node(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *nodes = NULL;
	pcmk_node_t *chosen = NULL;
	pcmk_node_t *best = NULL;
	const pcmk_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);

	if (prefer == NULL) {
	prefer = most_free_node;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// We've already finished assignment of resources to nodes
	return rsc->allocated_to != NULL;
	}

	// Sort allowed nodes by score
	nodes = sorted_allowed_nodes(rsc);
	if (nodes != NULL) {
	best = (pcmk_node_t *) nodes->data; // First node has best score
	}

	if ((prefer != NULL) && (nodes != NULL)) {
	// Get the allowed node version of prefer
	chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (chosen == NULL) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unknown",
	pe__node_name(prefer), rsc->id);

	/* Favor the preferred node as long as its score is at least as good as
	* the best allowed node's.
	*
	* An alternative would be to favor the preferred node even if the best
	* node is better, when the best node's score is less than INFINITY.
	*/
	} else if (chosen->weight < best->weight) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
	pe__node_name(chosen), rsc->id);
	chosen = NULL;

	} else if (!pcmk__node_available(chosen, true, false)) {
	pcmk__rsc_trace(rsc, "Preferred node %s for %s was unavailable",
	pe__node_name(chosen), rsc->id);
	chosen = NULL;

	} else {
	pcmk__rsc_trace(rsc,
	"Chose preferred node %s for %s "
	"(ignoring %d candidates)",
	pe__node_name(chosen), rsc->id,
	g_list_length(nodes));
	}
	}

	if ((chosen == NULL) && (best != NULL)) {
	/* Either there is no preferred node, or the preferred node is not
	* suitable, but another node is allowed to run the resource.
	*/

	chosen = best;

	if (!pe_rsc_is_unique_clone(rsc->parent)
	&& (chosen->weight > 0) // Zero not acceptable
	&& pcmk__node_available(chosen, false, false)) {
	/* If the resource is already running on a node, prefer that node if
	* it is just as good as the chosen node.
	*
	* We don't do this for unique clone instances, because
	* pcmk__assign_instances() has already assigned instances to their
	* running nodes when appropriate, and if we get here, we don't want
	* remaining unassigned instances to prefer a node that's already
	* running another instance.
	*/
	pcmk_node_t *running = pe__current_node(rsc);

	if (running == NULL) {
	// Nothing to do

	} else if (!pcmk__node_available(running, true, false)) {
	pcmk__rsc_trace(rsc,
	"Current node for %s (%s) can't run resources",
	rsc->id, pe__node_name(running));

	} else {
	int nodes_with_best_score = 1;

	for (GList *iter = nodes->next; iter; iter = iter->next) {
	pcmk_node_t allowed = (pcmk_node_t ) iter->data;

	if (allowed->weight != chosen->weight) {
	// The nodes are sorted by score, so no more are equal
	break;
	}
	if (pe__same_node(allowed, running)) {
	// Scores are equal, so prefer the current node
	chosen = allowed;
	}
	nodes_with_best_score++;
	}

	if (nodes_with_best_score > 1) {
	uint8_t log_level = LOG_INFO;

	if (chosen->weight >= INFINITY) {
	log_level = LOG_WARNING;
	}
	do_crm_log(log_level,
	"Chose %s for %s from %d nodes with score %s",
	pe__node_name(chosen), rsc->id,
	nodes_with_best_score,
	pcmk_readable_score(chosen->weight));
	}
	}
	}

	pcmk__rsc_trace(rsc, "Chose %s for %s from %d candidates",
	pe__node_name(chosen), rsc->id, g_list_length(nodes));
	}

	pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
	g_list_free(nodes);
	return rsc->allocated_to != NULL;
	}

	/*!
	* \internal
	* \brief Apply a "this with" colocation to a node's allowed node scores
	*
	* \param[in,out] colocation Colocation to apply
	* \param[in,out] rsc Resource being assigned
	*/
	static void
	apply_this_with(pcmk__colocation_t colocation, pcmk_resource_t rsc)
	{
	GHashTable *archive = NULL;
	pcmk_resource_t *other = colocation->primary;

	// In certain cases, we will need to revert the node scores
	if ((colocation->dependent_role >= pcmk_role_promoted)
	\|\| ((colocation->score < 0) && (colocation->score > -INFINITY))) {
	archive = pcmk__copy_node_table(rsc->allowed_nodes);
	}

	if (pcmk_is_set(other->flags, pcmk_rsc_unassigned)) {
	pcmk__rsc_trace(rsc,
	"%s: Assigning colocation %s primary %s first"
	"(score=%d role=%s)",
	rsc->id, colocation->id, other->id,
	colocation->score, role2text(colocation->dependent_role));
	other->cmds->assign(other, NULL, true);
	}

	// Apply the colocation score to this resource's allowed node scores
	rsc->cmds->apply_coloc_score(rsc, other, colocation, true);
	if ((archive != NULL)
	&& !pcmk__any_node_available(rsc->allowed_nodes)) {
	pcmk__rsc_info(rsc,
	"%s: Reverting scores from colocation with %s "
	"because no nodes allowed",
	rsc->id, other->id);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = archive;
	archive = NULL;
	}
	if (archive != NULL) {
	g_hash_table_destroy(archive);
	}
	}

	/*!
	* \internal
	* \brief Update a Pacemaker Remote node once its connection has been assigned
	*
	* \param[in] connection Connection resource that has been assigned
	*/
	static void
	remote_connection_assigned(const pcmk_resource_t *connection)
	{
	pcmk_node_t *remote_node = pe_find_node(connection->cluster->nodes,
	connection->id);

	CRM_CHECK(remote_node != NULL, return);

	if ((connection->allocated_to != NULL)
	&& (connection->next_role != pcmk_role_stopped)) {

	crm_trace("Pacemaker Remote node %s will be online",
	remote_node->details->id);
	remote_node->details->online = TRUE;
	if (remote_node->details->unseen) {
	// Avoid unnecessary fence, since we will attempt connection
	remote_node->details->unclean = FALSE;
	}

	} else {
	crm_trace("Pacemaker Remote node %s will be shut down "
	"(%sassigned connection's next role is %s)",
	remote_node->details->id,
	((connection->allocated_to == NULL)? "un" : ""),
	role2text(connection->next_role));
	remote_node->details->shutdown = TRUE;
	}
	}

	/*!
	* \internal
	* \brief Assign a primitive resource to a node
	*
	* \param[in,out] rsc Resource to assign to a node
	* \param[in] prefer Node to prefer, if all else is equal
	* \param[in] stop_if_fail If \c true and \p rsc can't be assigned to a
	* node, set next role to stopped and update
	* existing actions
	*
	* \return Node that \p rsc is assigned to, if assigned entirely to one node
	*
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	pcmk_node_t *
	pcmk__primitive_assign(pcmk_resource_t rsc, const pcmk_node_t prefer,
	bool stop_if_fail)
	{
	GList *this_with_colocations = NULL;
	GList *with_this_colocations = NULL;
	GList *iter = NULL;
	pcmk__colocation_t *colocation = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	// Never assign a child without parent being assigned first
	if ((rsc->parent != NULL)
	&& !pcmk_is_set(rsc->parent->flags, pcmk_rsc_assigning)) {
	pcmk__rsc_debug(rsc, "%s: Assigning parent %s first",
	rsc->id, rsc->parent->id);
	rsc->parent->cmds->assign(rsc->parent, prefer, stop_if_fail);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	// Assignment has already been done
	const char *node_name = "no node";

	if (rsc->allocated_to != NULL) {
	node_name = pe__node_name(rsc->allocated_to);
	}
	pcmk__rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
	return rsc->allocated_to;
	}

	// Ensure we detect assignment loops
	if (pcmk_is_set(rsc->flags, pcmk_rsc_assigning)) {
	pcmk__rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
	return NULL;
	}
	pcmk__set_rsc_flags(rsc, pcmk_rsc_assigning);

	pe__show_node_scores(true, rsc, "Pre-assignment", rsc->allowed_nodes,
	rsc->cluster);

	this_with_colocations = pcmk__this_with_colocations(rsc);
	with_this_colocations = pcmk__with_this_colocations(rsc);

	// Apply mandatory colocations first, to satisfy as many as possible
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score <= -CRM_SCORE_INFINITY)
	\|\| (colocation->score >= CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	pe__show_node_scores(true, rsc, "Mandatory-colocations",
	rsc->allowed_nodes, rsc->cluster);

	// Then apply optional colocations
	for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	apply_this_with(colocation, rsc);
	}
	}
	for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
	colocation = iter->data;

	if ((colocation->score > -CRM_SCORE_INFINITY)
	&& (colocation->score < CRM_SCORE_INFINITY)) {
	pcmk__add_dependent_scores(colocation, rsc);
	}
	}

	g_list_free(this_with_colocations);
	g_list_free(with_this_colocations);

	if (rsc->next_role == pcmk_role_stopped) {
	pcmk__rsc_trace(rsc,
	"Banning %s from all nodes because it will be stopped",
	rsc->id);
	resource_location(rsc, NULL, -INFINITY, PCMK_META_TARGET_ROLE,
	rsc->cluster);

	} else if ((rsc->next_role > rsc->role)
	&& !pcmk_is_set(rsc->cluster->flags, pcmk_sched_quorate)
	&& (rsc->cluster->no_quorum_policy == pcmk_no_quorum_freeze)) {
	crm_notice("Resource %s cannot be elevated from %s to %s due to "
	PCMK_OPT_NO_QUORUM_POLICY "=freeze",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role));
	pe__set_next_role(rsc, rsc->role, PCMK_OPT_NO_QUORUM_POLICY "=freeze");
	}

	pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_output_scores),
	rsc, __func__, rsc->allowed_nodes, rsc->cluster);

	// Unmanage resource if fencing is enabled but no device is configured
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(rsc->cluster->flags, pcmk_sched_have_fencing)) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	// Unmanaged resources stay on their current node
	const char *reason = NULL;
	pcmk_node_t *assign_to = NULL;

	pe__set_next_role(rsc, rsc->role, "unmanaged");
	assign_to = pe__current_node(rsc);
	if (assign_to == NULL) {
	reason = "inactive";
	} else if (rsc->role == pcmk_role_promoted) {
	reason = "promoted";
	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	reason = "failed";
	} else {
	reason = "active";
	}
	pcmk__rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
	(assign_to? assign_to->details->uname : "no node"),
	reason);
	pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);

	} else if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_stop_all)) {
	// Must stop at some point, but be consistent with stop_if_fail
	if (stop_if_fail) {
	pcmk__rsc_debug(rsc,
	"Forcing %s to stop: " PCMK_OPT_STOP_ALL_RESOURCES,
	rsc->id);
	}
	pcmk__assign_resource(rsc, NULL, true, stop_if_fail);

	} else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
	// Assignment failed
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	pcmk__rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
	} else if ((rsc->running_on != NULL) && stop_if_fail) {
	pcmk__rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
	}
	}

	pcmk__clear_rsc_flags(rsc, pcmk_rsc_assigning);

	if (rsc->is_remote_node) {
	remote_connection_assigned(rsc);
	}

	return rsc->allocated_to;
	}

	/*!
	* \internal
	* \brief Schedule actions to bring resource down and back to current role
	*
	* \param[in,out] rsc Resource to restart
	* \param[in,out] current Node that resource should be brought down on
	* \param[in] need_stop Whether the resource must be stopped
	* \param[in] need_promote Whether the resource must be promoted
	*
	* \return Role that resource would have after scheduled actions are taken
	*/
	static void
	schedule_restart_actions(pcmk_resource_t rsc, pcmk_node_t current,
	bool need_stop, bool need_promote)
	{
	enum rsc_role_e role = rsc->role;
	enum rsc_role_e next_role;
	rsc_transition_fn fn = NULL;

	pcmk__set_rsc_flags(rsc, pcmk_rsc_restarting);

	// Bring resource down to a stop on its current node
	while (role != pcmk_role_stopped) {
	next_role = rsc_state_matrix[role][pcmk_role_stopped];
	pcmk__rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
	(need_stop? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, current, !need_stop);
	role = next_role;
	}

	// Bring resource up to its next role on its next node
	while ((rsc->role <= rsc->next_role) && (role != rsc->role)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	bool required = need_stop;

	next_role = rsc_state_matrix[role][rsc->role];
	if ((next_role == pcmk_role_promoted) && need_promote) {
	required = true;
	}
	pcmk__rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
	(required? "required" : "optional"), rsc->id,
	role2text(role), role2text(next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, !required);
	role = next_role;
	}

	pcmk__clear_rsc_flags(rsc, pcmk_rsc_restarting);
	}

	/*!
	* \internal
	* \brief If a resource's next role is not explicitly specified, set a default
	*
	* \param[in,out] rsc Resource to set next role for
	*
	* \return "explicit" if next role was explicitly set, otherwise "implicit"
	*/
	static const char *
	set_default_next_role(pcmk_resource_t *rsc)
	{
	if (rsc->next_role != pcmk_role_unknown) {
	return "explicit";
	}

	if (rsc->allocated_to == NULL) {
	pe__set_next_role(rsc, pcmk_role_stopped, "assignment");
	} else {
	pe__set_next_role(rsc, pcmk_role_started, "assignment");
	}
	return "implicit";
	}

	/*!
	* \internal
	* \brief Create an action to represent an already pending start
	*
	* \param[in,out] rsc Resource to create start action for
	*/
	static void
	create_pending_start(pcmk_resource_t *rsc)
	{
	pcmk_action_t *start = NULL;

	pcmk__rsc_trace(rsc,
	"Creating action for %s to represent already pending start",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	pcmk__set_action_flags(start, pcmk_action_always_in_graph);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to take a resource to its next role
	*
	* \param[in,out] rsc Resource to schedule actions for
	*/
	static void
	schedule_role_transition_actions(pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	while (role != rsc->next_role) {
	enum rsc_role_e next_role = rsc_state_matrix[role][rsc->next_role];
	rsc_transition_fn fn = NULL;

	pcmk__rsc_trace(rsc,
	"Creating action to take %s from %s to %s "
	"(ending at %s)",
	rsc->id, role2text(role), role2text(next_role),
	role2text(rsc->next_role));
	fn = rsc_action_matrix[role][next_role];
	if (fn == NULL) {
	break;
	}
	fn(rsc, rsc->allocated_to, false);
	role = next_role;
	}
	}

	/*!
	* \internal
	* \brief Create all actions needed for a given primitive resource
	*
	* \param[in,out] rsc Primitive resource to create actions for
	*/
	void
	pcmk__primitive_create_actions(pcmk_resource_t *rsc)
	{
	bool need_stop = false;
	bool need_promote = false;
	bool is_moving = false;
	bool allow_migrate = false;
	bool multiply_active = false;

	pcmk_node_t *current = NULL;
	unsigned int num_all_active = 0;
	unsigned int num_clean_active = 0;
	const char *next_role_source = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	next_role_source = set_default_next_role(rsc);
	pcmk__rsc_trace(rsc,
	"Creating all actions for %s transition from %s to %s "
	"(%s) on %s",
	rsc->id, role2text(rsc->role), role2text(rsc->next_role),
	next_role_source, pe__node_name(rsc->allocated_to));

	current = rsc->fns->active_node(rsc, &num_all_active, &num_clean_active);

	g_list_foreach(rsc->dangling_migrations, pcmk__abort_dangling_migration,
	rsc);

	if ((current != NULL) && (rsc->allocated_to != NULL)
	&& !pe__same_node(current, rsc->allocated_to)
	&& (rsc->next_role >= pcmk_role_started)) {

	pcmk__rsc_trace(rsc, "Moving %s from %s to %s",
	rsc->id, pe__node_name(current),
	pe__node_name(rsc->allocated_to));
	is_moving = true;
	allow_migrate = pcmk__rsc_can_migrate(rsc, current);

	// This is needed even if migrating (though I'm not sure why ...)
	need_stop = true;
	}

	// Check whether resource is partially migrated and/or multiply active
	if ((rsc->partial_migration_source != NULL)
	&& (rsc->partial_migration_target != NULL)
	&& allow_migrate && (num_all_active == 2)
	&& pe__same_node(current, rsc->partial_migration_source)
	&& pe__same_node(rsc->allocated_to, rsc->partial_migration_target)) {
	/* A partial migration is in progress, and the migration target remains
	* the same as when the migration began.
	*/
	pcmk__rsc_trace(rsc,
	"Partial migration of %s from %s to %s will continue",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));

	} else if ((rsc->partial_migration_source != NULL)
	\|\| (rsc->partial_migration_target != NULL)) {
	// A partial migration is in progress but can't be continued

	if (num_all_active > 2) {
	// The resource is migrating and multiply active!
	crm_notice("Forcing recovery of %s because it is migrating "
	"from %s to %s and possibly active elsewhere",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));
	} else {
	// The migration source or target isn't available
	crm_notice("Forcing recovery of %s because it can no longer "
	"migrate from %s to %s",
	rsc->id, pe__node_name(rsc->partial_migration_source),
	pe__node_name(rsc->partial_migration_target));
	}
	need_stop = true;
	rsc->partial_migration_source = rsc->partial_migration_target = NULL;
	allow_migrate = false;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	multiply_active = (num_all_active > 1);
	} else {
	/* If a resource has "requires" set to nothing or quorum, don't consider
	* it active on unclean nodes (similar to how all resources behave when
	* PCMK_OPT_STONITH_ENABLED is false). We can start such resources
	* elsewhere before fencing completes, and if we considered the resource
	* active on the failed node, we would attempt recovery for being active
	* on multiple nodes.
	*/
	multiply_active = (num_clean_active > 1);
	}

	if (multiply_active) {
	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);

	// Resource was (possibly) incorrectly multiply active
	pcmk__sched_err("%s resource %s might be active on %u nodes (%s)",
	pcmk__s(class, "Untyped"), rsc->id, num_all_active,
	pcmk_multiply_active_text(rsc->recovery_type));
	crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ"
	"#Resource_is_Too_Active for more information");

	switch (rsc->recovery_type) {
	case pcmk_multiply_active_restart:
	need_stop = true;
	break;
	case pcmk_multiply_active_unexpected:
	need_stop = true; // stop_resource() will skip expected node
	pcmk__set_rsc_flags(rsc, pcmk_rsc_stop_unexpected);
	break;
	default:
	break;
	}

	} else {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_stop_unexpected);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	create_pending_start(rsc);
	}

	if (is_moving) {
	// Remaining tests are only for resources staying where they are

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_stop_if_failed)) {
	need_stop = true;
	pcmk__rsc_trace(rsc, "Recovering %s", rsc->id);
	} else {
	pcmk__rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
	if (rsc->next_role == pcmk_role_promoted) {
	need_promote = true;
	}
	}

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	pcmk__rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
	need_stop = true;

	} else if ((rsc->role > pcmk_role_started) && (current != NULL)
	&& (rsc->allocated_to != NULL)) {
	pcmk_action_t *start = NULL;

	pcmk__rsc_trace(rsc, "Creating start action for promoted resource %s",
	rsc->id);
	start = start_action(rsc, rsc->allocated_to, TRUE);
	if (!pcmk_is_set(start->flags, pcmk_action_optional)) {
	// Recovery of a promoted resource
	pcmk__rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
	need_stop = true;
	}
	}

	// Create any actions needed to bring resource down and back up to same role
	schedule_restart_actions(rsc, current, need_stop, need_promote);

	// Create any actions needed to take resource from this role to the next
	schedule_role_transition_actions(rsc);

	pcmk__create_recurring_actions(rsc);

	if (allow_migrate) {
	pcmk__create_migration_actions(rsc, current);
	}
	}

	/*!
	* \internal
	* \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
	*
	* \param[in] rsc Resource to check
	*/
	static void
	rsc_avoids_remote_nodes(const pcmk_resource_t *rsc)
	{
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	if (node->details->remote_rsc != NULL) {
	node->weight = -INFINITY;
	}
	}
	}

	/*!
	* \internal
	* \brief Return allowed nodes as (possibly sorted) list
	*
	* Convert a resource's hash table of allowed nodes to a list. If printing to
	* stdout, sort the list, to keep action ID numbers consistent for regression
	* test output (while avoiding the performance hit on a live cluster).
	*
	* \param[in] rsc Resource to check for allowed nodes
	*
	* \return List of resource's allowed nodes
	* \note Callers should take care not to rely on the list being sorted.
	*/
	static GList *
	allowed_nodes_as_list(const pcmk_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;

	if (rsc->allowed_nodes) {
	allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	if (!pcmk__is_daemon) {
	allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
	}

	return allowed_nodes;
	}

	/*!
	* \internal
	* \brief Create implicit constraints needed for a primitive resource
	*
	* \param[in,out] rsc Primitive resource to create implicit constraints for
	*/
	void
	pcmk__primitive_internal_constraints(pcmk_resource_t *rsc)
	{
	GList *allowed_nodes = NULL;
	bool check_unfencing = false;
	bool check_utilization = false;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__rsc_trace(rsc,
	"Skipping implicit constraints for unmanaged resource "
	"%s", rsc->id);
	return;
	}

	// Whether resource requires unfencing
	check_unfencing = !pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)
	&& pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing);

	// Whether a non-default placement strategy is used
	check_utilization = (g_hash_table_size(rsc->utilization) > 0)
	&& !pcmk__str_eq(rsc->cluster->placement_strategy,
	"default", pcmk__str_casei);

	// Order stops before starts (i.e. restart)
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0), NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0), NULL,
	pcmk__ar_ordered
	\|pcmk__ar_first_implies_then
	\|pcmk__ar_intermediate_stop,
	rsc->cluster);

	// Promotable ordering: demote before stop, start before promote
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)
	\|\| (rsc->role > pcmk_role_unpromoted)) {

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	pcmk__ar_promoted_then_implies_first, rsc->cluster);

	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	rsc, pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0),
	NULL,
	pcmk__ar_unrunnable_first_blocks, rsc->cluster);
	}

	// Don't clear resource history if probing on same node
	pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	pcmk__ar_if_on_same_node\|pcmk__ar_then_cancels_first,
	rsc->cluster);

	// Certain checks need allowed nodes
	if (check_unfencing \|\| check_utilization \|\| (rsc->container != NULL)) {
	allowed_nodes = allowed_nodes_as_list(rsc);
	}

	if (check_unfencing) {
	g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
	}

	if (check_utilization) {
	pcmk__create_utilization_constraints(rsc, allowed_nodes);
	}

	if (rsc->container != NULL) {
	pcmk_resource_t *remote_rsc = NULL;

	if (rsc->is_remote_node) {
	// rsc is the implicit remote connection for a guest or bundle node

	/* Guest resources are not allowed to run on Pacemaker Remote nodes,
	* to avoid nesting remotes. However, bundles are allowed.
	*/
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_remote_nesting_allowed)) {
	rsc_avoids_remote_nodes(rsc->container);
	}

	/* If someone cleans up a guest or bundle node's container, we will
	* likely schedule a (re-)probe of the container and recovery of the
	* connection. Order the connection stop after the container probe,
	* so that if we detect the container running, we will trigger a new
	* transition and avoid the unnecessary recovery.
	*/
	pcmk__order_resource_actions(rsc->container, PCMK_ACTION_MONITOR,
	rsc, PCMK_ACTION_STOP,
	pcmk__ar_ordered);

	/* A user can specify that a resource must start on a Pacemaker Remote
	* node by explicitly configuring it with the container=NODENAME
	* meta-attribute. This is of questionable merit, since location
	* constraints can accomplish the same thing. But we support it, so here
	* we check whether a resource (that is not itself a remote connection)
	* has container set to a remote node or guest node resource.
	*/
	} else if (rsc->container->is_remote_node) {
	remote_rsc = rsc->container;
	} else {
	remote_rsc = pe__resource_contains_guest_node(rsc->cluster,
	rsc->container);
	}

	if (remote_rsc != NULL) {
	/* Force the resource on the Pacemaker Remote node instead of
	* colocating the resource with the container resource.
	*/
	for (GList *item = allowed_nodes; item; item = item->next) {
	pcmk_node_t *node = item->data;

	if (node->details->remote_rsc != remote_rsc) {
	node->weight = -INFINITY;
	}
	}

	} else {
	/* This resource is either a filler for a container that does NOT
	* represent a Pacemaker Remote node, or a Pacemaker Remote
	* connection resource for a guest node or bundle.
	*/
	int score;

	crm_trace("Order and colocate %s relative to its container %s",
	rsc->id, rsc->container->id);

	pcmk__new_ordering(rsc->container,
	pcmk__op_key(rsc->container->id,
	PCMK_ACTION_START, 0),
	NULL, rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_first_implies_then
	\|pcmk__ar_unrunnable_first_blocks,
	rsc->cluster);

	pcmk__new_ordering(rsc,
	pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
	NULL,
	rsc->container,
	pcmk__op_key(rsc->container->id,
	PCMK_ACTION_STOP, 0),
	NULL, pcmk__ar_then_implies_first, rsc->cluster);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_remote_nesting_allowed)) {
	score = 10000; /* Highly preferred but not essential */
	} else {
	score = INFINITY; /* Force them to run on the same host */
	}
	pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
	rsc->container, NULL, NULL,
	pcmk__coloc_influence);
	}
	}

	if (rsc->is_remote_node
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	/* Remote connections and fencing devices are not allowed to run on
	* Pacemaker Remote nodes
	*/
	rsc_avoids_remote_nodes(rsc);
	}
	g_list_free(allowed_nodes);
	}

	/*!
	* \internal
	* \brief Apply a colocation's score to node scores or resource priority
	*
	* Given a colocation constraint, apply its score to the dependent's
	* allowed node scores (if we are still placing resources) or priority (if
	* we are choosing promotable clone instance roles).
	*
	* \param[in,out] dependent Dependent resource in colocation
	* \param[in] primary Primary resource in colocation
	* \param[in] colocation Colocation constraint to apply
	* \param[in] for_dependent true if called on behalf of dependent
	*/
	void
	pcmk__primitive_apply_coloc_score(pcmk_resource_t *dependent,
	const pcmk_resource_t *primary,
	const pcmk__colocation_t *colocation,
	bool for_dependent)
	{
	enum pcmk__coloc_affects filter_results;

	CRM_ASSERT((dependent != NULL) && (primary != NULL)
	&& (colocation != NULL));

	if (for_dependent) {
	// Always process on behalf of primary resource
	primary->cmds->apply_coloc_score(dependent, primary, colocation, false);
	return;
	}

	filter_results = pcmk__colocation_affects(dependent, primary, colocation,
	false);
	pcmk__rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
	((colocation->score > 0)? "Colocating" : "Anti-colocating"),
	dependent->id, primary->id, colocation->id,
	colocation->score,
	filter_results);

	switch (filter_results) {
	case pcmk__coloc_affects_role:
	pcmk__apply_coloc_to_priority(dependent, primary, colocation);
	break;
	case pcmk__coloc_affects_location:
	pcmk__apply_coloc_to_scores(dependent, primary, colocation);
	break;
	default: // pcmk__coloc_affects_nothing
	return;
	}
	}

	/* Primitive implementation of
	* pcmk_assignment_methods_t:with_this_colocations()
	*/
	void
	pcmk__with_primitive_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons_lhs; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_with_this(list, colocation, orig_rsc);
	}
	}
	}
	}

	/* Primitive implementation of
	* pcmk_assignment_methods_t:this_with_colocations()
	*/
	void
	pcmk__primitive_with_colocations(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (list != NULL));

	if (rsc == orig_rsc) {
	/* For the resource itself, add all of its own colocations and relevant
	* colocations from its parent (if any).
	*/
	pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
	if (rsc->parent != NULL) {
	rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc, list);
	}
	} else {
	// For an ancestor, add only explicitly configured constraints
	for (GList *iter = rsc->rsc_cons; iter != NULL; iter = iter->next) {
	pcmk__colocation_t *colocation = iter->data;

	if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
	pcmk__add_this_with(list, colocation, orig_rsc);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Return action flags for a given primitive resource action
	*
	* \param[in,out] action Action to get flags for
	* \param[in] node If not NULL, limit effects to this node (ignored)
	*
	* \return Flags appropriate to \p action on \p node
	*/
	uint32_t
	pcmk__primitive_action_flags(pcmk_action_t action, const pcmk_node_t node)
	{
	CRM_ASSERT(action != NULL);
	return (uint32_t) action->flags;
	}

	/*!
	* \internal
	* \brief Check whether a node is a multiply active resource's expected node
	*
	* \param[in] rsc Resource to check
	* \param[in] node Node to check
	*
	* \return \c true if \p rsc is multiply active with
	* \c PCMK_META_MULTIPLE_ACTIVE set to \c stop_unexpected, and \p node
	* is the node where it will remain active
	* \note This assumes that the resource's next role cannot be changed to stopped
	* after this is called, which should be reasonable if status has already
	* been unpacked and resources have been assigned to nodes.
	*/
	static bool
	is_expected_node(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	return pcmk_all_flags_set(rsc->flags,
	pcmk_rsc_stop_unexpected\|pcmk_rsc_restarting)
	&& (rsc->next_role > pcmk_role_stopped)
	&& pe__same_node(rsc->allocated_to, node);
	}

	/*!
	* \internal
	* \brief Schedule actions needed to stop a resource wherever it is active
	*
	* \param[in,out] rsc Resource being stopped
	* \param[in] node Node where resource is being stopped (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	stop_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t current = (pcmk_node_t ) iter->data;
	pcmk_action_t *stop = NULL;

	if (is_expected_node(rsc, current)) {
	/* We are scheduling restart actions for a multiply active resource
	* with PCMK_META_MULTIPLE_ACTIVE=stop_unexpected, and this is where
	* it should not be stopped.
	*/
	pcmk__rsc_trace(rsc,
	"Skipping stop of multiply active resource %s "
	"on expected node %s",
	rsc->id, pe__node_name(current));
	continue;
	}

	if (rsc->partial_migration_target != NULL) {
	// Continue migration if node originally was and remains target
	if (pe__same_node(current, rsc->partial_migration_target)
	&& pe__same_node(current, rsc->allocated_to)) {
	pcmk__rsc_trace(rsc,
	"Skipping stop of %s on %s "
	"because partial migration there will continue",
	rsc->id, pe__node_name(current));
	continue;
	} else {
	pcmk__rsc_trace(rsc,
	"Forcing stop of %s on %s "
	"because migration target changed",
	rsc->id, pe__node_name(current));
	optional = false;
	}
	}

	pcmk__rsc_trace(rsc, "Scheduling stop of %s on %s",
	rsc->id, pe__node_name(current));
	stop = stop_action(rsc, current, optional);

	if (rsc->allocated_to == NULL) {
	pe_action_set_reason(stop, "node availability", true);
	} else if (pcmk_all_flags_set(rsc->flags, pcmk_rsc_restarting
	\|pcmk_rsc_stop_unexpected)) {
	/* We are stopping a multiply active resource on a node that is
	* not its expected node, and we are still scheduling restart
	* actions, so the stop is for being multiply active.
	*/
	pe_action_set_reason(stop, "being multiply active", true);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__clear_action_flags(stop, pcmk_action_runnable);
	}

	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_remove_after_stop)) {
	pcmk__schedule_cleanup(rsc, current, optional);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_unfencing)) {
	pcmk_action_t *unfence = pe_fence_op(current, PCMK_ACTION_ON, true,
	NULL, false, rsc->cluster);

	order_actions(stop, unfence, pcmk__ar_then_implies_first);
	if (!pcmk__node_unfenced(current)) {
	pcmk__sched_err("Stopping %s until %s can be unfenced",
	rsc->id, pe__node_name(current));
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to start a resource on a node
	*
	* \param[in,out] rsc Resource being started
	* \param[in,out] node Node where resource should be started
	* \param[in] optional Whether actions should be optional
	*/
	static void
	start_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	pcmk_action_t *start = NULL;

	CRM_ASSERT(node != NULL);

	pcmk__rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(node), node->weight);
	start = start_action(rsc, node, TRUE);

	pcmk__order_vs_unfence(rsc, node, start, pcmk__ar_first_implies_then);

	if (pcmk_is_set(start->flags, pcmk_action_runnable) && !optional) {
	pcmk__clear_action_flags(start, pcmk_action_optional);
	}

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the start becomes necessary for other
	* reasons later.
	*/
	pcmk__rsc_trace(rsc,
	"Start of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pe__node_name(node));
	pcmk__set_action_flags(start, pcmk_action_pseudo);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to promote a resource on a node
	*
	* \param[in,out] rsc Resource being promoted
	* \param[in] node Node where resource should be promoted
	* \param[in] optional Whether actions should be optional
	*/
	static void
	promote_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	GList *iter = NULL;
	GList *action_list = NULL;
	bool runnable = true;

	CRM_ASSERT(node != NULL);

	// Any start must be runnable for promotion to be runnable
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_START, true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pcmk_action_t start = (pcmk_action_t ) iter->data;

	if (!pcmk_is_set(start->flags, pcmk_action_runnable)) {
	runnable = false;
	}
	}
	g_list_free(action_list);

	if (runnable) {
	pcmk_action_t *promote = promote_action(rsc, node, optional);

	pcmk__rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(node));

	if (is_expected_node(rsc, node)) {
	/* This could be a problem if the promote becomes necessary for
	* other reasons later.
	*/
	pcmk__rsc_trace(rsc,
	"Promotion of multiply active resouce %s "
	"on expected node %s will be a pseudo-action",
	rsc->id, pe__node_name(node));
	pcmk__set_action_flags(promote, pcmk_action_pseudo);
	}
	} else {
	pcmk__rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
	rsc->id, pe__node_name(node));
	action_list = pe__resource_actions(rsc, node, PCMK_ACTION_PROMOTE,
	true);
	for (iter = action_list; iter != NULL; iter = iter->next) {
	pcmk_action_t promote = (pcmk_action_t ) iter->data;

	pcmk__clear_action_flags(promote, pcmk_action_runnable);
	}
	g_list_free(action_list);
	}
	}

	/*!
	* \internal
	* \brief Schedule actions needed to demote a resource wherever it is active
	*
	* \param[in,out] rsc Resource being demoted
	* \param[in] node Node where resource should be demoted (ignored)
	* \param[in] optional Whether actions should be optional
	*/
	static void
	demote_resource(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	/* Since this will only be called for a primitive (possibly as an instance
	* of a collective resource), the resource is multiply active if it is
	* running on more than one node, so we want to demote on all of them as
	* part of recovery, regardless of which one is the desired node.
	*/
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t current = (pcmk_node_t ) iter->data;

	if (is_expected_node(rsc, current)) {
	pcmk__rsc_trace(rsc,
	"Skipping demote of multiply active resource %s "
	"on expected node %s",
	rsc->id, pe__node_name(current));
	} else {
	pcmk__rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
	(optional? "optional" : "required"), rsc->id,
	pe__node_name(current));
	demote_action(rsc, current, optional);
	}
	}
	}

	static void
	assert_role_error(pcmk_resource_t rsc, pcmk_node_t node, bool optional)
	{
	CRM_ASSERT(false);
	}

	/*!
	* \internal
	* \brief Schedule cleanup of a resource
	*
	* \param[in,out] rsc Resource to clean up
	* \param[in] node Node to clean up on
	* \param[in] optional Whether clean-up should be optional
	*/
	void
	pcmk__schedule_cleanup(pcmk_resource_t rsc, const pcmk_node_t node,
	bool optional)
	{
	/* If the cleanup is required, its orderings are optional, because they're
	* relevant only if both actions are required. Conversely, if the cleanup is
	* optional, the orderings make the then action required if the first action
	* becomes required.
	*/
	uint32_t flag = optional? pcmk__ar_first_implies_then : pcmk__ar_ordered;

	CRM_CHECK((rsc != NULL) && (node != NULL), return);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
	rsc->id, pe__node_name(node));
	return;
	}

	if (node->details->unclean \|\| !node->details->online) {
	pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
	rsc->id, pe__node_name(node));
	return;
	}

	crm_notice("Scheduling clean-up of %s on %s", rsc->id, pe__node_name(node));
	delete_action(rsc, node, optional);

	// stop -> clean-up -> start
	pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
	rsc, PCMK_ACTION_DELETE, flag);
	pcmk__order_resource_actions(rsc, PCMK_ACTION_DELETE,
	rsc, PCMK_ACTION_START, flag);
	}

	/*!
	* \internal
	* \brief Add primitive meta-attributes relevant to graph actions to XML
	*
	* \param[in] rsc Primitive resource whose meta-attributes should be added
	* \param[in,out] xml Transition graph action attributes XML to add to
	*/
	void
	pcmk__primitive_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	char *name = NULL;
	char *value = NULL;
	const pcmk_resource_t *parent = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (xml != NULL));

	/* Clone instance numbers get set internally as meta-attributes, and are
	* needed in the transition graph (for example, to tell unique clone
	* instances apart).
	*/
	value = g_hash_table_lookup(rsc->meta, PCMK__META_CLONE_INSTANCE_NUM);
	if (value != NULL) {
	name = crm_meta_name(PCMK__META_CLONE_INSTANCE_NUM);
	crm_xml_add(xml, name, value);
	free(name);
	}

	// Not sure if this one is really needed ...
	value = g_hash_table_lookup(rsc->meta, PCMK_META_REMOTE_NODE);
	if (value != NULL) {
	name = crm_meta_name(PCMK_META_REMOTE_NODE);
	crm_xml_add(xml, name, value);
	free(name);
	}

	/* The container meta-attribute can be set on the primitive itself or one of
	* its parents (for example, a group inside a container resource), so check
	* them all, and keep the highest one found.
	*/
	for (parent = rsc; parent != NULL; parent = parent->parent) {
	if (parent->container != NULL) {
	crm_xml_add(xml, CRM_META "_" PCMK__META_CONTAINER,
	parent->container->id);
	}
	}

	/* Bundle replica children will get their external-ip set internally as a
	* meta-attribute. The graph action needs it, but under a different naming
	* convention than other meta-attributes.
	*/
	value = g_hash_table_lookup(rsc->meta, "external-ip");
	if (value != NULL) {
	crm_xml_add(xml, "pcmk_external_ip", value);
	}
	}

	// Primitive implementation of pcmk_assignment_methods_t:add_utilization()
	void
	pcmk__primitive_add_utilization(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList all_rscs, GHashTable utilization)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
	&& (orig_rsc != NULL) && (utilization != NULL));

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
	return;
	}

	pcmk__rsc_trace(orig_rsc,
	"%s: Adding primitive %s as colocated utilization",
	orig_rsc->id, rsc->id);
	pcmk__release_node_capacity(utilization, rsc);
	}

	/*!
	* \internal
	* \brief Get epoch time of node's shutdown attribute (or now if none)
	*
	* \param[in,out] node Node to check
	*
	* \return Epoch time corresponding to shutdown attribute if set or now if not
	*/
	static time_t
	shutdown_time(pcmk_node_t *node)
	{
	const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
	time_t result = 0;

	if (shutdown != NULL) {
	long long result_ll;

	if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
	result = (time_t) result_ll;
	}
	}
	return (result == 0)? get_effective_time(node->details->data_set) : result;
	}

	/*!
	* \internal
	* \brief Ban a resource from a node if it's not locked to the node
	*
	* \param[in] data Node to check
	* \param[in,out] user_data Resource to check
	*/
	static void
	ban_if_not_locked(gpointer data, gpointer user_data)
	{
	const pcmk_node_t node = (const pcmk_node_t ) data;
	pcmk_resource_t rsc = (pcmk_resource_t ) user_data;

	if (strcmp(node->details->uname, rsc->lock_node->details->uname) != 0) {
	resource_location(rsc, node, -CRM_SCORE_INFINITY,
	PCMK_OPT_SHUTDOWN_LOCK, rsc->cluster);
	}
	}

	// Primitive implementation of pcmk_assignment_methods_t:shutdown_lock()
	void
	pcmk__primitive_shutdown_lock(pcmk_resource_t *rsc)
	{
	const char *class = NULL;

	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));

	- class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ class = crm_element_value(rsc->xml, PCMK_XA_CLASS);

	// Fence devices and remote connections can't be locked
	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
	\|\| pe__resource_is_remote_conn(rsc)) {
	return;
	}

	if (rsc->lock_node != NULL) {
	// The lock was obtained from resource history

	if (rsc->running_on != NULL) {
	/* The resource was started elsewhere even though it is now
	* considered locked. This shouldn't be possible, but as a
	* failsafe, we don't want to disturb the resource now.
	*/
	pcmk__rsc_info(rsc,
	"Cancelling shutdown lock "
	"because %s is already active", rsc->id);
	pe__clear_resource_history(rsc, rsc->lock_node);
	rsc->lock_node = NULL;
	rsc->lock_time = 0;
	}

	// Only a resource active on exactly one node can be locked
	} else if (pcmk__list_of_1(rsc->running_on)) {
	pcmk_node_t *node = rsc->running_on->data;

	if (node->details->shutdown) {
	if (node->details->unclean) {
	pcmk__rsc_debug(rsc,
	"Not locking %s to unclean %s for shutdown",
	rsc->id, pe__node_name(node));
	} else {
	rsc->lock_node = node;
	rsc->lock_time = shutdown_time(node);
	}
	}
	}

	if (rsc->lock_node == NULL) {
	// No lock needed
	return;
	}

	if (rsc->cluster->shutdown_lock > 0) {
	time_t lock_expiration = rsc->lock_time + rsc->cluster->shutdown_lock;

	pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
	rsc->id, pe__node_name(rsc->lock_node),
	(long long) lock_expiration);
	pe__update_recheck_time(++lock_expiration, rsc->cluster,
	"shutdown lock expiration");
	} else {
	pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown",
	rsc->id, pe__node_name(rsc->lock_node));
	}

	// If resource is locked to one node, ban it from all other nodes
	g_list_foreach(rsc->cluster->nodes, ban_if_not_locked, rsc);
	}
	diff --git a/lib/pacemaker/pcmk_sched_probes.c b/lib/pacemaker/pcmk_sched_probes.c
	index d80795daed..7ec9bb285d 100644
	--- a/lib/pacemaker/pcmk_sched_probes.c
	+++ b/lib/pacemaker/pcmk_sched_probes.c
	@@ -1,904 +1,904 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Add the expected result to a newly created probe
	*
	* \param[in,out] probe Probe action to add expected result to
	* \param[in] rsc Resource that probe is for
	* \param[in] node Node that probe will run on
	*/
	static void
	add_expected_result(pcmk_action_t probe, const pcmk_resource_t rsc,
	const pcmk_node_t *node)
	{
	// Check whether resource is currently active on node
	pcmk_node_t *running = pe_find_node_id(rsc->running_on, node->details->id);

	// The expected result is what we think the resource's current state is
	if (running == NULL) {
	pe__add_action_expected_result(probe, CRM_EX_NOT_RUNNING);

	} else if (rsc->role == pcmk_role_promoted) {
	pe__add_action_expected_result(probe, CRM_EX_PROMOTED);
	}
	}

	/*!
	* \internal
	* \brief Create any needed robes on a node for a list of resources
	*
	* \param[in,out] rscs List of resources to create probes for
	* \param[in,out] node Node to create probes on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_resource_list(GList rscs, pcmk_node_t node)
	{
	bool any_created = false;

	for (GList *iter = rscs; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->cmds->create_probe(rsc, node)) {
	any_created = true;
	}
	}
	return any_created;
	}

	/*!
	* \internal
	* \brief Order one resource's start after another's start-up probe
	*
	* \param[in,out] rsc1 Resource that might get start-up probe
	* \param[in] rsc2 Resource that might be started
	*/
	static void
	probe_then_start(pcmk_resource_t rsc1, pcmk_resource_t rsc2)
	{
	if ((rsc1->allocated_to != NULL)
	&& (g_hash_table_lookup(rsc1->known_on,
	rsc1->allocated_to->details->id) == NULL)) {

	pcmk__new_ordering(rsc1,
	pcmk__op_key(rsc1->id, PCMK_ACTION_MONITOR, 0),
	NULL,
	rsc2, pcmk__op_key(rsc2->id, PCMK_ACTION_START, 0),
	NULL,
	pcmk__ar_ordered, rsc1->cluster);
	}
	}

	/*!
	* \internal
	* \brief Check whether a guest resource will stop
	*
	* \param[in] node Guest node to check
	*
	* \return true if guest resource will likely stop, otherwise false
	*/
	static bool
	guest_resource_will_stop(const pcmk_node_t *node)
	{
	const pcmk_resource_t *guest_rsc = node->details->remote_rsc->container;

	/* Ideally, we'd check whether the guest has a required stop, but that
	* information doesn't exist yet, so approximate it ...
	*/
	return node->details->remote_requires_reset
	\|\| node->details->unclean
	\|\| pcmk_is_set(guest_rsc->flags, pcmk_rsc_failed)
	\|\| (guest_rsc->next_role == pcmk_role_stopped)

	// Guest is moving
	\|\| ((guest_rsc->role > pcmk_role_stopped)
	&& (guest_rsc->allocated_to != NULL)
	&& (pe_find_node(guest_rsc->running_on,
	guest_rsc->allocated_to->details->uname) == NULL));
	}

	/*!
	* \internal
	* \brief Create a probe action for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return Newly created probe action
	*/
	static pcmk_action_t *
	probe_action(pcmk_resource_t rsc, pcmk_node_t node)
	{
	pcmk_action_t *probe = NULL;
	char *key = pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0);

	crm_debug("Scheduling probe of %s %s on %s",
	role2text(rsc->role), rsc->id, pe__node_name(node));

	probe = custom_action(rsc, key, PCMK_ACTION_MONITOR, node, FALSE,
	rsc->cluster);
	pcmk__clear_action_flags(probe, pcmk_action_optional);

	pcmk__order_vs_unfence(rsc, node, probe, pcmk__ar_ordered);
	add_expected_result(probe, rsc, node);
	return probe;
	}

	/*!
	* \internal
	* \brief Create probes for a resource on a node, if needed
	*
	* \brief Schedule any probes needed for a resource on a node
	*
	* \param[in,out] rsc Resource to create probe for
	* \param[in,out] node Node to create probe on
	*
	* \return true if any probe was created, otherwise false
	*/
	bool
	pcmk__probe_rsc_on_node(pcmk_resource_t rsc, pcmk_node_t node)
	{
	uint32_t flags = pcmk__ar_ordered;
	pcmk_action_t *probe = NULL;
	pcmk_node_t *allowed = NULL;
	pcmk_resource_t *top = uber_parent(rsc);
	const char *reason = NULL;

	CRM_ASSERT((rsc != NULL) && (node != NULL));

	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_probe_resources)) {
	reason = "start-up probes are disabled";
	goto no_probe;
	}

	if (pe__is_guest_or_remote_node(node)) {
	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_none)) {
	reason = "Pacemaker Remote nodes cannot run stonith agents";
	goto no_probe;

	} else if (pe__is_guest_node(node)
	&& pe__resource_contains_guest_node(rsc->cluster, rsc)) {
	reason = "guest nodes cannot run resources containing guest nodes";
	goto no_probe;

	} else if (rsc->is_remote_node) {
	reason = "Pacemaker Remote nodes cannot host remote connections";
	goto no_probe;
	}
	}

	// If this is a collective resource, probes are created for its children
	if (rsc->children != NULL) {
	return pcmk__probe_resource_list(rsc->children, node);
	}

	if ((rsc->container != NULL) && !rsc->is_remote_node) {
	reason = "resource is inside a container";
	goto no_probe;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	reason = "resource is orphaned";
	goto no_probe;

	} else if (g_hash_table_lookup(rsc->known_on, node->details->id) != NULL) {
	reason = "resource state is already known";
	goto no_probe;
	}

	allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (rsc->exclusive_discover \|\| top->exclusive_discover) {
	// Exclusive discovery is enabled ...

	if (allowed == NULL) {
	// ... but this node is not allowed to run the resource
	reason = "resource has exclusive discovery but is not allowed "
	"on node";
	goto no_probe;

	} else if (allowed->rsc_discover_mode != pcmk_probe_exclusive) {
	// ... but no constraint marks this node for discovery of resource
	reason = "resource has exclusive discovery but is not enabled "
	"on node";
	goto no_probe;
	}
	}

	if (allowed == NULL) {
	allowed = node;
	}
	if (allowed->rsc_discover_mode == pcmk_probe_never) {
	reason = "node has discovery disabled";
	goto no_probe;
	}

	if (pe__is_guest_node(node)) {
	pcmk_resource_t *guest = node->details->remote_rsc->container;

	if (guest->role == pcmk_role_stopped) {
	// The guest is stopped, so we know no resource is active there
	reason = "node's guest is stopped";
	probe_then_start(guest, top);
	goto no_probe;

	} else if (guest_resource_will_stop(node)) {
	reason = "node's guest will stop";

	// Order resource start after guest stop (in case it's restarting)
	pcmk__new_ordering(guest,
	pcmk__op_key(guest->id, PCMK_ACTION_STOP, 0),
	NULL, top,
	pcmk__op_key(top->id, PCMK_ACTION_START, 0),
	NULL, pcmk__ar_ordered, rsc->cluster);
	goto no_probe;
	}
	}

	// We've eliminated all cases where a probe is not needed, so now it is
	probe = probe_action(rsc, node);

	/* Below, we will order the probe relative to start or reload. If this is a
	* clone instance, the start or reload is for the entire clone rather than
	* just the instance. Otherwise, the start or reload is for the resource
	* itself.
	*/
	if (!pe_rsc_is_clone(top)) {
	top = rsc;
	}

	/* Prevent a start if the resource can't be probed, but don't cause the
	* resource or entire clone to stop if already active.
	*/
	if (!pcmk_is_set(probe->flags, pcmk_action_runnable)
	&& (top->running_on == NULL)) {
	pcmk__set_relation_flags(flags, pcmk__ar_unrunnable_first_blocks);
	}

	// Start or reload after probing the resource
	pcmk__new_ordering(rsc, NULL, probe,
	top, pcmk__op_key(top->id, PCMK_ACTION_START, 0), NULL,
	flags, rsc->cluster);
	pcmk__new_ordering(rsc, NULL, probe, top, reload_key(rsc), NULL,
	pcmk__ar_ordered, rsc->cluster);

	return true;

	no_probe:
	pcmk__rsc_trace(rsc,
	"Skipping probe for %s on %s because %s",
	rsc->id, node->details->id, reason);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a probe should be ordered before another action
	*
	* \param[in] probe Probe action to check
	* \param[in] then Other action to check
	*
	* \return true if \p probe should be ordered before \p then, otherwise false
	*/
	static bool
	probe_needed_before_action(const pcmk_action_t *probe,
	const pcmk_action_t *then)
	{
	// Probes on a node are performed after unfencing it, not before
	if (pcmk__str_eq(then->task, PCMK_ACTION_STONITH, pcmk__str_none)
	&& pe__same_node(probe->node, then->node)) {
	const char *op = g_hash_table_lookup(then->meta, "stonith_action");

	if (pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	return false;
	}
	}

	// Probes should be done on a node before shutting it down
	if (pcmk__str_eq(then->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)
	&& (probe->node != NULL) && (then->node != NULL)
	&& !pe__same_node(probe->node, then->node)) {
	return false;
	}

	// Otherwise probes should always be done before any other action
	return true;
	}

	/*!
	* \internal
	* \brief Add implicit "probe then X" orderings for "stop then X" orderings
	*
	* If the state of a resource is not known yet, a probe will be scheduled,
	* expecting a "not running" result. If the probe fails, a stop will not be
	* scheduled until the next transition. Thus, if there are ordering constraints
	* like "stop this resource then do something else that's not for the same
	* resource", add implicit "probe this resource then do something" equivalents
	* so the relation is upheld until we know whether a stop is needed.
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	add_probe_orderings_for_stops(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->ordering_constraints; iter != NULL;
	iter = iter->next) {

	pcmk__action_relation_t *order = iter->data;
	uint32_t order_flags = pcmk__ar_ordered;
	GList *probes = NULL;
	GList *then_actions = NULL;
	pcmk_action_t *first = NULL;
	pcmk_action_t *then = NULL;

	// Skip disabled orderings
	if (order->flags == pcmk__ar_none) {
	continue;
	}

	// Skip non-resource orderings, and orderings for the same resource
	if ((order->rsc1 == NULL) \|\| (order->rsc1 == order->rsc2)) {
	continue;
	}

	// Skip invalid orderings (shouldn't be possible)
	first = order->action1;
	then = order->action2;
	if (((first == NULL) && (order->task1 == NULL))
	\|\| ((then == NULL) && (order->task2 == NULL))) {
	continue;
	}

	// Skip orderings for first actions other than stop
	if ((first != NULL) && !pcmk__str_eq(first->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((first == NULL)
	&& !pcmk__ends_with(order->task1,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}

	/* Do not imply a probe ordering for a resource inside of a stopping
	* container. Otherwise, it might introduce a transition loop, since a
	* probe could be scheduled after the container starts again.
	*/
	if ((order->rsc2 != NULL) && (order->rsc1->container == order->rsc2)) {

	if ((then != NULL) && pcmk__str_eq(then->task, PCMK_ACTION_STOP,
	pcmk__str_none)) {
	continue;
	} else if ((then == NULL)
	&& pcmk__ends_with(order->task2,
	"_" PCMK_ACTION_STOP "_0")) {
	continue;
	}
	}

	// Preserve certain order options for future filtering
	if (pcmk_is_set(order->flags, pcmk__ar_if_first_unmigratable)) {
	pcmk__set_relation_flags(order_flags,
	pcmk__ar_if_first_unmigratable);
	}
	if (pcmk_is_set(order->flags, pcmk__ar_if_on_same_node)) {
	pcmk__set_relation_flags(order_flags, pcmk__ar_if_on_same_node);
	}

	// Preserve certain order types for future filtering
	if ((order->flags == pcmk__ar_if_required_on_same_node)
	\|\| (order->flags == pcmk__ar_if_on_same_node_or_target)) {
	order_flags = order->flags;
	}

	// List all scheduled probes for the first resource
	probes = pe__resource_actions(order->rsc1, NULL, PCMK_ACTION_MONITOR,
	FALSE);
	if (probes == NULL) { // There aren't any
	continue;
	}

	// List all relevant "then" actions
	if (then != NULL) {
	then_actions = g_list_prepend(NULL, then);

	} else if (order->rsc2 != NULL) {
	then_actions = find_actions(order->rsc2->actions, order->task2,
	NULL);
	if (then_actions == NULL) { // There aren't any
	g_list_free(probes);
	continue;
	}
	}

	crm_trace("Implying 'probe then' orderings for '%s then %s' "
	"(id=%d, type=%.6x)",
	((first == NULL)? order->task1 : first->uuid),
	((then == NULL)? order->task2 : then->uuid),
	order->id, order->flags);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	for (GList *then_iter = then_actions; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk_action_t then = (pcmk_action_t ) then_iter->data;

	if (probe_needed_before_action(probe, then)) {
	order_actions(probe, then, order_flags);
	}
	}
	}

	g_list_free(then_actions);
	g_list_free(probes);
	}
	}

	/*!
	* \internal
	* \brief Add necessary orderings between probe and starts of clone instances
	*
	* , in additon to the ordering with the parent resource added upon creating
	* the probe.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action wrapper in the ordering
	*/
	static void
	add_start_orderings_for_probe(pcmk_action_t *probe,
	pcmk__related_action_t *after)
	{
	uint32_t flags = pcmk__ar_ordered\|pcmk__ar_unrunnable_first_blocks;

	/* Although the ordering between the probe of the clone instance and the
	* start of its parent has been added in pcmk__probe_rsc_on_node(), we
	* avoided enforcing `pcmk__ar_unrunnable_first_blocks` order type for that
	* as long as any of the clone instances are running to prevent them from
	* being unexpectedly stopped.
	*
	* On the other hand, we still need to prevent any inactive instances from
	* starting unless the probe is runnable so that we don't risk starting too
	* many instances before we know the state on all nodes.
	*/
	if ((after->action->rsc->variant <= pcmk_rsc_variant_group)
	\|\| pcmk_is_set(probe->flags, pcmk_action_runnable)
	// The order type is already enforced for its parent.
	\|\| pcmk_is_set(after->type, pcmk__ar_unrunnable_first_blocks)
	\|\| (pe__const_top_resource(probe->rsc, false) != after->action->rsc)
	\|\| !pcmk__str_eq(after->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	return;
	}

	crm_trace("Adding probe start orderings for 'unrunnable %s@%s "
	"then instances of %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->action->uuid, pe__node_name(after->action->node));

	for (GList *then_iter = after->action->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	if (then->action->rsc->running_on
	\|\| (pe__const_top_resource(then->action->rsc, false)
	!= after->action->rsc)
	\|\| !pcmk__str_eq(then->action->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	continue;
	}

	crm_trace("Adding probe start ordering for 'unrunnable %s@%s "
	"then %s@%s' (type=%#.6x)",
	probe->uuid, pe__node_name(probe->node),
	then->action->uuid, pe__node_name(then->action->node), flags);

	/* Prevent the instance from starting if the instance can't, but don't
	* cause any other intances to stop if already active.
	*/
	order_actions(probe, then->action, flags);
	}

	return;
	}

	/*!
	* \internal
	* \brief Order probes before restarts and re-promotes
	*
	* If a given ordering is a "probe then start" or "probe then promote" ordering,
	* add an implicit "probe then stop/demote" ordering in case the action is part
	* of a restart/re-promote, and do the same recursively for all actions ordered
	* after the "then" action.
	*
	* \param[in,out] probe Probe as 'first' action in an ordering
	* \param[in,out] after 'then' action in the ordering
	*/
	static void
	add_restart_orderings_for_probe(pcmk_action_t probe, pcmk_action_t after)
	{
	GList *iter = NULL;
	bool interleave = false;
	pcmk_resource_t *compatible_rsc = NULL;

	// Validate that this is a resource probe followed by some action
	if ((after == NULL) \|\| (probe == NULL) \|\| (probe->rsc == NULL)
	\|\| (probe->rsc->variant != pcmk_rsc_variant_primitive)
	\|\| !pcmk__str_eq(probe->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	return;
	}

	// Avoid running into any possible loop
	if (pcmk_is_set(after->flags, pcmk_action_detect_loop)) {
	return;
	}
	pcmk__set_action_flags(after, pcmk_action_detect_loop);

	crm_trace("Adding probe restart orderings for '%s@%s then %s@%s'",
	probe->uuid, pe__node_name(probe->node),
	after->uuid, pe__node_name(after->node));

	/* Add restart orderings if "then" is for a different primitive.
	* Orderings for collective resources will be added later.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant == pcmk_rsc_variant_primitive)
	&& (probe->rsc != after->rsc)) {

	GList *then_actions = NULL;

	if (pcmk__str_eq(after->task, PCMK_ACTION_START, pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_STOP, FALSE);

	} else if (pcmk__str_eq(after->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	then_actions = pe__resource_actions(after->rsc, NULL,
	PCMK_ACTION_DEMOTE, FALSE);
	}

	for (iter = then_actions; iter != NULL; iter = iter->next) {
	pcmk_action_t then = (pcmk_action_t ) iter->data;

	// Skip pseudo-actions (for example, those implied by fencing)
	if (!pcmk_is_set(then->flags, pcmk_action_pseudo)) {
	order_actions(probe, then, pcmk__ar_ordered);
	}
	}
	g_list_free(then_actions);
	}

	/* Detect whether "then" is an interleaved clone action. For these, we want
	* to add orderings only for the relevant instance.
	*/
	if ((after->rsc != NULL)
	&& (after->rsc->variant > pcmk_rsc_variant_group)) {
	const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
	PCMK_META_INTERLEAVE);

	interleave = crm_is_true(interleave_s);
	if (interleave) {
	compatible_rsc = pcmk__find_compatible_instance(probe->rsc,
	after->rsc,
	pcmk_role_unknown,
	false);
	}
	}

	/* Now recursively do the same for all actions ordered after "then". This
	* also handles collective resources since the collective action will be
	* ordered before its individual instances' actions.
	*/
	for (iter = after->actions_after; iter != NULL; iter = iter->next) {
	pcmk__related_action_t *after_wrapper = iter->data;

	/* pcmk__ar_first_implies_then is the reason why a required A.start
	* implies/enforces B.start to be required too, which is the cause of
	* B.restart/re-promote.
	*
	* Not sure about pcmk__ar_first_implies_same_node_then though. It's now
	* only used for unfencing case, which tends to introduce transition
	* loops...
	*/
	if (!pcmk_is_set(after_wrapper->type, pcmk__ar_first_implies_then)) {
	/* The order type between a group/clone and its child such as
	* B.start-> B_child.start is:
	* pcmk__ar_then_implies_first_graphed
	* \|pcmk__ar_unrunnable_first_blocks
	*
	* Proceed through the ordering chain and build dependencies with
	* its children.
	*/
	if ((after->rsc == NULL)
	\|\| (after->rsc->variant < pcmk_rsc_variant_group)
	\|\| (probe->rsc->parent == after->rsc)
	\|\| (after_wrapper->action->rsc == NULL)
	\|\| (after_wrapper->action->rsc->variant > pcmk_rsc_variant_group)
	\|\| (after->rsc != after_wrapper->action->rsc->parent)) {
	continue;
	}

	/* Proceed to the children of a group or a non-interleaved clone.
	* For an interleaved clone, proceed only to the relevant child.
	*/
	if ((after->rsc->variant > pcmk_rsc_variant_group) && interleave
	&& ((compatible_rsc == NULL)
	\|\| (compatible_rsc != after_wrapper->action->rsc))) {
	continue;
	}
	}

	crm_trace("Recursively adding probe restart orderings for "
	"'%s@%s then %s@%s' (type=%#.6x)",
	after->uuid, pe__node_name(after->node),
	after_wrapper->action->uuid,
	pe__node_name(after_wrapper->action->node),
	after_wrapper->type);

	add_restart_orderings_for_probe(probe, after_wrapper->action);
	}
	}

	/*!
	* \internal
	* \brief Clear the tracking flag on all scheduled actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	clear_actions_tracking_flag(pcmk_scheduler_t *scheduler)
	{
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t *action = iter->data;

	pcmk__clear_action_flags(action, pcmk_action_detect_loop);
	}
	}

	/*!
	* \internal
	* \brief Add start and restart orderings for probes scheduled for a resource
	*
	* \param[in,out] data Resource whose probes should be ordered
	* \param[in] user_data Unused
	*/
	static void
	add_start_restart_orderings_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	GList *probes = NULL;

	// For collective resources, order each instance recursively
	if (rsc->variant != pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, add_start_restart_orderings_for_rsc,
	NULL);
	return;
	}

	// Find all probes for given resource
	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	// Add probe restart orderings for each probe found
	for (GList *iter = probes; iter != NULL; iter = iter->next) {
	pcmk_action_t probe = (pcmk_action_t ) iter->data;

	for (GList *then_iter = probe->actions_after; then_iter != NULL;
	then_iter = then_iter->next) {

	pcmk__related_action_t *then = then_iter->data;

	add_start_orderings_for_probe(probe, then);
	add_restart_orderings_for_probe(probe, then->action);
	clear_actions_tracking_flag(rsc->cluster);
	}
	}

	g_list_free(probes);
	}

	/*!
	* \internal
	* \brief Add "A then probe B" orderings for "A then B" orderings
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This function is currently disabled (see next comment).
	*/
	static void
	order_then_probes(pcmk_scheduler_t *scheduler)
	{
	#if 0
	/* Given an ordering "A then B", we would prefer to wait for A to be started
	* before probing B.
	*
	* For example, if A is a filesystem which B can't even run without, it
	* would be helpful if the author of B's agent could assume that A is
	* running before B.monitor will be called.
	*
	* However, we can't _only_ probe after A is running, otherwise we wouldn't
	* detect the state of B if A could not be started. We can't even do an
	* opportunistic version of this, because B may be moving:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> B.start
	*
	* and if we add B.stop -> A.stop here, we get a loop:
	*
	* A.stop -> A.start -> B.probe -> B.stop -> A.stop
	*
	* We could kill the "B.probe -> B.stop" dependency, but that could mean
	* stopping B "too" soon, because B.start must wait for the probe, and
	* we don't want to stop B if we can't start it.
	*
	* We could add the ordering only if A is an anonymous clone with
	* clone-max == node-max (since we'll never be moving it). However, we could
	* still be stopping one instance at the same time as starting another.
	*
	* The complexity of checking for allowed conditions combined with the ever
	* narrowing use case suggests that this code should remain disabled until
	* someone gets smarter.
	*/
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	pcmk_action_t *start = NULL;
	GList *actions = NULL;
	GList *probes = NULL;

	actions = pe__resource_actions(rsc, NULL, PCMK_ACTION_START, FALSE);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if (start == NULL) {
	crm_debug("No start action for %s", rsc->id);
	continue;
	}

	probes = pe__resource_actions(rsc, NULL, PCMK_ACTION_MONITOR, FALSE);

	for (actions = start->actions_before; actions != NULL;
	actions = actions->next) {

	pcmk__related_action_t *before = actions->data;

	pcmk_action_t *first = before->action;
	pcmk_resource_t *first_rsc = first->rsc;

	if (first->required_runnable_before) {
	for (GList *clone_actions = first->actions_before;
	clone_actions != NULL;
	clone_actions = clone_actions->next) {

	before = clone_actions->data;

	crm_trace("Testing '%s then %s' for %s",
	first->uuid, before->action->uuid, start->uuid);

	CRM_ASSERT(before->action->rsc != NULL);
	first_rsc = before->action->rsc;
	break;
	}

	} else if (!pcmk__str_eq(first->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if (first_rsc == NULL) {
	continue;

	} else if (pe__const_top_resource(first_rsc, false)
	== pe__const_top_resource(start->rsc, false)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if (!pe_rsc_is_clone(pe__const_top_resource(first_rsc,
	false))) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_debug("Applying %s before %s %d", first->uuid, start->uuid,
	pe__const_top_resource(first_rsc, false)->variant);

	for (GList *probe_iter = probes; probe_iter != NULL;
	probe_iter = probe_iter->next) {

	pcmk_action_t probe = (pcmk_action_t ) probe_iter->data;

	crm_debug("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pcmk__ar_ordered);
	}
	}
	}
	#endif
	}

	void
	pcmk__order_probes(pcmk_scheduler_t *scheduler)
	{
	// Add orderings for "probe then X"
	g_list_foreach(scheduler->resources, add_start_restart_orderings_for_rsc,
	NULL);
	add_probe_orderings_for_stops(scheduler);

	order_then_probes(scheduler);
	}

	/*!
	* \internal
	* \brief Schedule any probes needed
	*
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing of failed remote nodes.
	*/
	void
	pcmk__schedule_probes(pcmk_scheduler_t *scheduler)
	{
	// Schedule probes on each node in the cluster as needed
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	const char *probed = NULL;

	if (!node->details->online) { // Don't probe offline nodes
	if (pcmk__is_failed_remote_node(node)) {
	pe_fence_node(scheduler, node,
	"the connection is unrecoverable", FALSE);
	}
	continue;

	} else if (node->details->unclean) { // ... or nodes that need fencing
	continue;

	} else if (!node->details->rsc_discovery_enabled) {
	// The user requested that probes not be done on this node
	continue;
	}

	/* This is no longer needed for live clusters, since the probe_complete
	* node attribute will never be in the CIB. However this is still useful
	* for processing old saved CIBs (< 1.1.14), including the
	* reprobe-target_rc regression test.
	*/
	probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
	if (probed != NULL && crm_is_true(probed) == FALSE) {
	pcmk_action_t *probe_op = NULL;

	probe_op = custom_action(NULL,
	crm_strdup_printf("%s-%s", CRM_OP_REPROBE,
	node->details->uname),
	CRM_OP_REPROBE, node, FALSE, scheduler);
	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT,
	XML_BOOLEAN_TRUE);
	continue;
	}

	// Probe each resource in the cluster on this node, as needed
	pcmk__probe_resource_list(scheduler->resources, node);
	}
	}
	diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c
	index 3c600942aa..ce1ed9faf2 100644
	--- a/lib/pacemaker/pcmk_sched_resource.c
	+++ b/lib/pacemaker/pcmk_sched_resource.c
	@@ -1,774 +1,774 @@
	/*
	* Copyright 2014-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <string.h>
	#include <crm/msg_xml.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	// Resource assignment methods by resource variant
	static pcmk_assignment_methods_t assignment_methods[] = {
	{
	pcmk__primitive_assign,
	pcmk__primitive_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__primitive_internal_constraints,
	pcmk__primitive_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_primitive_colocations,
	pcmk__primitive_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__apply_location,
	pcmk__primitive_action_flags,
	pcmk__update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__primitive_add_graph_meta,
	pcmk__primitive_add_utilization,
	pcmk__primitive_shutdown_lock,
	},
	{
	pcmk__group_assign,
	pcmk__group_create_actions,
	pcmk__probe_rsc_on_node,
	pcmk__group_internal_constraints,
	pcmk__group_apply_coloc_score,
	pcmk__group_colocated_resources,
	pcmk__with_group_colocations,
	pcmk__group_with_colocations,
	pcmk__group_add_colocated_node_scores,
	pcmk__group_apply_location,
	pcmk__group_action_flags,
	pcmk__group_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__add_rsc_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__group_add_utilization,
	pcmk__group_shutdown_lock,
	},
	{
	pcmk__clone_assign,
	pcmk__clone_create_actions,
	pcmk__clone_create_probe,
	pcmk__clone_internal_constraints,
	pcmk__clone_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_clone_colocations,
	pcmk__clone_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__clone_apply_location,
	pcmk__clone_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_resource_actions,
	pcmk__clone_add_actions_to_graph,
	pcmk__clone_add_graph_meta,
	pcmk__clone_add_utilization,
	pcmk__clone_shutdown_lock,
	},
	{
	pcmk__bundle_assign,
	pcmk__bundle_create_actions,
	pcmk__bundle_create_probe,
	pcmk__bundle_internal_constraints,
	pcmk__bundle_apply_coloc_score,
	pcmk__colocated_resources,
	pcmk__with_bundle_colocations,
	pcmk__bundle_with_colocations,
	pcmk__add_colocated_node_scores,
	pcmk__bundle_apply_location,
	pcmk__bundle_action_flags,
	pcmk__instance_update_ordered_actions,
	pcmk__output_bundle_actions,
	pcmk__bundle_add_actions_to_graph,
	pcmk__noop_add_graph_meta,
	pcmk__bundle_add_utilization,
	pcmk__bundle_shutdown_lock,
	}
	};

	/*!
	* \internal
	* \brief Check whether a resource's agent standard, provider, or type changed
	*
	* \param[in,out] rsc Resource to check
	* \param[in,out] node Node needing unfencing if agent changed
	* \param[in] rsc_entry XML with previously known agent information
	* \param[in] active_on_node Whether \p rsc is active on \p node
	*
	* \return true if agent for \p rsc changed, otherwise false
	*/
	bool
	pcmk__rsc_agent_changed(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *rsc_entry, bool active_on_node)
	{
	bool changed = false;
	const char *attr_list[] = {
	PCMK_XA_TYPE,
	- XML_AGENT_ATTR_CLASS,
	+ PCMK_XA_CLASS,
	PCMK_XA_PROVIDER,
	};

	for (int i = 0; i < PCMK__NELEM(attr_list); i++) {
	const char *value = crm_element_value(rsc->xml, attr_list[i]);
	const char *old_value = crm_element_value(rsc_entry, attr_list[i]);

	if (!pcmk__str_eq(value, old_value, pcmk__str_none)) {
	changed = true;
	trigger_unfencing(rsc, node, "Device definition changed", NULL,
	rsc->cluster);
	if (active_on_node) {
	crm_notice("Forcing restart of %s on %s "
	"because %s changed from '%s' to '%s'",
	rsc->id, pe__node_name(node), attr_list[i],
	pcmk__s(old_value, ""), pcmk__s(value, ""));
	}
	}
	}
	if (changed && active_on_node) {
	// Make sure the resource is restarted
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
	rsc->cluster);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_start_pending);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Add resource (and any matching children) to list if it matches ID
	*
	* \param[in] result List to add resource to
	* \param[in] rsc Resource to check
	* \param[in] id ID to match
	*
	* \return (Possibly new) head of list
	*/
	static GList *
	add_rsc_if_matching(GList result, pcmk_resource_t rsc, const char *id)
	{
	if ((strcmp(rsc->id, id) == 0)
	\|\| ((rsc->clone_name != NULL) && (strcmp(rsc->clone_name, id) == 0))) {
	result = g_list_prepend(result, rsc);
	}
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	result = add_rsc_if_matching(result, child, id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Find all resources matching a given ID by either ID or clone name
	*
	* \param[in] id Resource ID to check
	* \param[in] scheduler Scheduler data
	*
	* \return List of all resources that match \p id
	* \note The caller is responsible for freeing the return value with
	* g_list_free().
	*/
	GList *
	pcmk__rscs_matching_id(const char id, const pcmk_scheduler_t scheduler)
	{
	GList *result = NULL;

	CRM_CHECK((id != NULL) && (scheduler != NULL), return NULL);
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	result = add_rsc_if_matching(result, (pcmk_resource_t *) iter->data,
	id);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for a resource
	*
	* \param[in,out] data Resource to set assignment methods for
	* \param[in] user_data Ignored
	*/
	static void
	set_assignment_methods_for_rsc(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;

	rsc->cmds = &assignment_methods[rsc->variant];
	g_list_foreach(rsc->children, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Set the variant-appropriate assignment methods for all resources
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__set_assignment_methods(pcmk_scheduler_t *scheduler)
	{
	g_list_foreach(scheduler->resources, set_assignment_methods_for_rsc, NULL);
	}

	/*!
	* \internal
	* \brief Wrapper for colocated_resources() method for readability
	*
	* \param[in] rsc Resource to add to colocated list
	* \param[in] orig_rsc Resource originally requested
	* \param[in,out] list Pointer to list to add to
	*
	* \return (Possibly new) head of list
	*/
	static inline void
	add_colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t orig_rsc, GList *list)
	{
	list = rsc->cmds->colocated_resources(rsc, orig_rsc, list);
	}

	// Shared implementation of pcmk_assignment_methods_t:colocated_resources()
	GList *
	pcmk__colocated_resources(const pcmk_resource_t *rsc,
	const pcmk_resource_t *orig_rsc,
	GList *colocated_rscs)
	{
	const GList *iter = NULL;
	GList *colocations = NULL;

	if (orig_rsc == NULL) {
	orig_rsc = rsc;
	}

	if ((rsc == NULL) \|\| (g_list_find(colocated_rscs, rsc) != NULL)) {
	return colocated_rscs;
	}

	pcmk__rsc_trace(orig_rsc, "%s is in colocation chain with %s",
	rsc->id, orig_rsc->id);
	colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);

	// Follow colocations where this resource is the dependent resource
	colocations = pcmk__this_with_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *primary = constraint->primary;

	if (primary == orig_rsc) {
	continue; // Break colocation loop
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(rsc, primary, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(primary, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	// Follow colocations where this resource is the primary resource
	colocations = pcmk__with_this_colocations(rsc);
	for (iter = colocations; iter != NULL; iter = iter->next) {
	const pcmk__colocation_t *constraint = iter->data;
	const pcmk_resource_t *dependent = constraint->dependent;

	if (dependent == orig_rsc) {
	continue; // Break colocation loop
	}

	if (pe_rsc_is_clone(rsc) && !pe_rsc_is_clone(dependent)) {
	continue; // We can't be sure whether dependent will be colocated
	}

	if ((constraint->score == INFINITY) &&
	(pcmk__colocation_affects(dependent, rsc, constraint,
	true) == pcmk__coloc_affects_location)) {
	add_colocated_resources(dependent, orig_rsc, &colocated_rscs);
	}
	}
	g_list_free(colocations);

	return colocated_rscs;
	}

	// No-op function for variants that don't need to implement add_graph_meta()
	void
	pcmk__noop_add_graph_meta(const pcmk_resource_t rsc, xmlNode xml)
	{
	}

	/*!
	* \internal
	* \brief Output a summary of scheduled actions for a resource
	*
	* \param[in,out] rsc Resource to output actions for
	*/
	void
	pcmk__output_resource_actions(pcmk_resource_t *rsc)
	{
	pcmk_node_t *next = NULL;
	pcmk_node_t *current = NULL;
	pcmk__output_t *out = NULL;

	CRM_ASSERT(rsc != NULL);

	out = rsc->cluster->priv;
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) iter->data;

	child->cmds->output_actions(child);
	}
	return;
	}

	next = rsc->allocated_to;
	if (rsc->running_on) {
	current = pe__current_node(rsc);
	if (rsc->role == pcmk_role_stopped) {
	/* This can occur when resources are being recovered because
	* the current role can change in pcmk__primitive_create_actions()
	*/
	rsc->role = pcmk_role_started;
	}
	}

	if ((current == NULL) && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	/* Don't log stopped orphans */
	return;
	}

	out->message(out, "rsc-action", rsc, current, next);
	}

	/*!
	* \internal
	* \brief Add a resource to a node's list of assigned resources
	*
	* \param[in,out] node Node to add resource to
	* \param[in] rsc Resource to add
	*/
	static inline void
	add_assigned_resource(pcmk_node_t node, pcmk_resource_t rsc)
	{
	node->details->allocated_rsc = g_list_prepend(node->details->allocated_rsc,
	rsc);
	}

	/*!
	* \internal
	* \brief Assign a specified resource (of any variant) to a node
	*
	* Assign a specified resource and its children (if any) to a specified node, if
	* the node can run the resource (or unconditionally, if \p force is true). Mark
	* the resources as no longer provisional.
	*
	* If a resource can't be assigned (or \p node is \c NULL), unassign any
	* previous assignment. If \p stop_if_fail is \c true, set next role to stopped
	* and update any existing actions scheduled for the resource.
	*
	* \param[in,out] rsc Resource to assign
	* \param[in,out] node Node to assign \p rsc to
	* \param[in] force If true, assign to \p node even if unavailable
	* \param[in] stop_if_fail If \c true and either \p rsc can't be assigned
	* or \p chosen is \c NULL, set next role to
	* stopped and update existing actions (if \p rsc
	* is not a primitive, this applies to its
	* primitive descendants instead)
	*
	* \return \c true if the assignment of \p rsc changed, or \c false otherwise
	*
	* \note Assigning a resource to the NULL node using this function is different
	* from calling pcmk__unassign_resource(), in that it may also update any
	* actions created for the resource.
	* \note The \c pcmk_assignment_methods_t:assign() method is preferred, unless
	* a resource should be assigned to the \c NULL node or every resource in
	* a tree should be assigned to the same node.
	* \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
	* completely undo the assignment. A successful assignment can be either
	* undone or left alone as final. A failed assignment has the same effect
	* as calling pcmk__unassign_resource(); there are no side effects on
	* roles or actions.
	*/
	bool
	pcmk__assign_resource(pcmk_resource_t rsc, pcmk_node_t node, bool force,
	bool stop_if_fail)
	{
	bool changed = false;

	CRM_ASSERT(rsc != NULL);

	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk_resource_t *child_rsc = iter->data;

	changed \|= pcmk__assign_resource(child_rsc, node, force,
	stop_if_fail);
	}
	return changed;
	}

	// Assigning a primitive

	if (!force && (node != NULL)
	&& ((node->weight < 0)
	// Allow graph to assume that guest node connections will come up
	\|\| (!pcmk__node_available(node, true, false)
	&& !pe__is_guest_node(node)))) {

	pcmk__rsc_debug(rsc,
	"All nodes for resource %s are unavailable, unclean or "
	"shutting down (%s can%s run resources, with score %s)",
	rsc->id, pe__node_name(node),
	(pcmk__node_available(node, true, false)? "" : "not"),
	pcmk_readable_score(node->weight));

	if (stop_if_fail) {
	pe__set_next_role(rsc, pcmk_role_stopped, "node availability");
	}
	node = NULL;
	}

	if (rsc->allocated_to != NULL) {
	changed = !pe__same_node(rsc->allocated_to, node);
	} else {
	changed = (node != NULL);
	}
	pcmk__unassign_resource(rsc);
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unassigned);

	if (node == NULL) {
	char *rc_stopped = NULL;

	pcmk__rsc_debug(rsc, "Could not assign %s to a node", rsc->id);

	if (!stop_if_fail) {
	return changed;
	}
	pe__set_next_role(rsc, pcmk_role_stopped, "unable to assign");

	for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t op = (pcmk_action_t ) iter->data;

	pcmk__rsc_debug(rsc, "Updating %s for %s assignment failure",
	op->uuid, rsc->id);

	if (pcmk__str_eq(op->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	pcmk__clear_action_flags(op, pcmk_action_optional);

	} else if (pcmk__str_eq(op->task, PCMK_ACTION_START,
	pcmk__str_none)) {
	pcmk__clear_action_flags(op, pcmk_action_runnable);

	} else {
	// Cancel recurring actions, unless for stopped state
	const char *interval_ms_s = NULL;
	const char *target_rc_s = NULL;

	interval_ms_s = g_hash_table_lookup(op->meta,
	XML_LRM_ATTR_INTERVAL_MS);
	target_rc_s = g_hash_table_lookup(op->meta,
	XML_ATTR_TE_TARGET_RC);
	if (rc_stopped == NULL) {
	rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
	}

	if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
	&& !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) {

	pcmk__clear_action_flags(op, pcmk_action_runnable);
	}
	}
	}
	free(rc_stopped);
	return changed;
	}

	pcmk__rsc_debug(rsc, "Assigning %s to %s", rsc->id, pe__node_name(node));
	rsc->allocated_to = pe__copy_node(node);

	add_assigned_resource(node, rsc);
	node->details->num_resources++;
	node->count++;
	pcmk__consume_node_capacity(node->details->utilization, rsc);

	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_show_utilization)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->message(out, "resource-util", rsc, node, __func__);
	}
	return changed;
	}

	/*!
	* \internal
	* \brief Remove any node assignment from a specified resource and its children
	*
	* If a specified resource has been assigned to a node, remove that assignment
	* and mark the resource as provisional again.
	*
	* \param[in,out] rsc Resource to unassign
	*
	* \note This function is called recursively on \p rsc and its children.
	*/
	void
	pcmk__unassign_resource(pcmk_resource_t *rsc)
	{
	pcmk_node_t *old = rsc->allocated_to;

	if (old == NULL) {
	crm_info("Unassigning %s", rsc->id);
	} else {
	crm_info("Unassigning %s from %s", rsc->id, pe__node_name(old));
	}

	pcmk__set_rsc_flags(rsc, pcmk_rsc_unassigned);

	if (rsc->children == NULL) {
	if (old == NULL) {
	return;
	}
	rsc->allocated_to = NULL;

	/* We're going to free the pcmk_node_t, but its details member is shared
	* and will remain, so update that appropriately first.
	*/
	old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc,
	rsc);
	old->details->num_resources--;
	pcmk__release_node_capacity(old->details->utilization, rsc);
	free(old);
	return;
	}

	for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	pcmk__unassign_resource((pcmk_resource_t *) iter->data);
	}
	}

	/*!
	* \internal
	* \brief Check whether a resource has reached its migration threshold on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node Node to check
	* \param[out] failed If threshold has been reached, this will be set to
	* resource that failed (possibly a parent of \p rsc)
	*
	* \return true if the migration threshold has been reached, false otherwise
	*/
	bool
	pcmk__threshold_reached(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_resource_t **failed)
	{
	int fail_count, remaining_tries;
	pcmk_resource_t *rsc_to_ban = rsc;

	// Migration threshold of 0 means never force away
	if (rsc->migration_threshold == 0) {
	return false;
	}

	// If we're ignoring failures, also ignore the migration threshold
	if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	return false;
	}

	// If there are no failures, there's no need to force away
	fail_count = pe_get_failcount(node, rsc, NULL,
	pcmk__fc_effective\|pcmk__fc_fillers, NULL);
	if (fail_count <= 0) {
	return false;
	}

	// If failed resource is anonymous clone instance, we'll force clone away
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	rsc_to_ban = uber_parent(rsc);
	}

	// How many more times recovery will be tried on this node
	remaining_tries = rsc->migration_threshold - fail_count;

	if (remaining_tries <= 0) {
	pcmk__sched_warn("%s cannot run on %s due to reaching migration "
	"threshold (clean up resource to allow again)"
	CRM_XS " failures=%d "
	PCMK_META_MIGRATION_THRESHOLD "=%d",
	rsc_to_ban->id, pe__node_name(node), fail_count,
	rsc->migration_threshold);
	if (failed != NULL) {
	*failed = rsc_to_ban;
	}
	return true;
	}

	crm_info("%s can fail %d more time%s on "
	"%s before reaching migration threshold (%d)",
	rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries),
	pe__node_name(node), rsc->migration_threshold);
	return false;
	}

	/*!
	* \internal
	* \brief Get a node's score
	*
	* \param[in] node Node with ID to check
	* \param[in] nodes List of nodes to look for \p node score in
	*
	* \return Node's score, or -INFINITY if not found
	*/
	static int
	get_node_score(const pcmk_node_t node, GHashTable nodes)
	{
	pcmk_node_t *found_node = NULL;

	if ((node != NULL) && (nodes != NULL)) {
	found_node = g_hash_table_lookup(nodes, node->details->id);
	}
	return (found_node == NULL)? -INFINITY : found_node->weight;
	}

	/*!
	* \internal
	* \brief Compare two resources according to which should be assigned first
	*
	* \param[in] a First resource to compare
	* \param[in] b Second resource to compare
	* \param[in] data Sorted list of all nodes in cluster
	*
	* \return -1 if \p a should be assigned before \b, 0 if they are equal,
	* or +1 if \p a should be assigned after \b
	*/
	static gint
	cmp_resources(gconstpointer a, gconstpointer b, gpointer data)
	{
	/* GLib insists that this function require gconstpointer arguments, but we
	* make a small, temporary change to each argument (setting the
	* pe_rsc_merging flag) during comparison
	*/
	pcmk_resource_t resource1 = (pcmk_resource_t ) a;
	pcmk_resource_t resource2 = (pcmk_resource_t ) b;
	const GList *nodes = data;

	int rc = 0;
	int r1_score = -INFINITY;
	int r2_score = -INFINITY;
	pcmk_node_t *r1_node = NULL;
	pcmk_node_t *r2_node = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;
	const char *reason = NULL;

	// Resources with highest priority should be assigned first
	reason = "priority";
	r1_score = resource1->priority;
	r2_score = resource2->priority;
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// We need nodes to make any other useful comparisons
	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	// Calculate and log node scores
	resource1->cmds->add_colocated_node_scores(resource1, NULL, resource1->id,
	&r1_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	resource2->cmds->add_colocated_node_scores(resource2, NULL, resource2->id,
	&r2_nodes, NULL, 1,
	pcmk__coloc_select_this_with);
	pe__show_node_scores(true, NULL, resource1->id, r1_nodes,
	resource1->cluster);
	pe__show_node_scores(true, NULL, resource2->id, r2_nodes,
	resource2->cluster);

	// The resource with highest score on its current node goes first
	reason = "current location";
	if (resource1->running_on != NULL) {
	r1_node = pe__current_node(resource1);
	}
	if (resource2->running_on != NULL) {
	r2_node = pe__current_node(resource2);
	}
	r1_score = get_node_score(r1_node, r1_nodes);
	r2_score = get_node_score(r2_node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}

	// Otherwise a higher score on any node will do
	reason = "score";
	for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
	const pcmk_node_t node = (const pcmk_node_t ) iter->data;

	r1_score = get_node_score(node, r1_nodes);
	r2_score = get_node_score(node, r2_nodes);
	if (r1_score > r2_score) {
	rc = -1;
	goto done;
	}
	if (r1_score < r2_score) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d)%s%s %c %s (%d)%s%s: %s",
	resource1->id, r1_score,
	((r1_node == NULL)? "" : " on "),
	((r1_node == NULL)? "" : r1_node->details->id),
	((rc < 0)? '>' : ((rc > 0)? '<' : '=')),
	resource2->id, r2_score,
	((r2_node == NULL)? "" : " on "),
	((r2_node == NULL)? "" : r2_node->details->id),
	reason);
	if (r1_nodes != NULL) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes != NULL) {
	g_hash_table_destroy(r2_nodes);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Sort resources in the order they should be assigned to nodes
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__sort_resources(pcmk_scheduler_t *scheduler)
	{
	GList *nodes = g_list_copy(scheduler->nodes);

	nodes = pcmk__sort_nodes(nodes, NULL);
	scheduler->resources = g_list_sort_with_data(scheduler->resources,
	cmp_resources, nodes);
	g_list_free(nodes);
	}
	diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c
	index 8a5b571fa5..5591523f3c 100644
	--- a/lib/pacemaker/pcmk_simulate.c
	+++ b/lib/pacemaker/pcmk_simulate.c
	@@ -1,1006 +1,1006 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/common/scheduler.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#include <stdint.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include "libpacemaker_private.h"

	static pcmk__output_t *out = NULL;
	static cib_t *fake_cib = NULL;
	static GList *fake_resource_list = NULL;
	static const GList *fake_op_fail_list = NULL;

	static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
	const char *use_date);

	/*!
	* \internal
	* \brief Create an action name for use in a dot graph
	*
	* \param[in] action Action to create name for
	* \param[in] verbose If true, add action ID to name
	*
	* \return Newly allocated string with action name
	* \note It is the caller's responsibility to free the result.
	*/
	static char *
	create_action_name(const pcmk_action_t *action, bool verbose)
	{
	char *action_name = NULL;
	const char *prefix = "";
	const char *action_host = NULL;
	const char *clone_name = NULL;
	const char *task = action->task;

	if (action->node != NULL) {
	action_host = action->node->details->uname;
	} else if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	action_host = "<none>";
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
	prefix = "Cancel ";
	task = action->cancel_task;
	}

	if (action->rsc != NULL) {
	clone_name = action->rsc->clone_name;
	}

	if (clone_name != NULL) {
	char *key = NULL;
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta,
	XML_LRM_ATTR_INTERVAL_MS, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_NOTIFIED, NULL)) {
	const char *n_type = g_hash_table_lookup(action->meta,
	"notify_key_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_key_operation");

	CRM_ASSERT(n_type != NULL);
	CRM_ASSERT(n_task != NULL);
	key = pcmk__notify_key(clone_name, n_type, n_task);
	} else {
	key = pcmk__op_key(clone_name, task, interval_ms);
	}

	if (action_host != NULL) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, key, action_host);
	} else {
	action_name = crm_strdup_printf("%s%s", prefix, key);
	}
	free(key);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");

	action_name = crm_strdup_printf("%s%s '%s' %s",
	prefix, action->task, op, action_host);

	} else if (action->rsc && action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->uuid, action_host);

	} else if (action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->task, action_host);

	} else {
	action_name = crm_strdup_printf("%s", action->uuid);
	}

	if (verbose) {
	char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id);

	free(action_name);
	action_name = with_id;
	}
	return action_name;
	}

	/*!
	* \internal
	* \brief Display the status of a cluster
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] show_opts How to modify display (as pcmk_show_opt_e flags)
	* \param[in] section_opts Sections to display (as pcmk_section_e flags)
	* \param[in] title What to use as list title
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_cluster_status(pcmk_scheduler_t *scheduler, uint32_t show_opts,
	uint32_t section_opts, const char *title,
	bool print_spacer)
	{
	pcmk__output_t *out = scheduler->priv;
	GList *all = NULL;
	crm_exit_t stonith_rc = 0;
	enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;

	section_opts \|= pcmk_section_nodes \| pcmk_section_resources;
	show_opts \|= pcmk_show_inactive_rscs \| pcmk_show_failed_detail;

	all = g_list_prepend(all, (gpointer) "*");

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "%s", title);
	out->message(out, "cluster-status",
	scheduler, state, stonith_rc, NULL,
	false, section_opts, show_opts, NULL, all, all);
	out->end_list(out);

	g_list_free(all);
	}

	/*!
	* \internal
	* \brief Display a summary of all actions scheduled in a transition
	*
	* \param[in,out] scheduler Scheduler data (fully scheduled)
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_transition_summary(pcmk_scheduler_t *scheduler, bool print_spacer)
	{
	pcmk__output_t *out = scheduler->priv;

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "Transition Summary");
	pcmk__output_actions(scheduler);
	out->end_list(out);
	}

	/*!
	* \internal
	* \brief Reset scheduler input, output, date, and flags
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] input What to set as cluster input
	* \param[in] out What to set as cluster output object
	* \param[in] use_date What to set as cluster's current timestamp
	* \param[in] flags Group of enum pcmk_scheduler_flags to set
	*/
	static void
	reset(pcmk_scheduler_t scheduler, xmlNodePtr input, pcmk__output_t out,
	const char *use_date, unsigned int flags)
	{
	scheduler->input = input;
	scheduler->priv = out;
	set_effective_date(scheduler, true, use_date);
	if (pcmk_is_set(flags, pcmk_sim_sanitized)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_sanitized);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
	}
	}

	/*!
	* \brief Write out a file in dot(1) format describing the actions that will
	* be taken by the scheduler in response to an input CIB file.
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] dot_file The filename to write
	* \param[in] all_actions Write all actions, even those that are optional
	* or are on unmanaged resources
	* \param[in] verbose Add extra information, such as action IDs, to the
	* output
	*
	* \return Standard Pacemaker return code
	*/
	static int
	write_sim_dotfile(pcmk_scheduler_t scheduler, const char dot_file,
	bool all_actions, bool verbose)
	{
	GList *iter = NULL;
	FILE *dot_strm = fopen(dot_file, "w");

	if (dot_strm == NULL) {
	return errno;
	}

	fprintf(dot_strm, " digraph \"g\" {\n");
	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;
	const char *style = "dashed";
	const char *font = "black";
	const char *color = "black";
	char *action_name = create_action_name(action, verbose);

	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	font = "orange";
	}

	if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)) {
	style = "bold";
	color = "green";

	} else if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)) {
	color = "red";
	font = "purple";
	if (!all_actions) {
	goto do_not_write;
	}

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	color = "blue";
	if (!all_actions) {
	goto do_not_write;
	}

	} else {
	color = "red";
	CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pcmk_action_runnable));
	}

	pcmk__set_action_flags(action, pcmk_action_added_to_graph);
	fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
	action_name, style, color, font);
	do_not_write:
	free(action_name);
	}

	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	for (GList *before_iter = action->actions_before;
	before_iter != NULL; before_iter = before_iter->next) {

	pcmk__related_action_t *before = before_iter->data;

	char *before_name = NULL;
	char *after_name = NULL;
	const char *style = "dashed";
	bool optional = true;

	if (before->state == pe_link_dumped) {
	optional = false;
	style = "bold";
	} else if ((uint32_t) before->type == pcmk__ar_none) {
	continue;
	} else if (pcmk_is_set(before->action->flags,
	pcmk_action_added_to_graph)
	&& pcmk_is_set(action->flags, pcmk_action_added_to_graph)
	&& (uint32_t) before->type != pcmk__ar_if_on_same_node_or_target) {
	optional = false;
	}

	if (all_actions \|\| !optional) {
	before_name = create_action_name(before->action, verbose);
	after_name = create_action_name(action, verbose);
	fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
	before_name, after_name, style);
	free(before_name);
	free(after_name);
	}
	}
	}

	fprintf(dot_strm, "}\n");
	fflush(dot_strm);
	fclose(dot_strm);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Profile the configuration updates and scheduler actions in a single
	* CIB file, printing the profiling timings.
	*
	* \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in] xml_file The CIB file to profile
	* \param[in] repeat Number of times to run
	* \param[in,out] scheduler Scheduler data
	* \param[in] use_date The date to set the cluster's time to (may be NULL)
	*/
	static void
	profile_file(const char *xml_file, long long repeat,
	pcmk_scheduler_t scheduler, const char use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	xmlNode *cib_object = NULL;
	clock_t start = 0;
	clock_t end;
	unsigned long long scheduler_flags = pcmk_sched_no_compat;

	CRM_ASSERT(out != NULL);

	cib_object = filename2xml(xml_file);
	start = clock();

	if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
	create_xml_node(cib_object, XML_CIB_TAG_STATUS);
	}

	if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	free_xml(cib_object);
	return;
	}

	if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
	free_xml(cib_object);
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	scheduler_flags \|= pcmk_sched_output_scores;
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	scheduler_flags \|= pcmk_sched_show_utilization;
	}

	for (int i = 0; i < repeat; ++i) {
	xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object);

	scheduler->input = input;
	set_effective_date(scheduler, false, use_date);
	pcmk__schedule_actions(input, scheduler_flags, scheduler);
	pe_reset_working_set(scheduler);
	}

	end = clock();
	out->message(out, "profile", xml_file, start, end);
	}

	void
	pcmk__profile_dir(const char *dir, long long repeat,
	pcmk_scheduler_t scheduler, const char use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	struct dirent **namelist;

	int file_num = scandir(dir, &namelist, 0, alphasort);

	CRM_ASSERT(out != NULL);

	if (file_num > 0) {
	struct stat prop;
	char buffer[FILENAME_MAX];

	out->begin_list(out, NULL, NULL, "Timings");

	while (file_num--) {
	if ('.' == namelist[file_num]->d_name[0]) {
	free(namelist[file_num]);
	continue;

	} else if (!pcmk__ends_with_ext(namelist[file_num]->d_name,
	".xml")) {
	free(namelist[file_num]);
	continue;
	}
	snprintf(buffer, sizeof(buffer), "%s/%s",
	dir, namelist[file_num]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	profile_file(buffer, repeat, scheduler, use_date);
	}
	free(namelist[file_num]);
	}
	free(namelist);

	out->end_list(out);
	}
	}

	/*!
	* \brief Set the date of the cluster, either to the value given by
	* \p use_date, or to the "execution-date" value in the CIB.
	*
	* \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] print_original If \p true, the "execution-date" should
	* also be printed
	* \param[in] use_date The date to set the cluster's time to
	* (may be NULL)
	*/
	static void
	set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
	const char *use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	time_t original_date = 0;

	CRM_ASSERT(out != NULL);

	crm_element_value_epoch(scheduler->input, "execution-date", &original_date);

	if (use_date) {
	scheduler->now = crm_time_new(use_date);
	out->info(out, "Setting effective cluster time: %s", use_date);
	crm_time_log(LOG_NOTICE, "Pretending 'now' is", scheduler->now,
	crm_time_log_date \| crm_time_log_timeofday);

	} else if (original_date != 0) {
	scheduler->now = pcmk__copy_timet(original_date);

	if (print_original) {
	char *when = crm_time_as_string(scheduler->now,
	crm_time_log_date\|crm_time_log_timeofday);

	out->info(out, "Using the original execution date of: %s", when);
	free(when);
	}
	}
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a pseudo-action in a graph
	*
	* \param[in,out] graph Graph to update with pseudo-action result
	* \param[in,out] action Pseudo-action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_pseudo_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-pseudo-action", node, task);

	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate executing a resource action in a graph
	*
	* \param[in,out] graph Graph to update with resource action result
	* \param[in,out] action Resource action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_resource_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	int rc;
	lrmd_event_data_t *op = NULL;
	int target_outcome = PCMK_OCF_OK;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *resource = NULL;
	const char *rprovider = NULL;
	const char *resource_config_name = NULL;
	const char *operation = crm_element_value(action->xml, "operation");
	const char *target_rc_s = crm_meta_value(action->params,
	XML_ATTR_TE_TARGET_RC);

	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
	char *uuid = NULL;
	const char *router_node = crm_element_value(action->xml,
	XML_LRM_ATTR_ROUTER_NODE);

	// Certain actions don't need to be displayed or history entries
	if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
	crm_debug("No history injection for %s op on %s", operation, node);
	goto done; // Confirm action and update graph
	}

	if (action_rsc == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "Bad");
	free(node);
	return EPROTO;
	}

	/* A resource might be known by different names in the configuration and in
	* the action (for example, a clone instance). Grab the configuration name
	* (which is preferred when writing history), and if necessary, the instance
	* name.
	*/
	resource_config_name = crm_element_value(action_rsc, PCMK_XA_ID);
	if (resource_config_name == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "No ID");
	free(node);
	return EPROTO;
	}
	resource = resource_config_name;
	if (pe_find_resource(fake_resource_list, resource) == NULL) {
	const char *longname = crm_element_value(action_rsc, PCMK__XA_LONG_ID);

	if ((longname != NULL)
	&& (pe_find_resource(fake_resource_list, longname) != NULL)) {
	resource = longname;
	}
	}

	// Certain actions need to be displayed but don't need history entries
	if (pcmk__strcase_any_of(operation, PCMK_ACTION_DELETE,
	PCMK_ACTION_META_DATA, NULL)) {
	out->message(out, "inject-rsc-action", resource, operation, node,
	(guint) 0);
	goto done; // Confirm action and update graph
	}

	- rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
	+ rclass = crm_element_value(action_rsc, PCMK_XA_CLASS);
	rtype = crm_element_value(action_rsc, PCMK_XA_TYPE);
	rprovider = crm_element_value(action_rsc, PCMK_XA_PROVIDER);

	pcmk__scan_min_int(target_rc_s, &target_outcome, 0);

	CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL,
	cib_sync_call\|cib_scope_local) == pcmk_ok);

	// Ensure the action node is in the CIB
	uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
	cib_node = pcmk__inject_node(fake_cib, node,
	((router_node == NULL)? uuid: node));
	free(uuid);
	CRM_ASSERT(cib_node != NULL);

	// Add a history entry for the action
	cib_resource = pcmk__inject_resource_history(out, cib_node, resource,
	resource_config_name,
	rclass, rtype, rprovider);
	if (cib_resource == NULL) {
	crm_err("Could not simulate action %d history for resource %s",
	action->id, resource);
	free(node);
	free_xml(cib_node);
	return EINVAL;
	}

	// Simulate and display an executor event for the action result
	op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
	target_outcome, "User-injected result");
	out->message(out, "inject-rsc-action", resource, op->op_type, node,
	op->interval_ms);

	// Check whether action is in a list of desired simulated failures
	for (const GList *iter = fake_op_fail_list;
	iter != NULL; iter = iter->next) {
	const char spec = (const char ) iter->data;
	char *key = NULL;
	const char *match_name = NULL;

	// Allow user to specify anonymous clone with or without instance number
	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
	op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource;
	}
	free(key);

	// If not found, try the resource's name in the configuration
	if ((match_name == NULL)
	&& (strcmp(resource, resource_config_name) != 0)) {

	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name,
	op->op_type, op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource_config_name;
	}
	free(key);
	}

	if (match_name == NULL) {
	continue; // This failed action entry doesn't match
	}

	// ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
	rc = sscanf(spec, "%[^=]=%d", (int ) &op->rc);
	if (rc != 1) {
	out->err(out, "Invalid failed operation '%s' "
	"(result code must be integer)", spec);
	continue; // Keep checking other list entries
	}

	out->info(out, "Pretending action %d failed with rc=%d",
	action->id, op->rc);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
	graph->abort_priority = INFINITY;
	pcmk__inject_failcount(out, cib_node, match_name, op->op_type,
	op->interval_ms, op->rc);
	break;
	}

	pcmk__inject_action_result(cib_resource, op, target_outcome);
	lrmd_free_event(op);
	rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(node);
	free_xml(cib_node);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a cluster action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Cluster action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_cluster_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-cluster-action", node, task, rsc);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a fencing action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Fencing action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_fencing_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *op = crm_meta_value(action->params, "stonith_action");
	char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);

	out->message(out, "inject-fencing-action", target, op);

	if (!pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	int rc = pcmk_ok;
	GString *xpath = g_string_sized_new(512);

	// Set node state to offline
	xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target,
	false);

	CRM_ASSERT(cib_node != NULL);
	crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);
	rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	// Simulate controller clearing node's resource history and attributes
	pcmk__g_strcat(xpath,
	"//" XML_CIB_TAG_STATE
	"[@" XML_ATTR_UNAME "='", target, "']/" XML_CIB_TAG_LRM,
	NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	g_string_truncate(xpath, 0);
	pcmk__g_strcat(xpath,
	"//" XML_CIB_TAG_STATE
	"[@" XML_ATTR_UNAME "='", target, "']"
	"/" XML_TAG_TRANSIENT_NODEATTRS, NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	free_xml(cib_node);
	g_string_free(xpath, TRUE);
	}

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	free(target);
	return pcmk_rc_ok;
	}

	enum pcmk__graph_status
	pcmk__simulate_transition(pcmk_scheduler_t scheduler, cib_t cib,
	const GList *op_fail_list)
	{
	pcmk__graph_t *transition = NULL;
	enum pcmk__graph_status graph_rc;

	pcmk__graph_functions_t simulation_fns = {
	simulate_pseudo_action,
	simulate_resource_action,
	simulate_cluster_action,
	simulate_fencing_action,
	};

	out = scheduler->priv;

	fake_cib = cib;
	fake_op_fail_list = op_fail_list;

	if (!out->is_quiet(out)) {
	out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
	}

	pcmk__set_graph_functions(&simulation_fns);
	transition = pcmk__unpack_graph(scheduler->graph, crm_system_name);
	pcmk__log_graph(LOG_DEBUG, transition);

	fake_resource_list = scheduler->resources;
	do {
	graph_rc = pcmk__execute_graph(transition);
	} while (graph_rc == pcmk__graph_active);
	fake_resource_list = NULL;

	if (graph_rc != pcmk__graph_complete) {
	out->err(out, "Transition failed: %s",
	pcmk__graph_status2text(graph_rc));
	pcmk__log_graph(LOG_ERR, transition);
	out->err(out, "An invalid transition was produced");
	}
	pcmk__free_graph(transition);

	if (!out->is_quiet(out)) {
	// If not quiet, we'll need the resulting CIB for later display
	xmlNode *cib_object = NULL;
	int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object,
	cib_sync_call\|cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	pe_reset_working_set(scheduler);
	scheduler->input = cib_object;
	out->end_list(out);
	}
	return graph_rc;
	}

	int
	pcmk__simulate(pcmk_scheduler_t scheduler, pcmk__output_t out,
	const pcmk_injections_t *injections, unsigned int flags,
	uint32_t section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	int printed = pcmk_rc_no_output;
	int rc = pcmk_rc_ok;
	xmlNodePtr input = NULL;
	cib_t *cib = NULL;

	rc = cib__signon_query(out, &cib, &input);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}

	reset(scheduler, input, out, use_date, flags);
	cluster_status(scheduler);

	if ((cib->variant == cib_native)
	&& pcmk_is_set(section_opts, pcmk_section_times)) {
	if (pcmk__our_nodename == NULL) {
	// Currently used only in the times section
	pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0);
	}
	scheduler->localhost = pcmk__our_nodename;
	}

	if (!out->is_quiet(out)) {
	const bool show_pending = pcmk_is_set(flags, pcmk_sim_show_pending);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	printed = out->message(out, "maint-mode", scheduler->flags);
	}

	if (scheduler->disabled_resources \|\| scheduler->blocked_resources) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	printed = out->info(out,
	"%d of %d resource instances DISABLED and "
	"%d BLOCKED from further action due to failure",
	scheduler->disabled_resources,
	scheduler->ninstances,
	scheduler->blocked_resources);
	}

	/* Most formatted output headers use caps for each word, but this one
	* only has the first word capitalized for compatibility with pcs.
	*/
	print_cluster_status(scheduler, (show_pending? pcmk_show_pending : 0),
	section_opts, "Current cluster status",
	(printed == pcmk_rc_ok));
	printed = pcmk_rc_ok;
	}

	// If the user requested any injections, handle them
	if ((injections->node_down != NULL)
	\|\| (injections->node_fail != NULL)
	\|\| (injections->node_up != NULL)
	\|\| (injections->op_inject != NULL)
	\|\| (injections->ticket_activate != NULL)
	\|\| (injections->ticket_grant != NULL)
	\|\| (injections->ticket_revoke != NULL)
	\|\| (injections->ticket_standby != NULL)
	\|\| (injections->watchdog != NULL)) {

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	pcmk__inject_scheduler_input(scheduler, cib, injections);
	printed = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &input, cib_sync_call);
	if (rc != pcmk_rc_ok) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}

	cleanup_calculations(scheduler);
	reset(scheduler, input, out, use_date, flags);
	cluster_status(scheduler);
	}

	if (input_file != NULL) {
	rc = write_xml_file(input, input_file, FALSE);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}
	}

	if (pcmk_any_flags_set(flags, pcmk_sim_process \| pcmk_sim_simulate)) {
	pcmk__output_t *logger_out = NULL;
	unsigned long long scheduler_flags = pcmk_sched_no_compat;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	scheduler_flags \|= pcmk_sched_output_scores;
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	scheduler_flags \|= pcmk_sched_show_utilization;
	}

	if (pcmk_all_flags_set(scheduler->flags,
	pcmk_sched_output_scores
	\|pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL,
	"Assignment Scores and Utilization Information");
	printed = pcmk_rc_ok;

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Assignment Scores");
	printed = pcmk_rc_ok;

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Utilization Information");
	printed = pcmk_rc_ok;

	} else {
	rc = pcmk__log_output_new(&logger_out);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}
	pe__register_messages(logger_out);
	pcmk__register_lib_messages(logger_out);
	scheduler->priv = logger_out;
	}

	pcmk__schedule_actions(input, scheduler_flags, scheduler);

	if (logger_out == NULL) {
	out->end_list(out);
	} else {
	logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
	pcmk__output_free(logger_out);
	scheduler->priv = out;
	}

	input = NULL; /* Don't try and free it twice */

	if (graph_file != NULL) {
	rc = write_xml_file(scheduler->graph, graph_file, FALSE);
	if (rc < 0) {
	rc = pcmk_rc_graph_error;
	goto simulate_done;
	}
	}

	if (dot_file != NULL) {
	rc = write_sim_dotfile(scheduler, dot_file,
	pcmk_is_set(flags, pcmk_sim_all_actions),
	pcmk_is_set(flags, pcmk_sim_verbose));
	if (rc != pcmk_rc_ok) {
	rc = pcmk_rc_dot_error;
	goto simulate_done;
	}
	}

	if (!out->is_quiet(out)) {
	print_transition_summary(scheduler, printed == pcmk_rc_ok);
	}
	}

	rc = pcmk_rc_ok;

	if (!pcmk_is_set(flags, pcmk_sim_simulate)) {
	goto simulate_done;
	}

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	if (pcmk__simulate_transition(scheduler, cib, injections->op_fail)
	!= pcmk__graph_complete) {
	rc = pcmk_rc_invalid_transition;
	}

	if (out->is_quiet(out)) {
	goto simulate_done;
	}

	set_effective_date(scheduler, true, use_date);

	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
	}

	cluster_status(scheduler);
	print_cluster_status(scheduler, 0, section_opts, "Revised Cluster Status",
	true);

	simulate_done:
	cib__clean_up_connection(&cib);
	return rc;
	}

	int
	pcmk_simulate(xmlNodePtr xml, pcmk_scheduler_t scheduler,
	const pcmk_injections_t *injections, unsigned int flags,
	unsigned int section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	rc = pcmk__simulate(scheduler, out, injections, flags, section_opts,
	use_date, input_file, graph_file, dot_file);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	return rc;
	}
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index 68fd8a6240..c1a04dd1b6 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2229 +1,2229 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pcmk_resource_t *child;

	GList replicas; // pcmk__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) \
	CRM_ASSERT(rsc != NULL); \
	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle); \
	CRM_ASSERT(rsc->variant_opaque != NULL); \
	data = (pe__bundle_variant_data_t *) rsc->variant_opaque;

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__bundled_resource(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pcmk_resource_t *
	pe__get_rsc_in_container(const pcmk_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pcmk_resource_t *top = pe__const_top_resource(instance, true);

	if ((top == NULL) \|\| (top->variant != pcmk_rsc_variant_bundle)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pcmk__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (pe__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pcmk_resource_t *
	pe__first_container(const pcmk_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pcmk__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pcmk__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	crm_xml_add(rsc, PCMK_XA_ID, name);
	- crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
	+ crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(rsc, PCMK_XA_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	pcmk__config_err("Specifying the 'control-port' for %s requires "
	"'replicas-per-host=1'", data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	crm_xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d",
	data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = create_xml_node(xml_ip, "operations");
	crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->children = g_list_append(parent->children, replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pcmk_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	crm_xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
	crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
	crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
	env_opt, DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network, "host",
	pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = create_xml_node(xml_container, "operations");
	crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
	NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
	parent->children = g_list_append(parent->children, replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pcmk_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);

	if (match) {
	((pcmk_node_t *) match)->weight = -INFINITY;
	((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
	}
	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
	}
	}

	static int
	create_remote_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pcmk_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->cluster->resources, id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->cluster->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during scheduler data cleanup to
	* use as the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = ID(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pe_find_node(parent->cluster->nodes, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, "remote", "-INFINITY",
	parent->cluster);
	} else {
	node->weight = -INFINITY;
	}
	node->rsc_discover_mode = pcmk_probe_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pcmk_node_t entry. Ideally, it would do the same for bundle
	* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
	* obvious what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
	(gpointer) uname);

	replica->node = pe__copy_node(node);
	replica->node->weight = 500;
	replica->node->rsc_discover_mode = pcmk_probe_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->allowed_nodes);
	}
	replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->allowed_nodes,
	(gpointer) replica->node->details->id,
	pe__copy_node(replica->node));

	{
	pcmk_node_t *copy = pe__copy_node(replica->node);
	copy->weight = -INFINITY;
	g_hash_table_insert(replica->child->parent->allowed_nodes,
	(gpointer) replica->node->details->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->cluster) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pe__is_guest_or_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	node->weight = -INFINITY;
	}
	}

	replica->node->details->remote_rsc = replica->remote;

	// Ensure pe__is_guest_node() functions correctly immediately
	replica->remote->container = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	g_hash_table_insert(replica->node->details->attrs,
	strdup(CRM_ATTR_KIND), strdup("container"));

	/* One effect of this is that setup_container() will add
	* replica->remote to replica->container's fillers, which will make
	* pe__resource_contains_guest_node() true for replica->container.
	*
	* replica->child does NOT get added to replica->container's fillers.
	* The only noticeable effect if it did would be for its fail count to
	* be taken into account when checking replica->container's migration
	* threshold.
	*/
	parent->children = g_list_append(parent->children, replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pcmk_resource_t *parent,
	pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));

	CRM_ASSERT(mount != NULL);
	mount->source = strdup(source);
	mount->target = strdup(target);
	pcmk__str_update(&mount->options, options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pcmk__bundle_replica_t *
	replica_for_remote(pcmk_resource_t *remote)
	{
	pcmk_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}

	while (top->parent != NULL) {
	top = top->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->cluster);
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->xml);
	}

	const char *
	pe__add_bundle_remote_name(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	xmlNode xml, const char field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pcmk_node_t *node = NULL;
	pcmk__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->allocated_to;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pe__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pe__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->details->uname);
	}

	return node->details->uname;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", ID(mount_xml), flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid PCMK_META_PROMOTED_MAX
	value = crm_element_value(xml_obj, PCMK_META_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, "masters");
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	/* Default replicas to PCMK_META_PROMOTED_MAX if it was specified and 1
	* otherwise
	*/
	value = crm_element_value(xml_obj, "replicas");
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, "replicas-per-host");
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}

	bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
	bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
	bundle_data->image = crm_element_value_copy(xml_obj, "image");
	bundle_data->container_network = crm_element_value_copy(xml_obj, "network");

	xml_obj = first_named_child(rsc->xml, "network");
	if(xml_obj) {

	bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
	bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
	bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
	bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
	value = crm_element_value(xml_obj, "add-host");
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
	port->source = crm_element_value_copy(xml_child, "port");

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, "range");
	} else {
	port->target = crm_element_value_copy(xml_child, "internal-port");
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	pcmk__config_err("Invalid port directive %s", ID(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = first_named_child(rsc->xml, "storage");
	for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
	xml_child = pcmk__xe_next(xml_child)) {

	const char *source = crm_element_value(xml_child, "source-dir");
	const char *target = crm_element_value(xml_child, "target-dir");
	const char *options = crm_element_value(xml_child, "options");
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, "source-dir-root");
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	pcmk__config_err("Invalid mount directive %s", ID(xml_child));
	}
	}

	xml_obj = first_named_child(rsc->xml, "primitive");
	if (xml_obj && valid_network(bundle_data)) {
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
	(bundle_data->promoted_max? "master"
	: (const char *)xml_resource->name));

	xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
	crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_ORDERED, XML_BOOLEAN_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_PROMOTABLE, XML_BOOLEAN_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
	add_node_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	pcmk__config_err("Cannot control %s inside %s without either "
	"ip-range-start or control-port",
	rsc->id, ID(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	scheduler) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = calloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting control-port.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->children; childIter != NULL;
	childIter = childIter->next) {

	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->child = childIter->data;
	replica->child->exclusive_discover = TRUE;
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
	pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target =
	g_hash_table_lookup(replica->child->meta,
	PCMK_META_CONTAINER_ATTR_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	g_hash_table_replace(rsc->meta,
	strdup(PCMK_META_CONTAINER_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	g_hash_table_replace(bundle_data->child->meta,
	strdup(PCMK_META_CONTAINER_ATTR_TARGET),
	strdup(bundle_data->attribute_target));
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pcmk__bundle_replica_t *replica = NULL;

	replica = calloc(1, sizeof(pcmk__bundle_replica_t));
	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	pcmk__config_err("Failed unpacking resource %s", rsc->id);
	rsc->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->utilization;

	replica->container->utilization = replica->child->utilization;
	replica->child->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->children = g_list_append(rsc->children, bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pcmk_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pcmk_resource_t *
	pe__find_bundle_replica(const pcmk_resource_t bundle, const pcmk_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (replica->node->details == node->details) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_rsc_in_list(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	if (rsc != NULL) {
	if (options & pe_print_html) {
	status_print("<li>");
	}
	rsc->fns->print(rsc, pre_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	bundle_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (pre_text == NULL) {
	pre_text = "";
	}
	child_text = crm_strdup_printf("%s ", pre_text);

	get_bundle_variant_data(bundle_data, rsc);

	status_print("%s<bundle ", pre_text);
	status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
	status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
	status_print("image=\"%s\" ", bundle_data->image);
	status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print(">\n");

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	status_print("%s <replica " PCMK_XA_ID "=\"%d\">\n",
	pre_text, replica->offset);
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	status_print("%s </replica>\n", pre_text);
	}
	status_print("%s</bundle>\n", pre_text);
	free(child_text);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_TYPE, container_agent_str(bundle_data->agent_type),
	"image", bundle_data->image,
	"unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
	"maintenance",
	pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	PCMK_XA_DESCRIPTION, desc);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, PCMK_XA_ID, id);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
	replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml), show_opts,
	replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml), show_opts,
	replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
	replica->remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);

	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	const pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	get_bundle_variant_data(bundle_data, rsc);

	CRM_ASSERT(rsc != NULL);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
	continue;
	}

	print_ip = replica->ip != NULL &&
	!replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
	print_child = replica->child != NULL &&
	!replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
	print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
	print_remote = replica->remote != NULL &&
	!replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	out->message(out, crm_map_element_name(replica->ip->xml),
	new_show_opts, replica->ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, crm_map_element_name(replica->child->xml),
	new_show_opts, replica->child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, crm_map_element_name(replica->container->xml),
	new_show_opts, replica->container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, crm_map_element_name(replica->remote->xml),
	new_show_opts, replica->remote, only_node, only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	print_bundle_replica(pcmk__bundle_replica_t replica, const char pre_text,
	long options, void *print_data)
	{
	pcmk_node_t *node = NULL;
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	node = pe__current_node(replica->container);
	common_print(rsc, pre_text, buffer, node, options, print_data);
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	pe__print_bundle(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	char *child_text = NULL;
	CRM_CHECK(rsc != NULL, return);

	if (options & pe_print_xml) {
	bundle_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	get_bundle_variant_data(bundle_data, rsc);

	if (pre_text == NULL) {
	pre_text = " ";
	}

	status_print("%sContainer bundle%s: %s [%s]%s%s\n",
	pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}


	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica);
	if (options & pe_print_html) {
	status_print("<li>");
	}

	if (pcmk_is_set(options, pe_print_implicit)) {
	child_text = crm_strdup_printf(" %s", pre_text);
	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	status_print(" %sReplica[%d]\n", pre_text, replica->offset);
	}
	if (options & pe_print_html) {
	status_print("<br />\n<ul>\n");
	}
	print_rsc_in_list(replica->ip, child_text, options, print_data);
	print_rsc_in_list(replica->container, child_text, options, print_data);
	print_rsc_in_list(replica->remote, child_text, options, print_data);
	print_rsc_in_list(replica->child, child_text, options, print_data);
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	} else {
	child_text = crm_strdup_printf("%s ", pre_text);
	print_bundle_replica(replica, child_text, options, print_data);
	}
	free(child_text);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	if (options & pe_print_html) {
	status_print("</ul>\n");
	}
	}

	static void
	free_bundle_replica(pcmk__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	free_xml(replica->ip->xml);
	replica->ip->xml = NULL;
	replica->ip->fns->free(replica->ip);
	replica->ip = NULL;
	}
	if (replica->container) {
	free_xml(replica->container->xml);
	replica->container->xml = NULL;
	replica->container->fns->free(replica->container);
	replica->container = NULL;
	}
	if (replica->remote) {
	free_xml(replica->remote->xml);
	replica->remote->xml = NULL;
	replica->remote->fns->free(replica->remote);
	replica->remote = NULL;
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->children);

	if(bundle_data->child) {
	free_xml(bundle_data->child->xml);
	bundle_data->child->xml = NULL;
	bundle_data->child->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pcmk_resource_t *rsc)
	{
	if ((rsc == NULL) \|\| (rsc->variant != pcmk_rsc_variant_bundle)) {
	return 0;
	} else {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	}

	void
	pe__count_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pcmk__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	} else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pcmk_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of pcmk_rsc_methods_t:active_node()
	pcmk_node_t *
	pe__bundle_active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;
	pcmk_node_t *node = NULL;
	pcmk_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (replica->container->running_on != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->fns->active_node(container, count_all, count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;

	for (GList *node_iter = container->running_on; node_iter != NULL;
	node_iter = node_iter->next) {
	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
	index 2942bb2035..b368d19d0d 100644
	--- a/lib/pengine/complex.c
	+++ b/lib/pengine/complex.c
	@@ -1,1202 +1,1202 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/scheduler_internal.h>

	#include "pe_status_private.h"

	void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);

	static pcmk_node_t active_node(const pcmk_resource_t rsc,
	unsigned int *count_all,
	unsigned int *count_clean);

	pcmk_rsc_methods_t resource_class_functions[] = {
	{
	native_unpack,
	native_find_rsc,
	native_parameter,
	native_print,
	native_active,
	native_resource_state,
	native_location,
	native_free,
	pe__count_common,
	pe__native_is_filtered,
	active_node,
	pe__primitive_max_per_node,
	},
	{
	group_unpack,
	native_find_rsc,
	native_parameter,
	group_print,
	group_active,
	group_resource_state,
	native_location,
	group_free,
	pe__count_common,
	pe__group_is_filtered,
	active_node,
	pe__group_max_per_node,
	},
	{
	clone_unpack,
	native_find_rsc,
	native_parameter,
	clone_print,
	clone_active,
	clone_resource_state,
	native_location,
	clone_free,
	pe__count_common,
	pe__clone_is_filtered,
	active_node,
	pe__clone_max_per_node,
	},
	{
	pe__unpack_bundle,
	native_find_rsc,
	native_parameter,
	pe__print_bundle,
	pe__bundle_active,
	pe__bundle_resource_state,
	native_location,
	pe__free_bundle,
	pe__count_bundle,
	pe__bundle_is_filtered,
	pe__bundle_active_node,
	pe__bundle_max_per_node,
	}
	};

	static enum pe_obj_types
	get_resource_type(const char *name)
	{
	if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	return pcmk_rsc_variant_primitive;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
	return pcmk_rsc_variant_group;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
	// @COMPAT deprecated since 2.0.0
	return pcmk_rsc_variant_clone;

	} else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
	return pcmk_rsc_variant_bundle;
	}

	return pcmk_rsc_variant_unknown;
	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static void
	expand_parents_fixed_nvpairs(pcmk_resource_t *rsc,
	pe_rule_eval_data_t *rule_data,
	GHashTable meta_hash, pcmk_scheduler_t scheduler)
	{
	GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
	pcmk_resource_t *p = rsc->parent;

	if (p == NULL) {
	return ;
	}

	/* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
	/* The fixed value of the lower parent resource takes precedence and is not overwritten. */
	while(p != NULL) {
	/* A hash table for comparison is generated, including the id-ref. */
	pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS, rule_data,
	parent_orig_meta, NULL, FALSE, scheduler);
	p = p->parent;
	}

	/* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
	if (parent_orig_meta != NULL) {
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, parent_orig_meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) &key, (gpointer ) &value)) {
	/* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
	/* Attributes that already exist in the child lease are not updated. */
	dup_attr(key, value, meta_hash);
	}
	}

	if (parent_orig_meta != NULL) {
	g_hash_table_destroy(parent_orig_meta);
	}

	return ;

	}
	void
	get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rsc_eval_data_t rsc_rule_data = {
	- .standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	+ .standard = crm_element_value(rsc->xml, PCMK_XA_CLASS),
	.provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
	.agent = crm_element_value(rsc->xml, PCMK_XA_TYPE)
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
	const char prop_name = (const char ) a->name;
	const char *prop_value = pcmk__xml_attr_value(a);

	add_hash_param(meta_hash, prop_name, prop_value);
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
	/* If it is already explicitly set as a child, it will not be overwritten. */
	if (rsc->parent != NULL) {
	expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, scheduler);
	}

	/* check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_META_SETS,
	&rule_data, meta_hash, NULL, FALSE, scheduler);

	/* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
	/* The values already set up to this point will not be overwritten. */
	if (rsc->parent) {
	g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
	}
	}

	void
	get_rsc_attributes(GHashTable meta_hash, const pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	if (node) {
	rule_data.node_hash = node->details->attrs;
	}

	pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
	meta_hash, NULL, FALSE, scheduler);

	/* set anything else based on the parent */
	if (rsc->parent != NULL) {
	get_rsc_attributes(meta_hash, rsc->parent, node, scheduler);

	} else {
	/* and finally check the defaults */
	pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_ATTR_SETS,
	&rule_data, meta_hash, NULL, FALSE,
	scheduler);
	}
	}

	static char *
	template_op_key(xmlNode * op)
	{
	const char *name = crm_element_value(op, PCMK_XA_NAME);
	const char *role = crm_element_value(op, "role");
	char *key = NULL;

	if ((role == NULL)
	\|\| pcmk__strcase_any_of(role, PCMK__ROLE_STARTED, PCMK__ROLE_UNPROMOTED,
	PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
	role = PCMK__ROLE_UNKNOWN;
	}

	key = crm_strdup_printf("%s-%s", name, role);
	return key;
	}

	static gboolean
	unpack_template(xmlNode xml_obj, xmlNode *expanded_xml,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *cib_resources = NULL;
	xmlNode *template = NULL;
	xmlNode *new_xml = NULL;
	xmlNode *child_xml = NULL;
	xmlNode *rsc_ops = NULL;
	xmlNode *template_ops = NULL;
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for template unpacking");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	cib_resources = get_xpath_object("//" XML_CIB_TAG_RESOURCES,
	scheduler->input, LOG_TRACE);
	if (cib_resources == NULL) {
	pcmk__config_err("No resources configured");
	return FALSE;
	}

	template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
	PCMK_XA_ID, template_ref);
	if (template == NULL) {
	pcmk__config_err("No template named '%s'", template_ref);
	return FALSE;
	}

	new_xml = copy_xml(template);
	xmlNodeSetName(new_xml, xml_obj->name);
	crm_xml_add(new_xml, PCMK_XA_ID, id);
	crm_xml_add(new_xml, PCMK__META_CLONE_INSTANCE_NUM,
	crm_element_value(xml_obj, PCMK__META_CLONE_INSTANCE_NUM));

	template_ops = find_xml_node(new_xml, "operations", FALSE);

	for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
	child_xml = pcmk__xe_next(child_xml)) {
	xmlNode *new_child = NULL;

	new_child = add_node_copy(new_xml, child_xml);

	if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
	rsc_ops = new_child;
	}
	}

	if (template_ops && rsc_ops) {
	xmlNode *op = NULL;
	GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);

	for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	g_hash_table_insert(rsc_ops_hash, key, op);
	}

	for (op = pcmk__xe_first_child(template_ops); op != NULL;
	op = pcmk__xe_next(op)) {

	char *key = template_op_key(op);

	if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
	add_node_copy(rsc_ops, op);
	}

	free(key);
	}

	if (rsc_ops_hash) {
	g_hash_table_destroy(rsc_ops_hash);
	}

	free_xml(template_ops);
	}

	/free_xml(expanded_xml); */
	*expanded_xml = new_xml;

	#if 0 /* Disable multi-level templates for now */
	if (!unpack_template(new_xml, expanded_xml, scheduler)) {
	free_xml(*expanded_xml);
	*expanded_xml = NULL;
	return FALSE;
	}
	#endif

	return TRUE;
	}

	static gboolean
	add_template_rsc(xmlNode xml_obj, pcmk_scheduler_t scheduler)
	{
	const char *template_ref = NULL;
	const char *id = NULL;

	if (xml_obj == NULL) {
	pcmk__config_err("No resource object for processing resource list "
	"of template");
	return FALSE;
	}

	template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
	if (template_ref == NULL) {
	return TRUE;
	}

	id = ID(xml_obj);
	if (id == NULL) {
	pcmk__config_err("'%s' object must have a id", xml_obj->name);
	return FALSE;
	}

	if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
	pcmk__config_err("The resource object '%s' should not reference itself",
	id);
	return FALSE;
	}

	if (add_tag_ref(scheduler->template_rsc_sets, template_ref, id) == FALSE) {
	return FALSE;
	}

	return TRUE;
	}

	static bool
	detect_promotable(pcmk_resource_t *rsc)
	{
	const char *promotable = g_hash_table_lookup(rsc->meta,
	PCMK_META_PROMOTABLE);

	if (crm_is_true(promotable)) {
	return TRUE;
	}

	// @COMPAT deprecated since 2.0.0
	if (pcmk__xe_is(rsc->xml, PCMK_XE_PROMOTABLE_LEGACY)) {
	/* @TODO in some future version, pcmk__warn_once() here,
	* then drop support in even later version
	*/
	g_hash_table_insert(rsc->meta, strdup(PCMK_META_PROMOTABLE),
	strdup(XML_BOOLEAN_TRUE));
	return TRUE;
	}
	return FALSE;
	}

	static void
	free_params_table(gpointer data)
	{
	g_hash_table_destroy((GHashTable *) data);
	}

	/*!
	* \brief Get a table of resource parameters
	*
	* \param[in,out] rsc Resource to query
	* \param[in] node Node for evaluating rules (NULL for defaults)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Hash table containing resource parameter names and values
	* (or NULL if \p rsc or \p scheduler is NULL)
	* \note The returned table will be destroyed when the resource is freed, so
	* callers should not destroy it.
	*/
	GHashTable *
	pe_rsc_params(pcmk_resource_t rsc, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params_on_node = NULL;

	/* A NULL node is used to request the resource's default parameters
	* (not evaluated for node), but we always want something non-NULL
	* as a hash table key.
	*/
	const char *node_name = "";

	// Sanity check
	if ((rsc == NULL) \|\| (scheduler == NULL)) {
	return NULL;
	}
	if ((node != NULL) && (node->details->uname != NULL)) {
	node_name = node->details->uname;
	}

	// Find the parameter table for given node
	if (rsc->parameter_cache == NULL) {
	rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
	} else {
	params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
	}

	// If none exists yet, create one with parameters evaluated for node
	if (params_on_node == NULL) {
	params_on_node = pcmk__strkey_table(free, free);
	get_rsc_attributes(params_on_node, rsc, node, scheduler);
	g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
	params_on_node);
	}
	return params_on_node;
	}

	/*!
	* \internal
	* \brief Unpack a resource's "requires" meta-attribute
	*
	* \param[in,out] rsc Resource being unpacked
	* \param[in] value Value of "requires" meta-attribute
	* \param[in] is_default Whether \p value was selected by default
	*/
	static void
	unpack_requires(pcmk_resource_t rsc, const char value, bool is_default)
	{
	if (pcmk__str_eq(value, PCMK__VALUE_NOTHING, pcmk__str_casei)) {

	} else if (pcmk__str_eq(value, PCMK__VALUE_QUORUM, pcmk__str_casei)) {
	pcmk__set_rsc_flags(rsc, pcmk_rsc_needs_quorum);

	} else if (pcmk__str_eq(value, PCMK__VALUE_FENCING, pcmk__str_casei)) {
	pcmk__set_rsc_flags(rsc, pcmk_rsc_needs_fencing);
	if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("%s requires fencing but fencing is disabled",
	rsc->id);
	}

	} else if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"devices cannot require unfencing", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else if (!pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
	"to \"" PCMK__VALUE_QUORUM "\" because fencing "
	"is disabled", rsc->id);
	unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
	return;

	} else {
	pcmk__set_rsc_flags(rsc, pcmk_rsc_needs_fencing
	\|pcmk_rsc_needs_unfencing);
	}

	} else {
	const char *orig_value = value;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
	value = PCMK__VALUE_QUORUM;

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& xml_contains_remote_node(rsc->xml)) {
	value = PCMK__VALUE_QUORUM;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_enable_unfencing)) {
	value = PCMK__VALUE_UNFENCING;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	value = PCMK__VALUE_FENCING;

	} else if (rsc->cluster->no_quorum_policy == pcmk_no_quorum_ignore) {
	value = PCMK__VALUE_NOTHING;

	} else {
	value = PCMK__VALUE_QUORUM;
	}

	if (orig_value != NULL) {
	pcmk__config_err("Resetting '" PCMK_META_REQUIRES "' for %s "
	"to '%s' because '%s' is not valid",
	rsc->id, value, orig_value);
	}
	unpack_requires(rsc, value, true);
	return;
	}

	pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value,
	(is_default? " (default)" : ""));
	}

	#ifndef PCMK__COMPAT_2_0
	static void
	warn_about_deprecated_classes(pcmk_resource_t *rsc)
	{
	- const char *std = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *std = crm_element_value(rsc->xml, PCMK_XA_CLASS);

	if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_none)) {
	pcmk__warn_once(pcmk__wo_upstart,
	"Support for Upstart resources (such as %s) is "
	"deprecated and will be removed in a future release",
	rsc->id);

	} else if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_none)) {
	pcmk__warn_once(pcmk__wo_nagios,
	"Support for Nagios resources (such as %s) is "
	"deprecated and will be removed in a future release",
	rsc->id);
	}
	}
	#endif

	/*!
	* \internal
	* \brief Unpack configuration XML for a given resource
	*
	* Unpack the XML object containing a resource's configuration into a new
	* \c pcmk_resource_t object.
	*
	* \param[in] xml_obj XML node containing the resource's configuration
	* \param[out] rsc Where to store the unpacked resource information
	* \param[in] parent Resource's parent, if any
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code
	* \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
	* the caller is responsible for freeing it using its variant-specific
	* free() method. Otherwise, \p *rsc is guaranteed to be NULL.
	*/
	int
	pe__unpack_resource(xmlNode xml_obj, pcmk_resource_t *rsc,
	pcmk_resource_t parent, pcmk_scheduler_t scheduler)
	{
	xmlNode *expanded_xml = NULL;
	xmlNode *ops = NULL;
	const char *value = NULL;
	const char *id = NULL;
	bool guest_node = false;
	bool remote_node = false;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = NULL,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	CRM_CHECK(rsc != NULL, return EINVAL);
	CRM_CHECK((xml_obj != NULL) && (scheduler != NULL),
	*rsc = NULL;
	return EINVAL);

	rule_data.now = scheduler->now;

	crm_log_xml_trace(xml_obj, "[raw XML]");

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	if (id == NULL) {
	pcmk__config_err("Ignoring <%s> configuration without " PCMK_XA_ID,
	xml_obj->name);
	return pcmk_rc_unpack_error;
	}

	if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) {
	return pcmk_rc_unpack_error;
	}

	*rsc = calloc(1, sizeof(pcmk_resource_t));
	if (*rsc == NULL) {
	pcmk__sched_err("Unable to allocate memory for resource '%s'", id);
	return ENOMEM;
	}
	(*rsc)->cluster = scheduler;

	if (expanded_xml) {
	crm_log_xml_trace(expanded_xml, "[expanded XML]");
	(*rsc)->xml = expanded_xml;
	(*rsc)->orig_xml = xml_obj;

	} else {
	(*rsc)->xml = xml_obj;
	(*rsc)->orig_xml = NULL;
	}

	/* Do not use xml_obj from here on, use (rsc)->xml in case templates are involved /

	(*rsc)->parent = parent;

	ops = find_xml_node((*rsc)->xml, "operations", FALSE);
	(*rsc)->ops_xml = expand_idref(ops, scheduler->input);

	(rsc)->variant = get_resource_type((const char ) (*rsc)->xml->name);
	if ((*rsc)->variant == pcmk_rsc_variant_unknown) {
	pcmk__config_err("Ignoring resource '%s' of unknown type '%s'",
	id, (*rsc)->xml->name);
	common_free(*rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	#ifndef PCMK__COMPAT_2_0
	warn_about_deprecated_classes(*rsc);
	#endif

	(*rsc)->meta = pcmk__strkey_table(free, free);
	(*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
	(*rsc)->known_on = pcmk__strkey_table(NULL, free);

	value = crm_element_value((*rsc)->xml, PCMK__META_CLONE_INSTANCE_NUM);
	if (value) {
	(*rsc)->id = crm_strdup_printf("%s:%s", id, value);
	add_hash_param((*rsc)->meta, PCMK__META_CLONE_INSTANCE_NUM, value);

	} else {
	(*rsc)->id = strdup(id);
	}

	(rsc)->fns = &resource_class_functions[(rsc)->variant];

	get_meta_attributes((rsc)->meta, rsc, NULL, scheduler);
	(rsc)->parameters = pe_rsc_params(rsc, NULL, scheduler); // \deprecated

	(*rsc)->flags = 0;
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_runnable\|pcmk_rsc_unassigned);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_managed);
	}

	(*rsc)->rsc_cons = NULL;
	(*rsc)->rsc_tickets = NULL;
	(*rsc)->actions = NULL;
	(*rsc)->role = pcmk_role_stopped;
	(*rsc)->next_role = pcmk_role_unknown;

	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	(*rsc)->stickiness = 0;
	(*rsc)->migration_threshold = INFINITY;
	(*rsc)->failure_timeout = 0;

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_PRIORITY);
	(*rsc)->priority = char2score(value);

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_CRITICAL);
	if ((value == NULL) \|\| crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_critical);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_NOTIFY);
	if (crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_notify);
	}

	if (xml_contains_remote_node((*rsc)->xml)) {
	(*rsc)->is_remote_node = TRUE;
	if (g_hash_table_lookup((*rsc)->meta, PCMK__META_CONTAINER)) {
	guest_node = true;
	} else {
	remote_node = true;
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_ALLOW_MIGRATE);
	if (crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_migratable);
	} else if ((value == NULL) && remote_node) {
	/* By default, we want remote nodes to be able
	* to float around the cluster without having to stop all the
	* resources within the remote-node before moving. Allowing
	* migration support enables this feature. If this ever causes
	* problems, migration support can be explicitly turned off with
	* PCMK_META_ALLOW_MIGRATE=false.
	*/
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_migratable);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_IS_MANAGED);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	if (crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_managed);
	} else {
	pcmk__clear_rsc_flags(*rsc, pcmk_rsc_managed);
	}
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MAINTENANCE);
	if (crm_is_true(value)) {
	pcmk__clear_rsc_flags(*rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_maintenance);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	pcmk__clear_rsc_flags(*rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_maintenance);
	}

	if (pe_rsc_is_clone(pe__const_top_resource(*rsc, false))) {
	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_GLOBALLY_UNIQUE);
	if (crm_is_true(value)) {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_unique);
	}
	if (detect_promotable(*rsc)) {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_promotable);
	}
	} else {
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_unique);
	}

	// @COMPAT Deprecated meta-attribute
	value = g_hash_table_lookup((*rsc)->meta, PCMK__META_RESTART_TYPE);
	if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	(*rsc)->restart_type = pe_restart_restart;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: restart",
	(*rsc)->id);
	pcmk__warn_once(pcmk__wo_restart_type,
	"Support for " PCMK__META_RESTART_TYPE " is deprecated "
	"and will be removed in a future release");

	} else {
	(*rsc)->restart_type = pe_restart_ignore;
	pcmk__rsc_trace(*rsc, "%s dependency restart handling: ignore",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MULTIPLE_ACTIVE);
	if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_stop;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_block;
	pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block",
	(*rsc)->id);

	} else if (pcmk__str_eq(value, "stop_unexpected", pcmk__str_casei)) {
	(*rsc)->recovery_type = pcmk_multiply_active_unexpected;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: "
	"stop unexpected instances",
	(*rsc)->id);

	} else { // "stop_start"
	if (!pcmk__str_eq(value, "stop_start",
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_warn("%s is not a valid value for "
	PCMK_META_MULTIPLE_ACTIVE
	", using default of \"stop_start\"", value);
	}
	(*rsc)->recovery_type = pcmk_multiply_active_restart;
	pcmk__rsc_trace(*rsc,
	"%s multiple running resource recovery: stop/start",
	(*rsc)->id);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_RESOURCE_STICKINESS);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->stickiness = char2score(value);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MIGRATION_THRESHOLD);
	if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
	(*rsc)->migration_threshold = char2score(value);
	if ((*rsc)->migration_threshold < 0) {
	/* @TODO We use 1 here to preserve previous behavior, but this
	* should probably use the default (INFINITY) or 0 (to disable)
	* instead.
	*/
	pcmk__warn_once(pcmk__wo_neg_threshold,
	PCMK_META_MIGRATION_THRESHOLD
	" must be non-negative, using 1 instead");
	(*rsc)->migration_threshold = 1;
	}
	}

	- if (pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
	+ if (pcmk__str_eq(crm_element_value((*rsc)->xml, PCMK_XA_CLASS),
	PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_fencing);
	pcmk__set_rsc_flags(*rsc, pcmk_rsc_fence_device);
	}

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_REQUIRES);
	unpack_requires(*rsc, value, false);

	value = g_hash_table_lookup((*rsc)->meta, PCMK_META_FAILURE_TIMEOUT);
	if (value != NULL) {
	guint interval_ms = 0U;

	// Stored as seconds
	pcmk_parse_interval_spec(value, &interval_ms);
	(*rsc)->failure_timeout = (int) (interval_ms / 1000);
	}

	if (remote_node) {
	GHashTable params = pe_rsc_params(rsc, NULL, scheduler);

	/* Grabbing the value now means that any rules based on node attributes
	* will evaluate to false, so such rules should not be used with
	* PCMK_REMOTE_RA_RECONNECT_INTERVAL.
	*
	* @TODO Evaluate per node before using
	*/
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_RECONNECT_INTERVAL);
	if (value) {
	/* reconnect delay works by setting failure_timeout and preventing the
	* connection from starting until the failure is cleared. */
	pcmk_parse_interval_spec(value, &((*rsc)->remote_reconnect_ms));

	/* We want to override any default failure_timeout in use when remote
	* PCMK_REMOTE_RA_RECONNECT_INTERVAL is in use.
	*/
	(rsc)->failure_timeout = (rsc)->remote_reconnect_ms / 1000;
	}
	}

	get_target_role(rsc, &((rsc)->next_role));
	pcmk__rsc_trace(rsc, "%s desired next state: %s", (rsc)->id,
	(rsc)->next_role != pcmk_role_unknown? role2text((rsc)->next_role) : "default");

	if ((rsc)->fns->unpack(rsc, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	// This tag must stay exactly the same because it is tested elsewhere
	resource_location(*rsc, NULL, 0, "symmetric_default", scheduler);
	} else if (guest_node) {
	/* remote resources tied to a container resource must always be allowed
	* to opt-in to the cluster. Whether the connection resource is actually
	* allowed to be placed on a node is dependent on the container resource */
	resource_location(*rsc, NULL, 0, "remote_connection_default",
	scheduler);
	}

	pcmk__rsc_trace(rsc, "%s action notification: %s", (rsc)->id,
	pcmk_is_set((*rsc)->flags, pcmk_rsc_notify)? "required" : "not required");

	(*rsc)->utilization = pcmk__strkey_table(free, free);

	pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
	(*rsc)->utilization, NULL, FALSE, scheduler);

	if (expanded_xml) {
	if (add_template_rsc(xml_obj, scheduler) == FALSE) {
	(rsc)->fns->free(rsc);
	*rsc = NULL;
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	gboolean
	is_parent(pcmk_resource_t child, pcmk_resource_t rsc)
	{
	pcmk_resource_t *parent = child;

	if (parent == NULL \|\| rsc == NULL) {
	return FALSE;
	}
	while (parent->parent != NULL) {
	if (parent->parent == rsc) {
	return TRUE;
	}
	parent = parent->parent;
	}
	return FALSE;
	}

	pcmk_resource_t *
	uber_parent(pcmk_resource_t *rsc)
	{
	pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while ((parent->parent != NULL)
	&& (parent->parent->variant != pcmk_rsc_variant_bundle)) {
	parent = parent->parent;
	}
	return parent;
	}

	/*!
	* \internal
	* \brief Get the topmost parent of a resource as a const pointer
	*
	* \param[in] rsc Resource to check
	* \param[in] include_bundle If true, go all the way to bundle
	*
	* \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
	* the bundle if \p rsc is bundled and \p include_bundle is true,
	* otherwise the topmost parent of \p rsc up to a clone
	*/
	const pcmk_resource_t *
	pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle)
	{
	const pcmk_resource_t *parent = rsc;

	if (parent == NULL) {
	return NULL;
	}
	while (parent->parent != NULL) {
	if (!include_bundle
	&& (parent->parent->variant == pcmk_rsc_variant_bundle)) {
	break;
	}
	parent = parent->parent;
	}
	return parent;
	}

	void
	common_free(pcmk_resource_t * rsc)
	{
	if (rsc == NULL) {
	return;
	}

	pcmk__rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);

	g_list_free(rsc->rsc_cons);
	g_list_free(rsc->rsc_cons_lhs);
	g_list_free(rsc->rsc_tickets);
	g_list_free(rsc->dangling_migrations);

	if (rsc->parameter_cache != NULL) {
	g_hash_table_destroy(rsc->parameter_cache);
	}
	if (rsc->meta != NULL) {
	g_hash_table_destroy(rsc->meta);
	}
	if (rsc->utilization != NULL) {
	g_hash_table_destroy(rsc->utilization);
	}

	if ((rsc->parent == NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {

	free_xml(rsc->xml);
	rsc->xml = NULL;
	free_xml(rsc->orig_xml);
	rsc->orig_xml = NULL;

	/* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
	} else if (rsc->orig_xml) {
	free_xml(rsc->xml);
	rsc->xml = NULL;
	}
	if (rsc->running_on) {
	g_list_free(rsc->running_on);
	rsc->running_on = NULL;
	}
	if (rsc->known_on) {
	g_hash_table_destroy(rsc->known_on);
	rsc->known_on = NULL;
	}
	if (rsc->actions) {
	g_list_free(rsc->actions);
	rsc->actions = NULL;
	}
	if (rsc->allowed_nodes) {
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = NULL;
	}
	g_list_free(rsc->fillers);
	g_list_free(rsc->rsc_location);
	pcmk__rsc_trace(rsc, "Resource freed");
	free(rsc->id);
	free(rsc->clone_name);
	free(rsc->allocated_to);
	free(rsc->variant_opaque);
	free(rsc->pending_task);
	free(rsc);
	}

	/*!
	* \internal
	* \brief Count a node and update most preferred to it as appropriate
	*
	* \param[in] rsc An active resource
	* \param[in] node A node that \p rsc is active on
	* \param[in,out] active This will be set to \p node if \p node is more
	* preferred than the current value
	* \param[in,out] count_all If not NULL, this will be incremented
	* \param[in,out] count_clean If not NULL, this will be incremented if \p node
	* is online and clean
	*
	* \return true if the count should continue, or false if sufficiently known
	*/
	bool
	pe__count_active_node(const pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_node_t *active, unsigned int count_all,
	unsigned int *count_clean)
	{
	bool keep_looking = false;
	bool is_happy = false;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
	return false);

	is_happy = node->details->online && !node->details->unclean;

	if (count_all != NULL) {
	++*count_all;
	}
	if ((count_clean != NULL) && is_happy) {
	++*count_clean;
	}
	if ((count_all != NULL) \|\| (count_clean != NULL)) {
	keep_looking = true; // We're counting, so go through entire list
	}

	if (rsc->partial_migration_source != NULL) {
	if (node->details == rsc->partial_migration_source->details) {
	*active = node; // This is the migration source
	} else {
	keep_looking = true;
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	if (is_happy && ((active == NULL) \|\| !(active)->details->online
	\|\| (*active)->details->unclean)) {
	*active = node; // This is the first clean node
	} else {
	keep_looking = true;
	}
	}
	if (*active == NULL) {
	*active = node; // This is the first node checked
	}
	return keep_looking;
	}

	// Shared implementation of pcmk_rsc_methods_t:active_node()
	static pcmk_node_t *
	active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active,
	count_all, count_clean)) {
	break; // Don't waste time iterating if we don't have to
	}
	}
	return active;
	}

	/*!
	* \brief
	* \internal Find and count active nodes according to "requires"
	*
	* \param[in] rsc Resource to check
	* \param[out] count If not NULL, will be set to count of active nodes
	*
	* \return An active node (or NULL if resource is not active anywhere)
	*
	* \note This is a convenience wrapper for active_node() where the count of all
	* active nodes or only clean active nodes is desired according to the
	* "requires" meta-attribute.
	*/
	pcmk_node_t *
	pe__find_active_requires(const pcmk_resource_t rsc, unsigned int count)
	{
	if (rsc == NULL) {
	if (count != NULL) {
	*count = 0;
	}
	return NULL;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	return rsc->fns->active_node(rsc, count, NULL);

	} else {
	return rsc->fns->active_node(rsc, NULL, count);
	}
	}

	void
	pe__count_common(pcmk_resource_t *rsc)
	{
	if (rsc->children != NULL) {
	for (GList *item = rsc->children; item != NULL; item = item->next) {
	((pcmk_resource_t *) item->data)->fns->count(item->data);
	}

	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
	\|\| (rsc->role > pcmk_role_stopped)) {
	rsc->cluster->ninstances++;
	if (pe__resource_is_disabled(rsc)) {
	rsc->cluster->disabled_resources++;
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	rsc->cluster->blocked_resources++;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource's next role
	*
	* \param[in,out] rsc Resource to be updated
	* \param[in] role Resource's new next role
	* \param[in] why Human-friendly reason why role is changing (for logs)
	*/
	void
	pe__set_next_role(pcmk_resource_t rsc, enum rsc_role_e role, const char why)
	{
	CRM_ASSERT((rsc != NULL) && (why != NULL));
	if (rsc->next_role != role) {
	pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
	rsc->id, role2text(rsc->next_role), role2text(role),
	why);
	rsc->next_role = role;
	}
	}
	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index 8a3ac912d9..970cb5df22 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,1468 +1,1468 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/output.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <crm/msg_xml.h>
	#include <pe_status_private.h>

	#ifdef PCMK__COMPAT_2_0
	#define PROVIDER_SEP "::"
	#else
	#define PROVIDER_SEP ":"
	#endif

	/*!
	* \internal
	* \brief Check whether a resource is active on multiple nodes
	*/
	static bool
	is_multiply_active(const pcmk_resource_t *rsc)
	{
	unsigned int count = 0;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	pe__find_active_requires(rsc, &count);
	}
	return count > 1;
	}

	static void
	native_priority_to_node(pcmk_resource_t rsc, pcmk_node_t node,
	gboolean failed)
	{
	int priority = 0;

	if ((rsc->priority == 0) \|\| (failed == TRUE)) {
	return;
	}

	if (rsc->role == pcmk_role_promoted) {
	// Promoted instance takes base priority + 1
	priority = rsc->priority + 1;

	} else {
	priority = rsc->priority;
	}

	node->details->priority += priority;
	pcmk__rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s)",
	pe__node_name(node), node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "");

	/* Priority of a resource running on a guest node is added to the cluster
	* node as well. */
	if (node->details->remote_rsc
	&& node->details->remote_rsc->container) {
	GList *gIter = node->details->remote_rsc->container->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *a_node = gIter->data;

	a_node->details->priority += priority;
	pcmk__rsc_trace(rsc,
	"%s now has priority %d with %s'%s' "
	"(priority: %d%s) from guest node %s",
	pe__node_name(a_node), a_node->details->priority,
	(rsc->role == pcmk_role_promoted)? "promoted " : "",
	rsc->id, rsc->priority,
	(rsc->role == pcmk_role_promoted)? " + 1" : "",
	pe__node_name(node));
	}
	}
	}

	void
	native_add_running(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler, gboolean failed)
	{
	GList *gIter = rsc->running_on;

	CRM_CHECK(node != NULL, return);
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	CRM_CHECK(a_node != NULL, return);
	if (pcmk__str_eq(a_node->details->id, node->details->id, pcmk__str_casei)) {
	return;
	}
	}

	pcmk__rsc_trace(rsc, "Adding %s to %s %s", rsc->id, pe__node_name(node),
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : "(unmanaged)");

	rsc->running_on = g_list_append(rsc->running_on, node);
	if (rsc->variant == pcmk_rsc_variant_primitive) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);

	native_priority_to_node(rsc, node, failed);
	}

	if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& node->details->maintenance) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_maintenance);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk_resource_t *p = rsc->parent;

	pcmk__rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, INFINITY, "not_managed_default",
	scheduler);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->running_on = g_list_append(p->running_on, node);
	p = p->parent;
	}
	return;
	}

	if (is_multiply_active(rsc)) {
	switch (rsc->recovery_type) {
	case pcmk_multiply_active_stop:
	{
	GHashTableIter gIter;
	pcmk_node_t *local_node = NULL;

	/* make sure it doesn't come up again */
	if (rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(rsc->allowed_nodes);
	}
	rsc->allowed_nodes = pe__node_list2table(scheduler->nodes);
	g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	local_node->weight = -INFINITY;
	}
	}
	break;
	case pcmk_multiply_active_block:
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_blocked);

	/* If the resource belongs to a group or bundle configured with
	* PCMK_META_MULTIPLE_ACTIVE=block, block the entire entity.
	*/
	if (rsc->parent
	&& ((rsc->parent->variant == pcmk_rsc_variant_group)
	\|\| (rsc->parent->variant == pcmk_rsc_variant_bundle))
	&& (rsc->parent->recovery_type == pcmk_multiply_active_block)) {
	GList *gIter = rsc->parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *child = gIter->data;

	pcmk__clear_rsc_flags(child, pcmk_rsc_managed);
	pcmk__set_rsc_flags(child, pcmk_rsc_blocked);
	}
	}
	break;

	// pcmk_multiply_active_restart, pcmk_multiply_active_unexpected
	default:
	/* The scheduler will do the right thing because the relevant
	* variables and flags are set when unpacking the history.
	*/
	break;
	}
	crm_debug("%s is active on multiple nodes including %s: %s",
	rsc->id, pe__node_name(node),
	pcmk_multiply_active_text(rsc->recovery_type));

	} else {
	pcmk__rsc_trace(rsc, "Resource %s is active on %s",
	rsc->id, pe__node_name(node));
	}

	if (rsc->parent != NULL) {
	native_add_running(rsc->parent, node, scheduler, FALSE);
	}
	}

	static void
	recursive_clear_unique(pcmk_resource_t *rsc, gpointer user_data)
	{
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	add_hash_param(rsc->meta, PCMK_META_GLOBALLY_UNIQUE, XML_BOOLEAN_FALSE);
	g_list_foreach(rsc->children, (GFunc) recursive_clear_unique, NULL);
	}

	gboolean
	native_unpack(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *parent = uber_parent(rsc);
	- const char *standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *standard = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	uint32_t ra_caps = pcmk_get_ra_caps(standard);

	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	// Only some agent standards support unique and promotable clones
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& pe_rsc_is_clone(parent)) {

	/* @COMPAT We should probably reject this situation as an error (as we
	* do for promotable below) rather than warn and convert, but that would
	* be a backward-incompatible change that we should probably do with a
	* transform at a schema major version bump.
	*/
	pe__force_anon(standard, parent, rsc->id, scheduler);

	/* Clear PCMK_META_GLOBALLY_UNIQUE on the parent and all its descendants
	* unpacked so far (clearing the parent should make any future children
	* unpacking correct). We have to clear this resource explicitly because
	* it isn't hooked into the parent's children yet.
	*/
	recursive_clear_unique(parent, NULL);
	recursive_clear_unique(rsc, NULL);
	}
	if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
	&& pcmk_is_set(parent->flags, pcmk_rsc_promotable)) {

	pcmk__config_err("Resource %s is of type %s and therefore "
	"cannot be used as a promotable clone resource",
	rsc->id, standard);
	return FALSE;
	}
	return TRUE;
	}

	static bool
	rsc_is_on_node(pcmk_resource_t rsc, const pcmk_node_t node, int flags)
	{
	pcmk__rsc_trace(rsc, "Checking whether %s is on %s",
	rsc->id, pe__node_name(node));

	if (pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->running_on != NULL)) {

	for (GList *iter = rsc->running_on; iter; iter = iter->next) {
	pcmk_node_t loc = (pcmk_node_t ) iter->data;

	if (loc->details == node->details) {
	return true;
	}
	}

	} else if (pcmk_is_set(flags, pe_find_inactive) // @COMPAT deprecated
	&& (rsc->running_on == NULL)) {
	return true;

	} else if (!pcmk_is_set(flags, pcmk_rsc_match_current_node)
	&& (rsc->allocated_to != NULL)
	&& (rsc->allocated_to->details == node->details)) {
	return true;
	}
	return false;
	}

	pcmk_resource_t *
	native_find_rsc(pcmk_resource_t rsc, const char id,
	const pcmk_node_t *on_node, int flags)
	{
	bool match = false;
	pcmk_resource_t *result = NULL;

	CRM_CHECK(id && rsc && rsc->id, return NULL);

	if (pcmk_is_set(flags, pcmk_rsc_match_clone_only)) {
	const char *rid = ID(rsc->xml);

	if (!pe_rsc_is_clone(pe__const_top_resource(rsc, false))) {
	match = false;

	} else if (!strcmp(id, rsc->id) \|\| pcmk__str_eq(id, rid, pcmk__str_none)) {
	match = true;
	}

	} else if (!strcmp(id, rsc->id)) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_history)
	&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = true;

	} else if (pcmk_is_set(flags, pcmk_rsc_match_basename)
	\|\| (pcmk_is_set(flags, pcmk_rsc_match_anon_basename)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique))) {
	match = pe_base_name_eq(rsc, id);
	}

	if (match && on_node) {
	if (!rsc_is_on_node(rsc, on_node, flags)) {
	match = false;
	}
	}

	if (match) {
	return rsc;
	}

	for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	result = rsc->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	// create is ignored
	char *
	native_parameter(pcmk_resource_t rsc, pcmk_node_t node, gboolean create,
	const char name, pcmk_scheduler_t scheduler)
	{
	char *value_copy = NULL;
	const char *value = NULL;
	GHashTable *params = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pcmk__rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
	params = pe_rsc_params(rsc, node, scheduler);
	value = g_hash_table_lookup(params, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->meta, name);
	}
	pcmk__str_update(&value_copy, value);
	return value_copy;
	}

	gboolean
	native_active(pcmk_resource_t * rsc, gboolean all)
	{
	for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t a_node = (pcmk_node_t ) gIter->data;

	if (a_node->details->unclean) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is unclean",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	} else if (!a_node->details->online
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__rsc_trace(rsc, "Resource %s: %s is offline",
	rsc->id, pe__node_name(a_node));
	} else {
	pcmk__rsc_trace(rsc, "Resource %s active on %s",
	rsc->id, pe__node_name(a_node));
	return TRUE;
	}
	}
	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static const char *
	native_pending_state(const pcmk_resource_t *rsc)
	{
	const char *pending_state = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_START, pcmk__str_casei)) {
	pending_state = "Starting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_STOP,
	pcmk__str_casei)) {
	pending_state = "Stopping";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_casei)) {
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_casei)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_PROMOTE,
	pcmk__str_casei)) {
	pending_state = "Promoting";

	} else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_DEMOTE,
	pcmk__str_casei)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_task(const pcmk_resource_t *rsc)
	{
	const char *pending_task = NULL;

	if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
	pending_task = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	/*
	} else if (pcmk__str_eq(rsc->pending_task, "probe", pcmk__str_casei)) {
	pending_task = "Checking";
	*/
	}

	return pending_task;
	}

	static enum rsc_role_e
	native_displayable_role(const pcmk_resource_t *rsc)
	{
	enum rsc_role_e role = rsc->role;

	if ((role == pcmk_role_started)
	&& pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {

	role = pcmk_role_unpromoted;
	}
	return role;
	}

	static const char *
	native_displayable_state(const pcmk_resource_t *rsc, bool print_pending)
	{
	const char *rsc_state = NULL;

	if (print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(native_displayable_role(rsc));
	}
	return rsc_state;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	static void
	native_print_xml(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, pcmk_is_set(options, pe_print_pending));
	const char *target_role = NULL;

	/* resource information. */
	status_print("%s<resource ", pre_text);
	status_print(PCMK_XA_ID "=\"%s\" ", rsc_printable_id(rsc));
	status_print("resource_agent=\"%s%s%s:%s\" ", class,
	((prov == NULL)? "" : PROVIDER_SEP),
	((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, PCMK_XA_TYPE));

	status_print("role=\"%s\" ", rsc_state);
	if (rsc->meta) {
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}
	if (target_role) {
	status_print("target_role=\"%s\" ", target_role);
	}
	status_print("active=\"%s\" ", pcmk__btoa(rsc->fns->active(rsc, TRUE)));
	status_print("orphaned=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_removed));
	status_print("blocked=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_blocked));
	status_print("managed=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_managed));
	status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
	status_print("failure_ignored=\"%s\" ",
	pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure));
	status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	status_print("pending=\"%s\" ", pending_task);
	}
	}

	/* print out the nodes this resource is running on */
	if (options & pe_print_rsconly) {
	status_print("/>\n");
	/* do nothing */
	} else if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	status_print(">\n");
	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	status_print("%s <node name=\"%s\" " PCMK_XA_ID "=\"%s\" "
	"cached=\"%s\"/>\n",
	pre_text, pcmk__s(node->details->uname, ""),
	node->details->id, pcmk__btoa(!node->details->online));
	}
	status_print("%s</resource>\n", pre_text);
	} else {
	status_print("/>\n");
	}
	}

	// Append a flag to resource description string's flags list
	static bool
	add_output_flag(GString s, const char flag_desc, bool have_flags)
	{
	g_string_append(s, (have_flags? ", " : " ("));
	g_string_append(s, flag_desc);
	return true;
	}

	// Append a node name to resource description string's node list
	static bool
	add_output_node(GString s, const char node, bool have_nodes)
	{
	g_string_append(s, (have_nodes? " " : " [ "));
	g_string_append(s, node);
	return true;
	}

	/*!
	* \internal
	* \brief Create a string description of a resource
	*
	* \param[in] rsc Resource to describe
	* \param[in] name Desired identifier for the resource
	* \param[in] node If not NULL, node that resource is "on"
	* \param[in] show_opts Bitmask of pcmk_show_opt_e.
	* \param[in] target_role Resource's target role
	* \param[in] show_nodes Whether to display nodes when multiply active
	*
	* \return Newly allocated string description of resource
	* \note Caller must free the result with g_free().
	*/
	gchar *
	pcmk__native_output_string(const pcmk_resource_t rsc, const char name,
	const pcmk_node_t *node, uint32_t show_opts,
	const char *target_role, bool show_nodes)
	{
	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *provider = NULL;
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	GString *outstr = NULL;
	bool have_flags = false;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	return NULL;
	}

	CRM_CHECK(name != NULL, name = "unknown");
	CRM_CHECK(kind != NULL, kind = "unknown");
	CRM_CHECK(class != NULL, class = "unknown");

	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	}

	if ((node == NULL) && (rsc->lock_node != NULL)) {
	node = rsc->lock_node;
	}
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	node = NULL;
	}

	outstr = g_string_sized_new(128);

	// Resource name and agent
	pcmk__g_strcat(outstr,
	name, "\t(", class, ((provider == NULL)? "" : PROVIDER_SEP),
	pcmk__s(provider, ""), ":", kind, "):\t", NULL);

	// State on node
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	g_string_append(outstr, " ORPHANED");
	}
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	enum rsc_role_e role = native_displayable_role(rsc);

	g_string_append(outstr, " FAILED");
	if (role > pcmk_role_unpromoted) {
	pcmk__add_word(&outstr, 0, role2text(role));
	}
	} else {
	bool show_pending = pcmk_is_set(show_opts, pcmk_show_pending);

	pcmk__add_word(&outstr, 0, native_displayable_state(rsc, show_pending));
	}
	if (node) {
	pcmk__add_word(&outstr, 0, pe__node_name(node));
	}

	// Failed probe operation
	if (native_displayable_role(rsc) == pcmk_role_stopped) {
	xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
	if (probe_op != NULL) {
	int rc;

	pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__g_strcat(outstr, " (", services_ocf_exitcode_str(rc), ") ",
	NULL);
	}
	}

	// Flags, as: (<flag> [...])
	if (node && !(node->details->online) && node->details->unclean) {
	have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
	}
	if (node && (node == rsc->lock_node)) {
	have_flags = add_output_flag(outstr, "LOCKED", have_flags);
	}
	if (pcmk_is_set(show_opts, pcmk_show_pending)) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	have_flags = add_output_flag(outstr, pending_task, have_flags);
	}
	}
	if (target_role != NULL) {
	switch (text2role(target_role)) {
	case pcmk_role_unknown:
	pcmk__config_err("Invalid " PCMK_META_TARGET_ROLE
	" %s for resource %s", target_role, rsc->id);
	break;

	case pcmk_role_stopped:
	have_flags = add_output_flag(outstr, "disabled", have_flags);
	break;

	case pcmk_role_unpromoted:
	if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
	pcmk_rsc_promotable)) {
	have_flags = add_output_flag(outstr,
	PCMK_META_TARGET_ROLE ":",
	have_flags);
	g_string_append(outstr, target_role);
	}
	break;

	default:
	/* Only show target role if it limits our abilities (i.e. ignore
	* Started, as it is the default anyways, and doesn't prevent
	* the resource from becoming promoted).
	*/
	break;
	}
	}

	// Blocked or maintenance implies unmanaged
	if (pcmk_any_flags_set(rsc->flags,
	pcmk_rsc_blocked\|pcmk_rsc_maintenance)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
	have_flags = add_output_flag(outstr, "blocked", have_flags);

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	have_flags = add_output_flag(outstr, "maintenance", have_flags);
	}
	} else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	have_flags = add_output_flag(outstr, "unmanaged", have_flags);
	}

	if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	have_flags = add_output_flag(outstr, "failure ignored", have_flags);
	}


	if (have_flags) {
	g_string_append_c(outstr, ')');
	}

	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)
	\|\| pcmk__list_of_multiple(rsc->running_on)) {
	const char *desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);

	if (desc) {
	g_string_append(outstr, " (");
	g_string_append(outstr, desc);
	g_string_append(outstr, ")");

	}
	}

	if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
	&& pcmk__list_of_multiple(rsc->running_on)) {
	bool have_nodes = false;

	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t n = (pcmk_node_t ) iter->data;

	have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
	}
	if (have_nodes) {
	g_string_append(outstr, " ]");
	}
	}

	return g_string_free(outstr, FALSE);
	}

	int
	pe__common_output_html(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	const char *target_role = NULL;

	xmlNodePtr list_node = NULL;
	const char *cl = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	CRM_ASSERT(kind != NULL);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	PCMK__META_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	cl = "rsc-managed";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	cl = "rsc-failed";

	} else if ((rsc->variant == pcmk_rsc_variant_primitive)
	&& (rsc->running_on == NULL)) {
	cl = "rsc-failed";

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	cl = "rsc-multiple";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	cl = "rsc-failure-ignored";

	} else {
	cl = "rsc-ok";
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
	pcmk_create_html_node(list_node, "span", NULL, cl, s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	int
	pe__common_output_text(pcmk__output_t out, const pcmk_resource_t rsc,
	const char name, const pcmk_node_t node,
	uint32_t show_opts)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	PCMK__META_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return pcmk_rc_no_output;
	}
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	{
	gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
	target_role, true);

	out->list_item(out, NULL, "%s", s);
	g_free(s);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	common_print(pcmk_resource_t rsc, const char pre_text, const char *name,
	const pcmk_node_t node, long options, void print_data)
	{
	const char *target_role = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta,
	PCMK__META_INTERNAL_RSC);

	if (crm_is_true(is_internal)
	&& !pcmk_is_set(options, pe_print_implicit)) {

	crm_trace("skipping print of internal resource %s", rsc->id);
	return;
	}
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	if ((pre_text == NULL) && (options & pe_print_printf)) {
	pre_text = " ";
	}

	if (options & pe_print_html) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	status_print("<font color=\"yellow\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	status_print("<font color=\"red\">");

	} else if (rsc->running_on == NULL) {
	status_print("<font color=\"red\">");

	} else if (pcmk__list_of_multiple(rsc->running_on)) {
	status_print("<font color=\"orange\">");

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
	status_print("<font color=\"yellow\">");

	} else {
	status_print("<font color=\"green\">");
	}
	}

	{
	gchar *resource_s = pcmk__native_output_string(rsc, name, node, options,
	target_role, false);
	status_print("%s%s", (pre_text? pre_text : ""), resource_s);
	g_free(resource_s);
	}

	if (pcmk_is_set(options, pe_print_html)) {
	status_print(" </font> ");
	}

	if (!pcmk_is_set(options, pe_print_rsconly)
	&& pcmk__list_of_multiple(rsc->running_on)) {

	GList *gIter = rsc->running_on;
	int counter = 0;

	if (options & pe_print_html) {
	status_print("<ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print("[");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t n = (pcmk_node_t ) gIter->data;

	counter++;

	if (options & pe_print_html) {
	status_print("<li>\n%s", pe__node_name(n));

	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" %s", pe__node_name(n));

	} else if ((options & pe_print_log)) {
	status_print("\t%d : %s", counter, pe__node_name(n));

	} else {
	status_print("%s", pe__node_name(n));
	}
	if (options & pe_print_html) {
	status_print("</li>\n");

	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" ]");
	}
	}

	if (options & pe_print_html) {
	status_print("<br/>\n");
	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	native_print(pcmk_resource_t rsc, const char pre_text, long options,
	void *print_data)
	{
	const pcmk_node_t *node = NULL;

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	node = pe__current_node(rsc);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}

	common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *rsc_state = native_displayable_state(rsc, print_pending);

	const char *desc = NULL;
	char ra_name[LINE_MAX];
	char *nodes_running_on = NULL;
	const char *lock_node_name = NULL;
	int rc = pcmk_rc_no_output;
	const char *target_role = NULL;

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->meta != NULL) {
	target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	/* resource information. */
	snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
	((prov == NULL)? "" : PROVIDER_SEP), ((prov == NULL)? "" : prov),
	crm_element_value(rsc->xml, PCMK_XA_TYPE));

	nodes_running_on = pcmk__itoa(g_list_length(rsc->running_on));

	if (rsc->lock_node != NULL) {
	lock_node_name = rsc->lock_node->details->uname;
	}

	rc = pe__name_and_nvpairs_xml(out, true, "resource", 15,
	PCMK_XA_ID, rsc_printable_id(rsc),
	"resource_agent", ra_name,
	"role", rsc_state,
	"target_role", target_role,
	"active", pcmk__btoa(rsc->fns->active(rsc, TRUE)),
	"orphaned", pe__rsc_bool_str(rsc, pcmk_rsc_removed),
	"blocked", pe__rsc_bool_str(rsc, pcmk_rsc_blocked),
	"maintenance", pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
	"managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
	"failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
	"failure_ignored", pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure),
	"nodes_running_on", nodes_running_on,
	"pending", (print_pending? native_pending_task(rsc) : NULL),
	"locked_to", lock_node_name,
	PCMK_XA_DESCRIPTION, desc);
	free(nodes_running_on);

	CRM_ASSERT(rc == pcmk_rc_ok);

	if (rsc->running_on != NULL) {
	GList *gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	rc = pe__name_and_nvpairs_xml(out, false, "node", 3,
	PCMK_XA_NAME, node->details->uname,
	PCMK_XA_ID, node->details->id,
	"cached", pcmk__btoa(node->details->online));
	CRM_ASSERT(rc == pcmk_rc_ok);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return rc;
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pe__current_node(rsc);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__resource_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node G_GNUC_UNUSED = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const pcmk_node_t *node = pe__current_node(rsc);

	CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);

	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return pcmk_rc_no_output;
	}

	if (node == NULL) {
	// This is set only if a non-probe action is pending on this node
	node = rsc->pending_node;
	}
	return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
	}

	void
	native_free(pcmk_resource_t * rsc)
	{
	pcmk__rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const pcmk_resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->next_role;

	if (current) {
	role = rsc->role;
	}
	pcmk__rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief List nodes where a resource (or any of its children) is
	*
	* \param[in] rsc Resource to check
	* \param[out] list List to add result to
	* \param[in] current 0 = where allocated, 1 = where running,
	* 2 = where running or pending
	*
	* \return If list contains only one node, that node, or NULL otherwise
	*/
	pcmk_node_t *
	native_location(const pcmk_resource_t rsc, GList *list, int current)
	{
	// @COMPAT: Accept a pcmk__rsc_node argument instead of int current
	pcmk_node_t *one = NULL;
	GList *result = NULL;

	if (rsc->children) {
	GList *gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t child = (pcmk_resource_t ) gIter->data;

	child->fns->location(child, &result, current);
	}

	} else if (current) {

	if (rsc->running_on) {
	result = g_list_copy(rsc->running_on);
	}
	if ((current == 2) && rsc->pending_node
	&& !pe_find_node_id(result, rsc->pending_node->details->id)) {
	result = g_list_append(result, rsc->pending_node);
	}

	} else if (current == FALSE && rsc->allocated_to) {
	result = g_list_append(NULL, rsc->allocated_to);
	}

	if (result && (result->next == NULL)) {
	one = result->data;
	}

	if (list) {
	GList *gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GList rsc_list, GHashTable rsc_table, GHashTable * active_table)
	{
	GList *gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (rsc->variant != pcmk_rsc_variant_primitive) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
	const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);

	if (prov != NULL) {
	offset += snprintf(buffer + offset, LINE_MAX - offset,
	PROVIDER_SEP "%s", prov);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = calloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	GList *gIter2 = rsc->running_on;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE &&
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->details->uname);
	if (node_table == NULL) {
	node_table = pcmk__strkey_table(free, free);
	g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = calloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	/*!
	* \internal
	* \deprecated This function will be removed in a future release
	*/
	void
	print_rscs_brief(GList rsc_list, const char pre_text, long options,
	void *print_data, gboolean print_all)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GHashTableIter hash_iter;
	char *type = NULL;
	int *rsc_counter = NULL;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	g_hash_table_iter_init(&hash_iter, rsc_table);
	while (g_hash_table_iter_next(&hash_iter, (gpointer )&type, (gpointer )&rsc_counter)) {
	GHashTableIter hash_iter2;
	char *node_name = NULL;
	GHashTable *node_table = NULL;
	int active_counter_all = 0;

	g_hash_table_iter_init(&hash_iter2, active_table);
	while (g_hash_table_iter_next(&hash_iter2, (gpointer )&node_name, (gpointer )&node_table)) {
	int *active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (options & pe_print_rsconly) {
	node_name = NULL;
	}

	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	if (print_all) {
	status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0,
	rsc_counter ? *rsc_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	} else {
	status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	}

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}

	if (print_all && active_counter_all == 0) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	}

	int
	pe__rscs_brief_output(pcmk__output_t out, GList rsc_list, uint32_t show_opts)
	{
	GHashTable *rsc_table = pcmk__strkey_table(free, free);
	GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
	GList *sorted_rscs;
	int rc = pcmk_rc_no_output;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	/* Make a list of the rsc_table keys so that it can be sorted. This is to make sure
	* output order stays consistent between systems.
	*/
	sorted_rscs = g_hash_table_get_keys(rsc_table);
	sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);

	for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
	char type = (char ) gIter->data;
	int *rsc_counter = g_hash_table_lookup(rsc_table, type);

	GList *sorted_nodes = NULL;
	int active_counter_all = 0;

	/* Also make a list of the active_table keys so it can be sorted. If there's
	* more than one instance of a type of resource running, we need the nodes to
	* be sorted to make sure output order stays consistent between systems.
	*/
	sorted_nodes = g_hash_table_get_keys(active_table);
	sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);

	for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
	char node_name = (char ) gIter2->data;
	GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
	int *active_counter = NULL;

	if (node_table == NULL) {
	continue;
	}

	active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
	node_name = NULL;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
	*active_counter,
	rsc_counter ? *rsc_counter : 0, type,
	(*active_counter > 0) && node_name ? node_name : "");
	} else {
	out->list_item(out, NULL, "%d\t(%s):\tActive %s",
	*active_counter, type,
	(*active_counter > 0) && node_name ? node_name : "");
	}

	rc = pcmk_rc_ok;
	}

	if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
	out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);
	rc = pcmk_rc_ok;
	}

	if (sorted_nodes) {
	g_list_free(sorted_nodes);
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	if (sorted_rscs) {
	g_list_free(sorted_rscs);
	}

	return rc;
	}

	gboolean
	pe__native_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) \|\|
	pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
	return FALSE;
	} else if (check_parent && rsc->parent) {
	const pcmk_resource_t *up = pe__const_top_resource(rsc, true);

	return up->fns->is_filtered(up, only_rsc, FALSE);
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Get maximum primitive resource instances per node
	*
	* \param[in] rsc Primitive resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__primitive_max_per_node(const pcmk_resource_t *rsc)
	{
	CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
	return 1U;
	}
	diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
	index 694db9d537..20dd545ece 100644
	--- a/lib/pengine/pe_actions.c
	+++ b/lib/pengine/pe_actions.c
	@@ -1,1889 +1,1889 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml_internal.h>
	#include "pe_status_private.h"

	static void unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms);

	static void
	add_singleton(pcmk_scheduler_t scheduler, pcmk_action_t action)
	{
	if (scheduler->singletons == NULL) {
	scheduler->singletons = pcmk__strkey_table(NULL, NULL);
	}
	g_hash_table_insert(scheduler->singletons, action->uuid, action);
	}

	static pcmk_action_t *
	lookup_singleton(pcmk_scheduler_t scheduler, const char action_uuid)
	{
	if (scheduler->singletons == NULL) {
	return NULL;
	}
	return g_hash_table_lookup(scheduler->singletons, action_uuid);
	}

	/*!
	* \internal
	* \brief Find an existing action that matches arguments
	*
	* \param[in] key Action key to match
	* \param[in] rsc Resource to match (if any)
	* \param[in] node Node to match (if any)
	* \param[in] scheduler Scheduler data
	*
	* \return Existing action that matches arguments (or NULL if none)
	*/
	static pcmk_action_t *
	find_existing_action(const char key, const pcmk_resource_t rsc,
	const pcmk_node_t node, const pcmk_scheduler_t scheduler)
	{
	GList *matches = NULL;
	pcmk_action_t *action = NULL;

	/* When rsc is NULL, it would be quicker to check scheduler->singletons,
	* but checking all scheduler->actions takes the node into account.
	*/
	matches = find_actions(((rsc == NULL)? scheduler->actions : rsc->actions),
	key, node);
	if (matches == NULL) {
	return NULL;
	}
	CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));

	action = matches->data;
	g_list_free(matches);
	return action;
	}

	/*!
	* \internal
	* \brief Find the XML configuration corresponding to a specific action key
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] key "RSC_ACTION_INTERVAL" of action to find
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	static xmlNode *
	find_exact_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	operation != NULL; operation = crm_next_same_xml(operation)) {

	bool enabled = false;
	const char *config_name = NULL;
	const char *interval_spec = NULL;
	guint tmp_ms = 0U;

	// @TODO This does not consider meta-attributes, rules, defaults, etc.
	if (!include_disabled
	&& (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &tmp_ms);
	if (tmp_ms != interval_ms) {
	continue;
	}

	config_name = crm_element_value(operation, PCMK_XA_NAME);
	if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) {
	return operation;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Find the XML configuration of a resource action
	*
	* \param[in] rsc Resource to find action configuration for
	* \param[in] action_name Action name to search for
	* \param[in] interval_ms Action interval (in milliseconds) to search for
	* \param[in] include_disabled If false, do not return disabled actions
	*
	* \return XML configuration of desired action if any, otherwise NULL
	*/
	xmlNode *
	pcmk__find_action_config(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, bool include_disabled)
	{
	xmlNode *action_config = NULL;

	// Try requested action first
	action_config = find_exact_action_config(rsc, action_name, interval_ms,
	include_disabled);

	// For migrate_to and migrate_from actions, retry with "migrate"
	// @TODO This should be either documented or deprecated
	if ((action_config == NULL)
	&& pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	action_config = find_exact_action_config(rsc, "migrate", 0,
	include_disabled);
	}

	return action_config;
	}

	/*!
	* \internal
	* \brief Create a new action object
	*
	* \param[in] key Action key
	* \param[in] task Action name
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in] node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly allocated action
	* \note This function takes ownership of \p key. It is the caller's
	* responsibility to free the return value with pe_free_action().
	*/
	static pcmk_action_t *
	new_action(char key, const char task, pcmk_resource_t *rsc,
	const pcmk_node_t node, bool optional, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *action = calloc(1, sizeof(pcmk_action_t));

	CRM_ASSERT(action != NULL);

	action->rsc = rsc;
	action->task = strdup(task); CRM_ASSERT(action->task != NULL);
	action->uuid = key;

	if (node) {
	action->node = pe__copy_node(node);
	}

	if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
	// Resource history deletion for a node can be done on the DC
	pcmk__set_action_flags(action, pcmk_action_on_dc);
	}

	pcmk__set_action_flags(action, pcmk_action_runnable);
	if (optional) {
	pcmk__set_action_flags(action, pcmk_action_optional);
	} else {
	pcmk__clear_action_flags(action, pcmk_action_optional);
	}

	if (rsc == NULL) {
	action->meta = pcmk__strkey_table(free, free);
	} else {
	guint interval_ms = 0;

	parse_op_key(key, NULL, NULL, &interval_ms);
	action->op_entry = pcmk__find_action_config(rsc, task, interval_ms,
	true);

	/* If the given key is for one of the many notification pseudo-actions
	* (pre_notify_promote, etc.), the actual action name is "notify"
	*/
	if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) {
	action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY,
	0, true);
	}

	unpack_operation(action, action->op_entry, interval_ms);
	}

	pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
	(optional? "optional" : "required"),
	scheduler->action_id, key, task,
	((rsc == NULL)? "no resource" : rsc->id),
	pe__node_name(node));
	action->id = scheduler->action_id++;

	scheduler->actions = g_list_prepend(scheduler->actions, action);
	if (rsc == NULL) {
	add_singleton(scheduler, action);
	} else {
	rsc->actions = g_list_prepend(rsc->actions, action);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Unpack a resource's action-specific instance parameters
	*
	* \param[in] action_xml XML of action's configuration in CIB (if any)
	* \param[in,out] node_attrs Table of node attributes (for rule evaluation)
	* \param[in,out] scheduler Cluster working set (for rule evaluation)
	*
	* \return Newly allocated hash table of action-specific instance parameters
	*/
	GHashTable *
	pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
	GHashTable *node_attrs,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = node_attrs,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe__unpack_dataset_nvpairs(action_xml, XML_TAG_ATTR_SETS,
	&rule_data, params, NULL,
	FALSE, scheduler);
	return params;
	}

	/*!
	* \internal
	* \brief Update an action's optional flag
	*
	* \param[in,out] action Action to update
	* \param[in] optional Requested optional status
	*/
	static void
	update_action_optional(pcmk_action_t *action, gboolean optional)
	{
	// Force a non-recurring action to be optional if its resource is unmanaged
	if ((action->rsc != NULL) && (action->node != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_pseudo)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& (g_hash_table_lookup(action->meta,
	XML_LRM_ATTR_INTERVAL_MS) == NULL)) {
	pcmk__rsc_debug(action->rsc,
	"%s on %s is optional (%s is unmanaged)",
	action->uuid, pe__node_name(action->node),
	action->rsc->id);
	pcmk__set_action_flags(action, pcmk_action_optional);
	// We shouldn't clear runnable here because ... something

	// Otherwise require the action if requested
	} else if (!optional) {
	pcmk__clear_action_flags(action, pcmk_action_optional);
	}
	}

	static enum pe_quorum_policy
	effective_quorum_policy(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	enum pe_quorum_policy policy = scheduler->no_quorum_policy;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	policy = pcmk_no_quorum_ignore;

	} else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) {
	switch (rsc->role) {
	case pcmk_role_promoted:
	case pcmk_role_unpromoted:
	if (rsc->next_role > pcmk_role_unpromoted) {
	pe__set_next_role(rsc, pcmk_role_unpromoted,
	PCMK_OPT_NO_QUORUM_POLICY "=demote");
	}
	policy = pcmk_no_quorum_ignore;
	break;
	default:
	policy = pcmk_no_quorum_stop;
	break;
	}
	}
	return policy;
	}

	/*!
	* \internal
	* \brief Update a resource action's runnable flag
	*
	* \param[in,out] action Action to update
	* \param[in,out] scheduler Scheduler data
	*
	* \note This may also schedule fencing if a stop is unrunnable.
	*/
	static void
	update_resource_action_runnable(pcmk_action_t *action,
	pcmk_scheduler_t *scheduler)
	{
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	return;
	}

	if (action->node == NULL) {
	pcmk__rsc_trace(action->rsc, "%s is unrunnable (unallocated)",
	action->uuid);
	pcmk__clear_action_flags(action, pcmk_action_runnable);

	} else if (!pcmk_is_set(action->flags, pcmk_action_on_dc)
	&& !(action->node->details->online)
	&& (!pe__is_guest_node(action->node)
	\|\| action->node->details->remote_requires_reset)) {
	pcmk__clear_action_flags(action, pcmk_action_runnable);
	do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)",
	action->uuid, pe__node_name(action->node));
	if (pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
	&& pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& !(action->node->details->unclean)) {
	pe_fence_node(scheduler, action->node, "stop is unrunnable", false);
	}

	} else if (!pcmk_is_set(action->flags, pcmk_action_on_dc)
	&& action->node->details->pending) {
	pcmk__clear_action_flags(action, pcmk_action_runnable);
	do_crm_log(LOG_WARNING,
	"Action %s on %s is unrunnable (node is pending)",
	action->uuid, pe__node_name(action->node));

	} else if (action->needs == pcmk_requires_nothing) {
	pe_action_set_reason(action, NULL, TRUE);
	if (pe__is_guest_node(action->node)
	&& !pe_can_fence(scheduler, action->node)) {
	/* An action that requires nothing usually does not require any
	* fencing in order to be runnable. However, there is an exception:
	* such an action cannot be completed if it is on a guest node whose
	* host is unclean and cannot be fenced.
	*/
	pcmk__rsc_debug(action->rsc,
	"%s on %s is unrunnable "
	"(node's host cannot be fenced)",
	action->uuid, pe__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk_action_runnable);
	} else {
	pcmk__rsc_trace(action->rsc,
	"%s on %s does not require fencing or quorum",
	action->uuid, pe__node_name(action->node));
	pcmk__set_action_flags(action, pcmk_action_runnable);
	}

	} else {
	switch (effective_quorum_policy(action->rsc, scheduler)) {
	case pcmk_no_quorum_stop:
	pcmk__rsc_debug(action->rsc,
	"%s on %s is unrunnable (no quorum)",
	action->uuid, pe__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk_action_runnable);
	pe_action_set_reason(action, "no quorum", true);
	break;

	case pcmk_no_quorum_freeze:
	if (!action->rsc->fns->active(action->rsc, TRUE)
	\|\| (action->rsc->next_role > action->rsc->role)) {
	pcmk__rsc_debug(action->rsc,
	"%s on %s is unrunnable (no quorum)",
	action->uuid, pe__node_name(action->node));
	pcmk__clear_action_flags(action, pcmk_action_runnable);
	pe_action_set_reason(action, "quorum freeze", true);
	}
	break;

	default:
	//pe_action_set_reason(action, NULL, TRUE);
	pcmk__set_action_flags(action, pcmk_action_runnable);
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Update a resource object's flags for a new action on it
	*
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in] action New action
	*/
	static void
	update_resource_flags_for_action(pcmk_resource_t *rsc,
	const pcmk_action_t *action)
	{
	/* @COMPAT pcmk_rsc_starting and pcmk_rsc_stopping are deprecated and unused
	* within Pacemaker, and will eventually be removed
	*/
	if (pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)) {
	pcmk__set_rsc_flags(rsc, pcmk_rsc_stopping);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)) {
	if (pcmk_is_set(action->flags, pcmk_action_runnable)) {
	pcmk__set_rsc_flags(rsc, pcmk_rsc_starting);
	} else {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_starting);
	}
	}
	}

	static bool
	valid_stop_on_fail(const char *value)
	{
	return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL);
	}

	/*!
	* \internal
	* \brief Validate (and possibly reset) resource action's on_fail meta-attribute
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] action_config Action configuration XML from CIB (if any)
	* \param[in,out] meta Table of action meta-attributes
	*/
	static void
	validate_on_fail(const pcmk_resource_t rsc, const char action_name,
	const xmlNode action_config, GHashTable meta)
	{
	const char *name = NULL;
	const char *role = NULL;
	const char *interval_spec = NULL;
	const char *value = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);
	char *key = NULL;
	char *new_value = NULL;
	guint interval_ms = 0U;

	// Stop actions can only use certain on-fail values
	if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
	&& !valid_stop_on_fail(value)) {

	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
	"action to default value because '%s' is not "
	"allowed for stop", rsc->id, value);
	g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
	return;
	}

	/* Demote actions default on-fail to the on-fail value for the first
	* recurring monitor for the promoted role (if any).
	*/
	if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
	&& (value == NULL)) {

	/* @TODO This does not consider promote options set in a meta-attribute
	* block (which may have rules that need to be evaluated) rather than
	* XML properties.
	*/
	for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	operation != NULL; operation = crm_next_same_xml(operation)) {
	bool enabled = false;
	const char *promote_on_fail = NULL;

	/* We only care about explicit on-fail (if promote uses default, so
	* can demote)
	*/
	promote_on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
	if (promote_on_fail == NULL) {
	continue;
	}

	// We only care about recurring monitors for the promoted role
	name = crm_element_value(operation, PCMK_XA_NAME);
	role = crm_element_value(operation, "role");
	if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
	continue;
	}
	interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	if (interval_ms == 0U) {
	continue;
	}

	// We only care about enabled monitors
	if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	// Demote actions can't default to on-fail="demote"
	if (pcmk__str_eq(promote_on_fail, "demote", pcmk__str_casei)) {
	continue;
	}

	// Use value from first applicable promote action found
	key = strdup(XML_OP_ATTR_ON_FAIL);
	new_value = strdup(promote_on_fail);
	CRM_ASSERT((key != NULL) && (new_value != NULL));
	g_hash_table_insert(meta, key, new_value);
	}
	return;
	}

	if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
	&& !pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	key = strdup(XML_OP_ATTR_ON_FAIL);
	new_value = strdup("ignore");
	CRM_ASSERT((key != NULL) && (new_value != NULL));
	g_hash_table_insert(meta, key, new_value);
	return;
	}

	// on-fail="demote" is allowed only for certain actions
	if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	name = crm_element_value(action_config, PCMK_XA_NAME);
	role = crm_element_value(action_config, "role");
	interval_spec = crm_element_value(action_config,
	XML_LRM_ATTR_INTERVAL);
	pcmk_parse_interval_spec(interval_spec, &interval_ms);

	if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
	&& ((interval_ms == 0U)
	\|\| !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
	\|\| !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
	PCMK__ROLE_PROMOTED_LEGACY, NULL))) {

	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
	"action to default value because 'demote' is not "
	"allowed for it", rsc->id, name);
	g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
	return;
	}
	}
	}

	static int
	unpack_timeout(const char *value)
	{
	int timeout_ms = crm_get_msec(value);

	if (timeout_ms < 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}
	return timeout_ms;
	}

	// true if value contains valid, non-NULL interval origin for recurring op
	static bool
	unpack_interval_origin(const char value, const xmlNode xml_obj,
	guint interval_ms, const crm_time_t *now,
	long long *start_delay)
	{
	long long result = 0;
	guint interval_sec = interval_ms / 1000;
	crm_time_t *origin = NULL;

	// Ignore unspecified values and non-recurring operations
	if ((value == NULL) \|\| (interval_ms == 0) \|\| (now == NULL)) {
	return false;
	}

	// Parse interval origin from text
	origin = crm_time_new(value);
	if (origin == NULL) {
	pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
	"'%s' because '%s' is not valid",
	(ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
	return false;
	}

	// Get seconds since origin (negative if origin is in the future)
	result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
	crm_time_free(origin);

	// Calculate seconds from closest interval to now
	result = result % interval_sec;

	// Calculate seconds remaining until next interval
	result = ((result <= 0)? 0 : interval_sec) - result;
	crm_info("Calculated a start delay of %llds for operation '%s'",
	result,
	(ID(xml_obj)? ID(xml_obj) : "(unspecified)"));

	if (start_delay != NULL) {
	start_delay = result 1000; // milliseconds
	}
	return true;
	}

	static int
	unpack_start_delay(const char value, GHashTable meta)
	{
	int start_delay = 0;

	if (value != NULL) {
	start_delay = crm_get_msec(value);

	if (start_delay < 0) {
	start_delay = 0;
	}

	if (meta) {
	g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY),
	pcmk__itoa(start_delay));
	}
	}

	return start_delay;
	}

	/*!
	* \internal
	* \brief Find a resource's most frequent recurring monitor
	*
	* \param[in] rsc Resource to check
	*
	* \return Operation XML configured for most frequent recurring monitor for
	* \p rsc (if any)
	*/
	static xmlNode *
	most_frequent_monitor(const pcmk_resource_t *rsc)
	{
	guint min_interval_ms = G_MAXUINT;
	xmlNode *op = NULL;

	for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	operation != NULL; operation = crm_next_same_xml(operation)) {
	bool enabled = false;
	guint interval_ms = 0U;
	const char *interval_spec = crm_element_value(operation,
	XML_LRM_ATTR_INTERVAL);

	// We only care about enabled recurring monitors
	if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME),
	PCMK_ACTION_MONITOR, pcmk__str_none)) {
	continue;
	}

	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	if (interval_ms == 0U) {
	continue;
	}

	// @TODO This does not consider meta-attributes, rules, defaults, etc.
	if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
	&enabled) == pcmk_rc_ok) && !enabled) {
	continue;
	}

	if (interval_ms < min_interval_ms) {
	min_interval_ms = interval_ms;
	op = operation;
	}
	}
	return op;
	}

	/*!
	* \internal
	* \brief Unpack action meta-attributes
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] node Node that action is on
	* \param[in] action_name Action name
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] action_config Action XML configuration from CIB (if any)
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds) from its CIB XML
	* configuration (including defaults).
	*
	* \return Newly allocated hash table with normalized action meta-attributes
	*/
	GHashTable *
	pcmk__unpack_action_meta(pcmk_resource_t rsc, const pcmk_node_t node,
	const char *action_name, guint interval_ms,
	const xmlNode *action_config)
	{
	GHashTable *meta = NULL;
	char *name = NULL;
	char *value = NULL;
	const char *timeout_spec = NULL;
	const char *str = NULL;

	pe_rsc_eval_data_t rsc_rule_data = {
	- .standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
	+ .standard = crm_element_value(rsc->xml, PCMK_XA_CLASS),
	.provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
	.agent = crm_element_value(rsc->xml, PCMK_XA_TYPE),
	};

	pe_op_eval_data_t op_rule_data = {
	.op_name = action_name,
	.interval = interval_ms,
	};

	pe_rule_eval_data_t rule_data = {
	.node_hash = (node == NULL)? NULL : node->details->attrs,
	.role = pcmk_role_unknown,
	.now = rsc->cluster->now,
	.match_data = NULL,
	.rsc_data = &rsc_rule_data,
	.op_data = &op_rule_data,
	};

	meta = pcmk__strkey_table(free, free);

	// Cluster-wide <op_defaults> <meta_attributes>
	pe__unpack_dataset_nvpairs(rsc->cluster->op_defaults, XML_TAG_META_SETS,
	&rule_data, meta, NULL, FALSE, rsc->cluster);

	// Derive default timeout for probes from recurring monitor timeouts
	if (pcmk_is_probe(action_name, interval_ms)) {
	xmlNode *min_interval_mon = most_frequent_monitor(rsc);

	if (min_interval_mon != NULL) {
	/* @TODO This does not consider timeouts set in meta_attributes
	* blocks (which may also have rules that need to be evaluated).
	*/
	timeout_spec = crm_element_value(min_interval_mon,
	XML_ATTR_TIMEOUT);
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting default timeout for %s probe to "
	"most frequent monitor's timeout '%s'",
	rsc->id, timeout_spec);
	name = strdup(XML_ATTR_TIMEOUT);
	value = strdup(timeout_spec);
	CRM_ASSERT((name != NULL) && (value != NULL));
	g_hash_table_insert(meta, name, value);
	}
	}
	}

	if (action_config != NULL) {
	// <op> <meta_attributes> take precedence over defaults
	pe__unpack_dataset_nvpairs(action_config, XML_TAG_META_SETS, &rule_data,
	meta, NULL, TRUE, rsc->cluster);

	/* Anything set as an <op> XML property has highest precedence.
	* This ensures we use the name and interval from the <op> tag.
	* (See below for the only exception, fence device start/probe timeout.)
	*/
	for (xmlAttrPtr attr = action_config->properties;
	attr != NULL; attr = attr->next) {
	name = strdup((const char *) attr->name);
	value = strdup(pcmk__xml_attr_value(attr));

	CRM_ASSERT((name != NULL) && (value != NULL));
	g_hash_table_insert(meta, name, value);
	}
	}

	g_hash_table_remove(meta, PCMK_XA_ID);

	// Normalize interval to milliseconds
	if (interval_ms > 0) {
	name = strdup(XML_LRM_ATTR_INTERVAL);
	CRM_ASSERT(name != NULL);
	value = crm_strdup_printf("%u", interval_ms);
	g_hash_table_insert(meta, name, value);
	} else {
	g_hash_table_remove(meta, XML_LRM_ATTR_INTERVAL);
	}

	/* Timeout order of precedence (highest to lowest):
	* 1. pcmk_monitor_timeout resource parameter (only for starts and probes
	* when rsc has pcmk_ra_cap_fence_params; this gets used for recurring
	* monitors via the executor instead)
	* 2. timeout configured in <op> (with <op timeout> taking precedence over
	* <op> <meta_attributes>)
	* 3. timeout configured in <op_defaults> <meta_attributes>
	* 4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
	*/

	// Check for pcmk_monitor_timeout
	if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
	pcmk_ra_cap_fence_params)
	&& (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)
	\|\| pcmk_is_probe(action_name, interval_ms))) {

	GHashTable *params = pe_rsc_params(rsc, node, rsc->cluster);

	timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout");
	if (timeout_spec != NULL) {
	pcmk__rsc_trace(rsc,
	"Setting timeout for %s %s to "
	"pcmk_monitor_timeout (%s)",
	rsc->id, action_name, timeout_spec);
	name = strdup(XML_ATTR_TIMEOUT);
	value = strdup(timeout_spec);
	CRM_ASSERT((name != NULL) && (value != NULL));
	g_hash_table_insert(meta, name, value);
	}
	}

	// Normalize timeout to positive milliseconds
	name = strdup(XML_ATTR_TIMEOUT);
	CRM_ASSERT(name != NULL);
	timeout_spec = g_hash_table_lookup(meta, XML_ATTR_TIMEOUT);
	g_hash_table_insert(meta, name, pcmk__itoa(unpack_timeout(timeout_spec)));

	// Ensure on-fail has a valid value
	validate_on_fail(rsc, action_name, action_config, meta);

	// Normalize start-delay
	str = g_hash_table_lookup(meta, XML_OP_ATTR_START_DELAY);
	if (str != NULL) {
	unpack_start_delay(str, meta);
	} else {
	long long start_delay = 0;

	str = g_hash_table_lookup(meta, XML_OP_ATTR_ORIGIN);
	if (unpack_interval_origin(str, action_config, interval_ms,
	rsc->cluster->now, &start_delay)) {
	name = strdup(XML_OP_ATTR_START_DELAY);
	CRM_ASSERT(name != NULL);
	g_hash_table_insert(meta, name,
	crm_strdup_printf("%lld", start_delay));
	}
	}
	return meta;
	}

	/*!
	* \internal
	* \brief Determine an action's quorum and fencing dependency
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action being unpacked
	*
	* \return Quorum and fencing dependency appropriate to action
	*/
	enum rsc_start_requirement
	pcmk__action_requires(const pcmk_resource_t rsc, const char action_name)
	{
	const char *value = NULL;
	enum rsc_start_requirement requires = pcmk_requires_nothing;

	CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires);

	if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE, NULL)) {
	value = "nothing (not start or promote)";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
	requires = pcmk_requires_fencing;
	value = "fencing";

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_quorum)) {
	requires = pcmk_requires_quorum;
	value = "quorum";

	} else {
	value = "nothing";
	}
	pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value);
	return requires;
	}

	/*!
	* \internal
	* \brief Parse action failure response from a user-provided string
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Name of action
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in] value User-provided configuration value for on-fail
	*
	* \return Action failure response parsed from \p text
	*/
	enum action_fail_response
	pcmk__parse_on_fail(const pcmk_resource_t rsc, const char action_name,
	guint interval_ms, const char *value)
	{
	const char *desc = NULL;
	bool needs_remote_reset = false;
	enum action_fail_response on_fail = pcmk_on_fail_ignore;

	if (value == NULL) {
	// Use default

	} else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_block;
	desc = "block";

	} else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) {
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	on_fail = pcmk_on_fail_fence_node;
	desc = "node fencing";
	} else {
	pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
	"%s of %s to 'stop' because 'fence' is not "
	"valid when fencing is disabled",
	action_name, rsc->id);
	on_fail = pcmk_on_fail_stop;
	desc = "stop resource";
	}

	} else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_standby_node;
	desc = "node standby";

	} else if (pcmk__strcase_any_of(value, "ignore", PCMK__VALUE_NOTHING,
	NULL)) {
	desc = "ignore";

	} else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_ban;
	desc = "force migration";

	} else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_stop;
	desc = "stop resource";

	} else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_restart;
	desc = "restart (and possibly migrate)";

	} else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) {
	if (rsc->container == NULL) {
	pcmk__rsc_debug(rsc,
	"Using default " XML_OP_ATTR_ON_FAIL " for %s "
	"of %s because it does not have a container",
	action_name, rsc->id);
	} else {
	on_fail = pcmk_on_fail_restart_container;
	desc = "restart container (and possibly migrate)";
	}

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	on_fail = pcmk_on_fail_demote;
	desc = "demote instance";

	} else {
	pcmk__config_err("Using default '" XML_OP_ATTR_ON_FAIL "' for "
	"%s of %s because '%s' is not valid",
	action_name, rsc->id, value);
	}

	/* Remote node connections are handled specially. Failures that result
	* in dropping an active connection must result in fencing. The only
	* failures that don't are probes and starts. The user can explicitly set
	* on-fail="fence" to fence after start failures.
	*/
	if (pe__resource_is_remote_conn(rsc)
	&& !pcmk_is_probe(action_name, interval_ms)
	&& !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
	needs_remote_reset = true;
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	desc = NULL; // Force default for unmanaged connections
	}
	}

	if (desc != NULL) {
	// Explicit value used, default not needed

	} else if (rsc->container != NULL) {
	on_fail = pcmk_on_fail_restart_container;
	desc = "restart container (and possibly migrate) (default)";

	} else if (needs_remote_reset) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	desc = "fence remote node (default)";
	} else {
	desc = "recover remote node connection (default)";
	}
	on_fail = pcmk_on_fail_reset_remote;
	} else {
	on_fail = pcmk_on_fail_stop;
	desc = "stop unmanaged remote node (enforcing default)";
	}

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	on_fail = pcmk_on_fail_fence_node;
	desc = "resource fence (default)";
	} else {
	on_fail = pcmk_on_fail_block;
	desc = "resource block (default)";
	}

	} else {
	on_fail = pcmk_on_fail_restart;
	desc = "restart (and possibly migrate) (default)";
	}

	pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s",
	pcmk__readable_interval(interval_ms), action_name,
	rsc->id, desc);
	return on_fail;
	}

	/*!
	* \internal
	* \brief Determine a resource's role after failure of an action
	*
	* \param[in] rsc Resource that action is for
	* \param[in] action_name Action name
	* \param[in] on_fail Failure handling for action
	* \param[in] meta Unpacked action meta-attributes
	*
	* \return Resource role that results from failure of action
	*/
	enum rsc_role_e
	pcmk__role_after_failure(const pcmk_resource_t rsc, const char action_name,
	enum action_fail_response on_fail, GHashTable *meta)
	{
	const char *value = NULL;
	enum rsc_role_e role = pcmk_role_unknown;

	// Set default for role after failure specially in certain circumstances
	switch (on_fail) {
	case pcmk_on_fail_stop:
	role = pcmk_role_stopped;
	break;

	case pcmk_on_fail_reset_remote:
	if (rsc->remote_reconnect_ms != 0) {
	role = pcmk_role_stopped;
	}
	break;

	default:
	break;
	}

	// @COMPAT Check for explicitly configured role (deprecated)
	value = g_hash_table_lookup(meta, PCMK__META_ROLE_AFTER_FAILURE);
	if (value != NULL) {
	pcmk__warn_once(pcmk__wo_role_after,
	"Support for " PCMK__META_ROLE_AFTER_FAILURE " is "
	"deprecated and will be removed in a future release");
	if (role == pcmk_role_unknown) {
	role = text2role(value);
	if (role == pcmk_role_unknown) {
	pcmk__config_err("Ignoring invalid value %s for "
	PCMK__META_ROLE_AFTER_FAILURE,
	value);
	}
	}
	}

	if (role == pcmk_role_unknown) {
	// Use default
	if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	role = pcmk_role_unpromoted;
	} else {
	role = pcmk_role_started;
	}
	}
	pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s",
	rsc->id, action_name, role2text(role));
	return role;
	}

	/*!
	* \internal
	* \brief Unpack action configuration
	*
	* Unpack a resource action's meta-attributes (normalizing the interval,
	* timeout, and start delay values as integer milliseconds), requirements, and
	* failure policy from its CIB XML configuration (including defaults).
	*
	* \param[in,out] action Resource action to unpack into
	* \param[in] xml_obj Action configuration XML (NULL for defaults only)
	* \param[in] interval_ms How frequently to perform the operation
	*/
	static void
	unpack_operation(pcmk_action_t action, const xmlNode xml_obj,
	guint interval_ms)
	{
	const char *value = NULL;

	action->meta = pcmk__unpack_action_meta(action->rsc, action->node,
	action->task, interval_ms, xml_obj);
	action->needs = pcmk__action_requires(action->rsc, action->task);

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
	action->on_fail = pcmk__parse_on_fail(action->rsc, action->task,
	interval_ms, value);

	action->fail_role = pcmk__role_after_failure(action->rsc, action->task,
	action->on_fail, action->meta);
	}

	/*!
	* \brief Create or update an action object
	*
	* \param[in,out] rsc Resource that action is for (if any)
	* \param[in,out] key Action key (must be non-NULL)
	* \param[in] task Action name (must be non-NULL)
	* \param[in] on_node Node that action is on (if any)
	* \param[in] optional Whether action should be considered optional
	* \param[in,out] scheduler Scheduler data
	*
	* \return Action object corresponding to arguments (guaranteed not to be
	* \c NULL)
	* \note This function takes ownership of (and might free) \p key, and
	* \p scheduler takes ownership of the returned action (the caller should
	* not free it).
	*/
	pcmk_action_t *
	custom_action(pcmk_resource_t rsc, char key, const char *task,
	const pcmk_node_t *on_node, gboolean optional,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_action_t *action = NULL;

	CRM_ASSERT((key != NULL) && (task != NULL) && (scheduler != NULL));

	action = find_existing_action(key, rsc, on_node, scheduler);
	if (action == NULL) {
	action = new_action(key, task, rsc, on_node, optional, scheduler);
	} else {
	free(key);
	}

	update_action_optional(action, optional);

	if (rsc != NULL) {
	if ((action->node != NULL) && (action->op_entry != NULL)
	&& !pcmk_is_set(action->flags, pcmk_action_attrs_evaluated)) {

	GHashTable *attrs = action->node->details->attrs;

	if (action->extra != NULL) {
	g_hash_table_destroy(action->extra);
	}
	action->extra = pcmk__unpack_action_rsc_params(action->op_entry,
	attrs, scheduler);
	pcmk__set_action_flags(action, pcmk_action_attrs_evaluated);
	}

	update_resource_action_runnable(action, scheduler);
	update_resource_flags_for_action(rsc, action);
	}

	if (action->extra == NULL) {
	action->extra = pcmk__strkey_table(free, free);
	}

	return action;
	}

	pcmk_action_t *
	get_pseudo_op(const char name, pcmk_scheduler_t scheduler)
	{
	pcmk_action_t *op = lookup_singleton(scheduler, name);

	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler);
	pcmk__set_action_flags(op, pcmk_action_pseudo\|pcmk_action_runnable);
	}
	return op;
	}

	static GList *
	find_unfencing_devices(GList candidates, GList matches)
	{
	for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *candidate = gIter->data;

	if (candidate->children != NULL) {
	matches = find_unfencing_devices(candidate->children, matches);

	} else if (!pcmk_is_set(candidate->flags, pcmk_rsc_fence_device)) {
	continue;

	} else if (pcmk_is_set(candidate->flags, pcmk_rsc_needs_unfencing)) {
	matches = g_list_prepend(matches, candidate);

	} else if (pcmk__str_eq(g_hash_table_lookup(candidate->meta,
	PCMK_STONITH_PROVIDES),
	PCMK__VALUE_UNFENCING,
	pcmk__str_casei)) {
	matches = g_list_prepend(matches, candidate);
	}
	}
	return matches;
	}

	static int
	node_priority_fencing_delay(const pcmk_node_t *node,
	const pcmk_scheduler_t *scheduler)
	{
	int member_count = 0;
	int online_count = 0;
	int top_priority = 0;
	int lowest_priority = 0;
	GList *gIter = NULL;

	// PCMK_OPT_PRIORITY_FENCING_DELAY is disabled
	if (scheduler->priority_fencing_delay <= 0) {
	return 0;
	}

	/* No need to request a delay if the fencing target is not a normal cluster
	* member, for example if it's a remote node or a guest node. */
	if (node->details->type != pcmk_node_variant_cluster) {
	return 0;
	}

	// No need to request a delay if the fencing target is in our partition
	if (node->details->online) {
	return 0;
	}

	for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *n = gIter->data;

	if (n->details->type != pcmk_node_variant_cluster) {
	continue;
	}

	member_count ++;

	if (n->details->online) {
	online_count++;
	}

	if (member_count == 1
	\|\| n->details->priority > top_priority) {
	top_priority = n->details->priority;
	}

	if (member_count == 1
	\|\| n->details->priority < lowest_priority) {
	lowest_priority = n->details->priority;
	}
	}

	// No need to delay if we have more than half of the cluster members
	if (online_count > member_count / 2) {
	return 0;
	}

	/* All the nodes have equal priority.
	* Any configured corresponding `pcmk_delay_base/max` will be applied. */
	if (lowest_priority == top_priority) {
	return 0;
	}

	if (node->details->priority < top_priority) {
	return 0;
	}

	return scheduler->priority_fencing_delay;
	}

	pcmk_action_t *
	pe_fence_op(pcmk_node_t node, const char op, bool optional,
	const char *reason, bool priority_delay,
	pcmk_scheduler_t *scheduler)
	{
	char *op_key = NULL;
	pcmk_action_t *stonith_op = NULL;

	if(op == NULL) {
	op = scheduler->stonith_action;
	}

	op_key = crm_strdup_printf("%s-%s-%s",
	PCMK_ACTION_STONITH, node->details->uname, op);

	stonith_op = lookup_singleton(scheduler, op_key);
	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
	TRUE, scheduler);

	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
	add_hash_param(stonith_op->meta, "stonith_action", op);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) {
	/* Extra work to detect device changes
	*/
	GString *digests_all = g_string_sized_new(1024);
	GString *digests_secure = g_string_sized_new(1024);

	GList *matches = find_unfencing_devices(scheduler->resources, NULL);

	char *key = NULL;
	char *value = NULL;

	for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *match = gIter->data;
	const char *agent = g_hash_table_lookup(match->meta,
	PCMK_XA_TYPE);
	pcmk__op_digest_t *data = NULL;

	data = pe__compare_fencing_digest(match, agent, node,
	scheduler);
	if (data->rc == pcmk__digest_mismatch) {
	optional = FALSE;
	crm_notice("Unfencing node %s because the definition of "
	"%s changed", pe__node_name(node), match->id);
	if (!pcmk__is_daemon && scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;

	out->info(out,
	"notice: Unfencing node %s because the "
	"definition of %s changed",
	pe__node_name(node), match->id);
	}
	}

	pcmk__g_strcat(digests_all,
	match->id, ":", agent, ":",
	data->digest_all_calc, ",", NULL);
	pcmk__g_strcat(digests_secure,
	match->id, ":", agent, ":",
	data->digest_secure_calc, ",", NULL);
	}
	key = strdup(XML_OP_ATTR_DIGESTS_ALL);
	value = strdup((const char *) digests_all->str);
	CRM_ASSERT((key != NULL) && (value != NULL));
	g_hash_table_insert(stonith_op->meta, key, value);
	g_string_free(digests_all, TRUE);

	key = strdup(XML_OP_ATTR_DIGESTS_SECURE);
	value = strdup((const char *) digests_secure->str);
	CRM_ASSERT((key != NULL) && (value != NULL));
	g_hash_table_insert(stonith_op->meta, key, value);
	g_string_free(digests_secure, TRUE);
	}

	} else {
	free(op_key);
	}

	if (scheduler->priority_fencing_delay > 0

	/* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY
	* applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as
	* an indicator.
	*/
	&& (priority_delay

	/* The priority delay needs to be recalculated if this function has
	* been called by schedule_fencing_and_shutdowns() after node
	* priority has already been calculated by native_add_running().
	*/
	\|\| g_hash_table_lookup(stonith_op->meta,
	PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) {

	/* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if
	* it's 0 for the targeting node. So that it takes precedence over
	* any possible `pcmk_delay_base/max`.
	*/
	char *delay_s = pcmk__itoa(node_priority_fencing_delay(node,
	scheduler));

	g_hash_table_insert(stonith_op->meta,
	strdup(PCMK_OPT_PRIORITY_FENCING_DELAY),
	delay_s);
	}

	if(optional == FALSE && pe_can_fence(scheduler, node)) {
	pcmk__clear_action_flags(stonith_op, pcmk_action_optional);
	pe_action_set_reason(stonith_op, reason, false);

	} else if(reason && stonith_op->reason == NULL) {
	stonith_op->reason = strdup(reason);
	}

	return stonith_op;
	}

	void
	pe_free_action(pcmk_action_t *action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free);
	g_list_free_full(action->actions_after, free);
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	free(action->cancel_task);
	free(action->reason);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	int
	pe_get_configured_timeout(pcmk_resource_t rsc, const char action,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *child = NULL;
	GHashTable *action_meta = NULL;
	const char *timeout_spec = NULL;
	int timeout_ms = 0;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
	child != NULL; child = crm_next_same_xml(child)) {
	if (pcmk__str_eq(action, crm_element_value(child, PCMK_XA_NAME),
	pcmk__str_casei)) {
	timeout_spec = crm_element_value(child, XML_ATTR_TIMEOUT);
	break;
	}
	}

	if (timeout_spec == NULL && scheduler->op_defaults) {
	action_meta = pcmk__strkey_table(free, free);
	pe__unpack_dataset_nvpairs(scheduler->op_defaults, XML_TAG_META_SETS,
	&rule_data, action_meta, NULL, FALSE,
	scheduler);
	timeout_spec = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
	}

	// @TODO check meta-attributes
	// @TODO maybe use min-interval monitor timeout as default for monitors

	timeout_ms = crm_get_msec(timeout_spec);
	if (timeout_ms < 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	if (action_meta != NULL) {
	g_hash_table_destroy(action_meta);
	}
	return timeout_ms;
	}

	enum action_tasks
	get_complex_task(const pcmk_resource_t rsc, const char name)
	{
	enum action_tasks task = text2task(name);

	if ((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)) {
	switch (task) {
	case pcmk_action_stopped:
	case pcmk_action_started:
	case pcmk_action_demoted:
	case pcmk_action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s",
	name, rsc->id);
	--task;
	break;
	default:
	break;
	}
	}
	return task;
	}

	/*!
	* \internal
	* \brief Find first matching action in a list
	*
	* \param[in] input List of actions to search
	* \param[in] uuid If not NULL, action must have this UUID
	* \param[in] task If not NULL, action must have this action name
	* \param[in] on_node If not NULL, action must be on this node
	*
	* \return First action in list that matches criteria, or NULL if none
	*/
	pcmk_action_t *
	find_first_action(const GList input, const char uuid, const char *task,
	const pcmk_node_t *on_node)
	{
	CRM_CHECK(uuid \|\| task, return NULL);

	for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (on_node->details == action->node->details) {
	return action;
	}
	}

	return NULL;
	}

	GList *
	find_actions(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *gIter = input;
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
	continue;

	} else if (on_node == NULL) {
	crm_trace("Action %s matches (ignoring node)", key);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	crm_trace("Action %s matches (unallocated, assigning to %s)",
	key, pe__node_name(on_node));

	action->node = pe__copy_node(on_node);
	result = g_list_prepend(result, action);

	} else if (on_node->details == action->node->details) {
	crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	GList *
	find_actions_exact(GList input, const char key, const pcmk_node_t *on_node)
	{
	GList *result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	if (on_node == NULL) {
	return NULL;
	}

	for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t action = (pcmk_action_t ) gIter->data;

	if ((action->node != NULL)
	&& pcmk__str_eq(key, action->uuid, pcmk__str_casei)
	&& pcmk__str_eq(on_node->details->id, action->node->details->id,
	pcmk__str_casei)) {

	crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	/*!
	* \brief Find all actions of given type for a resource
	*
	* \param[in] rsc Resource to search
	* \param[in] node Find only actions scheduled on this node
	* \param[in] task Action name to search for
	* \param[in] require_node If TRUE, NULL node or action node will not match
	*
	* \return List of actions found (or NULL if none)
	* \note If node is not NULL and require_node is FALSE, matching actions
	* without a node will be assigned to node.
	*/
	GList *
	pe__resource_actions(const pcmk_resource_t rsc, const pcmk_node_t node,
	const char *task, bool require_node)
	{
	GList *result = NULL;
	char *key = pcmk__op_key(rsc->id, task, 0);

	if (require_node) {
	result = find_actions_exact(rsc->actions, key, node);
	} else {
	result = find_actions(rsc->actions, key, node);
	}
	free(key);
	return result;
	}

	/*!
	* \internal
	* \brief Create an action reason string based on the action itself
	*
	* \param[in] action Action to create reason string for
	* \param[in] flag Action flag that was cleared
	*
	* \return Newly allocated string suitable for use as action reason
	* \note It is the caller's responsibility to free() the result.
	*/
	char *
	pe__action2reason(const pcmk_action_t *action, enum pe_action_flags flag)
	{
	const char *change = NULL;

	switch (flag) {
	case pcmk_action_runnable:
	change = "unrunnable";
	break;
	case pcmk_action_migratable:
	change = "unmigrateable";
	break;
	case pcmk_action_optional:
	change = "required";
	break;
	default:
	// Bug: caller passed unsupported flag
	CRM_CHECK(change != NULL, change = "");
	break;
	}
	return crm_strdup_printf("%s%s%s %s", change,
	(action->rsc == NULL)? "" : " ",
	(action->rsc == NULL)? "" : action->rsc->id,
	action->task);
	}

	void pe_action_set_reason(pcmk_action_t action, const char reason,
	bool overwrite)
	{
	if (action->reason != NULL && overwrite) {
	pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
	action->uuid, action->reason,
	pcmk__s(reason, "(none)"));
	} else if (action->reason == NULL) {
	pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'",
	action->uuid, pcmk__s(reason, "(none)"));
	} else {
	// crm_assert(action->reason != NULL && !overwrite);
	return;
	}

	pcmk__str_update(&action->reason, reason);
	}

	/*!
	* \internal
	* \brief Create an action to clear a resource's history from CIB
	*
	* \param[in,out] rsc Resource to clear
	* \param[in] node Node to clear history on
	*/
	void
	pe__clear_resource_history(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	CRM_ASSERT((rsc != NULL) && (node != NULL));

	custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
	PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->cluster);
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	int
	pe__is_newer_op(const xmlNode xml_a, const xmlNode xml_b,
	bool same_node_default)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID);
	const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID);

	const char *a_node = crm_element_value(xml_a, XML_LRM_ATTR_TARGET);
	const char *b_node = crm_element_value(xml_b, XML_LRM_ATTR_TARGET);
	bool same_node = true;

	/* @COMPAT The on_node attribute was added to last_failure as of 1.1.13 (via
	* 8b3ca1c) and the other entries as of 1.1.12 (via 0b07b5c).
	*
	* In case that any of the lrm_rsc_op entries doesn't have on_node
	* attribute, we need to explicitly tell whether the two operations are on
	* the same node.
	*/
	if (a_node == NULL \|\| b_node == NULL) {
	same_node = same_node_default;

	} else {
	same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);
	}

	if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
	/* We have duplicate lrm_rsc_op entries in the status
	* section which is unlikely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why it's happening.
	*/
	pcmk__config_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
	crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesn't matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (a_call_id >= 0 && b_call_id >= 0
	&& (!same_node \|\| a_call_id == b_call_id)) {
	/*
	* The op and last_failed_op are the same
	* Order on last-rc-change
	*/
	time_t last_a = -1;
	time_t last_b = -1;

	crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
	crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);

	crm_trace("rc-change: %lld vs %lld",
	(long long) last_a, (long long) last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation
	* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
	*/

	int a_id = -1;
	int b_id = -1;

	const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic a");
	}
	if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
	NULL)) {
	sort_return(0, "bad magic b");
	}
	/* try to determine the relative age of the operation...
	* some pending operations (e.g. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means it's a shutdown operation and _always_ comes last
	*/
	if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesn't match then one better
	* be a pending operation.
	* pending operations don't survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));
	}

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	const xmlNode *xml_a = a;
	const xmlNode *xml_b = b;

	return pe__is_newer_op(xml_a, xml_b, true);
	}

	/*!
	* \internal
	* \brief Create a new pseudo-action for a resource
	*
	* \param[in,out] rsc Resource to create action for
	* \param[in] task Action name
	* \param[in] optional Whether action should be considered optional
	* \param[in] runnable Whethe action should be considered runnable
	*
	* \return New action object corresponding to arguments
	*/
	pcmk_action_t *
	pe__new_rsc_pseudo_action(pcmk_resource_t rsc, const char task, bool optional,
	bool runnable)
	{
	pcmk_action_t *action = NULL;

	CRM_ASSERT((rsc != NULL) && (task != NULL));

	action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
	optional, rsc->cluster);
	pcmk__set_action_flags(action, pcmk_action_pseudo);
	if (runnable) {
	pcmk__set_action_flags(action, pcmk_action_runnable);
	}
	return action;
	}

	/*!
	* \internal
	* \brief Add the expected result to an action
	*
	* \param[in,out] action Action to add expected result to
	* \param[in] expected_result Expected result to add
	*
	* \note This is more efficient than calling add_hash_param().
	*/
	void
	pe__add_action_expected_result(pcmk_action_t *action, int expected_result)
	{
	char *name = NULL;

	CRM_ASSERT((action != NULL) && (action->meta != NULL));

	name = strdup(XML_ATTR_TE_TARGET_RC);
	CRM_ASSERT (name != NULL);

	g_hash_table_insert(action->meta, name, pcmk__itoa(expected_result));
	}
	diff --git a/lib/pengine/pe_digest.c b/lib/pengine/pe_digest.c
	index 934b55bcca..7af913209d 100644
	--- a/lib/pengine/pe_digest.c
	+++ b/lib/pengine/pe_digest.c
	@@ -1,609 +1,609 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <stdbool.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/pengine/internal.h>
	#include "pe_status_private.h"

	extern bool pcmk__is_daemon;

	/*!
	* \internal
	* \brief Free an operation digest cache entry
	*
	* \param[in,out] ptr Pointer to cache entry to free
	*
	* \note The argument is a gpointer so this can be used as a hash table
	* free function.
	*/
	void
	pe__free_digests(gpointer ptr)
	{
	pcmk__op_digest_t *data = ptr;

	if (data != NULL) {
	free_xml(data->params_all);
	free_xml(data->params_secure);
	free_xml(data->params_restart);

	free(data->digest_all_calc);
	free(data->digest_restart_calc);
	free(data->digest_secure_calc);

	free(data);
	}
	}

	// Return true if XML attribute name is not substring of a given string
	static bool
	attr_not_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) == NULL) {
	crm_trace("Filtering %s (not found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	// Return true if XML attribute name is substring of a given string
	static bool
	attr_in_string(xmlAttrPtr a, void *user_data)
	{
	bool filter = false;
	char name = crm_strdup_printf(" %s ", (const char ) a->name);

	if (strstr((const char *) user_data, name) != NULL) {
	crm_trace("Filtering %s (found in '%s')",
	(const char ) a->name, (const char ) user_data);
	filter = true;
	}
	free(name);
	return filter;
	}

	/*!
	* \internal
	* \brief Add digest of all parameters to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in,out] rsc Resource that action was for
	* \param[in] node Node action was performed on
	* \param[in] params Resource parameters evaluated for node
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	* \param[in] xml_op Unused
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value table to override resource parameters
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	calculate_main_digest(pcmk__op_digest_t data, pcmk_resource_t rsc,
	const pcmk_node_t node, GHashTable params,
	const char task, guint interval_ms,
	const xmlNode xml_op, const char op_version,
	GHashTable overrides, pcmk_scheduler_t scheduler)
	{
	xmlNode *action_config = NULL;

	data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);

	/* REMOTE_CONTAINER_HACK: Allow Pacemaker Remote nodes to run containers
	* that themselves are Pacemaker Remote nodes
	*/
	(void) pe__add_bundle_remote_name(rsc, scheduler, data->params_all,
	PCMK_REMOTE_RA_ADDR);

	if (overrides != NULL) {
	// If interval was overridden, reset it
	const char *interval_s = g_hash_table_lookup(overrides, CRM_META "_"
	XML_LRM_ATTR_INTERVAL);

	if (interval_s != NULL) {
	long long value_ll;

	if ((pcmk__scan_ll(interval_s, &value_ll, 0LL) == pcmk_rc_ok)
	&& (value_ll >= 0) && (value_ll <= G_MAXUINT)) {
	*interval_ms = (guint) value_ll;
	}
	}

	// Add overrides to list of all parameters
	g_hash_table_foreach(overrides, hash2field, data->params_all);
	}

	// Add provided instance parameters
	g_hash_table_foreach(params, hash2field, data->params_all);

	// Find action configuration XML in CIB
	action_config = pcmk__find_action_config(rsc, task, *interval_ms, true);

	/* Add action-specific resource instance attributes to the digest list.
	*
	* If this is a one-time action with action-specific instance attributes,
	* enforce a restart instead of reload-agent in case the main digest doesn't
	* match, even if the restart digest does. This ensures any changes of the
	* action-specific parameters get applied for this specific action, and
	* digests calculated for the resulting history will be correct. Default the
	* result to RSC_DIGEST_RESTART for the case where the main digest doesn't
	* match.
	*/
	params = pcmk__unpack_action_rsc_params(action_config, node->details->attrs,
	scheduler);
	if ((*interval_ms == 0) && (g_hash_table_size(params) > 0)) {
	data->rc = pcmk__digest_restart;
	}
	g_hash_table_foreach(params, hash2field, data->params_all);
	g_hash_table_destroy(params);

	// Add action meta-attributes
	params = pcmk__unpack_action_meta(rsc, node, task, *interval_ms,
	action_config);
	g_hash_table_foreach(params, hash2metafield, data->params_all);
	g_hash_table_destroy(params);

	pcmk__filter_op_for_digest(data->params_all);

	data->digest_all_calc = calculate_operation_digest(data->params_all,
	op_version);
	}

	// Return true if XML attribute name is a Pacemaker-defined fencing parameter
	static bool
	is_fence_param(xmlAttrPtr attr, void *user_data)
	{
	return pcmk_stonith_param((const char *) attr->name);
	}

	/*!
	* \internal
	* \brief Add secure digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] rsc Resource that action was for
	* \param[in] params Resource parameters evaluated for node
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	* \param[in] overrides Key/value hash table to override resource parameters
	*/
	static void
	calculate_secure_digest(pcmk__op_digest_t data, const pcmk_resource_t rsc,
	GHashTable params, const xmlNode xml_op,
	const char op_version, GHashTable overrides)
	{
	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *secure_list = NULL;
	bool old_version = (compare_version(op_version, "3.16.0") < 0);

	if (xml_op == NULL) {
	secure_list = " passwd password user ";
	} else {
	secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
	}

	if (old_version) {
	data->params_secure = create_xml_node(NULL, XML_TAG_PARAMS);
	if (overrides != NULL) {
	g_hash_table_foreach(overrides, hash2field, data->params_secure);
	}

	g_hash_table_foreach(params, hash2field, data->params_secure);

	} else {
	// Start with a copy of all parameters
	data->params_secure = copy_xml(data->params_all);
	}

	if (secure_list != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_secure, attr_in_string,
	(void *) secure_list);
	}
	if (old_version
	&& pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_fence_params)) {
	/* For stonith resources, Pacemaker adds special parameters,
	* but these are not listed in fence agent meta-data, so with older
	* versions of DC, the controller will not hash them. That means we have
	* to filter them out before calculating our hash for comparison.
	*/
	pcmk__xe_remove_matching_attrs(data->params_secure, is_fence_param,
	NULL);
	}
	pcmk__filter_op_for_digest(data->params_secure);

	/* CRM_meta_timeout should be part of a digest for recurring operations.
	* However, with older versions of DC, the controller does not add timeout
	* to secure digests, because it only includes parameters declared by the
	* resource agent.
	* Remove any timeout that made it this far, to match.
	*/
	if (old_version) {
	xml_remove_prop(data->params_secure, CRM_META "_" XML_ATTR_TIMEOUT);
	}

	data->digest_secure_calc = calculate_operation_digest(data->params_secure,
	op_version);
	}

	/*!
	* \internal
	* \brief Add restart digest to a digest cache entry
	*
	* \param[out] data Digest cache entry to modify
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] op_version CRM feature set to use for digest calculation
	*
	* \note This function doesn't need to handle overrides because it starts with
	* data->params_all, which already has overrides applied.
	*/
	static void
	calculate_restart_digest(pcmk__op_digest_t data, const xmlNode xml_op,
	const char *op_version)
	{
	const char *value = NULL;

	// We must have XML of resource operation history
	if (xml_op == NULL) {
	return;
	}

	// And the history must have a restart digest to compare against
	if (crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) == NULL) {
	return;
	}

	// Start with a copy of all parameters
	data->params_restart = copy_xml(data->params_all);

	// Then filter out reloadable parameters, if any
	value = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
	if (value != NULL) {
	pcmk__xe_remove_matching_attrs(data->params_restart, attr_not_in_string,
	(void *) value);
	}

	value = crm_element_value(xml_op, PCMK_XA_CRM_FEATURE_SET);
	data->digest_restart_calc = calculate_operation_digest(data->params_restart,
	value);
	}

	/*!
	* \internal
	* \brief Create a new digest cache entry with calculated digests
	*
	* \param[in,out] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in,out] interval_ms Action's interval (will be reset if in overrides)
	* \param[in] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] overrides Key/value table to override resource parameters
	* \param[in] calc_secure Whether to calculate secure digest
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to new digest cache entry (or NULL on memory error)
	* \note It is the caller's responsibility to free the result using
	* pe__free_digests().
	*/
	pcmk__op_digest_t *
	pe__calculate_digests(pcmk_resource_t rsc, const char task,
	guint interval_ms, const pcmk_node_t node,
	const xmlNode xml_op, GHashTable overrides,
	bool calc_secure, pcmk_scheduler_t *scheduler)
	{
	pcmk__op_digest_t *data = calloc(1, sizeof(pcmk__op_digest_t));
	const char *op_version = NULL;
	GHashTable *params = NULL;

	if (data == NULL) {
	pcmk__sched_err("Could not allocate memory for operation digest");
	return NULL;
	}

	data->rc = pcmk__digest_match;

	if (xml_op != NULL) {
	op_version = crm_element_value(xml_op, PCMK_XA_CRM_FEATURE_SET);
	}

	if (op_version == NULL && scheduler != NULL && scheduler->input != NULL) {
	op_version = crm_element_value(scheduler->input,
	PCMK_XA_CRM_FEATURE_SET);
	}

	if (op_version == NULL) {
	op_version = CRM_FEATURE_SET;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	calculate_main_digest(data, rsc, node, params, task, interval_ms, xml_op,
	op_version, overrides, scheduler);
	if (calc_secure) {
	calculate_secure_digest(data, rsc, params, xml_op, op_version,
	overrides);
	}
	calculate_restart_digest(data, xml_op, op_version);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate action digests and store in node's digest cache
	*
	* \param[in,out] rsc Resource that action was for
	* \param[in] task Name of action performed
	* \param[in] interval_ms Action's interval
	* \param[in,out] node Node action was performed on
	* \param[in] xml_op XML of operation in CIB status (if available)
	* \param[in] calc_secure Whether to calculate secure digest
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to node's digest cache entry
	*/
	static pcmk__op_digest_t *
	rsc_action_digest(pcmk_resource_t rsc, const char task, guint interval_ms,
	pcmk_node_t node, const xmlNode xml_op,
	bool calc_secure, pcmk_scheduler_t *scheduler)
	{
	pcmk__op_digest_t *data = NULL;
	char *key = pcmk__op_key(rsc->id, task, interval_ms);

	data = g_hash_table_lookup(node->details->digest_cache, key);
	if (data == NULL) {
	data = pe__calculate_digests(rsc, task, &interval_ms, node, xml_op,
	NULL, calc_secure, scheduler);
	CRM_ASSERT(data != NULL);
	g_hash_table_insert(node->details->digest_cache, strdup(key), data);
	}
	free(key);
	return data;
	}

	/*!
	* \internal
	* \brief Calculate operation digests and compare against an XML history entry
	*
	* \param[in,out] rsc Resource to check
	* \param[in] xml_op Resource history XML
	* \param[in,out] node Node to use for digest calculation
	* \param[in,out] scheduler Scheduler data
	*
	* \return Pointer to node's digest cache entry, with comparison result set
	*/
	pcmk__op_digest_t *
	rsc_action_digest_cmp(pcmk_resource_t rsc, const xmlNode xml_op,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk__op_digest_t *data = NULL;
	guint interval_ms = 0;

	const char *op_version;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_all;
	const char *digest_restart;

	CRM_ASSERT(node != NULL);

	op_version = crm_element_value(xml_op, PCMK_XA_CRM_FEATURE_SET);
	digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
	digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	data = rsc_action_digest(rsc, task, interval_ms, node, xml_op,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_sanitized),
	scheduler);

	if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
	pcmk__rsc_info(rsc,
	"Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_restart, "missing"),
	data->digest_restart_calc, op_version,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
	data->rc = pcmk__digest_restart;

	} else if (digest_all == NULL) {
	/* it is unknown what the previous op digest was */
	data->rc = pcmk__digest_unknown;

	} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
	/* Given a non-recurring operation with extra parameters configured,
	* in case that the main digest doesn't match, even if the restart
	* digest matches, enforce a restart rather than a reload-agent anyway.
	* So that it ensures any changes of the extra parameters get applied
	* for this specific operation, and the digests calculated for the
	* resulting lrm_rsc_op will be correct.
	* Preserve the implied rc pcmk__digest_restart for the case that the
	* main digest doesn't match.
	*/
	if ((interval_ms == 0) && (data->rc == pcmk__digest_restart)) {
	pcmk__rsc_info(rsc,
	"Parameters containing extra ones to %ums-interval"
	" %s action for %s on %s "
	"changed: hash was %s vs. now %s (restart:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"),
	data->digest_all_calc, op_version,
	crm_element_value(xml_op,
	XML_ATTR_TRANSITION_MAGIC));

	} else {
	pcmk__rsc_info(rsc,
	"Parameters to %ums-interval %s action for %s on %s "
	"changed: hash was %s vs. now %s (%s:%s) %s",
	interval_ms, task, rsc->id, pe__node_name(node),
	pcmk__s(digest_all, "missing"),
	data->digest_all_calc,
	(interval_ms > 0)? "reschedule" : "reload",
	op_version,
	crm_element_value(xml_op,
	XML_ATTR_TRANSITION_MAGIC));
	data->rc = pcmk__digest_mismatch;
	}

	} else {
	data->rc = pcmk__digest_match;
	}
	return data;
	}

	/*!
	* \internal
	* \brief Create an unfencing summary for use in special node attribute
	*
	* Create a string combining a fence device's resource ID, agent type, and
	* parameter digest (whether for all parameters or just non-private parameters).
	* This can be stored in a special node attribute, allowing us to detect changes
	* in either the agent type or parameters, to know whether unfencing must be
	* redone or can be safely skipped when the device's history is cleaned.
	*
	* \param[in] rsc_id Fence device resource ID
	* \param[in] agent_type Fence device agent
	* \param[in] param_digest Fence device parameter digest
	*
	* \return Newly allocated string with unfencing digest
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	create_unfencing_summary(const char rsc_id, const char agent_type,
	const char *param_digest)
	{
	return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
	}

	/*!
	* \internal
	* \brief Check whether a node can skip unfencing
	*
	* Check whether a fence device's current definition matches a node's
	* stored summary of when it was last unfenced by the device.
	*
	* \param[in] rsc_id Fence device's resource ID
	* \param[in] agent Fence device's agent type
	* \param[in] digest_calc Fence device's current parameter digest
	* \param[in] node_summary Value of node's special unfencing node attribute
	* (a comma-separated list of unfencing summaries for
	* all devices that have unfenced this node)
	*
	* \return TRUE if digest matches, FALSE otherwise
	*/
	static bool
	unfencing_digest_matches(const char rsc_id, const char agent,
	const char digest_calc, const char node_summary)
	{
	bool matches = FALSE;

	if (rsc_id && agent && digest_calc && node_summary) {
	char *search_secure = create_unfencing_summary(rsc_id, agent,
	digest_calc);

	/* The digest was calculated including the device ID and agent,
	* so there is no risk of collision using strstr().
	*/
	matches = (strstr(node_summary, search_secure) != NULL);
	crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
	search_secure, matches? "" : "not ", node_summary);
	free(search_secure);
	}
	return matches;
	}

	/* Magic string to use as action name for digest cache entries used for
	* unfencing checks. This is not a real action name (i.e. "on"), so
	* pcmk__check_action_config() won't confuse these entries with real actions.
	*/
	#define STONITH_DIGEST_TASK "stonith-on"

	/*!
	* \internal
	* \brief Calculate fence device digests and digest comparison result
	*
	* \param[in,out] rsc Fence device resource
	* \param[in] agent Fence device's agent type
	* \param[in,out] node Node with digest cache to use
	* \param[in,out] scheduler Scheduler data
	*
	* \return Node's digest cache entry
	*/
	pcmk__op_digest_t *
	pe__compare_fencing_digest(pcmk_resource_t rsc, const char agent,
	pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	const char *node_summary = NULL;

	// Calculate device's current parameter digests
	pcmk__op_digest_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, 0U,
	node, NULL, TRUE, scheduler);

	// Check whether node has special unfencing summary node attribute
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
	if (node_summary == NULL) {
	data->rc = pcmk__digest_unknown;
	return data;
	}

	// Check whether full parameter digest matches
	if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
	node_summary)) {
	data->rc = pcmk__digest_match;
	return data;
	}

	// Check whether secure parameter digest matches
	node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
	if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
	node_summary)) {
	data->rc = pcmk__digest_match;
	if (!pcmk__is_daemon && scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;
	out->info(out, "Only 'private' parameters to %s "
	"for unfencing %s changed", rsc->id,
	pe__node_name(node));
	}
	return data;
	}

	// Parameters don't match
	data->rc = pcmk__digest_mismatch;
	if (pcmk_is_set(scheduler->flags, pcmk_sched_sanitized)
	&& (data->digest_secure_calc != NULL)) {

	if (scheduler->priv != NULL) {
	pcmk__output_t *out = scheduler->priv;
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	out->info(out, "Parameters to %s for unfencing "
	"%s changed, try '%s'", rsc->id,
	pe__node_name(node), digest);
	free(digest);
	} else if (!pcmk__is_daemon) {
	char *digest = create_unfencing_summary(rsc->id, agent,
	data->digest_secure_calc);

	printf("Parameters to %s for unfencing %s changed, try '%s'\n",
	rsc->id, pe__node_name(node), digest);
	free(digest);
	}
	}
	return data;
	}
	diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
	index 1a52a5cf40..e46a89da8f 100644
	--- a/lib/pengine/pe_output.c
	+++ b/lib/pengine/pe_output.c
	@@ -1,3179 +1,3179 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/xml_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/cib/util.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/internal.h>

	const char *
	pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
	{
	const char * desc = NULL;
	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)) {
	desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);
	}
	return desc;
	}

	/* Never display node attributes whose name starts with one of these prefixes */
	#define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX, \
	"shutdown", PCMK_NODE_ATTR_TERMINATE, "standby", "#", \
	NULL }

	static int
	compare_attribute(gconstpointer a, gconstpointer b)
	{
	int rc;

	rc = strcmp((const char )a, (const char )b);

	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether extended information about an attribute should be added.
	*
	* \param[in] node Node that ran this resource
	* \param[in,out] rsc_list List of resources for this node
	* \param[in,out] scheduler Scheduler data
	* \param[in] attrname Attribute to find
	* \param[out] expected_score Expected value for this attribute
	*
	* \return true if extended information should be printed, false otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static bool
	add_extra_info(const pcmk_node_t node, GList rsc_list,
	pcmk_scheduler_t scheduler, const char attrname,
	int *expected_score)
	{
	GList *gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->meta, PCMK_XA_TYPE);
	const char *name = NULL;
	GHashTable *params = NULL;

	if (rsc->children != NULL) {
	if (add_extra_info(node, rsc->children, scheduler, attrname,
	expected_score)) {
	return true;
	}
	}

	if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
	continue;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	name = g_hash_table_lookup(params, PCMK_XA_NAME);

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
	int host_list_num = 0;
	const char *hosts = g_hash_table_lookup(params, "host_list");
	const char *multiplier = g_hash_table_lookup(params, "multiplier");
	int multiplier_i;

	if (hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	if ((multiplier == NULL)
	\|\| (pcmk__scan_min_int(multiplier, &multiplier_i,
	INT_MIN) != pcmk_rc_ok)) {
	/* The ocf:pacemaker:ping resource agent defaults multiplier to
	* 1. The agent currently does not handle invalid text, but it
	* should, and this would be a reasonable choice ...
	*/
	multiplier_i = 1;
	}
	expected_score = host_list_num multiplier_i;

	return true;
	}
	}
	return false;
	}

	static GList *
	filter_attr_list(GList attr_list, char name)
	{
	int i;
	const char *filt_str[] = FILTER_STR;

	CRM_CHECK(name != NULL, return attr_list);

	/* filtering automatic attributes */
	for (i = 0; filt_str[i] != NULL; i++) {
	if (g_str_has_prefix(name, filt_str[i])) {
	return attr_list;
	}
	}

	return g_list_insert_sorted(attr_list, name, compare_attribute);
	}

	static GList *
	get_operation_list(xmlNode *rsc_entry) {
	GList *op_list = NULL;
	xmlNode *rsc_op = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
	rsc_op = pcmk__xe_next(rsc_op)) {
	const char *task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(rsc_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)
	\|\| (pcmk__str_eq(task, "probe", pcmk__str_none)
	&& (op_rc_i == CRM_EX_NOT_RUNNING))) {
	continue;
	}

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}

	op_list = g_list_sort(op_list, sort_op_by_callid);
	return op_list;
	}

	static void
	add_dump_node(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr node = user_data;
	pcmk_create_xml_text_node(node, (const char ) key, (const char ) value);
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	char *new_text = crm_strdup_printf("%s %s=%s",
	dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	#define XPATH_STACK "//" XML_CIB_TAG_NVPAIR \
	"[@" PCMK_XA_NAME "='" \
	PCMK_OPT_CLUSTER_INFRASTRUCTURE "']"

	static const char *
	get_cluster_stack(pcmk_scheduler_t *scheduler)
	{
	xmlNode *stack = get_xpath_object(XPATH_STACK, scheduler->input, LOG_DEBUG);
	return (stack != NULL)? crm_element_value(stack, PCMK_XA_VALUE) : "unknown";
	}

	static char *
	last_changed_string(const char last_written, const char user,
	const char client, const char origin) {
	if (last_written != NULL \|\| user != NULL \|\| client != NULL \|\| origin != NULL) {
	return crm_strdup_printf("%s%s%s%s%s%s%s",
	last_written ? last_written : "",
	user ? " by " : "",
	user ? user : "",
	client ? " via " : "",
	client ? client : "",
	origin ? " on " : "",
	origin ? origin : "");
	} else {
	return strdup("");
	}
	}

	static char *
	op_history_string(xmlNode xml_op, const char task, const char *interval_ms_s,
	int rc, bool print_timing) {
	const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	char *interval_str = NULL;
	char *buf = NULL;

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *pair = pcmk__format_nvpair("interval", interval_ms_s, "ms");
	interval_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	if (print_timing) {
	char *last_change_str = NULL;
	char *exec_str = NULL;
	char *queue_str = NULL;

	const char *value = NULL;

	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE, &epoch) == pcmk_ok)
	&& (epoch > 0)) {
	char *epoch_str = pcmk__epoch2str(&epoch, 0);

	last_change_str = crm_strdup_printf(" %s=\"%s\"",
	XML_RSC_OP_LAST_CHANGE,
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
	if (value) {
	char *pair = pcmk__format_nvpair(XML_RSC_OP_T_EXEC, value, "ms");
	exec_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	if (value) {
	char *pair = pcmk__format_nvpair(XML_RSC_OP_T_QUEUE, value, "ms");
	queue_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
	interval_str ? interval_str : "",
	last_change_str ? last_change_str : "",
	exec_str ? exec_str : "",
	queue_str ? queue_str : "",
	rc, services_ocf_exitcode_str(rc));

	if (last_change_str) {
	free(last_change_str);
	}

	if (exec_str) {
	free(exec_str);
	}

	if (queue_str) {
	free(queue_str);
	}
	} else {
	buf = crm_strdup_printf("(%s) %s%s%s", call, task,
	interval_str ? ":" : "",
	interval_str ? interval_str : "");
	}

	if (interval_str) {
	free(interval_str);
	}

	return buf;
	}

	static char *
	resource_history_string(pcmk_resource_t rsc, const char rsc_id, bool all,
	int failcount, time_t last_failure) {
	char *buf = NULL;

	if (rsc == NULL) {
	buf = crm_strdup_printf("%s: orphan", rsc_id);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *failcount_s = NULL;
	char *lastfail_s = NULL;

	if (failcount > 0) {
	failcount_s = crm_strdup_printf(" %s=%d", PCMK__FAIL_COUNT_PREFIX,
	failcount);
	} else {
	failcount_s = strdup("");
	}
	if (last_failure > 0) {
	buf = pcmk__epoch2str(&last_failure, 0);
	lastfail_s = crm_strdup_printf(" %s='%s'",
	PCMK__LAST_FAILURE_PREFIX, buf);
	free(buf);
	}

	buf = crm_strdup_printf("%s: " PCMK_META_MIGRATION_THRESHOLD "=%d%s%s",
	rsc_id, rsc->migration_threshold, failcount_s,
	lastfail_s? lastfail_s : "");
	free(failcount_s);
	free(lastfail_s);
	} else {
	buf = crm_strdup_printf("%s:", rsc_id);
	}

	return buf;
	}

	/*!
	* \internal
	* \brief Get a node's feature set for status display purposes
	*
	* \param[in] node Node to check
	*
	* \return String representation of feature set if the node is fully up (using
	* "<3.15.1" for older nodes that don't set the #feature-set attribute),
	* otherwise NULL
	*/
	static const char *
	get_node_feature_set(const pcmk_node_t *node)
	{
	if (node->details->online && node->details->expected_up
	&& !pe__is_guest_or_remote_node(node)) {

	const char *feature_set = g_hash_table_lookup(node->details->attrs,
	CRM_ATTR_FEATURE_SET);

	/* The feature set attribute is present since 3.15.1. If it is missing,
	* then the node must be running an earlier version.
	*/
	return pcmk__s(feature_set, "<3.15.1");
	}
	return NULL;
	}

	static bool
	is_mixed_version(pcmk_scheduler_t *scheduler)
	{
	const char *feature_set = NULL;
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;
	const char *node_feature_set = get_node_feature_set(node);
	if (node_feature_set != NULL) {
	if (feature_set == NULL) {
	feature_set = node_feature_set;
	} else if (strcmp(feature_set, node_feature_set) != 0) {
	return true;
	}
	}
	}
	return false;
	}

	static char *
	formatted_xml_buf(const pcmk_resource_t *rsc, bool raw)
	{
	if (raw) {
	return dump_xml_formatted(rsc->orig_xml ? rsc->orig_xml : rsc->xml);
	} else {
	return dump_xml_formatted(rsc->xml);
	}
	}

	#define XPATH_DC_VERSION "//" XML_CIB_TAG_NVPAIR \
	"[@" PCMK_XA_NAME "='" PCMK_OPT_DC_VERSION "']"

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	if (pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_ORIG);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	/* Always print DC if none, even if not requested */
	if ((scheduler->dc_node == NULL)
	\|\| pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	XML_ATTR_UPDATE_ORIG);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	/* Kind of a hack - close the list we may have opened earlier in this
	* function so we can put all the options into their own list. We
	* only want to do this on HTML output, though.
	*/
	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	out->begin_list(out, NULL, NULL, "Config Options");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	char *
	pe__node_display_name(pcmk_node_t *node, bool print_detail)
	{
	char *node_name;
	const char *node_host = NULL;
	const char *node_id = NULL;
	int name_len;

	CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));

	/* Host is displayed only if this is a guest node and detail is requested */
	if (print_detail && pe__is_guest_node(node)) {
	const pcmk_resource_t *container = node->details->remote_rsc->container;
	const pcmk_node_t *host_node = pe__current_node(container);

	if (host_node && host_node->details) {
	node_host = host_node->details->uname;
	}
	if (node_host == NULL) {
	node_host = ""; /* so we at least get "uname@" to indicate guest */
	}
	}

	/* Node ID is displayed if different from uname and detail is requested */
	if (print_detail && !pcmk__str_eq(node->details->uname, node->details->id, pcmk__str_casei)) {
	node_id = node->details->id;
	}

	/* Determine name length */
	name_len = strlen(node->details->uname) + 1;
	if (node_host) {
	name_len += strlen(node_host) + 1; /* "@node_host" */
	}
	if (node_id) {
	name_len += strlen(node_id) + 3; /* + " (node_id)" */
	}

	/* Allocate and populate display name */
	node_name = malloc(name_len);
	CRM_ASSERT(node_name != NULL);
	strcpy(node_name, node->details->uname);
	if (node_host) {
	strcat(node_name, "@");
	strcat(node_name, node_host);
	}
	if (node_id) {
	strcat(node_name, " (");
	strcat(node_name, node_id);
	strcat(node_name, ")");
	}
	return node_name;
	}

	int
	pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...)
	{
	xmlNodePtr xml_node = NULL;
	va_list args;

	CRM_ASSERT(tag_name != NULL);

	xml_node = pcmk__output_xml_peek_parent(out);
	CRM_ASSERT(xml_node != NULL);
	xml_node = create_xml_node(xml_node, tag_name);

	va_start(args, pairs_count);
	while(pairs_count--) {
	const char param_name = va_arg(args, const char );
	const char param_value = va_arg(args, const char );
	if (param_name && param_value) {
	crm_xml_add(xml_node, param_name, param_value);
	}
	};
	va_end(args);

	if (is_list) {
	pcmk__output_xml_push_parent(out, xml_node);
	}
	return pcmk_rc_ok;
	}

	static const char *
	role_desc(enum rsc_role_e role)
	{
	if (role == pcmk_role_promoted) {
	#ifdef PCMK__COMPAT_2_0
	return "as " PCMK__ROLE_PROMOTED_LEGACY " ";
	#else
	return "in " PCMK__ROLE_PROMOTED " role ";
	#endif
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	pcmk__output_create_html_node(out, "li", NULL, NULL, buf);

	free(node_name);
	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	free(node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
	char *weight_s = pcmk__itoa(pe_node->weight);

	pcmk__output_create_xml_node(out, "ban",
	PCMK_XA_ID, location->id,
	"resource", location->rsc->id,
	"node", pe_node->details->uname,
	"weight", weight_s,
	"promoted-only", promoted_only,
	/* This is a deprecated alias for
	* promoted_only. Removing it will break
	* backward compatibility of the API schema,
	* which will require an API schema major
	* version bump.
	*/
	"master_only", promoted_only,
	NULL);

	free(weight_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t ", "const char ", "GList *",
	"uint32_t", "bool")
	static int
	ban_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	const char prefix = va_arg(args, const char );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	GList gIter, gIter2;
	int rc = pcmk_rc_no_output;

	/* Print each ban */
	for (gIter = scheduler->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pcmk__location_t *location = gIter->data;
	const pcmk_resource_t *rsc = location->rsc;

	if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
	only_rsc, pcmk__str_star_matches)) {
	continue;
	}

	for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;

	if (node->weight < 0) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
	out->message(out, "ban", node, location, show_opts);
	}
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_html(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *nnodes_str = crm_strdup_printf("%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	pcmk_create_html_node(nodes_node, "span", NULL, NULL, nnodes_str);
	free(nnodes_str);

	if (ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");

	s = crm_strdup_printf(", %d ", nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else if (ndisabled && !nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL, ")");
	} else if (!ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else {
	char *s = crm_strdup_printf("%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_text(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	out->list_item(out, NULL, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED, %d BLOCKED from "
	"further action due to failure)",
	nresources, pcmk__plural_s(nresources), ndisabled,
	nblocked);
	} else if (ndisabled && !nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED)",
	nresources, pcmk__plural_s(nresources), ndisabled);
	} else if (!ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d BLOCKED from further action "
	"due to failure)",
	nresources, pcmk__plural_s(nresources), nblocked);
	} else {
	out->list_item(out, NULL, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_xml(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "nodes_configured", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "resources_configured", NULL);

	char *s = pcmk__itoa(nnodes);
	crm_xml_add(nodes_node, "number", s);
	free(s);

	s = pcmk__itoa(nresources);
	crm_xml_add(resources_node, "number", s);
	free(s);

	s = pcmk__itoa(ndisabled);
	crm_xml_add(resources_node, "disabled", s);
	free(s);

	s = pcmk__itoa(nblocked);
	crm_xml_add(resources_node, "blocked", s);
	free(s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Current DC: ");

	if (dc) {
	char *buf = crm_strdup_printf("%s (version %s) -", dc_name,
	dc_version_s ? dc_version_s : "unknown");
	pcmk_create_html_node(node, "span", NULL, NULL, buf);
	free(buf);

	if (mixed_version) {
	pcmk_create_html_node(node, "span", NULL, "warning",
	" MIXED-VERSION");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " partition");
	if (crm_is_true(quorum)) {
	pcmk_create_html_node(node, "span", NULL, NULL, " with");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", " WITHOUT");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " quorum");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	out->list_item(out, "Current DC",
	"%s (version %s) - %spartition %s quorum",
	dc_name, dc_version_s ? dc_version_s : "unknown",
	mixed_version ? "MIXED-VERSION " : "",
	crm_is_true(quorum) ? "with" : "WITHOUT");
	} else {
	out->list_item(out, "Current DC", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name G_GNUC_UNUSED = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	pcmk__output_create_xml_node(out, "current_dc",
	"present", "true",
	PCMK_XA_VERSION, pcmk__s(dc_version_s, ""),
	PCMK_XA_NAME, dc->details->uname,
	PCMK_XA_ID, dc->details->id,
	"with_quorum", pcmk__btoa(crm_is_true(quorum)),
	"mixed_version", pcmk__btoa(mixed_version),
	NULL);
	} else {
	pcmk__output_create_xml_node(out, "current_dc",
	"present", "false",
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
	static int
	cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
	unsigned long long flags = va_arg(args, unsigned long long);

	if (pcmk_is_set(flags, pcmk_sched_in_maintenance)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will not attempt to start, stop or recover services\n");
	return pcmk_rc_ok;
	} else if (pcmk_is_set(flags, pcmk_sched_stop_all)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will keep all resources stopped\n");
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will not attempt to start, stop, or recover services)");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "STOPPED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will keep all resources stopped)");
	} else {
	out->list_item(out, NULL, "Resource management: enabled");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_log(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	return out->info(out, "Resource management is DISABLED. The cluster will not attempt to start, stop or recover services.");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	return out->info(out, "Resource management is DISABLED. The cluster has stopped all resources.");
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_text(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	return pcmk_rc_ok;
	}

	#define bv(flag) pcmk__btoa(pcmk_is_set(scheduler->flags, (flag)))

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	const char *no_quorum_policy = NULL;
	char *stonith_timeout_str = pcmk__itoa(scheduler->stonith_timeout);
	char priority_fencing_delay_str = pcmk__itoa(scheduler->priority_fencing_delay 1000);

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	no_quorum_policy = "freeze";
	break;

	case pcmk_no_quorum_stop:
	no_quorum_policy = "stop";
	break;

	case pcmk_no_quorum_demote:
	no_quorum_policy = "demote";
	break;

	case pcmk_no_quorum_ignore:
	no_quorum_policy = "ignore";
	break;

	case pcmk_no_quorum_fence:
	no_quorum_policy = "suicide";
	break;
	}

	pcmk__output_create_xml_node(out, "cluster_options",
	PCMK_OPT_STONITH_ENABLED,
	bv(pcmk_sched_fencing_enabled),
	PCMK_OPT_SYMMETRIC_CLUSTER,
	bv(pcmk_sched_symmetric_cluster),
	PCMK_OPT_NO_QUORUM_POLICY, no_quorum_policy,
	PCMK_OPT_MAINTENANCE_MODE,
	bv(pcmk_sched_in_maintenance),
	PCMK_OPT_STOP_ALL_RESOURCES,
	bv(pcmk_sched_stop_all),
	PCMK_OPT_STONITH_TIMEOUT "-ms",
	stonith_timeout_str,
	PCMK_OPT_PRIORITY_FENCING_DELAY "-ms",
	priority_fencing_delay_str,
	NULL);
	free(stonith_timeout_str);
	free(priority_fencing_delay_str);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_html(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Stack: ");
	pcmk_create_html_node(node, "span", NULL, NULL, stack_s);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	pcmk_create_html_node(node, "span", NULL, NULL, " (");
	pcmk_create_html_node(node, "span", NULL, NULL,
	pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	pcmk_create_html_node(node, "span", NULL, NULL, ")");
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_text(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	out->list_item(out, "Stack", "%s (%s)",
	stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	} else {
	out->list_item(out, "Stack", "%s", stack_s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_xml(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	const char *state_s = NULL;

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
	}

	pcmk__output_create_xml_node(out, "stack",
	PCMK_XA_TYPE, stack_s,
	"pacemakerd-state", state_s,
	NULL);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_html(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk_create_html_node(updated_node, "span", NULL, "bold", "Last updated: ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, time_s);

	if (our_nodename != NULL) {
	pcmk_create_html_node(updated_node, "span", NULL, NULL, " on ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, our_nodename);
	}

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	pcmk_create_html_node(changed_node, "span", NULL, "bold", "Last change: ");
	pcmk_create_html_node(changed_node, "span", NULL, NULL, time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_xml(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk__output_create_xml_node(out, "last_update",
	"time", time_s,
	"origin", our_nodename,
	NULL);

	pcmk__output_create_xml_node(out, "last_change",
	"time", last_written ? last_written : "",
	"user", user ? user : "",
	"client", client ? client : "",
	"origin", origin ? origin : "",
	NULL);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_text(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	out->list_item(out, "Last updated", "%s%s%s",
	time_s, (our_nodename != NULL)? " on " : "",
	pcmk__s(our_nodename, ""));

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	out->list_item(out, "Last change", " %s", time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Display a failed action in less-technical natural language
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_friendly(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	char *rsc_id = NULL;
	char *task = NULL;
	guint interval_ms = 0;
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)
	\|\| !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {
	rsc_id = strdup("unknown resource");
	task = strdup("unknown action");
	interval_ms = 0;
	}
	CRM_ASSERT((rsc_id != NULL) && (task != NULL));

	str = g_string_sized_new(256); // Should be sufficient for most messages

	pcmk__g_strcat(str, rsc_id, " ", NULL);

	if (interval_ms != 0) {
	pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
	NULL);
	}
	pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
	node_name, NULL);

	if (status == PCMK_EXEC_DONE) {
	pcmk__g_strcat(str, " returned '", services_ocf_exitcode_str(rc), "'",
	NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
	}

	} else {
	pcmk__g_strcat(str, " could not be executed (",
	pcmk_exec_status_str(status), NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ": ", exit_reason, NULL);
	}
	g_string_append_c(str, ')');
	}


	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *s = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str, " at ", s, NULL);
	free(s);
	}
	if (!pcmk__str_empty(exec_time)) {
	int exec_time_ms = 0;

	if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
	&& (exec_time_ms > 0)) {

	pcmk__g_strcat(str, " after ",
	pcmk__readable_interval(exec_time_ms), NULL);
	}
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	free(rsc_id);
	free(task);
	}

	/*!
	* \internal
	* \brief Display a failed action with technical details
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_technical(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	const char *call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	const char *queue_time = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	const char *exit_status = services_ocf_exitcode_str(rc);
	const char *lrm_status = pcmk_exec_status_str(status);
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)) {
	op_key = "unknown operation";
	}
	if (pcmk__str_empty(exit_status)) {
	exit_status = "unknown exit status";
	}
	if (pcmk__str_empty(call_id)) {
	call_id = "unknown";
	}

	str = g_string_sized_new(256);

	g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
	op_key, node_name, exit_status, rc, call_id,
	lrm_status);

	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str,
	", " XML_RSC_OP_LAST_CHANGE "="
	"'", last_change_str, "'", NULL);
	free(last_change_str);
	}
	if (!pcmk__str_empty(queue_time)) {
	pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
	}
	if (!pcmk__str_empty(exec_time)) {
	pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_default(pcmk__output_t *out, va_list args)
	{
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *op_key = pe__xe_history_key(xml_op);
	const char *node_name = crm_element_value(xml_op, XML_ATTR_UNAME);
	const char *exit_reason = crm_element_value(xml_op,
	XML_LRM_ATTR_EXIT_REASON);
	const char *exec_time = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);

	int rc;
	int status;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, 0);

	if (pcmk__str_empty(node_name)) {
	node_name = "unknown node";
	}

	if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
	failed_action_technical(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	} else {
	failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *op_key = pe__xe_history_key(xml_op);
	const char *op_key_name = "op_key";
	int rc;
	int status;
	const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);

	time_t epoch = 0;
	char *rc_s = NULL;
	char *reason_s = crm_xml_escape(exit_reason ? exit_reason : "none");
	xmlNodePtr node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);
	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, 0);

	rc_s = pcmk__itoa(rc);
	if (crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY) == NULL) {
	op_key_name = PCMK_XA_ID;
	}
	node = pcmk__output_create_xml_node(out, "failure",
	op_key_name, op_key,
	"node", crm_element_value(xml_op, XML_ATTR_UNAME),
	"exitstatus", services_ocf_exitcode_str(rc),
	"exitreason", pcmk__s(reason_s, ""),
	"exitcode", rc_s,
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"status", pcmk_exec_status_str(status),
	NULL);
	free(rc_s);

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	guint interval_ms = 0;
	char *interval_ms_s = NULL;
	char *rc_change = pcmk__epoch2str(&epoch,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);

	crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);

	pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE, rc_change,
	"queued", crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
	"exec", crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
	"interval", interval_ms_s,
	"task", crm_element_value(xml_op, XML_LRM_ATTR_TASK),
	NULL);

	free(interval_ms_s);
	free(rc_change);
	}

	free(reason_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t ", "GList ",
	"GList *", "uint32_t", "bool")
	static int
	failed_action_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *xml_op = NULL;
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(scheduler->failed) == 0) {
	return rc;
	}

	for (xml_op = pcmk__xml_first_child(scheduler->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	char *rsc = NULL;

	if (!pcmk__str_in_list(crm_element_value(xml_op, XML_ATTR_UNAME), only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	if (pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	if (!parse_op_key(pe__xe_history_key(xml_op), &rsc, NULL, NULL)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
	free(rsc);
	continue;
	}

	free(rsc);

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
	out->message(out, "failed-action", xml_op, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
	{
	int health = pe__node_health(node);

	// Cluster membership
	if (node->details->online) {
	pcmk_create_html_node(parent, "span", NULL, "online", " online");
	} else {
	pcmk_create_html_node(parent, "span", NULL, "offline", " OFFLINE");
	}

	// Standby mode
	if (node->details->standby_onfail && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail,"
	" with active resources)");
	} else if (node->details->standby_onfail) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail)");
	} else if (node->details->standby && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby, with active resources)");
	} else if (node->details->standby) {
	pcmk_create_html_node(parent, "span", NULL, "standby", " (in standby)");
	}

	// Maintenance mode
	if (node->details->maintenance) {
	pcmk_create_html_node(parent, "span", NULL, "maint",
	" (in maintenance mode)");
	}

	// Node health
	if (health < 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_red",
	" (health is RED)");
	} else if (health == 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_yellow",
	" (health is YELLOW)");
	}

	// Feature set
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	char *buf = crm_strdup_printf(", feature set %s", feature_set);
	pcmk_create_html_node(parent, "span", NULL, NULL, buf);
	free(buf);
	}
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
	"GList ", "GList ")
	static int
	node_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (full) {
	xmlNodePtr item_node;

	if (pcmk_all_flags_set(show_opts, pcmk_show_brief \| pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "Resources");
	pe__rscs_brief_output(out, rscs, new_show_opts);
	out->end_list(out);
	}

	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc2 = NULL;
	int rc = pcmk_rc_no_output;

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc2->data;
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else {
	char *buf = crm_strdup_printf("%s:", node_name);

	item_node = pcmk__output_create_xml_node(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	status_node(node, item_node, show_opts);

	free(buf);
	}
	} else {
	out->begin_list(out, NULL, NULL, "%s:", node_name);
	}

	free(node_name);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get a human-friendly textual description of a node's status
	*
	* \param[in] node Node to check
	*
	* \return String representation of node's status
	*/
	static const char *
	node_text_status(const pcmk_node_t *node)
	{
	if (node->details->unclean) {
	if (node->details->online) {
	return "UNCLEAN (online)";

	} else if (node->details->pending) {
	return "UNCLEAN (pending)";

	} else {
	return "UNCLEAN (offline)";
	}

	} else if (node->details->pending) {
	return "pending";

	} else if (node->details->standby_onfail && node->details->online) {
	return "standby (on-fail)";

	} else if (node->details->standby) {
	if (node->details->online) {
	if (node->details->running_rsc) {
	return "standby (with active resources)";
	} else {
	return "standby";
	}
	} else {
	return "OFFLINE (standby)";
	}

	} else if (node->details->maintenance) {
	if (node->details->online) {
	return "maintenance";
	} else {
	return "OFFLINE (maintenance)";
	}

	} else if (node->details->online) {
	return "online";
	}

	return "OFFLINE";
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	GString *str = g_string_sized_new(64);
	int health = pe__node_health(node);

	// Create a summary line with node type, name, and status
	if (pe__is_guest_node(node)) {
	g_string_append(str, "GuestNode");
	} else if (pe__is_remote_node(node)) {
	g_string_append(str, "RemoteNode");
	} else {
	g_string_append(str, "Node");
	}
	pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);

	if (health < 0) {
	g_string_append(str, " (health is RED)");
	} else if (health == 0) {
	g_string_append(str, " (health is YELLOW)");
	}
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
	}
	}

	/* If we're grouping by node, print its resources */
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	pe__rscs_brief_output(out, rscs, new_show_opts);

	out->end_list(out);
	out->end_list(out);

	g_list_free(rscs);
	}

	} else {
	GList *gIter2 = NULL;

	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter2->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	out->end_list(out);
	out->end_list(out);
	}
	} else {
	out->list_item(out, NULL, "%s", str->str);
	}

	g_string_free(str, TRUE);
	free(node_name);
	} else {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	out->begin_list(out, NULL, NULL, "Node: %s", node_name);
	free(node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	const char *node_type = "unknown";
	char *length_s = pcmk__itoa(g_list_length(node->details->running_rsc));
	int health = pe__node_health(node);
	const char *health_s = NULL;
	const char *feature_set;

	switch (node->details->type) {
	case pcmk_node_variant_cluster:
	node_type = "member";
	break;
	case pcmk_node_variant_remote:
	node_type = "remote";
	break;
	case node_ping:
	node_type = "ping";
	break;
	}

	if (health < 0) {
	health_s = "red";
	} else if (health == 0) {
	health_s = "yellow";
	} else {
	health_s = "green";
	}

	feature_set = get_node_feature_set(node);

	pe__name_and_nvpairs_xml(out, true, "node", 15,
	PCMK_XA_NAME, node->details->uname,
	PCMK_XA_ID, node->details->id,
	"online", pcmk__btoa(node->details->online),
	"standby", pcmk__btoa(node->details->standby),
	"standby_onfail", pcmk__btoa(node->details->standby_onfail),
	"maintenance", pcmk__btoa(node->details->maintenance),
	"pending", pcmk__btoa(node->details->pending),
	"unclean", pcmk__btoa(node->details->unclean),
	"health", health_s,
	"feature_set", feature_set,
	"shutdown", pcmk__btoa(node->details->shutdown),
	"expected_up", pcmk__btoa(node->details->expected_up),
	"is_dc", pcmk__btoa(node->details->is_dc),
	"resources_running", length_s,
	PCMK_XA_TYPE, node_type);

	if (pe__is_guest_node(node)) {
	xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
	crm_xml_add(xml_node, "id_as_resource", node->details->remote_rsc->container->id);
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc = NULL;

	for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}
	}

	free(length_s);

	out->end_list(out);
	} else {
	pcmk__output_xml_create_parent(out, "node",
	PCMK_XA_NAME, node->details->uname,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_text(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}
	if (v <= 0) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
	} else if (v < expected_score) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_html(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;
	char *s = crm_strdup_printf("%s: %s", name, value);
	xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}

	pcmk_create_html_node(item_node, "span", NULL, NULL, s);
	free(s);

	if (v <= 0) {
	pcmk_create_html_node(item_node, "span", NULL, "bold", "(connectivity is lost)");
	} else if (v < expected_score) {
	char *buf = crm_strdup_printf("(connectivity is degraded -- expected %d", expected_score);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	free(buf);
	}
	} else {
	out->list_item(out, NULL, "%s: %s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	gchar *node_str = NULL;
	char *last_change_str = NULL;

	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, PCMK_EXEC_UNKNOWN);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const pcmk_node_t *node = pe__current_node(rsc);
	const char *target_role = g_hash_table_lookup(rsc->meta,
	PCMK_META_TARGET_ROLE);
	uint32_t show_opts = pcmk_show_rsc_only \| pcmk_show_pending;

	if (node == NULL) {
	node = rsc->pending_node;
	}

	node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
	show_opts, target_role, false);
	} else {
	node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change) == pcmk_ok) {
	last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
	XML_RSC_OP_LAST_CHANGE,
	pcmk__trim(ctime(&last_change)),
	crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
	}

	out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
	node_str, pe__xe_history_key(xml_op),
	crm_element_value(xml_op, XML_ATTR_UNAME),
	crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	crm_element_value(xml_op, XML_LRM_ATTR_RC),
	last_change_str ? last_change_str : "",
	pcmk_exec_status_str(status));

	g_free(node_str);
	free(last_change_str);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;
	xmlNode *node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
	&status, PCMK_EXEC_UNKNOWN);
	node = pcmk__output_create_xml_node(out, "operation",
	"op", pe__xe_history_key(xml_op),
	"node", crm_element_value(xml_op, XML_ATTR_UNAME),
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"rc", crm_element_value(xml_op, XML_LRM_ATTR_RC),
	"status", pcmk_exec_status_str(status),
	NULL);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	- const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	bool has_provider = pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_provider);

	char *agent_tuple = crm_strdup_printf("%s:%s:%s",
	class,
	(has_provider? provider : ""),
	kind);

	pcmk__xe_set_props(node, "rsc", rsc_printable_id(rsc),
	"agent", agent_tuple,
	NULL);
	free(agent_tuple);
	}

	if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&last_change) == pcmk_ok) {
	pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE,
	pcmk__trim(ctime(&last_change)),
	XML_RSC_OP_T_EXEC, crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_xml(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "attribute",
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (add_extra) {
	char *buf = pcmk__itoa(expected_score);
	crm_xml_add(node, "expected", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
	"bool", "GList ", "GList ")
	static int
	node_attribute_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	int rc = pcmk_rc_no_output;

	/* Display each node's attributes */
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;

	GList *attr_list = NULL;
	GHashTableIter iter;
	gpointer key;

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	g_hash_table_iter_init(&iter, node->details->attrs);
	while (g_hash_table_iter_next (&iter, &key, NULL)) {
	attr_list = filter_attr_list(attr_list, key);
	}

	if (attr_list == NULL) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node, pcmk__str_star_matches\|pcmk__str_casei)) {
	g_list_free(attr_list);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");

	out->message(out, "node", node, show_opts, false, only_node, only_rsc);

	for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
	const char *name = aIter->data;
	const char *value = NULL;
	int expected_score = 0;
	bool add_extra = false;

	value = pe_node_attribute_raw(node, name);

	add_extra = add_extra_info(node, node->details->running_rsc,
	scheduler, name, &expected_score);

	/* Print attribute name and value */
	out->message(out, "node-attribute", name, value, add_extra,
	expected_score);
	}

	g_list_free(attr_list);
	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s capacity:",
	comment, pe__node_name(node));

	g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "capacity",
	"node", node->details->uname,
	"comment", comment,
	NULL);
	g_hash_table_foreach(node->details->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t ", "pcmk_node_t ",
	"xmlNodePtr", "GList ", "GList ", "uint32_t", "uint32_t")
	static int
	node_history_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	xmlNode node_state = va_arg(args, xmlNode );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;
	int rc = pcmk_rc_no_output;

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	/* Print history of each of the node's resources */
	for (rsc_entry = first_named_child(lrm_rsc, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {
	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	/* We can't use is_filtered here to filter group resources. For is_filtered,
	* we have to decide whether to check the parent or not. If we check the
	* parent, all elements of a group will always be printed because that's how
	* is_filtered works for groups. If we do not check the parent, sometimes
	* this will filter everything out.
	*
	* For other resource types, is_filtered is okay.
	*/
	if (parent->variant == pcmk_rsc_variant_group) {
	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
	pcmk__str_star_matches)) {
	continue;
	}
	} else {
	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	continue;
	}
	}

	if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	if (failcount <= 0) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-history", rsc, rsc_id, false,
	failcount, last_failure, false);
	} else {
	GList *op_list = get_operation_list(rsc_entry);
	pcmk_resource_t *rsc = NULL;

	if (op_list == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources,
	crm_element_value(rsc_entry, PCMK_XA_ID));

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-operation-list", scheduler, rsc, node,
	op_list, show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_html(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_text(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	/* space-separated lists of node names */
	GString *online_nodes = NULL;
	GString *online_remote_nodes = NULL;
	GString *online_guest_nodes = NULL;
	GString *offline_nodes = NULL;
	GString *offline_remote_nodes = NULL;

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	free(node_name);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");

	// Determine whether to display node individually or in a list
	if (node->details->unclean \|\| node->details->pending
	\|\| (node->details->standby_onfail && node->details->online)
	\|\| node->details->standby \|\| node->details->maintenance
	\|\| pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
	\|\| pcmk_is_set(show_opts, pcmk_show_feature_set)
	\|\| (pe__node_health(node) <= 0)) {
	// Display node individually

	} else if (node->details->online) {
	// Display online node in a list
	if (pe__is_guest_node(node)) {
	pcmk__add_word(&online_guest_nodes, 1024, node_name);

	} else if (pe__is_remote_node(node)) {
	pcmk__add_word(&online_remote_nodes, 1024, node_name);

	} else {
	pcmk__add_word(&online_nodes, 1024, node_name);
	}
	free(node_name);
	continue;

	} else {
	// Display offline node in a list
	if (pe__is_remote_node(node)) {
	pcmk__add_word(&offline_remote_nodes, 1024, node_name);

	} else if (pe__is_guest_node(node)) {
	/* ignore offline guest nodes */

	} else {
	pcmk__add_word(&offline_nodes, 1024, node_name);
	}
	free(node_name);
	continue;
	}

	/* If we get here, node is in bad state, or we're grouping by node */
	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	free(node_name);
	}

	/* If we're not grouping by node, summarize nodes by status */
	if (online_nodes != NULL) {
	out->list_item(out, "Online", "[ %s ]",
	(const char *) online_nodes->str);
	g_string_free(online_nodes, TRUE);
	}
	if (offline_nodes != NULL) {
	out->list_item(out, "OFFLINE", "[ %s ]",
	(const char *) offline_nodes->str);
	g_string_free(offline_nodes, TRUE);
	}
	if (online_remote_nodes) {
	out->list_item(out, "RemoteOnline", "[ %s ]",
	(const char *) online_remote_nodes->str);
	g_string_free(online_remote_nodes, TRUE);
	}
	if (offline_remote_nodes) {
	out->list_item(out, "RemoteOFFLINE", "[ %s ]",
	(const char *) offline_remote_nodes->str);
	g_string_free(offline_remote_nodes, TRUE);
	}
	if (online_guest_nodes != NULL) {
	out->list_item(out, "GuestOnline", "[ %s ]",
	(const char *) online_guest_nodes->str);
	g_string_free(online_guest_nodes, TRUE);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_xml(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	out->begin_list(out, NULL, NULL, "nodes");
	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}
	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t ", "GList ", "GList *",
	"uint32_t", "uint32_t", "bool")
	static int
	node_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *node_state = NULL;
	xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_STATUS);
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(cib_status) == 0) {
	return rc;
	}

	for (node_state = first_named_child(cib_status, XML_CIB_TAG_STATE);
	node_state != NULL; node_state = crm_next_same_xml(node_state)) {
	pcmk_node_t *node = pe_find_node_id(scheduler->nodes, ID(node_state));

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
	pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");

	out->message(out, "node-history-list", scheduler, node, node_state,
	only_node, only_rsc, section_opts, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	if (rsc) {
	out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
	prefix, rsc->id, uname, score);
	} else {
	out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "node_weight",
	"function", prefix,
	"node", uname,
	"score", score,
	NULL);

	if (rsc) {
	crm_xml_add(node, PCMK_XA_ID, rsc->id);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_text(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
	pcmk_is_set(show_opts, pcmk_show_timing));

	out->list_item(out, NULL, "%s", buf);

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *rc_s = pcmk__itoa(rc);
	xmlNodePtr node = pcmk__output_create_xml_node(out, "operation_history",
	"call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
	"task", task,
	"rc", rc_s,
	"rc_text", services_ocf_exitcode_str(rc),
	NULL);
	free(rc_s);

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *s = crm_strdup_printf("%sms", interval_ms_s);
	crm_xml_add(node, "interval", s);
	free(s);
	}

	if (pcmk_is_set(show_opts, pcmk_show_timing)) {
	const char *value = NULL;
	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	char *s = pcmk__epoch2str(&epoch, 0);
	crm_xml_add(node, XML_RSC_OP_LAST_CHANGE, s);
	free(s);
	}

	value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, XML_RSC_OP_T_EXEC, s);
	free(s);
	}
	value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, XML_RSC_OP_T_QUEUE, s);
	free(s);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	out->list_item(out, NULL, "%s promotion score on %s: %s",
	child_rsc->id,
	chosen? chosen->details->uname : "none",
	score);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "promotion_score",
	PCMK_XA_ID, child_rsc->id,
	"score", score,
	NULL);

	if (chosen) {
	crm_xml_add(node, "node", chosen->details->uname);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config_text(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	pcmk__formatted_printf(out, "Resource XML:\n");
	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);

	if (as_header) {
	out->begin_list(out, NULL, NULL, "%s", buf);
	} else {
	out->list_item(out, NULL, "%s", buf);
	}

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource_history",
	PCMK_XA_ID, rsc_id,
	NULL);

	if (rsc == NULL) {
	pcmk__xe_set_bool_attr(node, "orphan", true);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *migration_s = pcmk__itoa(rsc->migration_threshold);

	pcmk__xe_set_props(node, "orphan", "false",
	PCMK_META_MIGRATION_THRESHOLD, migration_s,
	NULL);
	free(migration_s);

	if (failcount > 0) {
	char *s = pcmk__itoa(failcount);

	crm_xml_add(node, PCMK__FAIL_COUNT_PREFIX, s);
	free(s);
	}

	if (last_failure > 0) {
	char *s = pcmk__epoch2str(&last_failure, 0);

	crm_xml_add(node, PCMK__LAST_FAILURE_PREFIX, s);
	free(s);
	}
	}

	if (!as_header) {
	pcmk__output_xml_pop_parent(out);
	}

	return pcmk_rc_ok;
	}

	static void
	print_resource_header(pcmk__output_t *out, uint32_t show_opts)
	{
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	/* Active resources have already been printed by node */
	out->begin_list(out, NULL, NULL, "Inactive Resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->begin_list(out, NULL, NULL, "Full List of Resources");
	} else {
	out->begin_list(out, NULL, NULL, "Active Resources");
	}
	}


	PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
	"GList ", "GList ", "bool")
	static int
	resource_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_summary = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	bool print_spacer = va_arg(args, int);

	GList *rsc_iter;
	int rc = pcmk_rc_no_output;
	bool printed_header = false;

	/* If we already showed active resources by node, and
	* we're not showing inactive resources, we have nothing to do
	*/
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
	!pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	return rc;
	}

	/* If we haven't already printed resources grouped by node,
	* and brief output was requested, print resource summary */
	if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(scheduler->resources, only_rsc);

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;

	rc = pe__rscs_brief_output(out, rscs, show_opts);
	g_list_free(rscs);
	}

	/* For each resource, display it if appropriate */
	for (rsc_iter = scheduler->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rsc_iter->data;
	int x;

	/* Complex resources may have some sub-resources active and some inactive */
	gboolean is_active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	/* Skip inactive orphans (deleted but still in CIB) */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed) && !is_active) {
	continue;

	/* Skip active resources if we already displayed them by node */
	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (is_active) {
	continue;
	}

	/* Skip primitives already counted in a brief summary */
	} else if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& (rsc->variant == pcmk_rsc_variant_primitive)) {
	continue;

	/* Skip resources that aren't at least partially active,
	* unless we're displaying inactive resources
	*/
	} else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	continue;

	} else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
	continue;
	}

	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	/* Print this resource */
	x = out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc,
	only_node, only_rsc);
	if (x == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	if (print_summary && rc != pcmk_rc_ok) {
	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	out->list_item(out, NULL, "No inactive resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "No resources");
	} else {
	out->list_item(out, NULL, "No active resources");
	}
	}

	if (printed_header) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
	"pcmk_resource_t ", "pcmk_node_t ", "GList *", "uint32_t")
	static int
	resource_operation_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
	pcmk_scheduler_t *);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	GList op_list = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);

	GList *gIter = NULL;
	int rc = pcmk_rc_no_output;

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *interval_ms_s = crm_element_value(xml_op,
	XML_LRM_ATTR_INTERVAL_MS);
	const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* If this is the first printed operation, print heading for resource */
	if (rc == pcmk_rc_no_output) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
	failcount, last_failure, true);
	rc = pcmk_rc_ok;
	}

	/* Print the operation */
	out->message(out, "op-history", xml_op, task, interval_ms_s,
	op_rc_i, show_opts);
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
	fn, rsc->id, pe__node_name(node));

	g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "utilization",
	"resource", rsc->id,
	"node", node->details->uname,
	"function", fn,
	NULL);
	g_hash_table_foreach(rsc->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_html(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, NULL, "%s:\t%s%s %s=\"%s\"", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, NULL, "%s:\t%s%s", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_text(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, ticket->id, "%s%s %s=\"%s\"",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, ticket->id, "%s%s",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_xml(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, "ticket",
	PCMK_XA_ID, ticket->id,
	"status", ticket->granted ? "granted" : "revoked",
	"standby", pcmk__btoa(ticket->standby),
	NULL);

	if (ticket->last_granted > -1) {
	char *buf = pcmk__epoch2str(&ticket->last_granted, 0);

	crm_xml_add(node, "last-granted", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-list", "pcmk_scheduler_t *", "bool")
	static int
	ticket_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	bool print_spacer = va_arg(args, int);

	GHashTableIter iter;
	gpointer key, value;

	if (g_hash_table_size(scheduler->tickets) == 0) {
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);

	/* Print section heading */
	out->begin_list(out, NULL, NULL, "Tickets");

	/* Print each ticket */
	g_hash_table_iter_init(&iter, scheduler->tickets);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	pcmk_ticket_t ticket = (pcmk_ticket_t ) value;
	out->message(out, "ticket", ticket);
	}

	/* Close section */
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "ban", "default", ban_text },
	{ "ban", "html", ban_html },
	{ "ban", "xml", ban_xml },
	{ "ban-list", "default", ban_list },
	{ "bundle", "default", pe__bundle_text },
	{ "bundle", "xml", pe__bundle_xml },
	{ "bundle", "html", pe__bundle_html },
	{ "clone", "default", pe__clone_default },
	{ "clone", "xml", pe__clone_xml },
	{ "cluster-counts", "default", cluster_counts_text },
	{ "cluster-counts", "html", cluster_counts_html },
	{ "cluster-counts", "xml", cluster_counts_xml },
	{ "cluster-dc", "default", cluster_dc_text },
	{ "cluster-dc", "html", cluster_dc_html },
	{ "cluster-dc", "xml", cluster_dc_xml },
	{ "cluster-options", "default", cluster_options_text },
	{ "cluster-options", "html", cluster_options_html },
	{ "cluster-options", "log", cluster_options_log },
	{ "cluster-options", "xml", cluster_options_xml },
	{ "cluster-summary", "default", cluster_summary },
	{ "cluster-summary", "html", cluster_summary_html },
	{ "cluster-stack", "default", cluster_stack_text },
	{ "cluster-stack", "html", cluster_stack_html },
	{ "cluster-stack", "xml", cluster_stack_xml },
	{ "cluster-times", "default", cluster_times_text },
	{ "cluster-times", "html", cluster_times_html },
	{ "cluster-times", "xml", cluster_times_xml },
	{ "failed-action", "default", failed_action_default },
	{ "failed-action", "xml", failed_action_xml },
	{ "failed-action-list", "default", failed_action_list },
	{ "group", "default", pe__group_default},
	{ "group", "xml", pe__group_xml },
	{ "maint-mode", "text", cluster_maint_mode_text },
	{ "node", "default", node_text },
	{ "node", "html", node_html },
	{ "node", "xml", node_xml },
	{ "node-and-op", "default", node_and_op },
	{ "node-and-op", "xml", node_and_op_xml },
	{ "node-capacity", "default", node_capacity },
	{ "node-capacity", "xml", node_capacity_xml },
	{ "node-history-list", "default", node_history_list },
	{ "node-list", "default", node_list_text },
	{ "node-list", "html", node_list_html },
	{ "node-list", "xml", node_list_xml },
	{ "node-weight", "default", node_weight },
	{ "node-weight", "xml", node_weight_xml },
	{ "node-attribute", "default", node_attribute_text },
	{ "node-attribute", "html", node_attribute_html },
	{ "node-attribute", "xml", node_attribute_xml },
	{ "node-attribute-list", "default", node_attribute_list },
	{ "node-summary", "default", node_summary },
	{ "op-history", "default", op_history_text },
	{ "op-history", "xml", op_history_xml },
	{ "primitive", "default", pe__resource_text },
	{ "primitive", "xml", pe__resource_xml },
	{ "primitive", "html", pe__resource_html },
	{ "promotion-score", "default", promotion_score },
	{ "promotion-score", "xml", promotion_score_xml },
	{ "resource-config", "default", resource_config },
	{ "resource-config", "text", resource_config_text },
	{ "resource-history", "default", resource_history_text },
	{ "resource-history", "xml", resource_history_xml },
	{ "resource-list", "default", resource_list },
	{ "resource-operation-list", "default", resource_operation_list },
	{ "resource-util", "default", resource_util },
	{ "resource-util", "xml", resource_util_xml },
	{ "ticket", "default", ticket_text },
	{ "ticket", "html", ticket_html },
	{ "ticket", "xml", ticket_xml },
	{ "ticket-list", "default", ticket_list },

	{ NULL, NULL, NULL }
	};

	void
	pe__register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pengine/remote.c b/lib/pengine/remote.c
	index 0fcfdc0c9a..a8536f8533 100644
	--- a/lib/pengine/remote.c
	+++ b/lib/pengine/remote.c
	@@ -1,275 +1,275 @@
	/*
	* Copyright 2013-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>
	#include <glib.h>

	bool
	pe__resource_is_remote_conn(const pcmk_resource_t *rsc)
	{
	return (rsc != NULL) && rsc->is_remote_node
	&& pe__is_remote_node(pe_find_node(rsc->cluster->nodes, rsc->id));
	}

	bool
	pe__is_remote_node(const pcmk_node_t *node)
	{
	return (node != NULL) && (node->details->type == pcmk_node_variant_remote)
	&& ((node->details->remote_rsc == NULL)
	\|\| (node->details->remote_rsc->container == NULL));
	}

	bool
	pe__is_guest_node(const pcmk_node_t *node)
	{
	return (node != NULL) && (node->details->type == pcmk_node_variant_remote)
	&& (node->details->remote_rsc != NULL)
	&& (node->details->remote_rsc->container != NULL);
	}

	bool
	pe__is_guest_or_remote_node(const pcmk_node_t *node)
	{
	return (node != NULL) && (node->details->type == pcmk_node_variant_remote);
	}

	bool
	pe__is_bundle_node(const pcmk_node_t *node)
	{
	return pe__is_guest_node(node)
	&& pe_rsc_is_bundled(node->details->remote_rsc);
	}

	/*!
	* \internal
	* \brief Check whether a resource creates a guest node
	*
	* If a given resource contains a filler resource that is a remote connection,
	* return that filler resource (or NULL if none is found).
	*
	* \param[in] scheduler Scheduler data
	* \param[in] rsc Resource to check
	*
	* \return Filler resource with remote connection, or NULL if none found
	*/
	pcmk_resource_t *
	pe__resource_contains_guest_node(const pcmk_scheduler_t *scheduler,
	const pcmk_resource_t *rsc)
	{
	if ((rsc != NULL) && (scheduler != NULL)
	&& pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {

	for (GList *gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *filler = gIter->data;

	if (filler->is_remote_node) {
	return filler;
	}
	}
	}
	return NULL;
	}

	bool
	xml_contains_remote_node(xmlNode *xml)
	{
	const char *value = NULL;

	if (xml == NULL) {
	return false;
	}

	value = crm_element_value(xml, PCMK_XA_TYPE);
	if (!pcmk__str_eq(value, "remote", pcmk__str_casei)) {
	return false;
	}

	- value = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
	+ value = crm_element_value(xml, PCMK_XA_CLASS);
	if (!pcmk__str_eq(value, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
	return false;
	}

	value = crm_element_value(xml, PCMK_XA_PROVIDER);
	if (!pcmk__str_eq(value, "pacemaker", pcmk__str_casei)) {
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Execute a supplied function for each guest node running on a host
	*
	* \param[in] scheduler Scheduler data
	* \param[in] host Host node to check
	* \param[in] helper Function to call for each guest node
	* \param[in,out] user_data Pointer to pass to helper function
	*/
	void
	pe_foreach_guest_node(const pcmk_scheduler_t *scheduler,
	const pcmk_node_t *host,
	void (helper)(const pcmk_node_t, void*),
	void *user_data)
	{
	GList *iter;

	CRM_CHECK(scheduler && host && host->details && helper, return);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
	return;
	}
	for (iter = host->details->running_rsc; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->is_remote_node && (rsc->container != NULL)) {
	pcmk_node_t *guest_node = pe_find_node(scheduler->nodes, rsc->id);

	if (guest_node) {
	(*helper)(guest_node, user_data);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Create CIB XML for an implicit remote connection
	*
	* \param[in,out] parent If not \c NULL, use as parent XML element
	* \param[in] uname Name of Pacemaker Remote node
	* \param[in] container_id If not \c NULL, use this as connection container
	* \param[in] migrateable If not \c NULL, use as remote
	* \c PCMK_META_ALLOW_MIGRATE value
	* \param[in] is_managed If not \c NULL, use as remote
	* \c PCMK_META_IS_MANAGED value
	* \param[in] start_timeout If not \c NULL, use as remote connect timeout
	* \param[in] server If not \c NULL, use as \c PCMK_REMOTE_RA_ADDR
	* \param[in] port If not \c NULL, use as \c PCMK_REMOTE_RA_PORT
	*
	* \return Newly created XML
	*/
	xmlNode *
	pe_create_remote_xml(xmlNode parent, const char uname,
	const char container_id, const char migrateable,
	const char is_managed, const char start_timeout,
	const char server, const char port)
	{
	xmlNode *remote;
	xmlNode *xml_sub;

	remote = create_xml_node(parent, XML_CIB_TAG_RESOURCE);

	// Add identity
	crm_xml_add(remote, PCMK_XA_ID, uname);
	- crm_xml_add(remote, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
	+ crm_xml_add(remote, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(remote, PCMK_XA_PROVIDER, "pacemaker");
	crm_xml_add(remote, PCMK_XA_TYPE, "remote");

	// Add meta-attributes
	xml_sub = create_xml_node(remote, XML_TAG_META_SETS);
	crm_xml_set_id(xml_sub, "%s-%s", uname, XML_TAG_META_SETS);
	crm_create_nvpair_xml(xml_sub, NULL,
	PCMK__META_INTERNAL_RSC, XML_BOOLEAN_TRUE);
	if (container_id) {
	crm_create_nvpair_xml(xml_sub, NULL,
	PCMK__META_CONTAINER, container_id);
	}
	if (migrateable) {
	crm_create_nvpair_xml(xml_sub, NULL,
	PCMK_META_ALLOW_MIGRATE, migrateable);
	}
	if (is_managed) {
	crm_create_nvpair_xml(xml_sub, NULL, PCMK_META_IS_MANAGED, is_managed);
	}

	// Add instance attributes
	if (port \|\| server) {
	xml_sub = create_xml_node(remote, XML_TAG_ATTR_SETS);
	crm_xml_set_id(xml_sub, "%s-%s", uname, XML_TAG_ATTR_SETS);
	if (server) {
	crm_create_nvpair_xml(xml_sub, NULL, PCMK_REMOTE_RA_ADDR, server);
	}
	if (port) {
	crm_create_nvpair_xml(xml_sub, NULL, PCMK_REMOTE_RA_PORT, port);
	}
	}

	// Add operations
	xml_sub = create_xml_node(remote, "operations");
	crm_create_op_xml(xml_sub, uname, PCMK_ACTION_MONITOR, "30s", "30s");
	if (start_timeout) {
	crm_create_op_xml(xml_sub, uname, PCMK_ACTION_START, "0",
	start_timeout);
	}
	return remote;
	}

	// History entry to be checked for fail count clearing
	struct check_op {
	const xmlNode *rsc_op; // History entry XML
	pcmk_resource_t *rsc; // Known resource corresponding to history entry
	pcmk_node_t *node; // Known node corresponding to history entry
	enum pcmk__check_parameters check_type; // What needs checking
	};

	void
	pe__add_param_check(const xmlNode rsc_op, pcmk_resource_t rsc,
	pcmk_node_t *node, enum pcmk__check_parameters flag,
	pcmk_scheduler_t *scheduler)
	{
	struct check_op *check_op = NULL;

	CRM_CHECK(scheduler && rsc_op && rsc && node, return);

	check_op = calloc(1, sizeof(struct check_op));
	CRM_ASSERT(check_op != NULL);

	crm_trace("Deferring checks of %s until after allocation", ID(rsc_op));
	check_op->rsc_op = rsc_op;
	check_op->rsc = rsc;
	check_op->node = node;
	check_op->check_type = flag;
	scheduler->param_check = g_list_prepend(scheduler->param_check, check_op);
	}

	/*!
	* \internal
	* \brief Call a function for each action to be checked for addr substitution
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] cb Function to be called
	*/
	void
	pe__foreach_param_check(pcmk_scheduler_t *scheduler,
	void (cb)(pcmk_resource_t, pcmk_node_t*,
	const xmlNode*, enum pcmk__check_parameters))
	{
	CRM_CHECK(scheduler && cb, return);

	for (GList *item = scheduler->param_check;
	item != NULL; item = item->next) {
	struct check_op *check_op = item->data;

	cb(check_op->rsc, check_op->node, check_op->rsc_op,
	check_op->check_type);
	}
	}

	void
	pe__free_param_checks(pcmk_scheduler_t *scheduler)
	{
	if (scheduler && scheduler->param_check) {
	g_list_free_full(scheduler->param_check, free);
	scheduler->param_check = NULL;
	}
	}
	diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c
	index 402734963a..9bc5f70c3e 100644
	--- a/lib/pengine/rules.c
	+++ b/lib/pengine/rules.c
	@@ -1,1345 +1,1345 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <glib.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/rules_internal.h>
	#include <crm/pengine/internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <ctype.h>

	CRM_TRACE_INIT_DATA(pe_rules);

	/*!
	* \brief Evaluate any rules contained by given XML element
	*
	* \param[in,out] xml XML element to check for rules
	* \param[in] node_hash Node attributes to use to evaluate expressions
	* \param[in] now Time to use when evaluating expressions
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if no rules, or any of rules present is in effect, else FALSE
	*/
	gboolean
	pe_evaluate_rules(xmlNode ruleset, GHashTable node_hash, crm_time_t *now,
	crm_time_t *next_change)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_rules(ruleset, &rule_data, next_change);
	}

	gboolean
	pe_test_rule(xmlNode rule, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, crm_time_t next_change,
	pe_match_data_t *match_data)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = role,
	.now = now,
	.match_data = match_data,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_expr(rule, &rule_data, next_change);
	}

	/*!
	* \brief Evaluate one rule subelement (pass/fail)
	*
	* A rule element may contain another rule, a node attribute expression, or a
	* date expression. Given any one of those, evaluate it and return whether it
	* passed.
	*
	* \param[in,out] expr Rule subelement XML
	* \param[in] node_hash Node attributes to use when evaluating expression
	* \param[in] role Resource role to use when evaluating expression
	* \param[in] now Time to use when evaluating expression
	* \param[out] next_change If not NULL, set to when evaluation will change
	* \param[in] match_data If not NULL, resource back-references and params
	*
	* \return TRUE if expression is in effect under given conditions, else FALSE
	*/
	gboolean
	pe_test_expression(xmlNode expr, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, crm_time_t next_change,
	pe_match_data_t *match_data)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = role,
	.now = now,
	.match_data = match_data,
	.rsc_data = NULL,
	.op_data = NULL
	};

	return pe_eval_subexpr(expr, &rule_data, next_change);
	}

	enum expression_type
	find_expression_type(xmlNode * expr)
	{
	const char *attr = NULL;

	attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);

	if (pcmk__xe_is(expr, PCMK_XE_DATE_EXPRESSION)) {
	return pcmk__subexpr_datetime;

	} else if (pcmk__xe_is(expr, PCMK_XE_RSC_EXPRESSION)) {
	return pcmk__subexpr_resource;

	} else if (pcmk__xe_is(expr, PCMK_XE_OP_EXPRESSION)) {
	return pcmk__subexpr_operation;

	} else if (pcmk__xe_is(expr, XML_TAG_RULE)) {
	return pcmk__subexpr_rule;

	} else if (!pcmk__xe_is(expr, XML_TAG_EXPRESSION)) {
	return pcmk__subexpr_unknown;

	} else if (pcmk__str_any_of(attr, CRM_ATTR_UNAME, CRM_ATTR_KIND, CRM_ATTR_ID, NULL)) {
	return pcmk__subexpr_location;

	} else if (pcmk__str_eq(attr, CRM_ATTR_ROLE, pcmk__str_none)) {
	return pcmk__subexpr_role;
	}

	return pcmk__subexpr_attribute;
	}

	/* As per the nethack rules:
	*
	* moon period = 29.53058 days ~= 30, year = 365.2422 days
	* days moon phase advances on first day of year compared to preceding year
	* = 365.2422 - 12*29.53058 ~= 11
	* years in Metonic cycle (time until same phases fall on the same days of
	* the month) = 18.6 ~= 19
	* moon phase on first day of year (epact) ~= (11*(year%19) + 29) % 30
	* (29 as initial condition)
	* current phase in days = first day phase + days elapsed in year
	* 6 moons ~= 177 days
	* 177 ~= 8 reported phases * 22
	* + 11/22 for rounding
	*
	* 0-7, with 0: new, 4: full
	*/

	static int
	phase_of_the_moon(const crm_time_t *now)
	{
	uint32_t epact, diy, goldn;
	uint32_t y;

	crm_time_get_ordinal(now, &y, &diy);

	goldn = (y % 19) + 1;
	epact = (11 * goldn + 18) % 30;
	if ((epact == 25 && goldn > 11) \|\| epact == 24)
	epact++;

	return ((((((diy + epact) * 6) + 11) % 177) / 22) & 7);
	}

	static int
	check_one(const xmlNode cron_spec, const char xml_field, uint32_t time_field)
	{
	int rc = pcmk_rc_undetermined;
	const char *value = crm_element_value(cron_spec, xml_field);
	long long low, high;

	if (value == NULL) {
	/* Return pe_date_result_undetermined if the field is missing. */
	goto bail;
	}

	if (pcmk__parse_ll_range(value, &low, &high) != pcmk_rc_ok) {
	goto bail;
	} else if (low == high) {
	/* A single number was given, not a range. */
	if (time_field < low) {
	rc = pcmk_rc_before_range;
	} else if (time_field > high) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	}
	} else if (low != -1 && high != -1) {
	/* This is a range with both bounds. */
	if (time_field < low) {
	rc = pcmk_rc_before_range;
	} else if (time_field > high) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	}
	} else if (low == -1) {
	/* This is a range with no starting value. */
	rc = time_field <= high ? pcmk_rc_within_range : pcmk_rc_after_range;
	} else if (high == -1) {
	/* This is a range with no ending value. */
	rc = time_field >= low ? pcmk_rc_within_range : pcmk_rc_before_range;
	}

	bail:
	if (rc == pcmk_rc_within_range) {
	crm_debug("Condition '%s' in %s: passed", value, xml_field);
	} else {
	crm_debug("Condition '%s' in %s: failed", value, xml_field);
	}

	return rc;
	}

	static gboolean
	check_passes(int rc) {
	/* _within_range is obvious. _undetermined is a pass because
	* this is the return value if a field is not given. In this
	* case, we just want to ignore it and check other fields to
	* see if they place some restriction on what can pass.
	*/
	return rc == pcmk_rc_within_range \|\| rc == pcmk_rc_undetermined;
	}

	#define CHECK_ONE(spec, name, var) do { \
	int subpart_rc = check_one(spec, name, var); \
	if (check_passes(subpart_rc) == FALSE) { \
	return subpart_rc; \
	} \
	} while (0)

	int
	pe_cron_range_satisfied(const crm_time_t now, const xmlNode cron_spec)
	{
	uint32_t h, m, s, y, d, w;

	CRM_CHECK(now != NULL, return pcmk_rc_op_unsatisfied);

	crm_time_get_gregorian(now, &y, &m, &d);
	CHECK_ONE(cron_spec, "years", y);
	CHECK_ONE(cron_spec, "months", m);
	CHECK_ONE(cron_spec, "monthdays", d);

	crm_time_get_timeofday(now, &h, &m, &s);
	CHECK_ONE(cron_spec, "hours", h);
	CHECK_ONE(cron_spec, "minutes", m);
	CHECK_ONE(cron_spec, "seconds", s);

	crm_time_get_ordinal(now, &y, &d);
	CHECK_ONE(cron_spec, "yeardays", d);

	crm_time_get_isoweek(now, &y, &w, &d);
	CHECK_ONE(cron_spec, "weekyears", y);
	CHECK_ONE(cron_spec, "weeks", w);
	CHECK_ONE(cron_spec, "weekdays", d);

	CHECK_ONE(cron_spec, "moon", phase_of_the_moon(now));
	if (crm_element_value(cron_spec, "moon") != NULL) {
	pcmk__config_warn("Support for 'moon' in date_spec elements "
	"(such as %s) is deprecated and will be removed "
	"in a future release of Pacemaker", ID(cron_spec));
	}

	/* If we get here, either no fields were specified (which is success), or all
	* the fields that were specified had their conditions met (which is also a
	* success). Thus, the result is success.
	*/
	return pcmk_rc_ok;
	}

	static void
	update_field(crm_time_t t, const xmlNode xml, const char *attr,
	void (time_fn)(crm_time_t , int))
	{
	long long value;

	if ((pcmk__scan_ll(crm_element_value(xml, attr), &value, 0LL) == pcmk_rc_ok)
	&& (value != 0LL) && (value >= INT_MIN) && (value <= INT_MAX)) {
	time_fn(t, (int) value);
	}
	}

	static crm_time_t *
	parse_xml_duration(const crm_time_t start, const xmlNode duration_spec)
	{
	crm_time_t *end = pcmk_copy_time(start);

	update_field(end, duration_spec, "years", crm_time_add_years);
	update_field(end, duration_spec, "months", crm_time_add_months);
	update_field(end, duration_spec, "weeks", crm_time_add_weeks);
	update_field(end, duration_spec, "days", crm_time_add_days);
	update_field(end, duration_spec, "hours", crm_time_add_hours);
	update_field(end, duration_spec, "minutes", crm_time_add_minutes);
	update_field(end, duration_spec, "seconds", crm_time_add_seconds);

	return end;
	}

	// Set next_change to t if t is earlier
	static void
	crm_time_set_if_earlier(crm_time_t next_change, crm_time_t t)
	{
	if ((next_change != NULL) && (t != NULL)) {
	if (!crm_time_is_defined(next_change)
	\|\| (crm_time_compare(t, next_change) < 0)) {
	crm_time_set(next_change, t);
	}
	}
	}

	// Information about a block of nvpair elements
	typedef struct sorted_set_s {
	int score; // This block's score for sorting
	const char *name; // This block's ID
	const char *special_name; // ID that should sort first
	xmlNode *attr_set; // This block
	gboolean overwrite; // Whether existing values will be overwritten
	} sorted_set_t;

	static gint
	sort_pairs(gconstpointer a, gconstpointer b)
	{
	const sorted_set_t *pair_a = a;
	const sorted_set_t *pair_b = b;

	if (a == NULL && b == NULL) {
	return 0;
	} else if (a == NULL) {
	return 1;
	} else if (b == NULL) {
	return -1;
	}

	if (pcmk__str_eq(pair_a->name, pair_a->special_name, pcmk__str_casei)) {
	return -1;

	} else if (pcmk__str_eq(pair_b->name, pair_a->special_name, pcmk__str_casei)) {
	return 1;
	}

	/* If we're overwriting values, we want lowest score first, so the highest
	* score is processed last; if we're not overwriting values, we want highest
	* score first, so nothing else overwrites it.
	*/
	if (pair_a->score < pair_b->score) {
	return pair_a->overwrite? -1 : 1;
	} else if (pair_a->score > pair_b->score) {
	return pair_a->overwrite? 1 : -1;
	}
	return 0;
	}

	static void
	populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlNode * top)
	{
	const char *name = NULL;
	const char *value = NULL;
	const char *old_value = NULL;
	xmlNode *list = nvpair_list;
	xmlNode *an_attr = NULL;

	if (pcmk__xe_is(list->children, XML_TAG_ATTRS)) {
	list = list->children;
	}

	for (an_attr = pcmk__xe_first_child(list); an_attr != NULL;
	an_attr = pcmk__xe_next(an_attr)) {

	if (pcmk__str_eq((const char *)an_attr->name, XML_CIB_TAG_NVPAIR, pcmk__str_none)) {
	xmlNode *ref_nvpair = expand_idref(an_attr, top);

	name = crm_element_value(an_attr, PCMK_XA_NAME);
	if ((name == NULL) && (ref_nvpair != NULL)) {
	name = crm_element_value(ref_nvpair, PCMK_XA_NAME);
	}

	value = crm_element_value(an_attr, PCMK_XA_VALUE);
	if ((value == NULL) && (ref_nvpair != NULL)) {
	value = crm_element_value(ref_nvpair, PCMK_XA_VALUE);
	}

	if (name == NULL \|\| value == NULL) {
	continue;
	}

	old_value = g_hash_table_lookup(hash, name);

	if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
	if (old_value) {
	crm_trace("Letting %s default (removing explicit value \"%s\")",
	name, value);
	g_hash_table_remove(hash, name);
	}
	continue;

	} else if (old_value == NULL) {
	crm_trace("Setting %s=\"%s\"", name, value);
	g_hash_table_insert(hash, strdup(name), strdup(value));

	} else if (overwrite) {
	crm_trace("Setting %s=\"%s\" (overwriting old value \"%s\")",
	name, value, old_value);
	g_hash_table_replace(hash, strdup(name), strdup(value));
	}
	}
	}
	}

	typedef struct unpack_data_s {
	gboolean overwrite;
	void *hash;
	crm_time_t *next_change;
	const pe_rule_eval_data_t *rule_data;
	xmlNode *top;
	} unpack_data_t;

	static void
	unpack_attr_set(gpointer data, gpointer user_data)
	{
	sorted_set_t *pair = data;
	unpack_data_t *unpack_data = user_data;

	if (!pe_eval_rules(pair->attr_set, unpack_data->rule_data,
	unpack_data->next_change)) {
	return;
	}

	crm_trace("Adding attributes from %s (score %d) %s overwrite",
	pair->name, pair->score,
	(unpack_data->overwrite? "with" : "without"));
	populate_hash(pair->attr_set, unpack_data->hash, unpack_data->overwrite, unpack_data->top);
	}

	/*!
	* \internal
	* \brief Create a sorted list of nvpair blocks
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only get blocks of this element
	* \param[in] always_first If not NULL, sort block with this ID as first
	*
	* \return List of sorted_set_t entries for nvpair blocks
	*/
	static GList *
	make_pairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	const char *always_first, gboolean overwrite)
	{
	GList *unsorted = NULL;

	if (xml_obj == NULL) {
	return NULL;
	}
	for (xmlNode *attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (pcmk__str_eq(set_name, (const char *) attr_set->name,
	pcmk__str_null_matches)) {
	const char *score = NULL;
	sorted_set_t *pair = NULL;
	xmlNode *expanded_attr_set = expand_idref(attr_set, top);

	if (expanded_attr_set == NULL) {
	continue; // Not possible with schema validation enabled
	}

	pair = calloc(1, sizeof(sorted_set_t));
	pair->name = ID(expanded_attr_set);
	pair->special_name = always_first;
	pair->attr_set = expanded_attr_set;
	pair->overwrite = overwrite;

	score = crm_element_value(expanded_attr_set, XML_RULE_ATTR_SCORE);
	pair->score = char2score(score);

	unsorted = g_list_prepend(unsorted, pair);
	}
	}
	return g_list_sort(unsorted, sort_pairs);
	}

	/*!
	* \brief Extract nvpair blocks contained by an XML element into a hash table
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name If not NULL, only use blocks of this element
	* \param[in] rule_data Matching parameters to use when unpacking
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[out] next_change If not NULL, set to when evaluation will change
	*/
	void
	pe_eval_nvpairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	const pe_rule_eval_data_t rule_data, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t *next_change)
	{
	GList *pairs = make_pairs(top, xml_obj, set_name, always_first, overwrite);

	if (pairs) {
	unpack_data_t data = {
	.hash = hash,
	.overwrite = overwrite,
	.next_change = next_change,
	.top = top,
	.rule_data = rule_data
	};

	g_list_foreach(pairs, unpack_attr_set, &data);
	g_list_free_full(pairs, free);
	}
	}

	/*!
	* \brief Extract nvpair blocks contained by an XML element into a hash table
	*
	* \param[in,out] top XML document root (used to expand id-ref's)
	* \param[in] xml_obj XML element containing blocks of nvpair elements
	* \param[in] set_name Element name to identify nvpair blocks
	* \param[in] node_hash Node attributes to use when evaluating rules
	* \param[out] hash Where to store extracted name/value pairs
	* \param[in] always_first If not NULL, process block with this ID first
	* \param[in] overwrite Whether to replace existing values with same name
	* \param[in] now Time to use when evaluating rules
	* \param[out] next_change If not NULL, set to when evaluation will change
	*/
	void
	pe_unpack_nvpairs(xmlNode top, const xmlNode xml_obj, const char *set_name,
	GHashTable node_hash, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t now, crm_time_t next_change)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash,
	always_first, overwrite, next_change);
	}

	/*!
	* \brief Expand any regular expression submatches (%0-%9) in a string
	*
	* \param[in] string String possibly containing submatch variables
	* \param[in] match_data If not NULL, regular expression matches
	*
	* \return Newly allocated string identical to \p string with submatches
	* expanded, or NULL if there were no matches
	*/
	char *
	pe_expand_re_matches(const char string, const pe_re_match_data_t match_data)
	{
	size_t len = 0;
	int i;
	const char p, last_match_index;
	char p_dst, result = NULL;

	if (pcmk__str_empty(string) \|\| !match_data) {
	return NULL;
	}

	p = last_match_index = string;

	while (*p) {
	if (p == '%' && (p + 1) && isdigit(*(p + 1))) {
	i = *(p + 1) - '0';
	if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
	match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
	len += p - last_match_index + (match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so);
	last_match_index = p + 2;
	}
	p++;
	}
	p++;
	}
	len += p - last_match_index + 1;

	/* FIXME: Excessive? */
	if (len - 1 <= 0) {
	return NULL;
	}

	p_dst = result = calloc(1, len);
	p = string;

	while (*p) {
	if (p == '%' && (p + 1) && isdigit(*(p + 1))) {
	i = *(p + 1) - '0';
	if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
	match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
	/* rm_eo can be equal to rm_so, but then there is nothing to do */
	int match_len = match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so;
	memcpy(p_dst, match_data->string + match_data->pmatch[i].rm_so, match_len);
	p_dst += match_len;
	}
	p++;
	} else {
	(p_dst) = (p);
	p_dst++;
	}
	p++;
	}

	return result;
	}

	/*!
	* \brief Evaluate rules
	*
	* \param[in,out] ruleset XML possibly containing rule sub-elements
	* \param[in] rule_data
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if there are no rules or
	*/
	gboolean
	pe_eval_rules(xmlNode ruleset, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	// If there are no rules, pass by default
	gboolean ruleset_default = TRUE;

	for (xmlNode *rule = first_named_child(ruleset, XML_TAG_RULE);
	rule != NULL; rule = crm_next_same_xml(rule)) {

	ruleset_default = FALSE;
	if (pe_eval_expr(rule, rule_data, next_change)) {
	/* Only the deprecated "lifetime" element of location constraints
	* may contain more than one rule at the top level -- the schema
	* limits a block of nvpairs to a single top-level rule. So, this
	* effectively means that a lifetime is active if any rule it
	* contains is active.
	*/
	return TRUE;
	}
	}

	return ruleset_default;
	}

	/*!
	* \brief Evaluate all of a rule's expressions
	*
	* \param[in,out] rule XML containing a rule definition or its id-ref
	* \param[in] rule_data Matching parameters to check against rule
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if \p rule_data passes \p rule, otherwise FALSE
	*/
	gboolean
	pe_eval_expr(xmlNode rule, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	xmlNode *expr = NULL;
	gboolean test = TRUE;
	gboolean empty = TRUE;
	gboolean passed = TRUE;
	gboolean do_and = TRUE;
	const char *value = NULL;

	rule = expand_idref(rule, NULL);
	if (rule == NULL) {
	return FALSE; // Not possible with schema validation enabled
	}

	value = crm_element_value(rule, XML_RULE_ATTR_BOOLEAN_OP);
	if (pcmk__str_eq(value, "or", pcmk__str_casei)) {
	do_and = FALSE;
	passed = FALSE;
	}

	crm_trace("Testing rule %s", ID(rule));
	for (expr = pcmk__xe_first_child(rule); expr != NULL;
	expr = pcmk__xe_next(expr)) {

	test = pe_eval_subexpr(expr, rule_data, next_change);
	empty = FALSE;

	if (test && do_and == FALSE) {
	crm_trace("Expression %s/%s passed", ID(rule), ID(expr));
	return TRUE;

	} else if (test == FALSE && do_and) {
	crm_trace("Expression %s/%s failed", ID(rule), ID(expr));
	return FALSE;
	}
	}

	if (empty) {
	pcmk__config_err("Ignoring rule %s because it contains no expressions",
	ID(rule));
	}

	crm_trace("Rule %s %s", ID(rule), passed ? "passed" : "failed");
	return passed;
	}

	/*!
	* \brief Evaluate a single rule expression, including any subexpressions
	*
	* \param[in,out] expr XML containing a rule expression
	* \param[in] rule_data Matching parameters to check against expression
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return TRUE if \p rule_data passes \p expr, otherwise FALSE
	*/
	gboolean
	pe_eval_subexpr(xmlNode expr, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	gboolean accept = FALSE;
	const char *uname = NULL;

	switch (find_expression_type(expr)) {
	case pcmk__subexpr_rule:
	accept = pe_eval_expr(expr, rule_data, next_change);
	break;
	case pcmk__subexpr_attribute:
	case pcmk__subexpr_location:
	/* these expressions can never succeed if there is
	* no node to compare with
	*/
	if (rule_data->node_hash != NULL) {
	accept = pe__eval_attr_expr(expr, rule_data);
	}
	break;

	case pcmk__subexpr_datetime:
	switch (pe__eval_date_expr(expr, rule_data, next_change)) {
	case pcmk_rc_within_range:
	case pcmk_rc_ok:
	accept = TRUE;
	break;

	default:
	accept = FALSE;
	break;
	}
	break;

	case pcmk__subexpr_role:
	accept = pe__eval_role_expr(expr, rule_data);
	break;

	case pcmk__subexpr_resource:
	accept = pe__eval_rsc_expr(expr, rule_data);
	break;

	case pcmk__subexpr_operation:
	accept = pe__eval_op_expr(expr, rule_data);
	break;

	default:
	CRM_CHECK(FALSE /* bad type */ , return FALSE);
	accept = FALSE;
	}
	if (rule_data->node_hash) {
	uname = g_hash_table_lookup(rule_data->node_hash, CRM_ATTR_UNAME);
	}

	crm_trace("Expression %s %s on %s",
	ID(expr), accept ? "passed" : "failed", uname ? uname : "all nodes");
	return accept;
	}

	/*!
	* \internal
	* \brief Compare two values in a rule's node attribute expression
	*
	* \param[in] l_val Value on left-hand side of comparison
	* \param[in] r_val Value on right-hand side of comparison
	* \param[in] type How to interpret the values (allowed values:
	* \c "string", \c "integer", \c "number",
	* \c "version", \c NULL)
	* \param[in] op Type of comparison
	*
	* \return -1 if <tt>(l_val < r_val)</tt>,
	* 0 if <tt>(l_val == r_val)</tt>,
	* 1 if <tt>(l_val > r_val)</tt>
	*/
	static int
	compare_attr_expr_vals(const char l_val, const char r_val, const char *type,
	const char *op)
	{
	int cmp = 0;

	if (l_val != NULL && r_val != NULL) {
	if (type == NULL) {
	if (pcmk__strcase_any_of(op, "lt", "lte", "gt", "gte", NULL)) {
	if (pcmk__char_in_any_str('.', l_val, r_val, NULL)) {
	type = "number";
	} else {
	type = "integer";
	}

	} else {
	type = "string";
	}
	crm_trace("Defaulting to %s based comparison for '%s' op", type, op);
	}

	if (pcmk__str_eq(type, "string", pcmk__str_casei)) {
	cmp = strcasecmp(l_val, r_val);

	} else if (pcmk__str_eq(type, "integer", pcmk__str_casei)) {
	long long l_val_num;
	int rc1 = pcmk__scan_ll(l_val, &l_val_num, 0LL);

	long long r_val_num;
	int rc2 = pcmk__scan_ll(r_val, &r_val_num, 0LL);

	if ((rc1 == pcmk_rc_ok) && (rc2 == pcmk_rc_ok)) {
	if (l_val_num < r_val_num) {
	cmp = -1;
	} else if (l_val_num > r_val_num) {
	cmp = 1;
	} else {
	cmp = 0;
	}

	} else {
	crm_debug("Integer parse error. Comparing %s and %s as strings",
	l_val, r_val);
	cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
	}

	} else if (pcmk__str_eq(type, "number", pcmk__str_casei)) {
	double l_val_num;
	double r_val_num;

	int rc1 = pcmk__scan_double(l_val, &l_val_num, NULL, NULL);
	int rc2 = pcmk__scan_double(r_val, &r_val_num, NULL, NULL);

	if (rc1 == pcmk_rc_ok && rc2 == pcmk_rc_ok) {
	if (l_val_num < r_val_num) {
	cmp = -1;
	} else if (l_val_num > r_val_num) {
	cmp = 1;
	} else {
	cmp = 0;
	}

	} else {
	crm_debug("Floating-point parse error. Comparing %s and %s as "
	"strings", l_val, r_val);
	cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
	}

	} else if (pcmk__str_eq(type, "version", pcmk__str_casei)) {
	cmp = compare_version(l_val, r_val);

	}

	} else if (l_val == NULL && r_val == NULL) {
	cmp = 0;
	} else if (r_val == NULL) {
	cmp = 1;
	} else { // l_val == NULL && r_val != NULL
	cmp = -1;
	}

	return cmp;
	}

	/*!
	* \internal
	* \brief Check whether an attribute expression evaluates to \c true
	*
	* \param[in] l_val Value on left-hand side of comparison
	* \param[in] r_val Value on right-hand side of comparison
	* \param[in] type How to interpret the values (allowed values:
	* \c "string", \c "integer", \c "number",
	* \c "version", \c NULL)
	* \param[in] op Type of comparison.
	*
	* \return \c true if expression evaluates to \c true, \c false
	* otherwise
	*/
	static bool
	accept_attr_expr(const char l_val, const char r_val, const char *type,
	const char *op)
	{
	int cmp;

	if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
	return (l_val != NULL);

	} else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
	return (l_val == NULL);

	}

	cmp = compare_attr_expr_vals(l_val, r_val, type, op);

	if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
	return (cmp == 0);

	} else if (pcmk__str_eq(op, "ne", pcmk__str_casei)) {
	return (cmp != 0);

	} else if (l_val == NULL \|\| r_val == NULL) {
	// The comparison is meaningless from this point on
	return false;

	} else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
	return (cmp < 0);

	} else if (pcmk__str_eq(op, "lte", pcmk__str_casei)) {
	return (cmp <= 0);

	} else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
	return (cmp > 0);

	} else if (pcmk__str_eq(op, "gte", pcmk__str_casei)) {
	return (cmp >= 0);
	}

	return false; // Should never reach this point
	}

	/*!
	* \internal
	* \brief Get correct value according to value-source
	*
	* \param[in] value value given in rule expression
	* \param[in] value_source value-source given in rule expressions
	* \param[in] match_data If not NULL, resource back-references and params
	*/
	static const char *
	expand_value_source(const char value, const char value_source,
	const pe_match_data_t *match_data)
	{
	GHashTable *table = NULL;

	if (pcmk__str_empty(value)) {
	return NULL; // value_source is irrelevant

	} else if (pcmk__str_eq(value_source, "param", pcmk__str_casei)) {
	table = match_data->params;

	} else if (pcmk__str_eq(value_source, "meta", pcmk__str_casei)) {
	table = match_data->meta;

	} else { // literal
	return value;
	}

	if (table == NULL) {
	return NULL;
	}
	return (const char *) g_hash_table_lookup(table, value);
	}

	/*!
	* \internal
	* \brief Evaluate a node attribute expression based on #uname, #id, #kind,
	* or a generic node attribute
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data The match_data and node_hash members are used
	*
	* \return TRUE if rule_data satisfies the expression, FALSE otherwise
	*/
	gboolean
	pe__eval_attr_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	gboolean attr_allocated = FALSE;
	const char *h_val = NULL;

	const char *op = NULL;
	const char *type = NULL;
	const char *attr = NULL;
	const char *value = NULL;
	const char *value_source = NULL;

	attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);
	op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);
	value = crm_element_value(expr, PCMK_XA_VALUE);
	type = crm_element_value(expr, PCMK_XA_TYPE);
	value_source = crm_element_value(expr, XML_EXPR_ATTR_VALUE_SOURCE);

	if (attr == NULL) {
	pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_ATTRIBUTE
	" not specified", pcmk__s(ID(expr), "without ID"));
	return FALSE;
	} else if (op == NULL) {
	pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_OPERATION
	" not specified", pcmk__s(ID(expr), "without ID"));
	return FALSE;
	}

	if (rule_data->match_data != NULL) {
	// Expand any regular expression submatches (%0-%9) in attribute name
	if (rule_data->match_data->re != NULL) {
	char *resolved_attr = pe_expand_re_matches(attr, rule_data->match_data->re);

	if (resolved_attr != NULL) {
	attr = (const char *) resolved_attr;
	attr_allocated = TRUE;
	}
	}

	// Get value appropriate to value-source
	value = expand_value_source(value, value_source, rule_data->match_data);
	}

	if (rule_data->node_hash != NULL) {
	h_val = (const char *)g_hash_table_lookup(rule_data->node_hash, attr);
	}

	if (attr_allocated) {
	free((char *)attr);
	attr = NULL;
	}

	return accept_attr_expr(h_val, value, type, op);
	}

	/*!
	* \internal
	* \brief Evaluate a date_expression
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the now member is used
	* \param[out] next_change If not NULL, set to when evaluation will change
	*
	* \return Standard Pacemaker return code
	*/
	int
	pe__eval_date_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data,
	crm_time_t *next_change)
	{
	crm_time_t *start = NULL;
	crm_time_t *end = NULL;
	const char *value = NULL;
	const char *op = crm_element_value(expr, "operation");

	xmlNode *duration_spec = NULL;
	xmlNode *date_spec = NULL;

	// "undetermined" will also be returned for parsing errors
	int rc = pcmk_rc_undetermined;

	crm_trace("Testing expression: %s", ID(expr));

	duration_spec = first_named_child(expr, "duration");
	date_spec = first_named_child(expr, "date_spec");

	value = crm_element_value(expr, "start");
	if (value != NULL) {
	start = crm_time_new(value);
	}
	value = crm_element_value(expr, "end");
	if (value != NULL) {
	end = crm_time_new(value);
	}

	if (start != NULL && end == NULL && duration_spec != NULL) {
	end = parse_xml_duration(start, duration_spec);
	}

	if (pcmk__str_eq(op, "in_range", pcmk__str_null_matches \| pcmk__str_casei)) {
	if ((start == NULL) && (end == NULL)) {
	// in_range requires at least one of start or end
	} else if ((start != NULL) && (crm_time_compare(rule_data->now, start) < 0)) {
	rc = pcmk_rc_before_range;
	crm_time_set_if_earlier(next_change, start);
	} else if ((end != NULL) && (crm_time_compare(rule_data->now, end) > 0)) {
	rc = pcmk_rc_after_range;
	} else {
	rc = pcmk_rc_within_range;
	if (end && next_change) {
	// Evaluation doesn't change until second after end
	crm_time_add_seconds(end, 1);
	crm_time_set_if_earlier(next_change, end);
	}
	}

	} else if (pcmk__str_eq(op, "date_spec", pcmk__str_casei)) {
	rc = pe_cron_range_satisfied(rule_data->now, date_spec);
	// @TODO set next_change appropriately

	} else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
	if (start == NULL) {
	// gt requires start
	} else if (crm_time_compare(rule_data->now, start) > 0) {
	rc = pcmk_rc_within_range;
	} else {
	rc = pcmk_rc_before_range;

	// Evaluation doesn't change until second after start
	crm_time_add_seconds(start, 1);
	crm_time_set_if_earlier(next_change, start);
	}

	} else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
	if (end == NULL) {
	// lt requires end
	} else if (crm_time_compare(rule_data->now, end) < 0) {
	rc = pcmk_rc_within_range;
	crm_time_set_if_earlier(next_change, end);
	} else {
	rc = pcmk_rc_after_range;
	}
	}

	crm_time_free(start);
	crm_time_free(end);
	return rc;
	}

	gboolean
	pe__eval_op_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *name = crm_element_value(expr, PCMK_XA_NAME);
	const char *interval_s = crm_element_value(expr, XML_LRM_ATTR_INTERVAL);
	guint interval_ms = 0U;

	crm_trace("Testing op_defaults expression: %s", ID(expr));

	if (rule_data->op_data == NULL) {
	crm_trace("No operations data provided");
	return FALSE;
	}

	if (pcmk_parse_interval_spec(interval_s, &interval_ms) != pcmk_rc_ok) {
	crm_trace("Could not parse interval: %s", interval_s);
	return FALSE;
	}

	if ((interval_s != NULL) && (interval_ms != rule_data->op_data->interval)) {
	crm_trace("Interval doesn't match: %d != %d",
	interval_ms, rule_data->op_data->interval);
	return FALSE;
	}

	if (!pcmk__str_eq(name, rule_data->op_data->op_name, pcmk__str_none)) {
	crm_trace("Name doesn't match: %s != %s", name, rule_data->op_data->op_name);
	return FALSE;
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Check whether a resource role matches a rule role
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the role member is used
	*
	* \return true if role matches, otherwise false
	*/
	static bool
	role_matches(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *value = crm_element_value(expr, PCMK_XA_VALUE);
	enum rsc_role_e role = text2role(value);

	if (role == pcmk_role_unknown) {
	pcmk__config_err("Invalid role %s in rule expression", value);
	return false;
	}
	return role == rule_data->role;
	}

	/*!
	* \internal
	* \brief Evaluate a node attribute expression based on #role
	*
	* \param[in] expr XML of rule expression
	* \param[in] rule_data Only the role member is used
	*
	* \return TRUE if rule_data->role satisfies the expression, FALSE otherwise
	* \todo Drop this whole code. The #role attribute was never implemented
	* (rule_data->role is always pcmk_role_unknown), and it would be a poor
	* design anyway, since a unique promotable clone could have multiple
	* instances with different roles on a given node.
	*/
	gboolean
	pe__eval_role_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	const char *op = NULL;

	// A known role must be given to compare against
	if (rule_data->role == pcmk_role_unknown) {
	return FALSE;
	}

	op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);

	if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
	if (rule_data->role > pcmk_role_started) {
	return TRUE;
	}

	} else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
	if ((rule_data->role > pcmk_role_unknown)
	&& (rule_data->role < pcmk_role_unpromoted)) {
	return TRUE;
	}

	} else if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
	return role_matches(expr, rule_data)? TRUE : FALSE;

	} else if (pcmk__str_eq(op, "ne", pcmk__str_casei)
	// Test "ne" only with promotable clone roles
	&& (rule_data->role >= pcmk_role_unpromoted)) {
	return role_matches(expr, rule_data)? FALSE : TRUE;

	} else {
	pcmk__config_err("Operation '%s' is not valid with " CRM_ATTR_ROLE
	" comparisons", op);
	}
	return FALSE;
	}

	gboolean
	pe__eval_rsc_expr(const xmlNode expr, const pe_rule_eval_data_t rule_data)
	{
	- const char *class = crm_element_value(expr, XML_AGENT_ATTR_CLASS);
	+ const char *class = crm_element_value(expr, PCMK_XA_CLASS);
	const char *provider = crm_element_value(expr, PCMK_XA_PROVIDER);
	const char *type = crm_element_value(expr, PCMK_XA_TYPE);

	crm_trace("Testing rsc_defaults expression: %s", ID(expr));

	if (rule_data->rsc_data == NULL) {
	crm_trace("No resource data provided");
	return FALSE;
	}

	if (class != NULL &&
	!pcmk__str_eq(class, rule_data->rsc_data->standard, pcmk__str_none)) {
	crm_trace("Class doesn't match: %s != %s", class, rule_data->rsc_data->standard);
	return FALSE;
	}

	if ((provider == NULL && rule_data->rsc_data->provider != NULL) \|\|
	(provider != NULL && rule_data->rsc_data->provider == NULL) \|\|
	!pcmk__str_eq(provider, rule_data->rsc_data->provider, pcmk__str_none)) {
	crm_trace("Provider doesn't match: %s != %s", provider, rule_data->rsc_data->provider);
	return FALSE;
	}

	if (type != NULL &&
	!pcmk__str_eq(type, rule_data->rsc_data->agent, pcmk__str_none)) {
	crm_trace("Agent doesn't match: %s != %s", type, rule_data->rsc_data->agent);
	return FALSE;
	}

	return TRUE;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/pengine/rules_compat.h>

	gboolean
	test_ruleset(xmlNode ruleset, GHashTable node_hash, crm_time_t *now)
	{
	return pe_evaluate_rules(ruleset, node_hash, now, NULL);
	}

	gboolean
	test_rule(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
	{
	return pe_test_rule(rule, node_hash, role, now, NULL, NULL);
	}

	gboolean
	pe_test_rule_re(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
	{
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = NULL,
	.meta = NULL,
	};
	return pe_test_rule(rule, node_hash, role, now, NULL, &match_data);
	}

	gboolean
	pe_test_rule_full(xmlNode rule, GHashTable node_hash, enum rsc_role_e role,
	crm_time_t now, pe_match_data_t match_data)
	{
	return pe_test_rule(rule, node_hash, role, now, NULL, match_data);
	}

	gboolean
	test_expression(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
	{
	return pe_test_expression(expr, node_hash, role, now, NULL, NULL);
	}

	gboolean
	pe_test_expression_re(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
	{
	pe_match_data_t match_data = {
	.re = re_match_data,
	.params = NULL,
	.meta = NULL,
	};
	return pe_test_expression(expr, node_hash, role, now, NULL, &match_data);
	}

	gboolean
	pe_test_expression_full(xmlNode expr, GHashTable node_hash,
	enum rsc_role_e role, crm_time_t *now,
	pe_match_data_t *match_data)
	{
	return pe_test_expression(expr, node_hash, role, now, NULL, match_data);
	}

	void
	unpack_instance_attributes(xmlNode top, xmlNode xml_obj, const char *set_name,
	GHashTable node_hash, GHashTable hash,
	const char *always_first, gboolean overwrite,
	crm_time_t *now)
	{
	pe_rule_eval_data_t rule_data = {
	.node_hash = node_hash,
	.role = pcmk_role_unknown,
	.now = now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash, always_first,
	overwrite, NULL);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
	index a3369c6939..81c1521cbd 100644
	--- a/tools/crm_resource_print.c
	+++ b/tools/crm_resource_print.c
	@@ -1,822 +1,822 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm_resource.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>

	#define cons_string(x) x?x:"NA"
	static int
	print_constraint(xmlNode xml_obj, void userdata)
	{
	pcmk_scheduler_t scheduler = (pcmk_scheduler_t ) userdata;
	pcmk__output_t *out = scheduler->priv;
	xmlNode *lifetime = NULL;
	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);

	if (id == NULL) {
	return pcmk_rc_ok;
	}

	// @COMPAT lifetime is deprecated
	lifetime = first_named_child(xml_obj, "lifetime");
	if (pe_evaluate_rules(lifetime, NULL, scheduler->now, NULL) == FALSE) {
	return pcmk_rc_ok;
	}

	if (!pcmk__xe_is(xml_obj, XML_CONS_TAG_RSC_DEPEND)) {
	return pcmk_rc_ok;
	}

	out->info(out, "Constraint %s %s %s %s %s %s %s",
	xml_obj->name,
	cons_string(crm_element_value(xml_obj, PCMK_XA_ID)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
	cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
	cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));

	return pcmk_rc_ok;
	}

	void
	cli_resource_print_cts_constraints(pcmk_scheduler_t *scheduler)
	{
	pcmk__xe_foreach_child(pcmk_find_cib_element(scheduler->input,
	XML_CIB_TAG_CONSTRAINTS),
	NULL, print_constraint, scheduler);
	}

	void
	cli_resource_print_cts(pcmk_resource_t rsc, pcmk__output_t out)
	{
	const char *host = NULL;
	bool needs_quorum = TRUE;
	const char *rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	const char *rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	- const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ const char *rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	pcmk_node_t *node = pe__current_node(rsc);

	if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	needs_quorum = FALSE;
	} else {
	// @TODO check requires in resource meta-data and rsc_defaults
	}

	if (node != NULL) {
	host = node->details->uname;
	}

	out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld %#.16llx",
	rsc->xml->name, rsc->id,
	rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
	rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
	rsc->flags);

	g_list_foreach(rsc->children, (GFunc) cli_resource_print_cts, out);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print_operations(const char rsc_id, const char host_uname,
	bool active, pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_no_output;
	GList *ops = find_operations(rsc_id, host_uname, active, scheduler);

	if (!ops) {
	return rc;
	}

	out->begin_list(out, NULL, NULL, "Resource Operations");
	rc = pcmk_rc_ok;

	for (GList *lpc = ops; lpc != NULL; lpc = lpc->next) {
	xmlNode xml_op = (xmlNode ) lpc->data;
	out->message(out, "node-and-op", scheduler, xml_op);
	}

	out->end_list(out);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	bool expanded)
	{
	pcmk__output_t *out = scheduler->priv;
	uint32_t show_opts = pcmk_show_pending;
	GList *all = NULL;

	all = g_list_prepend(all, (gpointer) "*");

	out->begin_list(out, NULL, NULL, "Resource Config");
	out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc, all, all);
	out->message(out, "resource-config", rsc, !expanded);
	out->end_list(out);

	g_list_free(all);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Attributes");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_default(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	if (status == PCMK_EXEC_DONE) {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* returned <exit-code> (<exit-description>[: <exit-reason>])
	*/
	out->info(out, "Operation %s%s%s (%s%s%s:%s) returned %d (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, (int) rc, services_ocf_exitcode_str((int) rc),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	} else {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* could not be executed (<execution-status>[: <exit-reason>])
	*/
	out->err(out,
	"Operation %s%s%s (%s%s%s:%s) could not be executed (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, pcmk_exec_status_str(status),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_xml(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action G_GNUC_UNUSED = va_arg(args, const char );
	const char name G_GNUC_UNUSED = va_arg(args, const char );
	const char class G_GNUC_UNUSED = va_arg(args, const char );
	const char provider G_GNUC_UNUSED = va_arg(args, const char );
	const char type G_GNUC_UNUSED = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	char *exit_str = pcmk__itoa(rc);
	char *status_str = pcmk__itoa(status);

	pcmk__output_create_xml_node(out, "agent-status",
	"code", exit_str,
	"message", services_ocf_exitcode_str((int) rc),
	"execution_code", status_str,
	"execution_message", pcmk_exec_status_str(status),
	"reason", exit_reason,
	NULL);

	free(exit_str);
	free(status_str);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}
	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_default(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	if (rsc_name == NULL) {
	out->list_item(out, NULL, "Overriding the cluster configuration with '%s' = '%s'",
	name, value);
	} else {
	out->list_item(out, NULL, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
	rsc_name, name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_xml(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "override",
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (rsc_name != NULL) {
	crm_xml_add(node, "rsc", rsc_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Properties");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, const char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_default(pcmk__output_t *out, va_list args) {
	int verbose = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, "overrides");

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	/* hide output for validate-all if not in verbose */
	if ((verbose == 0)
	&& pcmk__str_eq(action, PCMK_ACTION_VALIDATE_ALL, pcmk__str_casei)) {
	return pcmk_rc_ok;
	}

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	doc = string2xml(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, "command", stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_xml(pcmk__output_t *out, va_list args) {
	int verbose G_GNUC_UNUSED = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource-agent-action",
	"action", action,
	- "class", class,
	+ PCMK_XA_CLASS, class,
	PCMK_XA_TYPE, type,
	NULL);

	if (rsc_name) {
	crm_xml_add(node, "rsc", rsc_name);
	}

	crm_xml_add(node, PCMK_XA_PROVIDER, provider);

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, "overrides");

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	doc = string2xml(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, "command", stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_default(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	if (checks->flags == 0) {
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Checks");

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
	parent->id, checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	out->list_item(out, "check",
	"'%s' cannot run on unhealthy nodes due to "
	PCMK_OPT_NODE_HEALTH_STRATEGY "='%s'",
	parent->id,
	pe_pref(checks->rsc->cluster->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY));
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_xml(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "check",
	PCMK_XA_ID, parent->id,
	NULL);

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	pcmk__xe_set_bool_attr(node, "remain_stopped", true);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	pcmk__xe_set_bool_attr(node, "promotable", false);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	pcmk__xe_set_bool_attr(node, "unmanaged", true);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	crm_xml_add(node, "locked-to", checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	pcmk__xe_set_bool_attr(node, "unhealthy", true);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const char ")
	static int
	resource_search_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const char requested_name = va_arg(args, const char );

	bool printed = false;
	int rc = pcmk_rc_no_output;

	if (!out->is_quiet(out) && nodes == NULL) {
	out->err(out, "resource %s is NOT running", requested_name);
	return rc;
	}

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;

	if (!printed) {
	out->begin_list(out, NULL, NULL, "Nodes");
	printed = true;
	rc = pcmk_rc_ok;
	}

	if (out->is_quiet(out)) {
	out->list_item(out, "node", "%s", ni->node_name);
	} else {
	const char *role_text = "";

	if (ni->promoted) {
	#ifdef PCMK__COMPAT_2_0
	role_text = " " PCMK__ROLE_PROMOTED_LEGACY;
	#else
	role_text = " " PCMK__ROLE_PROMOTED;
	#endif
	}
	out->list_item(out, "node", "resource %s is running on: %s%s",
	requested_name, ni->node_name, role_text);
	}
	}

	if (printed) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const char ")
	static int
	resource_search_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const char requested_name = va_arg(args, const char );

	pcmk__output_xml_create_parent(out, "nodes",
	"resource", requested_name,
	NULL);

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;
	xmlNodePtr sub_node = pcmk__output_create_xml_text_node(out, "node", ni->node_name);

	if (ni->promoted) {
	crm_xml_add(sub_node, "state", "promoted");
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_default(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	out->begin_list(out, NULL, NULL, "Resource Reasons");

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	rsc->fns->location(rsc, &hosts, TRUE);

	if (hosts == NULL) {
	out->list_item(out, "reason", "Resource %s is not running", rsc->id);
	} else {
	out->list_item(out, "reason", "Resource %s is running", rsc->id);
	}

	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	hosts = NULL;
	}

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	} else {
	out->list_item(out, "reason", "Resource %s is not running on host %s",
	rsc->id, host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	out->list_item(out, "reason", "Resource %s is %srunning",
	rsc->id, (hosts? "" : "not "));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_xml(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	xmlNodePtr xml_node = pcmk__output_xml_create_parent(out, "reason", NULL);

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	pcmk__output_xml_create_parent(out, "resources", NULL);

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	rsc->fns->location(rsc, &hosts, TRUE);

	pcmk__output_xml_create_parent(out, "resource",
	PCMK_XA_ID, rsc->id,
	"running", pcmk__btoa(hosts != NULL),
	NULL);

	cli_resource_check(out, rsc, NULL);
	pcmk__output_xml_pop_parent(out);
	g_list_free(hosts);
	hosts = NULL;
	}

	pcmk__output_xml_pop_parent(out);

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	crm_xml_add(xml_node, "running_on", host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	pcmk__output_xml_create_parent(out, "resources", NULL);

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, "resource",
	PCMK_XA_ID, rsc->id,
	"running", "true",
	"host", host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, "resource",
	PCMK_XA_ID, rsc->id,
	"running", "false",
	"host", host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	pcmk__output_xml_pop_parent(out);
	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	crm_xml_add(xml_node, "running", pcmk__btoa(hosts != NULL));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	return pcmk_rc_ok;
	}

	static void
	add_resource_name(pcmk_resource_t rsc, pcmk__output_t out)
	{
	if (rsc->children == NULL) {
	out->list_item(out, "resource", "%s", rsc->id);
	} else {
	g_list_foreach(rsc->children, (GFunc) add_resource_name, out);
	}
	}

	PCMK__OUTPUT_ARGS("resource-names-list", "GList *")
	static int
	resource_names(pcmk__output_t *out, va_list args) {
	GList resources = va_arg(args, GList );

	if (resources == NULL) {
	out->err(out, "NO resources configured\n");
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Names");
	g_list_foreach(resources, (GFunc) add_resource_name, out);
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "agent-status", "default", agent_status_default },
	{ "agent-status", "xml", agent_status_xml },
	{ "attribute-list", "default", attribute_list_default },
	{ "attribute-list", "text", attribute_list_text },
	{ "override", "default", override_default },
	{ "override", "xml", override_xml },
	{ "property-list", "default", property_list_default },
	{ "property-list", "text", property_list_text },
	{ "resource-agent-action", "default", resource_agent_action_default },
	{ "resource-agent-action", "xml", resource_agent_action_xml },
	{ "resource-check-list", "default", resource_check_list_default },
	{ "resource-check-list", "xml", resource_check_list_xml },
	{ "resource-search-list", "default", resource_search_list_default },
	{ "resource-search-list", "xml", resource_search_list_xml },
	{ "resource-reasons-list", "default", resource_reasons_list_default },
	{ "resource-reasons-list", "xml", resource_reasons_list_xml },
	{ "resource-names-list", "default", resource_names },

	{ NULL, NULL, NULL }
	};

	void
	crm_resource_register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index 1084e9f2a9..00648dfa83 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,2254 +1,2254 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>
	#include <crm/common/ipc_attrd_internal.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/lists_internal.h>
	#include <crm/services_internal.h>

	static GList *
	build_node_info_list(const pcmk_resource_t *rsc)
	{
	GList *retval = NULL;

	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;

	for (const GList *iter2 = child->running_on;
	iter2 != NULL; iter2 = iter2->next) {

	const pcmk_node_t node = (const pcmk_node_t ) iter2->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));

	ni->node_name = node->details->uname;
	ni->promoted = pcmk_is_set(rsc->flags, pcmk_rsc_promotable) &&
	child->fns->state(child, TRUE) == pcmk_role_promoted;

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	GList *
	cli_resource_search(pcmk_resource_t rsc, const char requested_name,
	pcmk_scheduler_t *scheduler)
	{
	GList *retval = NULL;
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	if (pe_rsc_is_clone(rsc)) {
	retval = build_node_info_list(rsc);

	/* The anonymous clone children's common ID is supplied */
	} else if (pe_rsc_is_clone(parent)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& rsc->clone_name
	&& pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
	&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {

	retval = build_node_info_list(parent);

	} else if (rsc->running_on != NULL) {
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));
	ni->node_name = node->details->uname;
	ni->promoted = (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted);

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	// \return Standard Pacemaker return code
	static int
	find_resource_attr(pcmk__output_t out, cib_t the_cib, const char *attr,
	const char rsc, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, char **value)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;
	GString *xpath = NULL;
	const char *xpath_base = NULL;

	if(value) {
	*value = NULL;
	}

	if(the_cib == NULL) {
	return ENOTCONN;
	}

	xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
	if (xpath_base == NULL) {
	crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
	return ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	pcmk__g_strcat(xpath,
	xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);

	if (attr_set_type != NULL) {
	pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
	if (set_name != NULL) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
	NULL);
	}
	}

	g_string_append(xpath, "//" XML_CIB_TAG_NVPAIR "[");
	if (attr_id != NULL) {
	pcmk__g_strcat(xpath, "@" PCMK_XA_ID "=\"", attr_id, "\"", NULL);
	}

	if (attr_name != NULL) {
	if (attr_id != NULL) {
	g_string_append(xpath, " and ");
	}
	pcmk__g_strcat(xpath, "@" PCMK_XA_NAME "=\"", attr_name, "\"", NULL);
	}
	g_string_append_c(xpath, ']');

	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto done;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_search->children != NULL) {
	xmlNode *child = NULL;

	rc = ENOTUNIQ;
	out->info(out, "Multiple attributes match name=%s", attr_name);

	for (child = pcmk__xml_first_child(xml_search); child != NULL;
	child = pcmk__xml_next(child)) {
	out->info(out, " Value: %s \t(id=%s)",
	crm_element_value(child, PCMK_XA_VALUE), ID(child));
	}

	out->spacer(out);

	} else if(value) {
	pcmk__str_update(value, crm_element_value(xml_search, attr));
	}

	done:
	g_string_free(xpath, TRUE);
	free_xml(xml_search);
	return rc;
	}

	/* PRIVATE. Use the find_matching_attr_resources instead. */
	static void
	find_matching_attr_resources_recursive(pcmk__output_t *out,
	GList /* <pcmk_resource_t> / **result,
	pcmk_resource_t rsc, const char rsc_id,
	const char * attr_set, const char * attr_set_type,
	const char * attr_id, const char * attr_name,
	cib_t * cib, const char * cmd, int depth)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = clone_strip(rsc->id);
	char *local_attr_id = NULL;

	/* visit the children */
	for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	find_matching_attr_resources_recursive(out, result,
	(pcmk_resource_t *) gIter->data,
	rsc_id, attr_set, attr_set_type,
	attr_id, attr_name, cib, cmd, depth+1);
	/* do it only once for clones */
	if (rsc->variant == pcmk_rsc_variant_clone) {
	break;
	}
	}

	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	/* Post-order traversal.
	* The root is always on the list and it is the last item. */
	if((0 == depth) \|\| (pcmk_rc_ok == rc)) {
	/* push the head */
	result = g_list_append(result, rsc);
	}

	free(local_attr_id);
	free(lookup_id);
	}


	/* The result is a linearized pre-ordered tree of resources. */
	static GList/<pcmk_resource_t>/
	find_matching_attr_resources(pcmk__output_t out, pcmk_resource_t rsc,
	const char * rsc_id, const char * attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd,
	gboolean force)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;
	GList * result = NULL;
	/* If --force is used, update only the requested resource (clone or primitive).
	* Otherwise, if the primitive has the attribute, use that.
	* Otherwise use the clone. */
	if(force == TRUE) {
	return g_list_append(result, rsc);
	}
	if ((rsc->parent != NULL)
	&& (rsc->parent->variant == pcmk_rsc_variant_clone)) {
	int rc = pcmk_rc_ok;
	char *local_attr_id = NULL;
	rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	free(local_attr_id);

	if(rc != pcmk_rc_ok) {
	rsc = rsc->parent;
	out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
	cmd, attr_name, rsc->id, rsc_id);
	}
	return g_list_append(result, rsc);

	} else if ((rsc->parent == NULL) && (rsc->children != NULL)
	&& (rsc->variant == pcmk_rsc_variant_clone)) {
	pcmk_resource_t *child = rsc->children->data;

	if (child->variant == pcmk_rsc_variant_primitive) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
	attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if(rc == pcmk_rc_ok) {
	rsc = child;
	out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
	attr_name, lookup_id, cmd, rsc_id);
	}

	free(local_attr_id);
	free(lookup_id);
	}
	return g_list_append(result, rsc);
	}
	/* If the resource is a group ==> children inherit the attribute if defined. */
	find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
	attr_set_type, attr_id, attr_name,
	cib, cmd, 0);
	return result;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_update_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char *attr_value, gboolean recursive,
	cib_t *cib, int cib_options, gboolean force)
	{
	pcmk__output_t *out = rsc->cluster->priv;
	int rc = pcmk_rc_ok;

	char *found_attr_id = NULL;

	GList/<pcmk_resource_t>/ resources = NULL;
	const char *top_id = pe__const_top_resource(rsc, false)->id;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
	attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	if (!force) {
	rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
	XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &found_attr_id);
	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	out->err(out,
	"WARNING: There is already a meta attribute "
	"for '%s' called '%s' (id=%s)",
	top_id, attr_name, found_attr_id);
	out->err(out,
	" Delete '%s' first or use the force option "
	"to override", found_attr_id);
	}
	free(found_attr_id);
	if (rc == pcmk_rc_ok) {
	return ENOTUNIQ;
	}
	}
	resources = g_list_append(resources, rsc);

	} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	crm_xml_add(rsc->xml, attr_name, attr_value);
	CRM_ASSERT(cib != NULL);
	rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
	cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Set attribute: name=%s value=%s",
	attr_name, attr_value);
	}
	return rc;

	} else {
	resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"update", force);
	}

	/* If the user specified attr_set or attr_id, the intent is to modify a
	* single resource, which will be the last item in the list.
	*/
	if ((attr_set != NULL) \|\| (attr_id != NULL)) {
	GList *last = g_list_last(resources);

	resources = g_list_remove_link(resources, last);
	g_list_free(resources);
	resources = last;
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	char *local_attr_set = NULL;
	const char *rsc_attr_id = attr_id;
	const char *rsc_attr_set = attr_set;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	found_attr_id = NULL;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &found_attr_id);

	switch (rc) {
	case pcmk_rc_ok:
	crm_debug("Found a match for name=%s: id=%s",
	attr_name, found_attr_id);
	rsc_attr_id = found_attr_id;
	break;

	case ENXIO:
	if (rsc_attr_set == NULL) {
	local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
	attr_set_type);
	rsc_attr_set = local_attr_set;
	}
	if (rsc_attr_id == NULL) {
	found_attr_id = crm_strdup_printf("%s-%s",
	rsc_attr_set, attr_name);
	rsc_attr_id = found_attr_id;
	}

	xml_top = create_xml_node(NULL, (const char *) rsc->xml->name);
	crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
	break;

	default:
	free(lookup_id);
	free(found_attr_id);
	g_list_free(resources);
	return rc;
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
	attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top,
	cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s",
	lookup_id, found_attr_id,
	((rsc_attr_set == NULL)? "" : " set="),
	pcmk__s(rsc_attr_set, ""),
	((attr_name == NULL)? "" : " name="),
	pcmk__s(attr_name, ""), attr_value);
	}

	free_xml(xml_top);

	free(lookup_id);
	free(found_attr_id);
	free(local_attr_set);

	if (recursive
	&& pcmk__str_eq(attr_set_type, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	GList *lpc = NULL;
	static bool need_init = true;

	if (need_init) {
	need_init = false;
	pcmk__unpack_constraints(rsc->cluster);
	pe__clear_resource_flags_on_all(rsc->cluster,
	pcmk_rsc_detect_loop);
	}

	/* We want to set the attribute only on resources explicitly
	* colocated with this one, so we use rsc->rsc_cons_lhs directly
	* rather than the with_this_colocations() method.
	*/
	pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if (!pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)
	&& (cons->score > 0)) {
	crm_debug("Setting %s=%s for dependent resource %s",
	attr_name, attr_value, cons->dependent->id);
	cli_resource_update_attribute(cons->dependent,
	cons->dependent->id, NULL,
	attr_set_type, NULL,
	attr_name, attr_value,
	recursive, cib, cib_options,
	force);
	}
	}
	}
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	cib_t *cib, int cib_options, gboolean force)
	{
	pcmk__output_t *out = rsc->cluster->priv;
	int rc = pcmk_rc_ok;
	GList/<pcmk_resource_t>/ resources = NULL;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID,
	pe__const_top_resource(rsc, false)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"delete", force);

	} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	xml_remove_prop(rsc->xml, attr_name);
	CRM_ASSERT(cib != NULL);
	rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
	cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted attribute: %s", attr_name);
	}
	return rc;

	} else {
	resources = g_list_append(resources, rsc);
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	xmlNode *xml_obj = NULL;
	char *found_attr_id = NULL;
	const char *rsc_attr_id = attr_id;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &found_attr_id);
	switch (rc) {
	case pcmk_rc_ok:
	break;

	case ENXIO:
	free(lookup_id);
	rc = pcmk_rc_ok;
	continue;

	default:
	free(lookup_id);
	g_list_free(resources);
	return rc;
	}

	if (rsc_attr_id == NULL) {
	rsc_attr_id = found_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj,
	cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted '%s' option: id=%s%s%s%s%s",
	lookup_id, found_attr_id,
	((attr_set == NULL)? "" : " set="),
	pcmk__s(attr_set, ""),
	((attr_name == NULL)? "" : " name="),
	pcmk__s(attr_name, ""));
	}

	free(lookup_id);
	free_xml(xml_obj);
	free(found_attr_id);
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
	const char host_uname, const char rsc_id,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	const char *router_node = host_uname;
	const char *rsc_api_id = NULL;
	const char *rsc_long_id = NULL;
	const char *rsc_class = NULL;
	const char *rsc_provider = NULL;
	const char *rsc_type = NULL;
	bool cib_only = false;
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	out->err(out, "Resource %s not found", rsc_id);
	return ENXIO;

	} else if (rsc->variant != pcmk_rsc_variant_primitive) {
	out->err(out, "We can only process primitive resources, not %s", rsc_id);
	return EINVAL;
	}

	- rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ rsc_class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	rsc_provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
	rsc_type = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	if ((rsc_class == NULL) \|\| (rsc_type == NULL)) {
	out->err(out, "Resource %s does not have a class and type", rsc_id);
	return EINVAL;
	}

	{
	pcmk_node_t *node = pe_find_node(scheduler->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Node %s not found", host_uname);
	return pcmk_rc_node_unknown;
	}

	if (!(node->details->online)) {
	if (do_fail_resource) {
	out->err(out, "Node %s is not online", host_uname);
	return ENOTCONN;
	} else {
	cib_only = true;
	}
	}
	if (!cib_only && pe__is_guest_or_remote_node(node)) {
	node = pe__current_node(node->details->remote_rsc);
	if (node == NULL) {
	out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return ENOTCONN;
	}
	router_node = node->details->uname;
	}
	}

	if (rsc->clone_name) {
	rsc_api_id = rsc->clone_name;
	rsc_long_id = rsc->id;
	} else {
	rsc_api_id = rsc->id;
	}
	if (do_fail_resource) {
	return pcmk_controld_api_fail(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id,
	rsc_class, rsc_provider, rsc_type);
	} else {
	return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id, rsc_class,
	rsc_provider, rsc_type, cib_only);
	}
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(const pcmk_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	return strdup(name);
	}
	return clone_strip(name);
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_history(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_trace("Processing %d mainloop inputs",
	pcmk_controld_api_replies_expected(controld_api));
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	pcmk_controld_api_replies_expected(controld_api));
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_failures(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char rsc_id, const char *operation,
	const char interval_spec, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = pcmk__strkey_table(NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	guint interval_ms = 0U;

	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(scheduler->failed);
	xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {

	failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	pcmk_resource_t *fail_rsc = NULL;

	fail_rsc = pe_find_resource_with_flags(scheduler->resources,
	failed_id,
	pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename);
	if (!fail_rsc \|\| !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
	if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
	if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
	continue;
	}
	}

	g_hash_table_add(rscs, (gpointer) failed_id);
	}

	free(interval_ms_s);

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_fail_attrs(const pcmk_resource_t rsc, const char operation,
	const char interval_spec, const pcmk_node_t node)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
	operation, interval_spec, NULL,
	attr_options);
	free(rsc_name);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete(pcmk_ipc_api_t controld_api, const char host_uname,
	const pcmk_resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pcmk_scheduler_t *scheduler, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	pcmk_node_t *node = NULL;

	if (rsc == NULL) {
	return ENXIO;

	} else if (rsc->children) {

	for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) lpc->data;

	rc = cli_resource_delete(controld_api, host_uname, child, operation,
	interval_spec, just_failures, scheduler,
	force);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	return pcmk_rc_ok;

	} else if (host_uname == NULL) {
	GList *lpc = NULL;
	GList *nodes = g_hash_table_get_values(rsc->known_on);

	if(nodes == NULL && force) {
	nodes = pcmk__copy_node_list(scheduler->nodes, false);

	} else if(nodes == NULL && rsc->exclusive_discover) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	if(node->weight >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (pcmk_node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(controld_api, node->details->uname, rsc,
	operation, interval_spec, just_failures,
	scheduler, force);
	}
	if (rc != pcmk_rc_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_rc_ok;
	}

	node = pe_find_node(scheduler->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Unable to clean up %s because node %s not found",
	rsc->id, host_uname);
	return ENODEV;
	}

	if (!node->details->rsc_discovery_enabled) {
	out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
	rsc->id, host_uname);
	return EOPNOTSUPP;
	}

	if (controld_api == NULL) {
	out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
	rsc->id, host_uname);
	return pcmk_rc_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up %s failures on %s: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
	interval_spec, scheduler);
	} else {
	rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
	}
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	} else {
	out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_cleanup_all(pcmk_ipc_api_t controld_api, const char node_name,
	const char operation, const char interval_spec,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	display_name);
	return rc;
	}

	if (node_name) {
	pcmk_node_t *node = pe_find_node(scheduler->nodes, node_name);

	if (node == NULL) {
	out->err(out, "Unknown node: %s", node_name);
	return ENXIO;
	}
	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
	interval_spec, NULL, attr_options);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up all failures on %s: %s",
	display_name, pcmk_rc_str(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(out, controld_api, node_name, NULL,
	operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
	node_name, pcmk_rc_str(rc));
	return rc;
	}
	} else {
	for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
	operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
	pcmk_rc_str(rc));
	return rc;
	}
	}
	}

	out->info(out, "Cleaned up all resources on %s", display_name);
	return rc;
	}

	static void
	check_role(resource_checks_t *checks)
	{
	const char *role_s = g_hash_table_lookup(checks->rsc->meta,
	PCMK_META_TARGET_ROLE);

	if (role_s == NULL) {
	return;
	}
	switch (text2role(role_s)) {
	case pcmk_role_stopped:
	checks->flags \|= rsc_remain_stopped;
	break;

	case pcmk_role_unpromoted:
	if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
	pcmk_rsc_promotable)) {
	checks->flags \|= rsc_unpromotable;
	}
	break;

	default:
	break;
	}
	}

	static void
	check_managed(resource_checks_t *checks)
	{
	const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
	PCMK_META_IS_MANAGED);

	if ((managed_s != NULL) && !crm_is_true(managed_s)) {
	checks->flags \|= rsc_unmanaged;
	}
	}

	static void
	check_locked(resource_checks_t *checks)
	{
	if (checks->rsc->lock_node != NULL) {
	checks->flags \|= rsc_locked;
	checks->lock_node = checks->rsc->lock_node->details->uname;
	}
	}

	static bool
	node_is_unhealthy(pcmk_node_t *node)
	{
	switch (pe__health_strategy(node->details->data_set)) {
	case pcmk__health_strategy_none:
	break;

	case pcmk__health_strategy_no_red:
	if (pe__node_health(node) < 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_only_green:
	if (pe__node_health(node) <= 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_progressive:
	case pcmk__health_strategy_custom:
	/* @TODO These are finite scores, possibly with rules, and possibly
	* combining with other scores, so attributing these as a cause is
	* nontrivial.
	*/
	break;
	}
	return false;
	}

	static void
	check_node_health(resource_checks_t checks, pcmk_node_t node)
	{
	if (node == NULL) {
	GHashTableIter iter;
	bool allowed = false;
	bool all_nodes_unhealthy = true;

	g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	allowed = true;
	if (!node_is_unhealthy(node)) {
	all_nodes_unhealthy = false;
	break;
	}
	}
	if (allowed && all_nodes_unhealthy) {
	checks->flags \|= rsc_node_health;
	}

	} else if (node_is_unhealthy(node)) {
	checks->flags \|= rsc_node_health;
	}
	}

	int
	cli_resource_check(pcmk__output_t out, pcmk_resource_t rsc, pcmk_node_t *node)
	{
	resource_checks_t checks = { .rsc = rsc };

	check_role(&checks);
	check_managed(&checks);
	check_locked(&checks);
	check_node_health(&checks, node);

	return out->message(out, "resource-check-list", &checks);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_fail(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	crm_notice("Failing %s on %s", rsc_id, host_uname);
	return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
	}

	static GHashTable *
	generate_resource_params(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	combined = pcmk__strkey_table(free, free);

	params = pe_rsc_params(rsc, node, scheduler);
	if (params != NULL) {
	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	}

	meta = pcmk__strkey_table(free, free);
	get_meta_attributes(meta, rsc, node, scheduler);
	if (meta != NULL) {
	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	bool resource_is_running_on(pcmk_resource_t rsc, const char host)
	{
	bool found = true;
	GList *hIter = NULL;
	GList *hosts = NULL;

	if (rsc == NULL) {
	return false;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pcmk_node_t node = (pcmk_node_t ) hIter->data;

	if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if (host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = false;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = false;
	}

	done:
	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (rsc->variant == pcmk_rsc_variant_group) {
	active = g_list_concat(active,
	get_active_resources(host, rsc->children));
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(pcmk__output_t out, GList items, const char *tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	out->info(out, "%s%s", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as scheduler input
	*
	* This also updates the scheduler timestamp to the current time.
	*
	* \param[in,out] scheduler Scheduler data to update
	* \param[in,out] xml XML to use as input
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns ENOKEY ("Required key not available") if that fails,
	* but perhaps pcmk_rc_schema_validation would be better in that case.
	*/
	int
	update_scheduler_input(pcmk_scheduler_t scheduler, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return ENOKEY;
	}
	scheduler->input = *xml;
	scheduler->now = crm_time_new(NULL);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Update scheduler XML input based on a CIB query
	*
	* \param[in] scheduler Scheduler data to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->input and scheduler->now.
	*/
	static int
	update_scheduler_input_to_cib(pcmk__output_t out, pcmk_scheduler_t scheduler,
	cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
	return rc;
	}
	rc = update_scheduler_input(scheduler, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not upgrade the current CIB XML");
	free_xml(cib_xml_copy);
	return rc;
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	update_dataset(cib_t cib, pcmk_scheduler_t scheduler, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc = pcmk_rc_ok;

	pcmk__output_t *out = scheduler->priv;

	pe_reset_working_set(scheduler);
	pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if(simulate) {
	bool prev_quiet = false;

	pid = pcmk__getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	out->err(out, "Could not create shadow cib: '%s'", pid);
	rc = ENXIO;
	goto done;
	}

	rc = write_xml_file(scheduler->input, shadow_file, FALSE);

	if (rc < 0) {
	out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_rc_str(rc), rc);
	goto done;
	}

	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	prev_quiet = out->is_quiet(out);
	out->quiet = true;
	pcmk__simulate_transition(scheduler, shadow_cib, NULL);
	out->quiet = prev_quiet;

	rc = update_dataset(shadow_cib, scheduler, false);

	} else {
	cluster_status(scheduler);
	}

	done:
	// Do not free scheduler->input here, we need rsc->xml to be valid later on
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	/*!
	* \internal
	* \brief Find the maximum stop timeout of a resource and its children (if any)
	*
	* \param[in,out] rsc Resource to get timeout for
	*
	* \return Maximum stop timeout for \p rsc (in milliseconds)
	*/
	static int
	max_rsc_stop_timeout(pcmk_resource_t *rsc)
	{
	long long result_ll;
	int max_delay = 0;
	xmlNode *config = NULL;
	GHashTable *meta = NULL;

	if (rsc == NULL) {
	return 0;
	}

	// If resource is collective, use maximum of its children's stop timeouts
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	int delay = max_rsc_stop_timeout(child);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Maximum stop timeout for %s is now %s "
	"due to %s", rsc->id,
	pcmk__readable_interval(delay), child->id);
	max_delay = delay;
	}
	}
	return max_delay;
	}

	// Get resource's stop action configuration from CIB
	config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);

	/* Get configured timeout for stop action (fully evaluated for rules,
	* defaults, etc.).
	*
	* @TODO This currently ignores node (which might matter for rules)
	*/
	meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
	if ((pcmk__scan_ll(g_hash_table_lookup(meta, XML_ATTR_TIMEOUT),
	&result_ll, -1LL) == pcmk_rc_ok)
	&& (result_ll >= 0) && (result_ll <= INT_MAX)) {
	max_delay = (int) result_ll;
	}
	g_hash_table_destroy(meta);

	return max_delay;
	}

	/*!
	* \internal
	* \brief Find a reasonable waiting time for stopping any one resource in a list
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] resources List of names of resources that will be stopped
	*
	* \return Rough estimate of a reasonable time to wait (in seconds) to stop any
	* one resource in \p resources
	* \note This estimate is very rough, simply the maximum stop timeout of all
	* given resources and their children, plus a small fudge factor. It does
	* not account for children that must be stopped in sequence, action
	* throttling, or any demotions needed. It checks the stop timeout, even
	* if the resources in question are actually being started.
	*/
	static int
	wait_time_estimate(pcmk_scheduler_t scheduler, const GList resources)
	{
	int max_delay = 0;

	// Find maximum stop timeout in milliseconds
	for (const GList *item = resources; item != NULL; item = item->next) {
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
	(const char *) item->data);
	int delay = max_rsc_stop_timeout(rsc);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Wait time is now %s due to %s",
	pcmk__readable_interval(delay), rsc->id);
	max_delay = delay;
	}
	}

	return (max_delay / 1000) + 5;
	}

	#define waiting_for_starts(d, r, h) ((d != NULL) \|\| \
	(!resource_is_running_on((r), (h))))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in,out] out Output object
	* \param[in,out] rsc The resource to restart
	* \param[in] node Node to restart resource on (NULL for all)
	* \param[in] move_lifetime If not NULL, how long constraint should
	* remain in effect (as ISO 8601 string)
	* \param[in] timeout_ms Consider failed if actions do not complete
	* in this time (specified in milliseconds,
	* but a two-second granularity is actually
	* used; if 0, it will be calculated based on
	* the resource timeout)
	* \param[in,out] cib Connection to the CIB manager
	* \param[in] cib_options Group of enum cib_call_options flags to
	* use with CIB calls
	* \param[in] promoted_role_only If true, limit to promoted instances
	* \param[in] force If true, apply only to requested instance
	* if part of a collective resource
	*
	* \return Standard Pacemaker return code (exits on certain failures)
	*/
	int
	cli_resource_restart(pcmk__output_t out, pcmk_resource_t rsc,
	const pcmk_node_t node, const char move_lifetime,
	int timeout_ms, cib_t *cib, int cib_options,
	gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	bool stop_via_ban = false;
	char *rsc_id = NULL;
	char *lookup_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pcmk_scheduler_t *scheduler = NULL;
	pcmk_resource_t *parent = uber_parent(rsc);

	bool running = false;
	const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
	const char *host = node ? node->details->uname : NULL;

	/* If the implicit resource or primitive resource of a bundle is given, operate on the
	* bundle itself instead.
	*/
	if (pe_rsc_is_bundled(rsc)) {
	rsc = parent->parent;
	}

	running = resource_is_running_on(rsc, host);

	if (pe_rsc_is_clone(parent) && !running) {
	if (pe_rsc_is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	rsc = parent->fns->find_rsc(parent, lookup_id, node,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node);
	free(lookup_id);
	running = resource_is_running_on(rsc, host);
	}

	if (!running) {
	if (host) {
	out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
	} else {
	out->err(out, "%s is not running anywhere and so cannot be restarted", id);
	}
	return ENXIO;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	out->err(out, "Unmanaged resources cannot be restarted.");
	return EAGAIN;
	}

	rsc_id = strdup(rsc->id);

	if (pe_rsc_is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	if (host) {
	if (pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) {
	stop_via_ban = true;
	} else if (pe_rsc_is_clone(parent)) {
	stop_via_ban = true;
	free(lookup_id);
	lookup_id = strdup(parent->id);
	}
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = errno;
	out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}

	scheduler->priv = out;
	rc = update_dataset(cib, scheduler, false);

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
	goto done;
	}

	restart_target_active = get_active_resources(host, scheduler->resources);
	current_active = get_active_resources(host, scheduler->resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	out->quiet = true;
	rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
	cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);
	} else {
	/* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
	* Remember any existing PCMK_META_TARGET_ROLE so we can restore it
	* later (though it only makes any difference if it's Unpromoted).
	*/

	find_resource_attr(out, cib, PCMK_XA_VALUE, lookup_id, NULL, NULL, NULL,
	PCMK_META_TARGET_ROLE, &orig_target_role);
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE,
	PCMK_ACTION_STOPPED, FALSE, cib,
	cib_options, force);
	}
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	current_active = NULL;
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	restart_target_active = NULL;
	}
	goto done;
	}

	rc = update_dataset(cib, scheduler, true);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources would be stopped");
	goto failure;
	}

	target_active = get_active_resources(host, scheduler->resources);
	dump_list(target_active, "Target");

	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (list_delta != NULL) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(cib, scheduler, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were stopped");
	goto failure;
	}

	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(host, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);

	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE,
	orig_target_role, FALSE, cib,
	cib_options, force);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE, cib,
	cib_options, force);
	}

	if(rc != pcmk_rc_ok) {
	out->err(out,
	"Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	goto done;
	}

	if (target_active != NULL) {
	g_list_free_full(target_active, free);
	}
	target_active = restart_target_active;

	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(cib, scheduler, false);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were started");
	goto failure;
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(NULL, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	rc = pcmk_rc_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
	} else if (orig_target_role) {
	cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	PCMK_META_TARGET_ROLE, orig_target_role,
	FALSE, cib, cib_options, force);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE, cib,
	cib_options, force);
	}

	done:
	if (list_delta != NULL) {
	g_list_free(list_delta);
	}
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	if (target_active != NULL && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	free(lookup_id);
	pe_free_working_set(scheduler);
	return rc;
	}

	static inline bool
	action_is_pending(const pcmk_action_t *action)
	{
	if (pcmk_any_flags_set(action->flags,
	pcmk_action_optional\|pcmk_action_pseudo)
	\|\| !pcmk_is_set(action->flags, pcmk_action_runnable)
	\|\| pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return true if any actions in the list are pending, otherwise false
	*/
	static bool
	actions_are_pending(const GList *actions)
	{
	for (const GList *action = actions; action != NULL; action = action->next) {
	const pcmk_action_t a = (const pcmk_action_t ) action->data;

	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
	return true;
	}
	}
	return false;
	}

	static void
	print_pending_actions(pcmk__output_t out, GList actions)
	{
	GList *action;

	out->info(out, "Pending actions:");
	for (action = actions; action != NULL; action = action->next) {
	pcmk_action_t a = (pcmk_action_t ) action->data;

	if (!action_is_pending(a)) {
	continue;
	}

	if (a->node) {
	out->info(out, "\tAction %d: %s\ton %s",
	a->id, a->uuid, pe__node_name(a->node));
	} else {
	out->info(out, "\tAction %d: %s", a->id, a->uuid);
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in,out] out Output object
	* \param[in] timeout_ms Consider failed if actions do not complete in
	* this time (specified in milliseconds, but
	* one-second granularity is actually used; if 0, a
	* default will be used)
	* \param[in,out] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	*/
	int
	wait_till_stable(pcmk__output_t out, int timeout_ms, cib_t cib)
	{
	pcmk_scheduler_t *scheduler = NULL;
	xmlXPathObjectPtr search;
	int rc = pcmk_rc_ok;
	bool pending_unknown_state_resources;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;
	bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	return ENOMEM;
	}

	do {
	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %lld seconds for cluster actions to complete", (long long) time_diff);
	} else {
	print_pending_actions(out, scheduler->actions);
	pe_free_working_set(scheduler);
	return ETIME;
	}
	if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	pe_reset_working_set(scheduler);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	pe_free_working_set(scheduler);
	return rc;
	}
	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_DC_VERSION);

	if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
	out->info(out, "warning: wait option may not work properly in "
	"mixed-version cluster");
	printed_version_warning = true;
	}
	}

	search = xpath_search(scheduler->input, "/cib/status/node_state/lrm/lrm_resources/lrm_resource/"
	XML_LRM_TAG_RSC_OP "[@" XML_LRM_ATTR_RC "='193']");
	pending_unknown_state_resources = (numXpathResults(search) > 0);
	freeXpathObject(search);
	} while (actions_are_pending(scheduler->actions) \|\| pending_unknown_state_resources);

	pe_free_working_set(scheduler);
	return rc;
	}

	static const char *
	get_action(const char *rsc_action) {
	const char *action = NULL;

	if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
	action = PCMK_ACTION_VALIDATE_ALL;

	} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
	action = PCMK_ACTION_MONITOR;

	} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
	"force-demote", "force-promote", NULL)) {
	action = rsc_action+6;
	} else {
	action = rsc_action;
	}

	return action;
	}

	/*!
	* \brief Set up environment variables as expected by resource agents
	*
	* When the cluster executes resource agents, it adds certain environment
	* variables (directly or via resource meta-attributes) expected by some
	* resource agents. Add the essential ones that many resource agents expect, so
	* the behavior is the same for command-line execution.
	*
	* \param[in,out] params Resource parameters that will be passed to agent
	* \param[in] timeout_ms Action timeout (in milliseconds)
	* \param[in] check_level OCF check level
	* \param[in] verbosity Verbosity level
	*/
	static void
	set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
	int verbosity)
	{
	g_hash_table_insert(params, strdup("CRM_meta_timeout"),
	crm_strdup_printf("%d", timeout_ms));

	g_hash_table_insert(params, strdup(PCMK_XA_CRM_FEATURE_SET),
	strdup(CRM_FEATURE_SET));

	if (check_level >= 0) {
	char *level = crm_strdup_printf("%d", check_level);

	setenv("OCF_CHECK_LEVEL", level, 1);
	free(level);
	}

	pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
	if (verbosity > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	/* A resource agent using the standard ocf-shellfuncs library will not print
	* messages to stderr if it doesn't have a controlling terminal (e.g. if
	* crm_resource is called via script or ssh). This forces it to do so.
	*/
	setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
	}

	/*!
	* \internal
	* \brief Apply command-line overrides to resource parameters
	*
	* \param[in,out] params Parameters to be passed to agent
	* \param[in] overrides Parameters to override (or NULL if none)
	*/
	static void
	apply_overrides(GHashTable params, GHashTable overrides)
	{
	if (overrides != NULL) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer *) &name,
	(gpointer *) &value)) {
	g_hash_table_replace(params, strdup(name), strdup(value));
	}
	}
	}

	crm_exit_t
	cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char rsc_action,
	GHashTable params, GHashTable override_hash,
	int timeout_ms, int resource_verbose, gboolean force,
	int check_level)
	{
	const char *class = rsc_class;
	const char *action = get_action(rsc_action);
	crm_exit_t exit_code = CRM_EX_OK;
	svc_action_t *op = NULL;

	// If no timeout was provided, use the same default as the cluster
	if (timeout_ms == 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	set_agent_environment(params, timeout_ms, check_level, resource_verbose);
	apply_overrides(params, override_hash);

	op = services__create_resource_action(rsc_name? rsc_name : "test",
	rsc_class, rsc_prov, rsc_type, action,
	0, timeout_ms, params, 0);
	if (op == NULL) {
	out->err(out, "Could not execute %s using %s%s%s:%s: %s",
	action, rsc_class, (rsc_prov? ":" : ""),
	(rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
	g_hash_table_destroy(params);
	return CRM_EX_OSERR;
	}

	if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
	class = resources_find_service_class(rsc_type);
	}
	if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
	"Manual execution of the %s standard is "
	"unsupported", pcmk__s(class, "unspecified"));
	}

	if (op->rc != PCMK_OCF_UNKNOWN) {
	exit_code = op->rc;
	goto done;
	}

	services_action_sync(op);

	// Map results to OCF codes for consistent reporting to user
	{
	enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);

	// Cast variable instead of function return to keep compilers happy
	exit_code = (crm_exit_t) ocf_code;
	}

	done:
	out->message(out, "resource-agent-action", resource_verbose, rsc_class,
	rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
	exit_code, op->status, services__exit_reason(op),
	op->stdout_data, op->stderr_data);
	services_action_free(op);
	return exit_code;
	}

	crm_exit_t
	cli_resource_execute(pcmk_resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	int timeout_ms, cib_t cib, pcmk_scheduler_t scheduler,
	int resource_verbose, gboolean force, int check_level)
	{
	pcmk__output_t *out = scheduler->priv;
	crm_exit_t exit_code = CRM_EX_OK;
	const char *rid = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	GHashTable *params = NULL;

	if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
	"force-promote", NULL)) {
	if(pe_rsc_is_clone(rsc)) {
	GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
	if(nodes != NULL && force == FALSE) {
	out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
	rsc_action, rsc->id);
	out->err(out,
	"Try setting "
	PCMK_META_TARGET_ROLE "=" PCMK__ROLE_STOPPED
	" first or specifying the force option");
	return CRM_EX_UNSAFE;
	}

	g_list_free_full(nodes, free);
	}
	}

	if(pe_rsc_is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->children->data;
	}

	if (rsc->variant == pcmk_rsc_variant_group) {
	out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	} else if (pe_rsc_is_bundled(rsc)) {
	out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	}

	- rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	+ rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);

	params = generate_resource_params(rsc, NULL /* @TODO use local node */,
	scheduler);

	if (timeout_ms == 0) {
	timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action),
	scheduler);
	}

	rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;

	exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
	params, override_hash, timeout_ms,
	resource_verbose, force, check_level);
	return exit_code;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_move(const pcmk_resource_t rsc, const char rsc_id,
	const char host_name, const char move_lifetime, cib_t *cib,
	int cib_options, pcmk_scheduler_t *scheduler,
	gboolean promoted_role_only, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	unsigned int count = 0;
	pcmk_node_t *current = NULL;
	pcmk_node_t *dest = pe_find_node(scheduler->nodes, host_name);
	bool cur_is_dest = false;

	if (dest == NULL) {
	return pcmk_rc_node_unknown;
	}

	if (promoted_role_only
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {

	const pcmk_resource_t *p = pe__const_top_resource(rsc, false);

	if (pcmk_is_set(p->flags, pcmk_rsc_promotable)) {
	out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	out->info(out, "Ignoring --promoted option: %s is not promotable",
	rsc_id);
	promoted_role_only = FALSE;
	}
	}

	current = pe__find_active_requires(rsc, &count);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	unsigned int promoted_count = 0;
	pcmk_node_t *promoted_node = NULL;

	for (const GList *iter = rsc->children; iter; iter = iter->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if (child_role == pcmk_role_promoted) {
	rsc = child;
	promoted_node = pe__current_node(child);
	promoted_count++;
	}
	}
	if (promoted_role_only \|\| (promoted_count != 0)) {
	count = promoted_count;
	current = promoted_node;
	}

	}

	if (count > 1) {
	if (pe_rsc_is_clone(rsc)) {
	current = NULL;
	} else {
	return pcmk_rc_multiple;
	}
	}

	if (current && (current->details == dest->details)) {
	cur_is_dest = true;
	if (force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, promoted_role_only?"promoted":"active",
	pe__node_name(dest));
	} else {
	return pcmk_rc_already;
	}
	}

	/* Clear any previous prefer constraints across all nodes. */
	cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, cib_options, false,
	force);

	/* Clear any previous ban constraints on 'dest'. */
	cli_resource_clear(rsc_id, dest->details->uname, scheduler->nodes, cib,
	cib_options, TRUE, force);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);

	crm_trace("%s%s now prefers %s%s",
	rsc->id, (promoted_role_only? " (promoted)" : ""),
	pe__node_name(dest), force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if (force && !cur_is_dest) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);
	} else if(count > 1) {
	out->info(out, "Resource '%s' is currently %s in %d locations. "
	"One may now move to %s",
	rsc_id, (promoted_role_only? "promoted" : "active"),
	count, pe__node_name(dest));
	out->info(out, "To prevent '%s' from being %s at a specific location, "
	"specify a node.",
	rsc_id, (promoted_role_only? "promoted" : "active"));

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Oct 16, 12:21 AM (16 h, 51 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2530815
Default Alt Text: (1 MB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions