No OneTemporary
Actions

Size

983 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c
	index 54fcd1f612..f9f2ad3e2a 100644
	--- a/daemons/attrd/attrd_cib.c
	+++ b/daemons/attrd/attrd_cib.c
	@@ -1,702 +1,704 @@
	/*
	* Copyright 2013-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <errno.h>
	#include <inttypes.h> // PRIu32
	#include <stdbool.h>
	#include <stdlib.h>
	#include <glib.h>

	#include <crm/msg_xml.h>
	#include <crm/common/logging.h>
	#include <crm/common/results.h>
	#include <crm/common/strings_internal.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h> // pcmk__get_node()

	#include "pacemaker-attrd.h"

	static int last_cib_op_done = 0;

	static void write_attribute(attribute_t *a, bool ignore_delay);

	static void
	attrd_cib_destroy_cb(gpointer user_data)
	{
	cib_t *cib = user_data;

	cib->cmds->signoff(cib);

	if (attrd_shutting_down(false)) {
	crm_info("Disconnected from the CIB manager");

	} else {
	// @TODO This should trigger a reconnect, not a shutdown
	crm_crit("Lost connection to the CIB manager, shutting down");
	attrd_exit_status = CRM_EX_DISCONNECT;
	attrd_shutdown(0);
	}
	}

	static void
	attrd_cib_updated_cb(const char event, xmlNode msg)
	{
	const xmlNode *patchset = NULL;
	const char *client_name = NULL;

	if (attrd_shutting_down(true)) {
	return;
	}

	if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
	return;
	}

	if (cib__element_in_patchset(patchset, PCMK_XE_ALERTS)) {
	mainloop_set_trigger(attrd_config_read);
	}

	if (!attrd_election_won()) {
	// Don't write attributes if we're not the writer
	return;
	}

	client_name = crm_element_value(msg, F_CIB_CLIENTNAME);
	if (!cib__client_triggers_refresh(client_name)) {
	// The CIB is still accurate
	return;
	}

	if (cib__element_in_patchset(patchset, PCMK_XE_NODES)
	\|\| cib__element_in_patchset(patchset, PCMK_XE_STATUS)) {

	/* An unsafe client modified the PCMK_XE_NODES or PCMK_XE_STATUS
	* section. Write transient attributes to ensure they're up-to-date in
	* the CIB.
	*/
	if (client_name == NULL) {
	client_name = crm_element_value(msg, F_CIB_CLIENTID);
	}
	crm_notice("Updating all attributes after %s event triggered by %s",
	event, pcmk__s(client_name, "(unidentified client)"));

	attrd_write_attributes(attrd_write_all);
	}
	}

	int
	attrd_cib_connect(int max_retry)
	{
	static int attempts = 0;

	int rc = -ENOTCONN;

	the_cib = cib_new();
	if (the_cib == NULL) {
	return -ENOTCONN;
	}

	do {
	if (attempts > 0) {
	sleep(attempts);
	}
	attempts++;
	crm_debug("Connection attempt %d to the CIB manager", attempts);
	rc = the_cib->cmds->signon(the_cib, T_ATTRD, cib_command);

	} while ((rc != pcmk_ok) && (attempts < max_retry));

	if (rc != pcmk_ok) {
	crm_err("Connection to the CIB manager failed: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	goto cleanup;
	}

	crm_debug("Connected to the CIB manager after %d attempts", attempts);

	rc = the_cib->cmds->set_connection_dnotify(the_cib, attrd_cib_destroy_cb);
	if (rc != pcmk_ok) {
	crm_err("Could not set disconnection callback");
	goto cleanup;
	}

	rc = the_cib->cmds->add_notify_callback(the_cib, T_CIB_DIFF_NOTIFY,
	attrd_cib_updated_cb);
	if (rc != pcmk_ok) {
	crm_err("Could not set CIB notification callback");
	goto cleanup;
	}

	return pcmk_ok;

	cleanup:
	cib__clean_up_connection(&the_cib);
	return -ENOTCONN;
	}

	void
	attrd_cib_disconnect(void)
	{
	CRM_CHECK(the_cib != NULL, return);
	the_cib->cmds->del_notify_callback(the_cib, T_CIB_DIFF_NOTIFY,
	attrd_cib_updated_cb);
	cib__clean_up_connection(&the_cib);
	}

	static void
	attrd_erase_cb(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	const char node = pcmk__s((const char ) user_data, "a node");

	if (rc == pcmk_ok) {
	crm_info("Cleared transient node attributes for %s from CIB", node);
	} else {
	crm_err("Unable to clear transient node attributes for %s from CIB: %s",
	node, pcmk_strerror(rc));
	}
	}

	-#define XPATH_TRANSIENT "//node_state[@uname='%s']/" XML_TAG_TRANSIENT_NODEATTRS
	+#define XPATH_TRANSIENT "//" PCMK__XE_NODE_STATE \
	+ "[@" PCMK_XA_UNAME "='%s']" \
	+ "/" XML_TAG_TRANSIENT_NODEATTRS

	/*!
	* \internal
	* \brief Wipe all transient node attributes for a node from the CIB
	*
	* \param[in] node Node to clear attributes for
	*/
	void
	attrd_cib_erase_transient_attrs(const char *node)
	{
	int call_id = 0;
	char *xpath = NULL;

	CRM_CHECK(node != NULL, return);

	xpath = crm_strdup_printf(XPATH_TRANSIENT, node);

	crm_debug("Clearing transient node attributes for %s from CIB using %s",
	node, xpath);

	call_id = the_cib->cmds->remove(the_cib, xpath, NULL, cib_xpath);
	free(xpath);

	// strdup() is just for logging here, so ignore failure
	the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE,
	strdup(node), "attrd_erase_cb",
	attrd_erase_cb, free);
	}

	/*!
	* \internal
	* \brief Prepare the CIB after cluster is connected
	*/
	void
	attrd_cib_init(void)
	{
	/* We have no attribute values in memory, so wipe the CIB to match. This is
	* normally done by the DC's controller when this node leaves the cluster, but
	* this handles the case where the node restarted so quickly that the
	* cluster layer didn't notice.
	*
	* \todo If pacemaker-attrd respawns after crashing (see PCMK_ENV_RESPAWNED),
	* ideally we'd skip this and sync our attributes from the writer.
	* However, currently we reject any values for us that the writer has, in
	* attrd_peer_update().
	*/
	attrd_cib_erase_transient_attrs(attrd_cluster->uname);

	// Set a trigger for reading the CIB (for the alerts section)
	attrd_config_read = mainloop_add_trigger(G_PRIORITY_HIGH, attrd_read_options, NULL);

	// Always read the CIB at start-up
	mainloop_set_trigger(attrd_config_read);
	}

	static gboolean
	attribute_timer_cb(gpointer data)
	{
	attribute_t *a = data;
	crm_trace("Dampen interval expired for %s", a->id);
	attrd_write_or_elect_attribute(a);
	return FALSE;
	}

	static void
	attrd_cib_callback(xmlNode msg, int call_id, int rc, xmlNode output, void *user_data)
	{
	int level = LOG_ERR;
	GHashTableIter iter;
	const char *peer = NULL;
	attribute_value_t *v = NULL;

	char *name = user_data;
	attribute_t *a = g_hash_table_lookup(attributes, name);

	if(a == NULL) {
	crm_info("Attribute %s no longer exists", name);
	return;
	}

	a->update = 0;
	if (rc == pcmk_ok && call_id < 0) {
	rc = call_id;
	}

	switch (rc) {
	case pcmk_ok:
	level = LOG_INFO;
	last_cib_op_done = call_id;
	if (a->timer && !a->timeout_ms) {
	// Remove temporary dampening for failed writes
	mainloop_timer_del(a->timer);
	a->timer = NULL;
	}
	break;

	case -pcmk_err_diff_failed: /* When an attr changes while the CIB is syncing */
	case -ETIME: /* When an attr changes while there is a DC election */
	case -ENXIO: /* When an attr changes while the CIB is syncing a
	* newer config from a node that just came up
	*/
	level = LOG_WARNING;
	break;
	}

	do_crm_log(level, "CIB update %d result for %s: %s " CRM_XS " rc=%d",
	call_id, a->id, pcmk_strerror(rc), rc);

	g_hash_table_iter_init(&iter, a->values);
	while (g_hash_table_iter_next(&iter, (gpointer ) & peer, (gpointer ) & v)) {
	do_crm_log(level, "* %s[%s]=%s",
	a->id, peer, pcmk__s(v->requested, "(null)"));
	if (rc == pcmk_ok) {
	pcmk__str_update(&(v->requested), NULL);
	} else {
	a->changed = true; // Reattempt write below if we are still writer
	}
	}

	if (a->changed && attrd_election_won()) {
	if (rc == pcmk_ok) {
	/* We deferred a write of a new update because this update was in
	* progress. Write out the new value without additional delay.
	*/
	write_attribute(a, false);

	/* We're re-attempting a write because the original failed; delay
	* the next attempt so we don't potentially flood the CIB manager
	* and logs with a zillion attempts per second.
	*
	* @TODO We could elect a new writer instead. However, we'd have to
	* somehow downgrade our vote, and we'd still need something like this
	* if all peers similarly fail to write this attribute (which may
	* indicate a corrupted attribute entry rather than a CIB issue).
	*/
	} else if (a->timer) {
	// Attribute has a dampening value, so use that as delay
	if (!mainloop_timer_running(a->timer)) {
	crm_trace("Delayed re-attempted write for %s by %s",
	name, pcmk__readable_interval(a->timeout_ms));
	mainloop_timer_start(a->timer);
	}
	} else {
	/* Set a temporary dampening of 2 seconds (timer will continue
	* to exist until the attribute's dampening gets set or the
	* write succeeds).
	*/
	a->timer = attrd_add_timer(a->id, 2000, a);
	mainloop_timer_start(a->timer);
	}
	}
	}

	/*!
	* \internal
	* \brief Add a set-attribute update request to the current CIB transaction
	*
	* \param[in] attr Attribute to update
	* \param[in] attr_id ID of attribute to update
	* \param[in] node_id ID of node for which to update attribute value
	* \param[in] set_id ID of attribute set
	* \param[in] value New value for attribute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_set_attr_update(const attribute_t attr, const char attr_id,
	const char node_id, const char set_id, const char *value)
	{
	- xmlNode *update = create_xml_node(NULL, XML_CIB_TAG_STATE);
	+ xmlNode *update = create_xml_node(NULL, PCMK__XE_NODE_STATE);
	xmlNode *child = update;
	int rc = ENOMEM;

	if (child == NULL) {
	goto done;
	}
	crm_xml_add(child, PCMK_XA_ID, node_id);

	child = create_xml_node(child, XML_TAG_TRANSIENT_NODEATTRS);
	if (child == NULL) {
	goto done;
	}
	crm_xml_add(child, PCMK_XA_ID, node_id);

	child = create_xml_node(child, attr->set_type);
	if (child == NULL) {
	goto done;
	}
	crm_xml_add(child, PCMK_XA_ID, set_id);

	child = create_xml_node(child, XML_CIB_TAG_NVPAIR);
	if (child == NULL) {
	goto done;
	}
	crm_xml_add(child, PCMK_XA_ID, attr_id);
	crm_xml_add(child, PCMK_XA_NAME, attr->id);
	crm_xml_add(child, PCMK_XA_VALUE, value);

	rc = the_cib->cmds->modify(the_cib, PCMK_XE_STATUS, update,
	cib_can_create\|cib_transaction);
	rc = pcmk_legacy2rc(rc);

	done:
	free_xml(update);
	return rc;
	}

	/*!
	* \internal
	* \brief Add an unset-attribute update request to the current CIB transaction
	*
	* \param[in] attr Attribute to update
	* \param[in] attr_id ID of attribute to update
	* \param[in] node_id ID of node for which to update attribute value
	* \param[in] set_id ID of attribute set
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_unset_attr_update(const attribute_t attr, const char attr_id,
	const char node_id, const char set_id)
	{
	char *xpath = crm_strdup_printf("/" PCMK_XE_CIB
	"/" PCMK_XE_STATUS
	- "/" XML_CIB_TAG_STATE
	+ "/" PCMK__XE_NODE_STATE
	"[@" PCMK_XA_ID "='%s']"
	"/" XML_TAG_TRANSIENT_NODEATTRS
	"[@" PCMK_XA_ID "='%s']"
	"/%s[@" PCMK_XA_ID "='%s']"
	"/" XML_CIB_TAG_NVPAIR
	"[@" PCMK_XA_ID "='%s' "
	"and @" PCMK_XA_NAME "='%s']",
	node_id, node_id, attr->set_type, set_id,
	attr_id, attr->id);

	int rc = the_cib->cmds->remove(the_cib, xpath, NULL,
	cib_xpath\|cib_transaction);

	free(xpath);
	return pcmk_legacy2rc(rc);
	}

	/*!
	* \internal
	* \brief Add an attribute update request to the current CIB transaction
	*
	* \param[in] attr Attribute to update
	* \param[in] value New value for attribute
	* \param[in] node_id ID of node for which to update attribute value
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_attr_update(const attribute_t attr, const char value, const char *node_id)
	{
	char *set_id = NULL;
	char *attr_id = NULL;
	int rc = pcmk_rc_ok;

	if (attr->set_id != NULL) {
	pcmk__str_update(&set_id, attr->set_id);
	} else {
	set_id = crm_strdup_printf("%s-%s", PCMK_XE_STATUS, node_id);
	}
	crm_xml_sanitize_id(set_id);

	if (attr->uuid != NULL) {
	pcmk__str_update(&attr_id, attr->uuid);
	} else {
	attr_id = crm_strdup_printf("%s-%s", set_id, attr->id);
	}
	crm_xml_sanitize_id(attr_id);

	if (value != NULL) {
	rc = add_set_attr_update(attr, attr_id, node_id, set_id, value);
	} else {
	rc = add_unset_attr_update(attr, attr_id, node_id, set_id);
	}
	free(set_id);
	free(attr_id);
	return rc;
	}

	static void
	send_alert_attributes_value(attribute_t a, GHashTable t)
	{
	int rc = 0;
	attribute_value_t *at = NULL;
	GHashTableIter vIter;

	g_hash_table_iter_init(&vIter, t);

	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & at)) {
	rc = attrd_send_attribute_alert(at->nodename, at->nodeid,
	a->id, at->current);
	crm_trace("Sent alerts for %s[%s]=%s: nodeid=%d rc=%d",
	a->id, at->nodename, at->current, at->nodeid, rc);
	}
	}

	static void
	set_alert_attribute_value(GHashTable t, attribute_value_t v)
	{
	attribute_value_t *a_v = NULL;
	a_v = calloc(1, sizeof(attribute_value_t));
	CRM_ASSERT(a_v != NULL);

	a_v->nodeid = v->nodeid;
	a_v->nodename = strdup(v->nodename);
	pcmk__str_update(&a_v->current, v->current);

	g_hash_table_replace(t, a_v->nodename, a_v);
	}

	mainloop_timer_t *
	attrd_add_timer(const char id, int timeout_ms, attribute_t attr)
	{
	return mainloop_timer_add(id, timeout_ms, FALSE, attribute_timer_cb, attr);
	}

	/*!
	* \internal
	* \brief Write an attribute's values to the CIB if appropriate
	*
	* \param[in,out] a Attribute to write
	* \param[in] ignore_delay If true, write attribute now regardless of any
	* configured delay
	*/
	static void
	write_attribute(attribute_t *a, bool ignore_delay)
	{
	int private_updates = 0, cib_updates = 0;
	attribute_value_t *v = NULL;
	GHashTableIter iter;
	GHashTable *alert_attribute_value = NULL;
	int rc = pcmk_ok;

	if (a == NULL) {
	return;
	}

	/* If this attribute will be written to the CIB ... */
	if (!stand_alone && !a->is_private) {
	/* Defer the write if now's not a good time */
	if (a->update && (a->update < last_cib_op_done)) {
	crm_info("Write out of '%s' continuing: update %d considered lost",
	a->id, a->update);
	a->update = 0; // Don't log this message again

	} else if (a->update) {
	crm_info("Write out of '%s' delayed: update %d in progress",
	a->id, a->update);
	goto done;

	} else if (mainloop_timer_running(a->timer)) {
	if (ignore_delay) {
	mainloop_timer_stop(a->timer);
	crm_debug("Overriding '%s' write delay", a->id);
	} else {
	crm_info("Delaying write of '%s'", a->id);
	goto done;
	}
	}

	// Initiate a transaction for all the peer value updates
	CRM_CHECK(the_cib != NULL, goto done);
	the_cib->cmds->set_user(the_cib, a->user);
	rc = the_cib->cmds->init_transaction(the_cib);
	if (rc != pcmk_ok) {
	crm_err("Failed to write %s (id %s, set %s): Could not initiate "
	"CIB transaction",
	a->id, pcmk__s(a->uuid, "n/a"), pcmk__s(a->set_id, "n/a"));
	goto done;
	}
	}

	/* Attribute will be written shortly, so clear changed flag */
	a->changed = false;

	/* We will check all peers' uuids shortly, so initialize this to false */
	a->unknown_peer_uuids = false;

	/* Attribute will be written shortly, so clear forced write flag */
	a->force_write = FALSE;

	/* Make the table for the attribute trap */
	alert_attribute_value = pcmk__strikey_table(NULL,
	attrd_free_attribute_value);

	/* Iterate over each peer value of this attribute */
	g_hash_table_iter_init(&iter, a->values);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &v)) {
	const char *uuid = NULL;

	if (pcmk_is_set(v->flags, attrd_value_remote)) {
	/* If this is a Pacemaker Remote node, the node's UUID is the same
	* as its name, which we already have.
	*/
	uuid = v->nodename;

	} else {
	// This will create a cluster node cache entry if none exists
	crm_node_t *peer = pcmk__get_node(v->nodeid, v->nodename, NULL,
	pcmk__node_search_any);

	uuid = peer->uuid;

	// Remember peer's node ID if we're just now learning it
	if ((peer->id != 0) && (v->nodeid == 0)) {
	crm_trace("Learned ID %u for node %s", peer->id, v->nodename);
	v->nodeid = peer->id;
	}
	}

	/* If this is a private attribute, no update needs to be sent */
	if (stand_alone \|\| a->is_private) {
	private_updates++;
	continue;
	}

	// Defer write if this is a cluster node that's never been seen
	if (uuid == NULL) {
	a->unknown_peer_uuids = true;
	crm_notice("Cannot update %s[%s]='%s' now because node's UUID is "
	"unknown (will retry if learned)",
	a->id, v->nodename, v->current);
	continue;
	}

	// Update this value as part of the CIB transaction we're building
	rc = add_attr_update(a, v->current, uuid);
	if (rc != pcmk_rc_ok) {
	crm_err("Failed to update %s[%s]='%s': %s "
	CRM_XS " node uuid=%s id=%" PRIu32,
	a->id, v->nodename, v->current, pcmk_rc_str(rc),
	uuid, v->nodeid);
	continue;
	}

	crm_debug("Updating %s[%s]=%s (node uuid=%s id=%" PRIu32 ")",
	a->id, v->nodename, v->current, uuid, v->nodeid);
	cib_updates++;

	/* Preservation of the attribute to transmit alert */
	set_alert_attribute_value(alert_attribute_value, v);

	// Save this value so we can log it when write completes
	pcmk__str_update(&(v->requested), v->current);
	}

	if (private_updates) {
	crm_info("Processed %d private change%s for %s, id=%s, set=%s",
	private_updates, pcmk__plural_s(private_updates),
	a->id, pcmk__s(a->uuid, "n/a"), pcmk__s(a->set_id, "n/a"));
	}
	if (cib_updates > 0) {
	char *id = NULL;

	// Commit transaction
	a->update = the_cib->cmds->end_transaction(the_cib, true, cib_none);

	crm_info("Sent CIB request %d with %d change%s for %s (id %s, set %s)",
	a->update, cib_updates, pcmk__plural_s(cib_updates),
	a->id, pcmk__s(a->uuid, "n/a"), pcmk__s(a->set_id, "n/a"));

	pcmk__str_update(&id, a->id);
	if (the_cib->cmds->register_callback_full(the_cib, a->update,
	CIB_OP_TIMEOUT_S, FALSE, id,
	"attrd_cib_callback",
	attrd_cib_callback, free)) {
	// Transmit alert of the attribute
	send_alert_attributes_value(a, alert_attribute_value);
	}
	}

	done:
	// Discard transaction (if any)
	if (the_cib != NULL) {
	the_cib->cmds->end_transaction(the_cib, false, cib_none);
	the_cib->cmds->set_user(the_cib, NULL);
	}

	if (alert_attribute_value != NULL) {
	g_hash_table_destroy(alert_attribute_value);
	}
	}

	/*!
	* \internal
	* \brief Write out attributes
	*
	* \param[in] options Group of enum attrd_write_options
	*/
	void
	attrd_write_attributes(uint32_t options)
	{
	GHashTableIter iter;
	attribute_t *a = NULL;

	crm_debug("Writing out %s attributes",
	pcmk_is_set(options, attrd_write_all)? "all" : "changed");
	g_hash_table_iter_init(&iter, attributes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) {
	if (!pcmk_is_set(options, attrd_write_all) && a->unknown_peer_uuids) {
	// Try writing this attribute again, in case peer ID was learned
	a->changed = true;
	} else if (a->force_write) {
	/* If the force_write flag is set, write the attribute. */
	a->changed = true;
	}

	if (pcmk_is_set(options, attrd_write_all) \|\| a->changed) {
	bool ignore_delay = pcmk_is_set(options, attrd_write_no_delay);

	if (a->force_write) {
	// Always ignore delay when forced write flag is set
	ignore_delay = true;
	}
	write_attribute(a, ignore_delay);
	} else {
	crm_trace("Skipping unchanged attribute %s", a->id);
	}
	}
	}

	void
	attrd_write_or_elect_attribute(attribute_t *a)
	{
	if (attrd_election_won()) {
	write_attribute(a, false);
	} else {
	attrd_start_election_if_needed();
	}
	}
	diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c
	index 03261a3864..279a6225f7 100644
	--- a/daemons/controld/controld_cib.c
	+++ b/daemons/controld/controld_cib.c
	@@ -1,1070 +1,1069 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h> /* sleep */

	#include <crm/common/alerts_internal.h>
	#include <crm/common/xml.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/lrmd_internal.h>

	#include <pacemaker-controld.h>

	// Call ID of the most recent in-progress CIB resource update (or 0 if none)
	static int pending_rsc_update = 0;

	/*!
	* \internal
	* \brief Respond to a dropped CIB connection
	*
	* \param[in] user_data CIB connection that dropped
	*/
	static void
	handle_cib_disconnect(gpointer user_data)
	{
	CRM_LOG_ASSERT(user_data == controld_globals.cib_conn);

	controld_trigger_fsa();
	controld_globals.cib_conn->state = cib_disconnected;

	if (pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	// @TODO This should trigger a reconnect, not a shutdown
	crm_crit("Lost connection to the CIB manager, shutting down");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	} else { // Expected
	crm_info("Disconnected from the CIB manager");
	}
	}

	static void
	do_cib_updated(const char event, xmlNode msg)
	{
	const xmlNode *patchset = NULL;
	const char *client_name = NULL;

	crm_debug("Received CIB diff notification: DC=%s", pcmk__btoa(AM_I_DC));

	if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
	return;
	}

	if (cib__element_in_patchset(patchset, PCMK_XE_ALERTS)
	\|\| cib__element_in_patchset(patchset, PCMK_XE_CRM_CONFIG)) {

	controld_trigger_config();
	}

	if (!AM_I_DC) {
	// We're not in control of the join sequence
	return;
	}

	client_name = crm_element_value(msg, F_CIB_CLIENTNAME);
	if (!cib__client_triggers_refresh(client_name)) {
	// The CIB is still accurate
	return;
	}

	if (cib__element_in_patchset(patchset, PCMK_XE_NODES)
	\|\| cib__element_in_patchset(patchset, PCMK_XE_STATUS)) {

	/* An unsafe client modified the PCMK_XE_NODES or PCMK_XE_STATUS
	* section. Ensure the node list is up-to-date, and start the join
	* process again so we get everyone's current resource history.
	*/
	if (client_name == NULL) {
	client_name = crm_element_value(msg, F_CIB_CLIENTID);
	}
	crm_notice("Populating nodes and starting an election after %s event "
	"triggered by %s",
	event, pcmk__s(client_name, "(unidentified client)"));

	populate_cib_nodes(node_update_quick\|node_update_all, __func__);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	void
	controld_disconnect_cib_manager(void)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	CRM_ASSERT(cib_conn != NULL);

	crm_debug("Disconnecting from the CIB manager");

	controld_clear_fsa_input_flags(R_CIB_CONNECTED);

	cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
	do_cib_updated);
	cib_free_callbacks(cib_conn);

	if (cib_conn->state != cib_disconnected) {
	cib_conn->cmds->set_secondary(cib_conn,
	cib_scope_local\|cib_discard_reply);
	cib_conn->cmds->signoff(cib_conn);
	}
	}

	/* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
	void
	do_cib_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static int cib_retries = 0;

	cib_t *cib_conn = controld_globals.cib_conn;

	void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect;
	void (update_cb) (const char event, xmlNodePtr msg) = do_cib_updated;

	int rc = pcmk_ok;

	CRM_ASSERT(cib_conn != NULL);

	if (pcmk_is_set(action, A_CIB_STOP)) {
	if ((cib_conn->state != cib_disconnected)
	&& (pending_rsc_update != 0)) {

	crm_info("Waiting for resource update %d to complete",
	pending_rsc_update);
	crmd_fsa_stall(FALSE);
	return;
	}
	controld_disconnect_cib_manager();
	}

	if (!pcmk_is_set(action, A_CIB_START)) {
	return;
	}

	if (cur_state == S_STOPPING) {
	crm_err("Ignoring request to connect to the CIB manager after "
	"shutdown");
	return;
	}

	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);

	if (rc != pcmk_ok) {
	// A short wait that usually avoids stalling the FSA
	sleep(1);
	rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
	cib_command_nonblocking);
	}

	if (rc != pcmk_ok) {
	crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));

	} else if (cib_conn->cmds->set_connection_dnotify(cib_conn,
	dnotify_fn) != pcmk_ok) {
	crm_err("Could not set dnotify callback");

	} else if (cib_conn->cmds->add_notify_callback(cib_conn,
	T_CIB_DIFF_NOTIFY,
	update_cb) != pcmk_ok) {
	crm_err("Could not set CIB notification callback (update)");

	} else {
	controld_set_fsa_input_flags(R_CIB_CONNECTED);
	cib_retries = 0;
	}

	if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
	cib_retries++;

	if (cib_retries < 30) {
	crm_warn("Couldn't complete CIB registration %d times... "
	"pause and retry", cib_retries);
	controld_start_wait_timer();
	crmd_fsa_stall(FALSE);

	} else {
	crm_err("Could not complete CIB registration %d times... "
	"hard error", cib_retries);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}
	}

	#define MIN_CIB_OP_TIMEOUT (30)

	/*!
	* \internal
	* \brief Get the timeout (in seconds) that should be used with CIB operations
	*
	* \return The maximum of 30 seconds, the value of the PCMK_cib_timeout
	* environment variable, or 10 seconds times one more than the number of
	* nodes in the cluster.
	*/
	unsigned int
	cib_op_timeout(void)
	{
	// @COMPAT: Drop env_timeout at 3.0.0
	static int env_timeout = -1;
	unsigned int calculated_timeout = 0;

	if (env_timeout == -1) {
	const char *env = pcmk__env_option(PCMK__ENV_CIB_TIMEOUT);

	pcmk__scan_min_int(env, &env_timeout, MIN_CIB_OP_TIMEOUT);
	crm_trace("Minimum CIB op timeout: %ds (environment: %s)",
	env_timeout, (env? env : "none"));
	}

	calculated_timeout = 1 + crm_active_peers();
	if (crm_remote_peer_cache) {
	calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
	}
	calculated_timeout *= 10;

	calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
	crm_trace("Calculated timeout: %us", calculated_timeout);

	if (controld_globals.cib_conn) {
	controld_globals.cib_conn->call_timeout = calculated_timeout;
	}
	return calculated_timeout;
	}

	/*!
	* \internal
	* \brief Get CIB call options to use local scope if primary is unavailable
	*
	* \return CIB call options
	*/
	int
	crmd_cib_smart_opt(void)
	{
	int call_opt = cib_none;

	if ((controld_globals.fsa_state == S_ELECTION)
	\|\| (controld_globals.fsa_state == S_PENDING)) {
	crm_info("Sending update to local CIB in state: %s",
	fsa_state2string(controld_globals.fsa_state));
	cib__set_call_options(call_opt, "update", cib_scope_local);
	}
	return call_opt;
	}

	static void
	cib_delete_callback(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	char *desc = user_data;

	if (rc == 0) {
	crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
	} else {
	crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
	desc, call_id, pcmk_strerror(rc), rc);
	}
	}

	-// Searches for various portions of node_state to delete
	+// Searches for various portions of PCMK__XE_NODE_STATE to delete

	-// Match a particular node's node_state (takes node name 1x)
	-#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE \
	- "[@" PCMK_XA_UNAME "='%s']"
	+// Match a particular node's PCMK__XE_NODE_STATE (takes node name 1x)
	+#define XPATH_NODE_STATE "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']"

	// Node's lrm section (name 1x)
	#define XPATH_NODE_LRM XPATH_NODE_STATE "/" XML_CIB_TAG_LRM

	/* Node's lrm_rsc_op entries and lrm_resource entries without unexpired lock
	* (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x)
	*/
	#define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP \
	"\|" XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE \
	"[not(@" PCMK_OPT_SHUTDOWN_LOCK ") " \
	"or " PCMK_OPT_SHUTDOWN_LOCK "<%lld]"

	// Node's transient_attributes section (name 1x)
	#define XPATH_NODE_ATTRS XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS

	-// Everything under node_state (name 1x)
	+// Everything under PCMK__XE_NODE_STATE (name 1x)
	#define XPATH_NODE_ALL XPATH_NODE_STATE "/*"

	/* Unlocked history + transient attributes
	* (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x, name 1x)
	*/
	#define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "\|" XPATH_NODE_ATTRS

	/*!
	* \internal
	* \brief Get the XPath and description of a node state section to be deleted
	*
	* \param[in] uname Desired node
	- * \param[in] section Subsection of node_state to be deleted
	+ * \param[in] section Subsection of \c PCMK__XE_NODE_STATE to be deleted
	* \param[out] xpath Where to store XPath of \p section
	* \param[out] desc If not \c NULL, where to store description of \p section
	*/
	void
	controld_node_state_deletion_strings(const char *uname,
	enum controld_section_e section,
	char xpath, char desc)
	{
	const char *desc_pre = NULL;

	// Shutdown locks that started before this time are expired
	long long expire = (long long) time(NULL)
	- controld_globals.shutdown_lock_limit;

	switch (section) {
	case controld_section_lrm:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
	desc_pre = "resource history";
	break;
	case controld_section_lrm_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED,
	uname, uname, expire);
	desc_pre = "resource history (other than shutdown locks)";
	break;
	case controld_section_attrs:
	*xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
	desc_pre = "transient attributes";
	break;
	case controld_section_all:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
	desc_pre = "all state";
	break;
	case controld_section_all_unlocked:
	*xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
	uname, uname, expire, uname);
	desc_pre = "all state (other than shutdown locks)";
	break;
	default:
	// We called this function incorrectly
	CRM_ASSERT(false);
	break;
	}

	if (desc != NULL) {
	*desc = crm_strdup_printf("%s for node %s", desc_pre, uname);
	}
	}

	/*!
	* \internal
	- * \brief Delete subsection of a node's CIB node_state
	+ * \brief Delete subsection of a node's CIB \c PCMK__XE_NODE_STATE
	*
	* \param[in] uname Desired node
	- * \param[in] section Subsection of node_state to delete
	+ * \param[in] section Subsection of \c PCMK__XE_NODE_STATE to delete
	* \param[in] options CIB call options to use
	*/
	void
	controld_delete_node_state(const char *uname, enum controld_section_e section,
	int options)
	{
	cib_t *cib = controld_globals.cib_conn;
	char *xpath = NULL;
	char *desc = NULL;
	int cib_rc = pcmk_ok;

	CRM_ASSERT((uname != NULL) && (cib != NULL));

	controld_node_state_deletion_strings(uname, section, &xpath, &desc);

	cib__set_call_options(options, "node state deletion",
	cib_xpath\|cib_multiple);
	cib_rc = cib->cmds->remove(cib, xpath, NULL, options);
	fsa_register_cib_callback(cib_rc, desc, cib_delete_callback);
	crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, cib_rc, xpath);

	// CIB library handles freeing desc
	free(xpath);
	}

	// Takes node name and resource ID
	-#define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE \
	+#define XPATH_RESOURCE_HISTORY "//" PCMK__XE_NODE_STATE \
	"[@" PCMK_XA_UNAME "='%s']/" \
	XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE \
	"[@" PCMK_XA_ID "='%s']"
	// @TODO could add "and @PCMK_OPT_SHUTDOWN_LOCK" to limit to locks

	/*!
	* \internal
	* \brief Clear resource history from CIB for a given resource and node
	*
	* \param[in] rsc_id ID of resource to be cleared
	* \param[in] node Node whose resource history should be cleared
	* \param[in] user_name ACL user name to use
	* \param[in] call_options CIB call options
	*
	* \return Standard Pacemaker return code
	*/
	int
	controld_delete_resource_history(const char rsc_id, const char node,
	const char *user_name, int call_options)
	{
	char *desc = NULL;
	char *xpath = NULL;
	int rc = pcmk_rc_ok;
	cib_t *cib = controld_globals.cib_conn;

	CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);

	desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
	if (cib == NULL) {
	crm_err("Unable to clear %s: no CIB connection", desc);
	free(desc);
	return ENOTCONN;
	}

	// Ask CIB to delete the entry
	xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);

	cib->cmds->set_user(cib, user_name);
	rc = cib->cmds->remove(cib, xpath, NULL, call_options\|cib_xpath);
	cib->cmds->set_user(cib, NULL);

	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_err("Could not delete resource status of %s on %s%s%s: %s "
	CRM_XS " rc=%d", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""),
	pcmk_rc_str(rc), rc);
	free(desc);
	free(xpath);
	return rc;
	}

	if (pcmk_is_set(call_options, cib_sync_call)) {
	if (pcmk_is_set(call_options, cib_dryrun)) {
	crm_debug("Deletion of %s would succeed", desc);
	} else {
	crm_debug("Deletion of %s succeeded", desc);
	}
	free(desc);

	} else {
	crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
	desc, rc, xpath);
	fsa_register_cib_callback(rc, desc, cib_delete_callback);
	// CIB library handles freeing desc
	}

	free(xpath);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Build XML and string of parameters meeting some criteria, for digest
	*
	* \param[in] op Executor event with parameter table to use
	* \param[in] metadata Parsed meta-data for executed resource agent
	* \param[in] param_type Flag used for selection criteria
	* \param[out] result Will be set to newly created XML with selected
	* parameters as attributes
	*
	* \return Newly allocated space-separated string of parameter names
	* \note Selection criteria varies by param_type: for the restart digest, we
	* want parameters that are not marked reloadable (OCF 1.1) or that
	* are marked unique (pre-1.1), for both string and XML results; for the
	* secure digest, we want parameters that are marked private for the
	* string, but parameters that are not marked private for the XML.
	* \note It is the caller's responsibility to free the string return value with
	* \p g_string_free() and the XML result with \p free_xml().
	*/
	static GString *
	build_parameter_list(const lrmd_event_data_t *op,
	const struct ra_metadata_s *metadata,
	enum ra_param_flags_e param_type, xmlNode **result)
	{
	GString *list = NULL;

	*result = create_xml_node(NULL, PCMK_XE_PARAMETERS);

	/* Consider all parameters only except private ones to be consistent with
	* what scheduler does with calculate_secure_digest().
	*/
	if (param_type == ra_param_private
	&& compare_version(controld_globals.dc_version, "3.16.0") >= 0) {
	g_hash_table_foreach(op->params, hash2field, *result);
	pcmk__filter_op_for_digest(*result);
	}

	for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
	struct ra_param_s param = (struct ra_param_s ) iter->data;

	bool accept_for_list = false;
	bool accept_for_xml = false;

	switch (param_type) {
	case ra_param_reloadable:
	accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_unique:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = accept_for_list;
	break;

	case ra_param_private:
	accept_for_list = pcmk_is_set(param->rap_flags, param_type);
	accept_for_xml = !accept_for_list;
	break;
	}

	if (accept_for_list) {
	crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));

	if (list == NULL) {
	// We will later search for " WORD ", so start list with a space
	pcmk__add_word(&list, 256, " ");
	}
	pcmk__add_word(&list, 0, param->rap_name);

	} else {
	crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
	}

	if (accept_for_xml) {
	const char *v = g_hash_table_lookup(op->params, param->rap_name);

	if (v != NULL) {
	crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
	crm_xml_add(*result, param->rap_name, v);
	}

	} else {
	crm_trace("Removing attr %s from the xml result", param->rap_name);
	xml_remove_prop(*result, param->rap_name);
	}
	}

	if (list != NULL) {
	// We will later search for " WORD ", so end list with a space
	pcmk__add_word(&list, 0, " ");
	}
	return list;
	}

	static void
	append_restart_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *restart = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	if (op->interval_ms > 0) {
	/* monitors are not reloadable */
	return;
	}

	if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
	/* Add parameters not marked reloadable to the PCMK__XA_OP_FORCE_RESTART
	* list
	*/
	list = build_parameter_list(op, metadata, ra_param_reloadable,
	&restart);

	} else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
	/* @COMPAT pre-OCF-1.1 resource agents
	*
	* Before OCF 1.1, Pacemaker abused "unique=0" to indicate
	* reloadability. Add any parameters with unique="1" to the
	* PCMK__XA_OP_FORCE_RESTART list.
	*/
	list = build_parameter_list(op, metadata, ra_param_unique, &restart);

	} else {
	// Resource does not support agent reloads
	return;
	}

	digest = calculate_operation_digest(restart, version);
	/* Add PCMK__XA_OP_FORCE_RESTART and PCMK__XA_OP_RESTART_DIGEST to indicate
	* the resource supports reload, no matter if it actually supports any
	* reloadable parameters
	*/
	crm_xml_add(update, PCMK__XA_OP_FORCE_RESTART,
	(list == NULL)? "" : (const char *) list->str);
	crm_xml_add(update, PCMK__XA_OP_RESTART_DIGEST, digest);

	if ((list != NULL) && (list->len > 0)) {
	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	} else {
	crm_trace("%s: %s", op->rsc_id, digest);
	}

	if (list != NULL) {
	g_string_free(list, TRUE);
	}
	free_xml(restart);
	free(digest);
	}

	static void
	append_secure_list(lrmd_event_data_t op, struct ra_metadata_s metadata,
	xmlNode update, const char version)
	{
	GString *list = NULL;
	char *digest = NULL;
	xmlNode *secure = NULL;

	CRM_LOG_ASSERT(op->params != NULL);

	/* To keep PCMK__XA_OP_SECURE_PARAMS short, we want it to contain the secure
	* parameters but PCMK__XA_OP_SECURE_DIGEST to be based on the insecure ones
	*/
	list = build_parameter_list(op, metadata, ra_param_private, &secure);

	if (list != NULL) {
	digest = calculate_operation_digest(secure, version);
	crm_xml_add(update, PCMK__XA_OP_SECURE_PARAMS,
	(const char *) list->str);
	crm_xml_add(update, PCMK__XA_OP_SECURE_DIGEST, digest);

	crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
	g_string_free(list, TRUE);
	} else {
	crm_trace("%s: no secure parameters", op->rsc_id);
	}

	free_xml(secure);
	free(digest);
	}

	/*!
	* \internal
	* \brief Create XML for a resource history entry
	*
	* \param[in] func Function name of caller
	* \param[in,out] parent XML to add entry to
	* \param[in] rsc Affected resource
	* \param[in,out] op Action to add an entry for (or NULL to do nothing)
	* \param[in] node_name Node where action occurred
	*/
	void
	controld_add_resource_history_xml_as(const char func, xmlNode parent,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op,
	const char *node_name)
	{
	int target_rc = 0;
	xmlNode *xml_op = NULL;
	struct ra_metadata_s *metadata = NULL;
	const char *caller_version = NULL;
	lrm_state_t *lrm_state = NULL;

	if (op == NULL) {
	return;
	}

	target_rc = rsc_op_expected_rc(op);

	caller_version = g_hash_table_lookup(op->params, PCMK_XA_CRM_FEATURE_SET);
	CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);

	xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
	controld_globals.our_nodename, func);
	if (xml_op == NULL) {
	return;
	}

	if ((rsc == NULL) \|\| (op->params == NULL)
	\|\| !crm_op_needs_metadata(rsc->standard, op->op_type)) {

	crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
	op->op_type, op->rsc_id, op->params, rsc);
	return;
	}

	lrm_state = lrm_state_find(node_name);
	if (lrm_state == NULL) {
	crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
	" because we have no connection to executor for %s",
	op->rsc_id, op->op_type, op->interval_ms, node_name);
	return;
	}

	/* Ideally the metadata is cached, and the agent is just a fallback.
	*
	* @TODO Go through all callers and ensure they get metadata asynchronously
	* first.
	*/
	metadata = controld_get_rsc_metadata(lrm_state, rsc,
	controld_metadata_from_agent
	\|controld_metadata_from_cache);
	if (metadata == NULL) {
	return;
	}

	crm_trace("Including additional digests for %s:%s:%s",
	rsc->standard, rsc->provider, rsc->type);
	append_restart_list(op, metadata, xml_op, caller_version);
	append_secure_list(op, metadata, xml_op, caller_version);

	return;
	}

	/*!
	* \internal
	* \brief Record an action as pending in the CIB, if appropriate
	*
	* \param[in] node_name Node where the action is pending
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Pending action
	*
	* \return true if action was recorded in CIB, otherwise false
	*/
	bool
	controld_record_pending_op(const char node_name, const lrmd_rsc_info_t rsc,
	lrmd_event_data_t *op)
	{
	const char *record_pending = NULL;

	CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL),
	return false);

	// Never record certain operation types as pending
	if ((op->op_type == NULL) \|\| (op->params == NULL)
	\|\| !controld_action_is_recordable(op->op_type)) {
	return false;
	}

	// Check action's PCMK_META_RECORD_PENDING meta-attribute (defaults to true)
	record_pending = crm_meta_value(op->params, PCMK_META_RECORD_PENDING);
	if ((record_pending != NULL) && !crm_is_true(record_pending)) {
	return false;
	}

	op->call_id = -1;
	op->t_run = time(NULL);
	op->t_rcchange = op->t_run;

	lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);

	crm_debug("Recording pending %s-interval %s for %s on %s in the CIB",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	node_name);
	controld_update_resource_history(node_name, rsc, op, 0);
	return true;
	}

	static void
	cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_diff_failed:
	case -pcmk_err_diff_resync:
	crm_trace("Resource history update completed (call=%d rc=%d)",
	call_id, rc);
	break;
	default:
	if (call_id > 0) {
	crm_warn("Resource history update %d failed: %s "
	CRM_XS " rc=%d", call_id, pcmk_strerror(rc), rc);
	} else {
	crm_warn("Resource history update failed: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	}
	}

	if (call_id == pending_rsc_update) {
	pending_rsc_update = 0;
	controld_trigger_fsa();
	}
	}

	/* Only successful stops, and probes that found the resource inactive, get locks
	* recorded in the history. This ensures the resource stays locked to the node
	* until it is active there again after the node comes back up.
	*/
	static bool
	should_preserve_lock(lrmd_event_data_t *op)
	{
	if (!pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
	return false;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_STOP) && (op->rc == PCMK_OCF_OK)) {
	return true;
	}
	if (!strcmp(op->op_type, PCMK_ACTION_MONITOR)
	&& (op->rc == PCMK_OCF_NOT_RUNNING)) {
	return true;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Request a CIB update
	*
	* \param[in] section Section of CIB to update
	* \param[in] data New XML of CIB section to update
	* \param[in] options CIB call options
	* \param[in] callback If not \c NULL, set this as the operation callback
	*
	* \return Standard Pacemaker return code
	*
	* \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is
	* stored in \p pending_rsc_update on success.
	*/
	int
	controld_update_cib(const char section, xmlNode data, int options,
	void (callback)(xmlNode , int, int, xmlNode , void ))
	{
	cib_t *cib = controld_globals.cib_conn;
	int cib_rc = -ENOTCONN;

	CRM_ASSERT(data != NULL);

	if (cib != NULL) {
	cib_rc = cib->cmds->modify(cib, section, data, options);
	if (cib_rc >= 0) {
	crm_debug("Submitted CIB update %d for %s section",
	cib_rc, section);
	}
	}

	if (callback == NULL) {
	if (cib_rc < 0) {
	crm_err("Failed to update CIB %s section: %s",
	section, pcmk_rc_str(pcmk_legacy2rc(cib_rc)));
	}

	} else {
	if ((cib_rc >= 0) && (callback == cib_rsc_callback)) {
	/* Checking for a particular callback is a little hacky, but it
	* didn't seem worth adding an output argument for cib_rc for just
	* one use case.
	*/
	pending_rsc_update = cib_rc;
	}
	fsa_register_cib_callback(cib_rc, NULL, callback);
	}

	return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc);
	}

	/*!
	* \internal
	* \brief Update resource history entry in CIB
	*
	* \param[in] node_name Node where action occurred
	* \param[in] rsc Resource that action is for
	* \param[in,out] op Action to record
	* \param[in] lock_time If nonzero, when resource was locked to node
	*
	* \note On success, the CIB update's call ID will be stored in
	* pending_rsc_update.
	*/
	void
	controld_update_resource_history(const char *node_name,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op, time_t lock_time)
	{
	xmlNode *update = NULL;
	xmlNode *xml = NULL;
	int call_opt = crmd_cib_smart_opt();
	const char *node_id = NULL;
	const char *container = NULL;

	CRM_CHECK((node_name != NULL) && (op != NULL), return);

	if (rsc == NULL) {
	crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
	controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
	return;
	}

	// <status>
	update = create_xml_node(NULL, PCMK_XE_STATUS);

	// <node_state ...>
	- xml = create_xml_node(update, XML_CIB_TAG_STATE);
	+ xml = create_xml_node(update, PCMK__XE_NODE_STATE);
	if (pcmk__str_eq(node_name, controld_globals.our_nodename,
	pcmk__str_casei)) {
	node_id = controld_globals.our_uuid;
	} else {
	node_id = node_name;
	pcmk__xe_set_bool_attr(xml, XML_NODE_IS_REMOTE, true);
	}
	crm_xml_add(xml, PCMK_XA_ID, node_id);
	crm_xml_add(xml, PCMK_XA_UNAME, node_name);
	crm_xml_add(xml, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);

	// <lrm ...>
	xml = create_xml_node(xml, XML_CIB_TAG_LRM);
	crm_xml_add(xml, PCMK_XA_ID, node_id);

	// <lrm_resources>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCES);

	// <lrm_resource ...>
	xml = create_xml_node(xml, XML_LRM_TAG_RESOURCE);
	crm_xml_add(xml, PCMK_XA_ID, op->rsc_id);
	crm_xml_add(xml, PCMK_XA_CLASS, rsc->standard);
	crm_xml_add(xml, PCMK_XA_PROVIDER, rsc->provider);
	crm_xml_add(xml, PCMK_XA_TYPE, rsc->type);
	if (lock_time != 0) {
	/* Actions on a locked resource should either preserve the lock by
	* recording it with the action result, or clear it.
	*/
	if (!should_preserve_lock(op)) {
	lock_time = 0;
	}
	crm_xml_add_ll(xml, PCMK_OPT_SHUTDOWN_LOCK, (long long) lock_time);
	}
	if (op->params != NULL) {
	container = g_hash_table_lookup(op->params,
	CRM_META "_" PCMK__META_CONTAINER);
	if (container != NULL) {
	crm_trace("Resource %s is a part of container resource %s",
	op->rsc_id, container);
	crm_xml_add(xml, PCMK__META_CONTAINER, container);
	}
	}

	// <lrm_resource_op ...> (possibly more than one)
	controld_add_resource_history_xml(xml, rsc, op, node_name);

	/* Update CIB asynchronously. Even if it fails, the resource state should be
	* discovered during the next election. Worst case, the node is wrongly
	* fenced for running a resource it isn't.
	*/
	crm_log_xml_trace(update, __func__);
	controld_update_cib(PCMK_XE_STATUS, update, call_opt, cib_rsc_callback);
	free_xml(update);
	}

	/*!
	* \internal
	* \brief Erase an LRM history entry from the CIB, given the operation data
	*
	* \param[in] op Operation whose history should be deleted
	*/
	void
	controld_delete_action_history(const lrmd_event_data_t *op)
	{
	xmlNode *xml_top = NULL;

	CRM_CHECK(op != NULL, return);

	xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
	crm_xml_add_int(xml_top, PCMK__XA_CALL_ID, op->call_id);
	crm_xml_add(xml_top, PCMK__XA_TRANSITION_KEY, op->user_data);

	if (op->interval_ms > 0) {
	char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);

	/* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
	crm_xml_add(xml_top, PCMK_XA_ID, op_id);
	free(op_id);
	}

	crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
	op->rsc_id, op->op_type, op->interval_ms, op->call_id);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn,
	PCMK_XE_STATUS, xml_top, cib_none);
	crm_log_xml_trace(xml_top, "op:cancel");
	free_xml(xml_top);
	}

	/* Define xpath to find LRM resource history entry by node and resource */
	#define XPATH_HISTORY \
	"/" PCMK_XE_CIB "/" PCMK_XE_STATUS \
	- "/" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']" \
	+ "/" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']" \
	"/" XML_LRM_TAG_RSC_OP

	/* ... and also by operation key */
	#define XPATH_HISTORY_ID XPATH_HISTORY "[@" PCMK_XA_ID "='%s']"

	/* ... and also by operation key and operation call ID */
	#define XPATH_HISTORY_CALL XPATH_HISTORY \
	"[@" PCMK_XA_ID "='%s' and @" PCMK__XA_CALL_ID "='%d']"

	/* ... and also by operation key and original operation key */
	#define XPATH_HISTORY_ORIG XPATH_HISTORY \
	"[@" PCMK_XA_ID "='%s' and @" PCMK__XA_OPERATION_KEY "='%s']"

	/*!
	* \internal
	* \brief Delete a last_failure resource history entry from the CIB
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] action If specified, delete only if this was failed action
	* \param[in] interval_ms If \p action is specified, it has this interval
	*/
	void
	controld_cib_delete_last_failure(const char rsc_id, const char node,
	const char *action, guint interval_ms)
	{
	char *xpath = NULL;
	char *last_failure_key = NULL;
	CRM_CHECK((rsc_id != NULL) && (node != NULL), return);

	// Generate XPath to match desired entry
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
	if (action == NULL) {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id,
	last_failure_key);
	} else {
	char *action_key = pcmk__op_key(rsc_id, action, interval_ms);

	xpath = crm_strdup_printf(XPATH_HISTORY_ORIG, node, rsc_id,
	last_failure_key, action_key);
	free(action_key);
	}
	free(last_failure_key);

	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}

	/*!
	* \internal
	* \brief Delete resource history entry from the CIB, given operation key
	*
	* \param[in] rsc_id Name of resource to clear history for
	* \param[in] node Name of node to clear history for
	* \param[in] key Operation key of operation to clear history for
	* \param[in] call_id If specified, delete entry only if it has this call ID
	*/
	void
	controld_delete_action_history_by_key(const char rsc_id, const char node,
	const char *key, int call_id)
	{
	char *xpath = NULL;

	CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return);

	if (call_id > 0) {
	xpath = crm_strdup_printf(XPATH_HISTORY_CALL, node, rsc_id, key,
	call_id);
	} else {
	xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id, key);
	}
	controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
	NULL, cib_xpath);
	free(xpath);
	}
	diff --git a/daemons/controld/controld_cib.h b/daemons/controld/controld_cib.h
	index dcc5a48db0..5835391a76 100644
	--- a/daemons/controld/controld_cib.h
	+++ b/daemons/controld/controld_cib.h
	@@ -1,123 +1,123 @@
	/*
	- * Copyright 2004-2023 the Pacemaker project contributors
	+ * Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CONTROLD_CIB__H
	#define PCMK__CONTROLD_CIB__H

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h> // PCMK__CIB_REQUEST_MODIFY
	#include "controld_globals.h" // controld_globals.cib_conn

	static inline void
	fsa_cib_anon_update(const char section, xmlNode data) {
	if (controld_globals.cib_conn == NULL) {
	crm_err("No CIB connection available");
	} else {
	controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn,
	section, data,
	cib_scope_local\|cib_can_create);
	}
	}

	static inline void
	fsa_cib_anon_update_discard_reply(const char section, xmlNode data) {
	if (controld_globals.cib_conn == NULL) {
	crm_err("No CIB connection available");
	} else {
	controld_globals.cib_conn->cmds->modify(controld_globals.cib_conn,
	section, data,
	cib_scope_local
	\|cib_can_create
	\|cib_discard_reply);
	}
	}

	int controld_update_cib(const char section, xmlNode data, int options,
	void (callback)(xmlNode , int, int, xmlNode *,
	void *));
	unsigned int cib_op_timeout(void);

	-// Subsections of node_state
	+// Subsections of PCMK__XE_NODE_STATE
	enum controld_section_e {
	controld_section_lrm,
	controld_section_lrm_unlocked,
	controld_section_attrs,
	controld_section_all,
	controld_section_all_unlocked
	};

	void controld_node_state_deletion_strings(const char *uname,
	enum controld_section_e section,
	char xpath, char desc);
	void controld_delete_node_state(const char *uname,
	enum controld_section_e section, int options);
	int controld_delete_resource_history(const char rsc_id, const char node,
	const char *user_name, int call_options);

	/* Convenience macro for registering a CIB callback
	* (assumes that data can be freed with free())
	*/
	# define fsa_register_cib_callback(id, data, fn) do { \
	cib_t *cib_conn = controld_globals.cib_conn; \
	\
	CRM_ASSERT(cib_conn != NULL); \
	cib_conn->cmds->register_callback_full(cib_conn, id, cib_op_timeout(), \
	FALSE, data, #fn, fn, free); \
	} while(0)

	void controld_add_resource_history_xml_as(const char func, xmlNode parent,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op,
	const char *node_name);

	#define controld_add_resource_history_xml(parent, rsc, op, node_name) \
	controld_add_resource_history_xml_as(__func__, (parent), (rsc), \
	(op), (node_name))

	bool controld_record_pending_op(const char *node_name,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op);

	void controld_update_resource_history(const char *node_name,
	const lrmd_rsc_info_t *rsc,
	lrmd_event_data_t *op, time_t lock_time);

	void controld_delete_action_history(const lrmd_event_data_t *op);

	void controld_cib_delete_last_failure(const char rsc_id, const char node,
	const char *action, guint interval_ms);

	void controld_delete_action_history_by_key(const char rsc_id, const char node,
	const char *key, int call_id);

	void controld_disconnect_cib_manager(void);

	int crmd_cib_smart_opt(void);

	/*!
	* \internal
	* \brief Check whether an action type should be recorded in the CIB
	*
	* \param[in] action Action type
	*
	* \return true if action should be recorded, false otherwise
	*/
	static inline bool
	controld_action_is_recordable(const char *action)
	{
	return !pcmk__str_any_of(action, PCMK_ACTION_CANCEL, PCMK_ACTION_DELETE,
	PCMK_ACTION_NOTIFY, PCMK_ACTION_META_DATA, NULL);
	}

	#endif // PCMK__CONTROLD_CIB__H
	diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c
	index d87adbfcc1..1e1fcc7ab8 100644
	--- a/daemons/controld/controld_join_client.c
	+++ b/daemons/controld/controld_join_client.c
	@@ -1,363 +1,363 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);

	extern ha_msg_input_t copy_ha_msg_input(ha_msg_input_t orig);

	/*!
	* \internal
	* \brief Remember if DC is shutting down as we join
	*
	* If we're joining while the current DC is shutting down, update its expected
	* state, so we don't fence it if we become the new DC. (We weren't a peer
	* when it broadcast its shutdown request.)
	*
	* \param[in] msg A join message from the DC
	*/
	static void
	update_dc_expected(const xmlNode *msg)
	{
	if ((controld_globals.dc_name != NULL)
	&& pcmk__xe_attr_is_true(msg, PCMK__XA_DC_LEAVING)) {
	crm_node_t *dc_node = pcmk__get_node(0, controld_globals.dc_name, NULL,
	pcmk__node_search_cluster);

	pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_DOWN);
	}
	}

	/* A_CL_JOIN_QUERY */
	/* is there a DC out there? */
	void
	do_cl_join_query(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
	CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);

	sleep(1); // Give the cluster layer time to propagate to the DC
	update_dc(NULL); /* Unset any existing value so that the result is not discarded */
	crm_debug("Querying for a DC");
	send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
	free_xml(req);
	}

	/* A_CL_JOIN_ANNOUNCE */

	/* this is kind of a workaround for the fact that we may not be around or
	* are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
	*/
	void
	do_cl_join_announce(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* don't announce if we're in one of these states */
	if (cur_state != S_PENDING) {
	crm_warn("Not announcing cluster join because in state %s",
	fsa_state2string(cur_state));
	return;
	}

	if (!pcmk_is_set(controld_globals.fsa_input_register, R_STARTING)) {
	/* send as a broadcast */
	xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
	CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);

	crm_debug("Announcing availability");
	update_dc(NULL);
	send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
	free_xml(req);

	} else {
	/* Delay announce until we have finished local startup */
	crm_warn("Delaying announce of cluster join until local startup is complete");
	return;
	}
	}

	static int query_call_id = 0;

	/* A_CL_JOIN_REQUEST */
	/* aka. accept the welcome offer */
	void
	do_cl_join_offer_respond(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *welcome_from;
	const char *join_id;

	CRM_CHECK(input != NULL, return);

	welcome_from = crm_element_value(input->msg, PCMK__XA_SRC);
	join_id = crm_element_value(input->msg, PCMK__XA_JOIN_ID);
	crm_trace("Accepting cluster join offer from node %s "CRM_XS" join-%s",
	welcome_from, crm_element_value(input->msg, PCMK__XA_JOIN_ID));

	/* we only ever want the last one */
	if (query_call_id > 0) {
	crm_trace("Cancelling previous join query: %d", query_call_id);
	remove_cib_op_callback(query_call_id, FALSE);
	query_call_id = 0;
	}

	if (update_dc(input->msg) == FALSE) {
	crm_warn("Discarding cluster join offer from node %s (expected %s)",
	welcome_from, controld_globals.dc_name);
	return;
	}

	update_dc_expected(input->msg);

	query_call_id = cib_conn->cmds->query(cib_conn, NULL, NULL,
	cib_scope_local\|cib_no_children);
	fsa_register_cib_callback(query_call_id, strdup(join_id),
	join_query_callback);
	crm_trace("Registered join query callback: %d", query_call_id);

	controld_set_fsa_action_flags(A_DC_TIMER_STOP);
	controld_trigger_fsa();
	}

	void
	join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	char *join_id = user_data;
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	CRM_LOG_ASSERT(join_id != NULL);

	if (query_call_id != call_id) {
	crm_trace("Query %d superseded", call_id);
	goto done;
	}

	query_call_id = 0;
	if(rc != pcmk_ok \|\| output == NULL) {
	crm_err("Could not retrieve version details for join-%s: %s (%d)",
	join_id, pcmk_strerror(rc), rc);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);

	} else if (controld_globals.dc_name == NULL) {
	crm_debug("Membership is in flux, not continuing join-%s", join_id);

	} else {
	xmlNode *reply = NULL;

	crm_debug("Respond to join offer join-%s from %s",
	join_id, controld_globals.dc_name);
	copy_in_properties(generation, output);

	reply = create_request(CRM_OP_JOIN_REQUEST, generation,
	controld_globals.dc_name, CRM_SYSTEM_DC,
	CRM_SYSTEM_CRMD, NULL);

	crm_xml_add(reply, PCMK__XA_JOIN_ID, join_id);
	crm_xml_add(reply, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
	send_cluster_message(pcmk__get_node(0, controld_globals.dc_name, NULL,
	pcmk__node_search_cluster),
	crm_msg_crmd, reply, TRUE);
	free_xml(reply);
	}

	done:
	free_xml(generation);
	}

	void
	set_join_state(const char start_state, const char node_name, const char *node_uuid,
	bool remote)
	{
	if (pcmk__str_eq(start_state, "standby", pcmk__str_casei)) {
	crm_notice("Forcing node %s to join in %s state per configured "
	"environment", node_name, start_state);
	cib__update_node_attr(controld_globals.logger_out,
	controld_globals.cib_conn, cib_sync_call,
	PCMK_XE_NODES, node_uuid,
	NULL, NULL, NULL, "standby", "on", NULL,
	remote ? "remote" : NULL);

	} else if (pcmk__str_eq(start_state, "online", pcmk__str_casei)) {
	crm_notice("Forcing node %s to join in %s state per configured "
	"environment", node_name, start_state);
	cib__update_node_attr(controld_globals.logger_out,
	controld_globals.cib_conn, cib_sync_call,
	PCMK_XE_NODES, node_uuid,
	NULL, NULL, NULL, "standby", "off", NULL,
	remote ? "remote" : NULL);

	} else if (pcmk__str_eq(start_state, "default", pcmk__str_casei)) {
	crm_debug("Not forcing a starting state on node %s", node_name);

	} else {
	crm_warn("Unrecognized start state '%s', using 'default' (%s)",
	start_state, node_name);
	}
	}

	static int
	update_conn_host_cache(xmlNode node, void userdata)
	{
	const char *remote = crm_element_value(node, PCMK_XA_ID);
	const char *conn_host = crm_element_value(node, PCMK__XA_CONN_HOST);
	- const char *state = crm_element_value(node, XML_CIB_TAG_STATE);
	+ const char *state = crm_element_value(node, "node_state");

	crm_node_t *remote_peer = crm_remote_peer_get(remote);

	if (remote_peer == NULL) {
	return pcmk_rc_ok;
	}

	if (conn_host != NULL) {
	pcmk__str_update(&remote_peer->conn_host, conn_host);
	}

	if (state != NULL) {
	pcmk__update_peer_state(__func__, remote_peer, state, 0);
	}

	return pcmk_rc_ok;
	}

	/* A_CL_JOIN_RESULT */
	/* aka. this is notification that we have (or have not) been accepted */
	void
	do_cl_join_finalize_respond(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *tmp1 = NULL;
	gboolean was_nack = TRUE;
	static gboolean first_join = TRUE;
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
	const char *start_state = pcmk__env_option(PCMK__ENV_NODE_START_STATE);

	int join_id = -1;
	const char *op = crm_element_value(input->msg, PCMK__XA_CRM_TASK);
	const char *welcome_from = crm_element_value(input->msg, PCMK__XA_SRC);

	if (!pcmk__str_eq(op, CRM_OP_JOIN_ACKNAK, pcmk__str_casei)) {
	crm_trace("Ignoring op=%s message", op);
	return;
	}

	/* calculate if it was an ack or a nack */
	if (pcmk__xe_attr_is_true(input->msg, CRM_OP_JOIN_ACKNAK)) {
	was_nack = FALSE;
	}

	crm_element_value_int(input->msg, PCMK__XA_JOIN_ID, &join_id);

	if (was_nack) {
	crm_err("Shutting down because cluster join with leader %s failed "
	CRM_XS" join-%d NACK'd", welcome_from, join_id);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	controld_set_fsa_input_flags(R_STAYDOWN);
	return;
	}

	if (!AM_I_DC
	&& pcmk__str_eq(welcome_from, controld_globals.our_nodename,
	pcmk__str_casei)) {
	crm_warn("Discarding our own welcome - we're no longer the DC");
	return;
	}

	if (update_dc(input->msg) == FALSE) {
	crm_warn("Discarding %s from node %s (expected from %s)",
	op, welcome_from, controld_globals.dc_name);
	return;
	}

	update_dc_expected(input->msg);

	/* record the node's feature set as a transient attribute */
	update_attrd(controld_globals.our_nodename, CRM_ATTR_FEATURE_SET,
	CRM_FEATURE_SET, NULL, FALSE);

	/* send our status section to the DC */
	tmp1 = controld_query_executor_state();
	if (tmp1 != NULL) {
	xmlNode *remotes = NULL;
	xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1,
	controld_globals.dc_name, CRM_SYSTEM_DC,
	CRM_SYSTEM_CRMD, NULL);

	crm_xml_add_int(reply, PCMK__XA_JOIN_ID, join_id);

	crm_debug("Confirming join-%d: sending local operation history to %s",
	join_id, controld_globals.dc_name);

	/*
	* If this is the node's first join since the controller started on it,
	* set its initial state (standby or member) according to the user's
	* preference.
	*
	* We do not clear the LRM history here. Even if the DC failed to do it
	* when we last left, removing them here creates a race condition if the
	* controller is being recovered. Instead of a list of active resources
	* from the executor, we may end up with a blank status section. If we
	* are _NOT_ lucky, we will probe for the "wrong" instance of anonymous
	* clones and end up with multiple active instances on the machine.
	*/
	if (first_join
	&& !pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {

	first_join = FALSE;
	if (start_state) {
	set_join_state(start_state, controld_globals.our_nodename,
	controld_globals.our_uuid, false);
	}
	}

	send_cluster_message(pcmk__get_node(0, controld_globals.dc_name, NULL,
	pcmk__node_search_cluster),
	crm_msg_crmd, reply, TRUE);
	free_xml(reply);

	if (AM_I_DC == FALSE) {
	register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE,
	__func__);
	}

	free_xml(tmp1);

	/* Update the remote node cache with information about which node
	* is hosting the connection.
	*/
	remotes = pcmk__xe_match(input->msg, PCMK_XE_NODES, NULL, NULL);
	if (remotes != NULL) {
	pcmk__xe_foreach_child(remotes, XML_CIB_TAG_NODE, update_conn_host_cache, NULL);
	}

	} else {
	crm_err("Could not confirm join-%d with %s: Local operation history "
	"failed", join_id, controld_globals.dc_name);
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}
	diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
	index ecbd25011f..bfdcfcb1de 100644
	--- a/daemons/controld/controld_join_dc.c
	+++ b/daemons/controld/controld_join_dc.c
	@@ -1,1035 +1,1035 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/crm.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>

	#include <pacemaker-controld.h>

	static char *max_generation_from = NULL;
	static xmlNodePtr max_generation_xml = NULL;

	/*!
	* \internal
	* \brief Nodes from which a CIB sync has failed since the peer joined
	*
	* This table is of the form (<tt>node_name -> join_id</tt>). \p node_name is
	* the name of a client node from which a CIB \p sync_from() call has failed in
	* \p do_dc_join_finalize() since the client joined the cluster as a peer.
	* \p join_id is the ID of the join round in which the \p sync_from() failed,
	* and is intended for use in nack log messages.
	*/
	static GHashTable *failed_sync_nodes = NULL;

	void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
	void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
	gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	/* Numeric counter used to identify join rounds (an unsigned int would be
	* appropriate, except we get and set it in XML as int)
	*/
	static int current_join_id = 0;

	/*!
	* \internal
	* \brief Destroy the hash table containing failed sync nodes
	*/
	void
	controld_destroy_failed_sync_table(void)
	{
	if (failed_sync_nodes != NULL) {
	g_hash_table_destroy(failed_sync_nodes);
	failed_sync_nodes = NULL;
	}
	}

	/*!
	* \internal
	* \brief Remove a node from the failed sync nodes table if present
	*
	* \param[in] node_name Node name to remove
	*/
	void
	controld_remove_failed_sync_node(const char *node_name)
	{
	if (failed_sync_nodes != NULL) {
	g_hash_table_remove(failed_sync_nodes, (gchar *) node_name);
	}
	}

	/*!
	* \internal
	* \brief Add to a hash table a node whose CIB failed to sync
	*
	* \param[in] node_name Name of node whose CIB failed to sync
	* \param[in] join_id Join round when the failure occurred
	*/
	static void
	record_failed_sync_node(const char *node_name, gint join_id)
	{
	if (failed_sync_nodes == NULL) {
	failed_sync_nodes = pcmk__strikey_table(g_free, NULL);
	}

	/* If the node is already in the table then we failed to nack it during the
	* filter offer step
	*/
	CRM_LOG_ASSERT(g_hash_table_insert(failed_sync_nodes, g_strdup(node_name),
	GINT_TO_POINTER(join_id)));
	}

	/*!
	* \internal
	* \brief Look up a node name in the failed sync table
	*
	* \param[in] node_name Name of node to look up
	* \param[out] join_id Where to store the join ID of when the sync failed
	*
	* \return Standard Pacemaker return code. Specifically, \p pcmk_rc_ok if the
	* node name was found, or \p pcmk_rc_node_unknown otherwise.
	* \note \p *join_id is set to -1 if the node is not found.
	*/
	static int
	lookup_failed_sync_node(const char node_name, gint join_id)
	{
	*join_id = -1;

	if (failed_sync_nodes != NULL) {
	gpointer result = g_hash_table_lookup(failed_sync_nodes,
	(gchar *) node_name);
	if (result != NULL) {
	*join_id = GPOINTER_TO_INT(result);
	return pcmk_rc_ok;
	}
	}
	return pcmk_rc_node_unknown;
	}

	void
	crm_update_peer_join(const char source, crm_node_t node, enum crm_join_phase phase)
	{
	enum crm_join_phase last = 0;

	CRM_CHECK(node != NULL, return);

	/* Remote nodes do not participate in joins */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return;
	}

	last = node->join;

	if(phase == last) {
	crm_trace("Node %s join-%d phase is still %s "
	CRM_XS " nodeid=%u source=%s",
	node->uname, current_join_id, crm_join_phase_str(last),
	node->id, source);

	} else if ((phase <= crm_join_none) \|\| (phase == (last + 1))) {
	node->join = phase;
	crm_trace("Node %s join-%d phase is now %s (was %s) "
	CRM_XS " nodeid=%u source=%s",
	node->uname, current_join_id, crm_join_phase_str(phase),
	crm_join_phase_str(last), node->id, source);

	} else {
	crm_warn("Rejecting join-%d phase update for node %s because "
	"can't go from %s to %s " CRM_XS " nodeid=%u source=%s",
	current_join_id, node->uname, crm_join_phase_str(last),
	crm_join_phase_str(phase), node->id, source);
	}
	}

	static void
	start_join_round(void)
	{
	GHashTableIter iter;
	crm_node_t *peer = NULL;

	crm_debug("Starting new join round join-%d", current_join_id);

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	crm_update_peer_join(__func__, peer, crm_join_none);
	}
	if (max_generation_from != NULL) {
	free(max_generation_from);
	max_generation_from = NULL;
	}
	if (max_generation_xml != NULL) {
	free_xml(max_generation_xml);
	max_generation_xml = NULL;
	}
	controld_clear_fsa_input_flags(R_HAVE_CIB);
	}

	/*!
	* \internal
	* \brief Create a join message from the DC
	*
	* \param[in] join_op Join operation name
	* \param[in] host_to Recipient of message
	*/
	static xmlNode *
	create_dc_message(const char join_op, const char host_to)
	{
	xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
	CRM_SYSTEM_DC, NULL);

	/* Identify which election this is a part of */
	crm_xml_add_int(msg, PCMK__XA_JOIN_ID, current_join_id);

	/* Add a field specifying whether the DC is shutting down. This keeps the
	* joining node from fencing the old DC if it becomes the new DC.
	*/
	pcmk__xe_set_bool_attr(msg, PCMK__XA_DC_LEAVING,
	pcmk_is_set(controld_globals.fsa_input_register,
	R_SHUTDOWN));
	return msg;
	}

	static void
	join_make_offer(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *offer = NULL;
	crm_node_t member = (crm_node_t )value;

	CRM_ASSERT(member != NULL);
	if (crm_is_peer_active(member) == FALSE) {
	crm_info("Not making join-%d offer to inactive node %s",
	current_join_id,
	(member->uname? member->uname : "with unknown name"));
	if(member->expected == NULL && pcmk__str_eq(member->state, CRM_NODE_LOST, pcmk__str_casei)) {
	/* You would think this unsafe, but in fact this plus an
	* active resource is what causes it to be fenced.
	*
	* Yes, this does mean that any node that dies at the same
	* time as the old DC and is not running resource (still)
	* won't be fenced.
	*
	* I'm not happy about this either.
	*/
	pcmk__update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
	}
	return;
	}

	if (member->uname == NULL) {
	crm_info("Not making join-%d offer to node uuid %s with unknown name",
	current_join_id, member->uuid);
	return;
	}

	if (controld_globals.membership_id != crm_peer_seq) {
	controld_globals.membership_id = crm_peer_seq;
	crm_info("Making join-%d offers based on membership event %llu",
	current_join_id, crm_peer_seq);
	}

	if(user_data && member->join > crm_join_none) {
	crm_info("Not making join-%d offer to already known node %s (%s)",
	current_join_id, member->uname,
	crm_join_phase_str(member->join));
	return;
	}

	crm_update_peer_join(__func__, (crm_node_t*)member, crm_join_none);

	offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname);

	// Advertise our feature set so the joining node can bail if not compatible
	crm_xml_add(offer, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);

	crm_info("Sending join-%d offer to %s", current_join_id, member->uname);
	send_cluster_message(member, crm_msg_crmd, offer, TRUE);
	free_xml(offer);

	crm_update_peer_join(__func__, member, crm_join_welcomed);
	}

	/* A_DC_JOIN_OFFER_ALL */
	void
	do_dc_join_offer_all(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int count;

	/* Reset everyone's status back to down or in_ccm in the CIB.
	* Any nodes that are active in the CIB but not in the cluster membership
	* will be seen as offline by the scheduler anyway.
	*/
	current_join_id++;
	start_join_round();

	update_dc(NULL);
	if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
	crm_info("A new node joined the cluster");
	}
	g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);

	count = crmd_join_phase_count(crm_join_welcomed);
	crm_info("Waiting on join-%d requests from %d outstanding node%s",
	current_join_id, count, pcmk__plural_s(count));

	// Don't waste time by invoking the scheduler yet
	}

	/* A_DC_JOIN_OFFER_ONE */
	void
	do_dc_join_offer_one(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_node_t *member;
	ha_msg_input_t *welcome = NULL;
	int count;
	const char *join_to = NULL;

	if (msg_data->data == NULL) {
	crm_info("Making join-%d offers to any unconfirmed nodes "
	"because an unknown node joined", current_join_id);
	g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
	check_join_state(cur_state, __func__);
	return;
	}

	welcome = fsa_typed_data(fsa_dt_ha_msg);
	if (welcome == NULL) {
	// fsa_typed_data() already logged an error
	return;
	}

	join_to = crm_element_value(welcome->msg, PCMK__XA_SRC);
	if (join_to == NULL) {
	crm_err("Can't make join-%d offer to unknown node", current_join_id);
	return;
	}
	member = pcmk__get_node(0, join_to, NULL, pcmk__node_search_cluster);

	/* It is possible that a node will have been sick or starting up when the
	* original offer was made. However, it will either re-announce itself in
	* due course, or we can re-store the original offer on the client.
	*/

	crm_update_peer_join(__func__, member, crm_join_none);
	join_make_offer(NULL, member, NULL);

	/* If the offer isn't to the local node, make an offer to the local node as
	* well, to ensure the correct value for max_generation_from.
	*/
	if (strcasecmp(join_to, controld_globals.our_nodename) != 0) {
	member = pcmk__get_node(0, controld_globals.our_nodename, NULL,
	pcmk__node_search_cluster);
	join_make_offer(NULL, member, NULL);
	}

	/* This was a genuine join request; cancel any existing transition and
	* invoke the scheduler.
	*/
	abort_transition(INFINITY, pcmk__graph_restart, "Node join", NULL);

	count = crmd_join_phase_count(crm_join_welcomed);
	crm_info("Waiting on join-%d requests from %d outstanding node%s",
	current_join_id, count, pcmk__plural_s(count));

	// Don't waste time by invoking the scheduler yet
	}

	static int
	compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
	{
	const char *elem_l = crm_element_value(left, field);
	const char *elem_r = crm_element_value(right, field);

	long long int_elem_l;
	long long int_elem_r;

	pcmk__scan_ll(elem_l, &int_elem_l, -1LL);
	pcmk__scan_ll(elem_r, &int_elem_r, -1LL);

	if (int_elem_l < int_elem_r) {
	return -1;

	} else if (int_elem_l > int_elem_r) {
	return 1;
	}

	return 0;
	}

	/* A_DC_JOIN_PROCESS_REQ */
	void
	do_dc_join_filter_offer(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *generation = NULL;

	int cmp = 0;
	int join_id = -1;
	int count = 0;
	gint value = 0;
	gboolean ack_nack_bool = TRUE;
	ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);

	const char *join_from = crm_element_value(join_ack->msg, PCMK__XA_SRC);
	const char *ref = crm_element_value(join_ack->msg, PCMK_XA_REFERENCE);
	const char *join_version = crm_element_value(join_ack->msg,
	PCMK_XA_CRM_FEATURE_SET);
	crm_node_t *join_node = NULL;

	if (join_from == NULL) {
	crm_err("Ignoring invalid join request without node name");
	return;
	}
	join_node = pcmk__get_node(0, join_from, NULL, pcmk__node_search_cluster);

	crm_element_value_int(join_ack->msg, PCMK__XA_JOIN_ID, &join_id);
	if (join_id != current_join_id) {
	crm_debug("Ignoring join-%d request from %s because we are on join-%d",
	join_id, join_from, current_join_id);
	check_join_state(cur_state, __func__);
	return;
	}

	generation = join_ack->xml;
	if (max_generation_xml != NULL && generation != NULL) {
	int lpc = 0;

	const char *attributes[] = {
	PCMK_XA_ADMIN_EPOCH,
	PCMK_XA_EPOCH,
	PCMK_XA_NUM_UPDATES,
	};

	for (lpc = 0; cmp == 0 && lpc < PCMK__NELEM(attributes); lpc++) {
	cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
	}
	}

	if (ref == NULL) {
	ref = "none"; // for logging only
	}

	if (lookup_failed_sync_node(join_from, &value) == pcmk_rc_ok) {
	crm_err("Rejecting join-%d request from node %s because we failed to "
	"sync its CIB in join-%d " CRM_XS " ref=%s",
	join_id, join_from, value, ref);
	ack_nack_bool = FALSE;

	} else if (!crm_is_peer_active(join_node)) {
	if (match_down_event(join_from) != NULL) {
	/* The join request was received after the node was fenced or
	* otherwise shutdown in a way that we're aware of. No need to log
	* an error in this rare occurrence; we know the client was recently
	* shut down, and receiving a lingering in-flight request is not
	* cause for alarm.
	*/
	crm_debug("Rejecting join-%d request from inactive node %s "
	CRM_XS " ref=%s", join_id, join_from, ref);
	} else {
	crm_err("Rejecting join-%d request from inactive node %s "
	CRM_XS " ref=%s", join_id, join_from, ref);
	}
	ack_nack_bool = FALSE;

	} else if (generation == NULL) {
	crm_err("Rejecting invalid join-%d request from node %s "
	"missing CIB generation " CRM_XS " ref=%s",
	join_id, join_from, ref);
	ack_nack_bool = FALSE;

	} else if ((join_version == NULL)
	\|\| !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
	crm_err("Rejecting join-%d request from node %s because feature set %s"
	" is incompatible with ours (%s) " CRM_XS " ref=%s",
	join_id, join_from, (join_version? join_version : "pre-3.1.0"),
	CRM_FEATURE_SET, ref);
	ack_nack_bool = FALSE;

	} else if (max_generation_xml == NULL) {
	const char *validation = crm_element_value(generation,
	PCMK_XA_VALIDATE_WITH);

	if (get_schema_version(validation) < 0) {
	crm_err("Rejecting join-%d request from %s (with first CIB "
	"generation) due to unknown schema version %s "
	CRM_XS " ref=%s",
	join_id, join_from, validation, ref);
	ack_nack_bool = FALSE;

	} else {
	crm_debug("Accepting join-%d request from %s (with first CIB "
	"generation) " CRM_XS " ref=%s",
	join_id, join_from, ref);
	max_generation_xml = copy_xml(generation);
	pcmk__str_update(&max_generation_from, join_from);
	}

	} else if ((cmp < 0)
	\|\| ((cmp == 0)
	&& pcmk__str_eq(join_from, controld_globals.our_nodename,
	pcmk__str_casei))) {
	const char *validation = crm_element_value(generation,
	PCMK_XA_VALIDATE_WITH);

	if (get_schema_version(validation) < 0) {
	crm_err("Rejecting join-%d request from %s (with better CIB "
	"generation than current best from %s) due to unknown "
	"schema version %s " CRM_XS " ref=%s",
	join_id, join_from, max_generation_from, validation, ref);
	ack_nack_bool = FALSE;

	} else {
	crm_debug("Accepting join-%d request from %s (with better CIB "
	"generation than current best from %s) " CRM_XS " ref=%s",
	join_id, join_from, max_generation_from, ref);
	crm_log_xml_debug(max_generation_xml, "Old max generation");
	crm_log_xml_debug(generation, "New max generation");

	free_xml(max_generation_xml);
	max_generation_xml = copy_xml(join_ack->xml);
	pcmk__str_update(&max_generation_from, join_from);
	}

	} else {
	crm_debug("Accepting join-%d request from %s " CRM_XS " ref=%s",
	join_id, join_from, ref);
	}

	if (!ack_nack_bool) {
	if (compare_version(join_version, "3.17.0") < 0) {
	/* Clients with CRM_FEATURE_SET < 3.17.0 may respawn infinitely
	* after a nack message, don't send one
	*/
	crm_update_peer_join(__func__, join_node, crm_join_nack_quiet);
	} else {
	crm_update_peer_join(__func__, join_node, crm_join_nack);
	}
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);

	} else {
	crm_update_peer_join(__func__, join_node, crm_join_integrated);
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
	}

	count = crmd_join_phase_count(crm_join_integrated);
	crm_debug("%d node%s currently integrated in join-%d",
	count, pcmk__plural_s(count), join_id);

	if (check_join_state(cur_state, __func__) == FALSE) {
	// Don't waste time by invoking the scheduler yet
	count = crmd_join_phase_count(crm_join_welcomed);
	crm_debug("Waiting on join-%d requests from %d outstanding node%s",
	join_id, count, pcmk__plural_s(count));
	}
	}

	/* A_DC_JOIN_FINALIZE */
	void
	do_dc_join_finalize(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	char *sync_from = NULL;
	int rc = pcmk_ok;
	int count_welcomed = crmd_join_phase_count(crm_join_welcomed);
	int count_finalizable = crmd_join_phase_count(crm_join_integrated)
	+ crmd_join_phase_count(crm_join_nack)
	+ crmd_join_phase_count(crm_join_nack_quiet);

	/* This we can do straight away and avoid clients timing us out
	* while we compute the latest CIB
	*/
	if (count_welcomed != 0) {
	crm_debug("Waiting on join-%d requests from %d outstanding node%s "
	"before finalizing join", current_join_id, count_welcomed,
	pcmk__plural_s(count_welcomed));
	crmd_join_phase_log(LOG_DEBUG);
	/* crmd_fsa_stall(FALSE); Needed? */
	return;

	} else if (count_finalizable == 0) {
	crm_debug("Finalization not needed for join-%d at the current time",
	current_join_id);
	crmd_join_phase_log(LOG_DEBUG);
	check_join_state(controld_globals.fsa_state, __func__);
	return;
	}

	controld_clear_fsa_input_flags(R_HAVE_CIB);
	if (pcmk__str_eq(max_generation_from, controld_globals.our_nodename,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	controld_set_fsa_input_flags(R_HAVE_CIB);
	}

	if (!controld_globals.transition_graph->complete) {
	crm_warn("Delaying join-%d finalization while transition in progress",
	current_join_id);
	crmd_join_phase_log(LOG_DEBUG);
	crmd_fsa_stall(FALSE);
	return;
	}

	if (pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
	// Send our CIB out to everyone
	pcmk__str_update(&sync_from, controld_globals.our_nodename);
	crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
	current_join_id, count_finalizable,
	pcmk__plural_s(count_finalizable));
	crm_log_xml_debug(max_generation_xml, "Requested CIB version");

	} else {
	// Ask for the agreed best CIB
	pcmk__str_update(&sync_from, max_generation_from);
	crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
	current_join_id, count_finalizable,
	pcmk__plural_s(count_finalizable), sync_from);
	crm_log_xml_notice(max_generation_xml, "Requested CIB version");
	}
	crmd_join_phase_log(LOG_DEBUG);

	rc = controld_globals.cib_conn->cmds->sync_from(controld_globals.cib_conn,
	sync_from, NULL, cib_none);
	fsa_register_cib_callback(rc, sync_from, finalize_sync_callback);
	}

	void
	free_max_generation(void)
	{
	free(max_generation_from);
	max_generation_from = NULL;

	free_xml(max_generation_xml);
	max_generation_xml = NULL;
	}

	void
	finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	CRM_LOG_ASSERT(-EPERM != rc);

	if (rc != pcmk_ok) {
	const char sync_from = (const char ) user_data;

	do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
	"Could not sync CIB from %s in join-%d: %s",
	sync_from, current_join_id, pcmk_strerror(rc));

	if (rc != -pcmk_err_old_data) {
	record_failed_sync_node(sync_from, current_join_id);
	}

	/* restart the whole join process */
	register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL,
	__func__);

	} else if (!AM_I_DC) {
	crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);

	} else if (controld_globals.fsa_state != S_FINALIZE_JOIN) {
	crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN "
	"(%s)", current_join_id,
	fsa_state2string(controld_globals.fsa_state));

	} else {
	controld_set_fsa_input_flags(R_HAVE_CIB);

	/* make sure dc_uuid is re-set to us */
	if (!check_join_state(controld_globals.fsa_state, __func__)) {
	int count_finalizable = 0;

	count_finalizable = crmd_join_phase_count(crm_join_integrated)
	+ crmd_join_phase_count(crm_join_nack)
	+ crmd_join_phase_count(crm_join_nack_quiet);

	crm_debug("Notifying %d node%s of join-%d results",
	count_finalizable, pcmk__plural_s(count_finalizable),
	current_join_id);
	g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
	}
	}
	}

	static void
	join_node_state_commit_callback(xmlNode *msg, int call_id, int rc,
	xmlNode output, void user_data)
	{
	const char *node = user_data;

	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL; // for register_fsa_error() macro

	crm_crit("join-%d node history update (via CIB call %d) for node %s "
	"failed: %s",
	current_join_id, call_id, node, pcmk_strerror(rc));
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}

	crm_debug("join-%d node history update (via CIB call %d) for node %s "
	"complete",
	current_join_id, call_id, node);
	check_join_state(controld_globals.fsa_state, __func__);
	}

	/* A_DC_JOIN_PROCESS_ACK */
	void
	do_dc_join_ack(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int join_id = -1;
	ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);

	const char *op = crm_element_value(join_ack->msg, PCMK__XA_CRM_TASK);
	char *join_from = crm_element_value_copy(join_ack->msg, PCMK__XA_SRC);
	crm_node_t *peer = NULL;

	enum controld_section_e section = controld_section_lrm;
	char *xpath = NULL;
	xmlNode *state = join_ack->xml;
	xmlNode *execd_state = NULL;

	cib_t *cib = controld_globals.cib_conn;
	int rc = pcmk_ok;

	// Sanity checks
	if (join_from == NULL) {
	crm_warn("Ignoring message received without node identification");
	goto done;
	}
	if (op == NULL) {
	crm_warn("Ignoring message received from %s without task", join_from);
	goto done;
	}

	if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
	crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
	op, join_from, CRM_OP_JOIN_CONFIRM);
	goto done;
	}

	if (crm_element_value_int(join_ack->msg, PCMK__XA_JOIN_ID, &join_id) != 0) {
	crm_warn("Ignoring join confirmation from %s without valid join ID",
	join_from);
	goto done;
	}

	peer = pcmk__get_node(0, join_from, NULL, pcmk__node_search_cluster);
	if (peer->join != crm_join_finalized) {
	crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
	"(currently %s not %s)",
	join_id, join_from, crm_join_phase_str(peer->join),
	crm_join_phase_str(crm_join_finalized));
	goto done;
	}

	if (join_id != current_join_id) {
	crm_err("Rejecting join-%d confirmation from %s "
	"because currently on join-%d",
	join_id, join_from, current_join_id);
	crm_update_peer_join(__func__, peer, crm_join_nack);
	goto done;
	}

	crm_update_peer_join(__func__, peer, crm_join_confirmed);

	/* Update CIB with node's current executor state. A new transition will be
	* triggered later, when the CIB manager notifies us of the change.
	*
	* The delete and modify requests are part of an atomic transaction.
	*/
	rc = cib->cmds->init_transaction(cib);
	if (rc != pcmk_ok) {
	goto done;
	}

	// Delete relevant parts of node's current executor state from CIB
	if (pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
	section = controld_section_lrm_unlocked;
	}
	controld_node_state_deletion_strings(join_from, section, &xpath, NULL);

	rc = cib->cmds->remove(cib, xpath, NULL,
	cib_scope_local
	\|cib_xpath
	\|cib_multiple
	\|cib_transaction);
	if (rc != pcmk_ok) {
	goto done;
	}

	// Update CIB with node's latest known executor state
	if (pcmk__str_eq(join_from, controld_globals.our_nodename,
	pcmk__str_casei)) {

	// Use the latest possible state if processing our own join ack
	execd_state = controld_query_executor_state();

	if (execd_state != NULL) {
	crm_debug("Updating local node history for join-%d from query "
	"result",
	current_join_id);
	state = execd_state;

	} else {
	crm_warn("Updating local node history from join-%d confirmation "
	"because query failed",
	current_join_id);
	}

	} else {
	crm_debug("Updating node history for %s from join-%d confirmation",
	join_from, current_join_id);
	}

	rc = cib->cmds->modify(cib, PCMK_XE_STATUS, state,
	cib_scope_local\|cib_can_create\|cib_transaction);
	free_xml(execd_state);
	if (rc != pcmk_ok) {
	goto done;
	}

	// Commit the transaction
	rc = cib->cmds->end_transaction(cib, true, cib_scope_local);
	fsa_register_cib_callback(rc, join_from, join_node_state_commit_callback);

	if (rc > 0) {
	// join_from will be freed after callback
	join_from = NULL;
	rc = pcmk_ok;
	}

	done:
	if (rc != pcmk_ok) {
	crm_crit("join-%d node history update for node %s failed: %s",
	current_join_id, join_from, pcmk_strerror(rc));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	free(join_from);
	free(xpath);
	}

	void
	finalize_join_for(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *acknak = NULL;
	xmlNode *tmp1 = NULL;
	crm_node_t *join_node = value;
	const char *join_to = join_node->uname;
	bool integrated = false;

	switch (join_node->join) {
	case crm_join_integrated:
	integrated = true;
	break;
	case crm_join_nack:
	case crm_join_nack_quiet:
	break;
	default:
	crm_trace("Not updating non-integrated and non-nacked node %s (%s) "
	"for join-%d", join_to,
	crm_join_phase_str(join_node->join), current_join_id);
	return;
	}

	/* Update the <node> element with the node's name and UUID, in case they
	* weren't known before
	*/
	crm_trace("Updating node name and UUID in CIB for %s", join_to);
	tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
	crm_xml_add(tmp1, PCMK_XA_ID, crm_peer_uuid(join_node));
	crm_xml_add(tmp1, PCMK_XA_UNAME, join_to);
	fsa_cib_anon_update(PCMK_XE_NODES, tmp1);
	free_xml(tmp1);

	if (join_node->join == crm_join_nack_quiet) {
	crm_trace("Not sending nack message to node %s with feature set older "
	"than 3.17.0", join_to);
	return;
	}

	join_node = pcmk__get_node(0, join_to, NULL, pcmk__node_search_cluster);
	if (!crm_is_peer_active(join_node)) {
	/*
	* NACK'ing nodes that the membership layer doesn't know about yet
	* simply creates more churn
	*
	* Better to leave them waiting and let the join restart when
	* the new membership event comes in
	*
	* All other NACKs (due to versions etc) should still be processed
	*/
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
	return;
	}

	// Acknowledge or nack node's join request
	crm_debug("%sing join-%d request from %s",
	integrated? "Acknowledg" : "Nack", current_join_id, join_to);
	acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
	pcmk__xe_set_bool_attr(acknak, CRM_OP_JOIN_ACKNAK, integrated);

	if (integrated) {
	// No change needed for a nacked node
	crm_update_peer_join(__func__, join_node, crm_join_finalized);
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);

	/* Iterate through the remote peer cache and add information on which
	* node hosts each to the ACK message. This keeps new controllers in
	* sync with what has already happened.
	*/
	if (crm_remote_peer_cache_size() != 0) {
	GHashTableIter iter;
	crm_node_t *node = NULL;
	xmlNode *remotes = create_xml_node(acknak, PCMK_XE_NODES);

	g_hash_table_iter_init(&iter, crm_remote_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *remote = NULL;

	if (!node->conn_host) {
	continue;
	}

	remote = create_xml_node(remotes, XML_CIB_TAG_NODE);
	pcmk__xe_set_props(remote,
	PCMK_XA_ID, node->uname,
	- XML_CIB_TAG_STATE, node->state,
	+ "node_state", node->state,
	PCMK__XA_CONN_HOST, node->conn_host,
	NULL);
	}
	}
	}
	send_cluster_message(join_node, crm_msg_crmd, acknak, TRUE);
	free_xml(acknak);
	return;
	}

	gboolean
	check_join_state(enum crmd_fsa_state cur_state, const char *source)
	{
	static unsigned long long highest_seq = 0;

	if (controld_globals.membership_id != crm_peer_seq) {
	crm_debug("join-%d: Membership changed from %llu to %llu "
	CRM_XS " highest=%llu state=%s for=%s",
	current_join_id, controld_globals.membership_id, crm_peer_seq,
	highest_seq, fsa_state2string(cur_state), source);
	if(highest_seq < crm_peer_seq) {
	/* Don't spam the FSA with duplicates */
	highest_seq = crm_peer_seq;
	register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
	}

	} else if (cur_state == S_INTEGRATION) {
	if (crmd_join_phase_count(crm_join_welcomed) == 0) {
	int count = crmd_join_phase_count(crm_join_integrated);

	crm_debug("join-%d: Integration of %d peer%s complete "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
	return TRUE;
	}

	} else if (cur_state == S_FINALIZE_JOIN) {
	if (!pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
	crm_debug("join-%d: Delaying finalization until we have CIB "
	CRM_XS " state=%s for=%s",
	current_join_id, fsa_state2string(cur_state), source);
	return TRUE;

	} else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
	int count = crmd_join_phase_count(crm_join_welcomed);

	crm_debug("join-%d: Still waiting on %d welcomed node%s "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else if (crmd_join_phase_count(crm_join_integrated) != 0) {
	int count = crmd_join_phase_count(crm_join_integrated);

	crm_debug("join-%d: Still waiting on %d integrated node%s "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else if (crmd_join_phase_count(crm_join_finalized) != 0) {
	int count = crmd_join_phase_count(crm_join_finalized);

	crm_debug("join-%d: Still waiting on %d finalized node%s "
	CRM_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else {
	crm_debug("join-%d: Complete " CRM_XS " state=%s for=%s",
	current_join_id, fsa_state2string(cur_state), source);
	register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
	return TRUE;
	}
	}

	return FALSE;
	}

	void
	do_dc_join_final(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
	crm_update_quorum(crm_have_quorum, TRUE);
	}

	int crmd_join_phase_count(enum crm_join_phase phase)
	{
	int count = 0;
	crm_node_t *peer;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	if(peer->join == phase) {
	count++;
	}
	}
	return count;
	}

	void crmd_join_phase_log(int level)
	{
	crm_node_t *peer;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->uname,
	crm_join_phase_str(peer->join));
	}
	}
	diff --git a/daemons/controld/controld_membership.c b/daemons/controld/controld_membership.c
	index 3ea748c402..a25e0a5fd5 100644
	--- a/daemons/controld/controld_membership.c
	+++ b/daemons/controld/controld_membership.c
	@@ -1,468 +1,468 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	/* put these first so that uuid_t is defined without conflicts */
	#include <crm_internal.h>

	#include <string.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/cluster/internal.h>

	#include <pacemaker-controld.h>

	void post_cache_update(int instance);

	extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	static void
	reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
	{
	crm_node_t *node = value;

	if (crm_is_peer_active(node) == FALSE) {
	crm_update_peer_join(__func__, node, crm_join_none);

	if(node && node->uname) {
	if (pcmk__str_eq(controld_globals.our_nodename, node->uname,
	pcmk__str_casei)) {
	crm_err("We're not part of the cluster anymore");
	register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);

	} else if (!AM_I_DC
	&& pcmk__str_eq(node->uname, controld_globals.dc_name,
	pcmk__str_casei)) {
	crm_warn("Our DC node (%s) left the cluster", node->uname);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	}
	}

	if ((controld_globals.fsa_state == S_INTEGRATION)
	\|\| (controld_globals.fsa_state == S_FINALIZE_JOIN)) {
	check_join_state(controld_globals.fsa_state, __func__);
	}
	if ((node != NULL) && (node->uuid != NULL)) {
	fail_incompletable_actions(controld_globals.transition_graph,
	node->uuid);
	}
	}
	}

	void
	post_cache_update(int instance)
	{
	xmlNode *no_op = NULL;

	crm_peer_seq = instance;
	crm_debug("Updated cache after membership event %d.", instance);

	g_hash_table_foreach(crm_peer_cache, reap_dead_nodes, NULL);
	controld_set_fsa_input_flags(R_MEMBERSHIP);

	if (AM_I_DC) {
	populate_cib_nodes(node_update_quick \| node_update_cluster \| node_update_peer \|
	node_update_expected, __func__);
	}

	/*
	* If we lost nodes, we should re-check the election status
	* Safe to call outside of an election
	*/
	controld_set_fsa_action_flags(A_ELECTION_CHECK);
	controld_trigger_fsa();

	/* Membership changed, remind everyone we're here.
	* This will aid detection of duplicate DCs
	*/
	no_op = create_request(CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD,
	AM_I_DC ? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD, NULL);
	send_cluster_message(NULL, crm_msg_crmd, no_op, FALSE);
	free_xml(no_op);
	}

	static void
	crmd_node_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if (rc == pcmk_ok) {
	crm_trace("Node update %d complete", call_id);

	} else if(call_id < pcmk_ok) {
	crm_err("Node update failed: %s (%d)", pcmk_strerror(call_id), call_id);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	} else {
	crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	/*!
	* \internal
	* \brief Create an XML node state tag with updates
	*
	* \param[in,out] node Node whose state will be used for update
	* \param[in] flags Bitmask of node_update_flags indicating what to update
	* \param[in,out] parent XML node to contain update (or NULL)
	* \param[in] source Who requested the update (only used for logging)
	*
	* \return Pointer to created node state tag
	*/
	xmlNode *
	create_node_state_update(crm_node_t node, int flags, xmlNode parent,
	const char *source)
	{
	const char *value = NULL;
	xmlNode *node_state;

	if (!node->state) {
	crm_info("Node update for %s cancelled: no state, not seen yet", node->uname);
	return NULL;
	}

	- node_state = create_xml_node(parent, XML_CIB_TAG_STATE);
	+ node_state = create_xml_node(parent, PCMK__XE_NODE_STATE);

	if (pcmk_is_set(node->flags, crm_remote_node)) {
	pcmk__xe_set_bool_attr(node_state, XML_NODE_IS_REMOTE, true);
	}

	if (crm_xml_add(node_state, PCMK_XA_ID, crm_peer_uuid(node)) == NULL) {
	crm_info("Node update for %s cancelled: no ID", node->uname);
	free_xml(node_state);
	return NULL;
	}

	crm_xml_add(node_state, PCMK_XA_UNAME, node->uname);

	if ((flags & node_update_cluster) && node->state) {
	if (compare_version(controld_globals.dc_version, "3.18.0") >= 0) {
	// A value 0 means the node is not a cluster member.
	crm_xml_add_ll(node_state, PCMK__XA_IN_CCM, node->when_member);

	} else {
	pcmk__xe_set_bool_attr(node_state, PCMK__XA_IN_CCM,
	pcmk__str_eq(node->state, CRM_NODE_MEMBER,
	pcmk__str_casei));
	}
	}

	if (!pcmk_is_set(node->flags, crm_remote_node)) {
	if (flags & node_update_peer) {
	if (compare_version(controld_globals.dc_version, "3.18.0") >= 0) {
	// A value 0 means the peer is offline in CPG.
	crm_xml_add_ll(node_state, PCMK__XA_CRMD, node->when_online);

	} else {
	// @COMPAT DCs < 2.1.7 use online/offline rather than timestamp
	value = OFFLINESTATUS;
	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	value = ONLINESTATUS;
	}
	crm_xml_add(node_state, PCMK__XA_CRMD, value);
	}
	}

	if (flags & node_update_join) {
	if (node->join <= crm_join_none) {
	value = CRMD_JOINSTATE_DOWN;
	} else {
	value = CRMD_JOINSTATE_MEMBER;
	}
	crm_xml_add(node_state, PCMK__XA_JOIN, value);
	}

	if (flags & node_update_expected) {
	crm_xml_add(node_state, PCMK__XA_EXPECTED, node->expected);
	}
	}

	crm_xml_add(node_state, PCMK_XA_CRM_DEBUG_ORIGIN, source);

	return node_state;
	}

	static void
	remove_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
	xmlNode * output, void *user_data)
	{
	char *node_uuid = user_data;

	do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
	"Deletion of the unknown conflicting node \"%s\": %s (rc=%d)",
	node_uuid, pcmk_strerror(rc), rc);
	}

	static void
	search_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
	xmlNode * output, void *user_data)
	{
	char *new_node_uuid = user_data;
	xmlNode *node_xml = NULL;

	if (rc != pcmk_ok) {
	if (rc != -ENXIO) {
	crm_notice("Searching conflicting nodes for %s failed: %s (%d)",
	new_node_uuid, pcmk_strerror(rc), rc);
	}
	return;

	} else if (output == NULL) {
	return;
	}

	if (pcmk__xe_is(output, XML_CIB_TAG_NODE)) {
	node_xml = output;

	} else {
	node_xml = pcmk__xml_first_child(output);
	}

	for (; node_xml != NULL; node_xml = pcmk__xml_next(node_xml)) {
	const char *node_uuid = NULL;
	const char *node_uname = NULL;
	GHashTableIter iter;
	crm_node_t *node = NULL;
	gboolean known = FALSE;

	if (!pcmk__xe_is(node_xml, XML_CIB_TAG_NODE)) {
	continue;
	}

	node_uuid = crm_element_value(node_xml, PCMK_XA_ID);
	node_uname = crm_element_value(node_xml, PCMK_XA_UNAME);

	if (node_uuid == NULL \|\| node_uname == NULL) {
	continue;
	}

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (node->uuid
	&& pcmk__str_eq(node->uuid, node_uuid, pcmk__str_casei)
	&& node->uname
	&& pcmk__str_eq(node->uname, node_uname, pcmk__str_casei)) {

	known = TRUE;
	break;
	}
	}

	if (known == FALSE) {
	cib_t *cib_conn = controld_globals.cib_conn;
	int delete_call_id = 0;
	xmlNode *node_state_xml = NULL;

	crm_notice("Deleting unknown node %s/%s which has conflicting uname with %s",
	node_uuid, node_uname, new_node_uuid);

	delete_call_id = cib_conn->cmds->remove(cib_conn, PCMK_XE_NODES,
	node_xml, cib_scope_local);
	fsa_register_cib_callback(delete_call_id, strdup(node_uuid),
	remove_conflicting_node_callback);

	- node_state_xml = create_xml_node(NULL, XML_CIB_TAG_STATE);
	+ node_state_xml = create_xml_node(NULL, PCMK__XE_NODE_STATE);
	crm_xml_add(node_state_xml, PCMK_XA_ID, node_uuid);
	crm_xml_add(node_state_xml, PCMK_XA_UNAME, node_uname);

	delete_call_id = cib_conn->cmds->remove(cib_conn, PCMK_XE_STATUS,
	node_state_xml,
	cib_scope_local);
	fsa_register_cib_callback(delete_call_id, strdup(node_uuid),
	remove_conflicting_node_callback);
	free_xml(node_state_xml);
	}
	}
	}

	static void
	node_list_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if(call_id < pcmk_ok) {
	crm_err("Node list update failed: %s (%d)", pcmk_strerror(call_id), call_id);
	crm_log_xml_debug(msg, "update:failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	} else if(rc < pcmk_ok) {
	crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "update:failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	void
	populate_cib_nodes(enum node_update_flags flags, const char *source)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	int call_id = 0;
	gboolean from_hashtable = TRUE;
	xmlNode *node_list = create_xml_node(NULL, PCMK_XE_NODES);

	#if SUPPORT_COROSYNC
	if (!pcmk_is_set(flags, node_update_quick) && is_corosync_cluster()) {
	from_hashtable = pcmk__corosync_add_nodes(node_list);
	}
	#endif

	if (from_hashtable) {
	GHashTableIter iter;
	crm_node_t *node = NULL;
	GString *xpath = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *new_node = NULL;

	if ((node->uuid != NULL) && (node->uname != NULL)) {
	crm_trace("Creating node entry for %s/%s", node->uname, node->uuid);
	if (xpath == NULL) {
	xpath = g_string_sized_new(512);
	} else {
	g_string_truncate(xpath, 0);
	}

	/* We need both to be valid */
	new_node = create_xml_node(node_list, XML_CIB_TAG_NODE);
	crm_xml_add(new_node, PCMK_XA_ID, node->uuid);
	crm_xml_add(new_node, PCMK_XA_UNAME, node->uname);

	/* Search and remove unknown nodes with the conflicting uname from CIB */
	pcmk__g_strcat(xpath,
	"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION
	"/" PCMK_XE_NODES "/" XML_CIB_TAG_NODE
	"[@" PCMK_XA_UNAME "='", node->uname, "']"
	"[@" PCMK_XA_ID "!='", node->uuid, "']", NULL);

	call_id = cib_conn->cmds->query(cib_conn,
	(const char *) xpath->str,
	NULL,
	cib_scope_local\|cib_xpath);
	fsa_register_cib_callback(call_id, strdup(node->uuid),
	search_conflicting_node_callback);
	}
	}

	if (xpath != NULL) {
	g_string_free(xpath, TRUE);
	}
	}

	crm_trace("Populating <nodes> section from %s", from_hashtable ? "hashtable" : "cluster");

	if ((controld_update_cib(PCMK_XE_NODES, node_list, cib_scope_local,
	node_list_update_callback) == pcmk_rc_ok)
	&& (crm_peer_cache != NULL) && AM_I_DC) {
	/*
	* There is no need to update the local CIB with our values if
	* we've not seen valid membership data
	*/
	GHashTableIter iter;
	crm_node_t *node = NULL;

	free_xml(node_list);
	node_list = create_xml_node(NULL, PCMK_XE_STATUS);

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	create_node_state_update(node, flags, node_list, source);
	}

	if (crm_remote_peer_cache) {
	g_hash_table_iter_init(&iter, crm_remote_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	create_node_state_update(node, flags, node_list, source);
	}
	}

	controld_update_cib(PCMK_XE_STATUS, node_list, cib_scope_local,
	crmd_node_update_complete);
	}
	free_xml(node_list);
	}

	static void
	cib_quorum_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	fsa_data_t *msg_data = NULL;

	if (rc == pcmk_ok) {
	crm_trace("Quorum update %d complete", call_id);

	} else {
	crm_err("Quorum update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	void
	crm_update_quorum(gboolean quorum, gboolean force_update)
	{
	bool has_quorum = pcmk_is_set(controld_globals.flags, controld_has_quorum);

	if (quorum) {
	controld_set_global_flags(controld_ever_had_quorum);

	} else if (pcmk_all_flags_set(controld_globals.flags,
	controld_ever_had_quorum
	\|controld_no_quorum_suicide)) {
	pcmk__panic(__func__);
	}

	if (AM_I_DC
	&& ((has_quorum && !quorum) \|\| (!has_quorum && quorum)
	\|\| force_update)) {
	xmlNode *update = NULL;

	update = create_xml_node(NULL, PCMK_XE_CIB);
	crm_xml_add_int(update, PCMK_XA_HAVE_QUORUM, quorum);
	crm_xml_add(update, PCMK_XA_DC_UUID, controld_globals.our_uuid);

	crm_debug("Updating quorum status to %s", pcmk__btoa(quorum));
	controld_update_cib(PCMK_XE_CIB, update, cib_scope_local,
	cib_quorum_update_complete);
	free_xml(update);

	/* Quorum changes usually cause a new transition via other activity:
	* quorum gained via a node joining will abort via the node join,
	* and quorum lost via a node leaving will usually abort via resource
	* activity and/or fencing.
	*
	* However, it is possible that nothing else causes a transition (e.g.
	* someone forces quorum via corosync-cmaptcl, or quorum is lost due to
	* a node in standby shutting down cleanly), so here ensure a new
	* transition is triggered.
	*/
	if (quorum) {
	/* If quorum was gained, abort after a short delay, in case multiple
	* nodes are joining around the same time, so the one that brings us
	* to quorum doesn't cause all the remaining ones to be fenced.
	*/
	abort_after_delay(INFINITY, pcmk__graph_restart, "Quorum gained",
	5000);
	} else {
	abort_transition(INFINITY, pcmk__graph_restart, "Quorum lost",
	NULL);
	}
	}

	if (quorum) {
	controld_set_global_flags(controld_has_quorum);
	} else {
	controld_clear_global_flags(controld_has_quorum);
	}
	}
	diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c
	index afcaf12950..7b6aefedbc 100644
	--- a/daemons/controld/controld_te_actions.c
	+++ b/daemons/controld/controld_te_actions.c
	@@ -1,760 +1,760 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/lrmd.h> // lrmd_event_data_t, lrmd_free_event()
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>

	#include <pacemaker-internal.h>
	#include <pacemaker-controld.h>

	static GHashTable *te_targets = NULL;
	void send_rsc_command(pcmk__graph_action_t *action);
	static void te_update_job_count(pcmk__graph_action_t *action, int offset);

	static void
	te_start_action_timer(const pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	action->timer = g_timeout_add(action->timeout + graph->network_delay,
	action_timer_callback, (void *) action);
	CRM_ASSERT(action->timer != 0);
	}

	/*!
	* \internal
	* \brief Execute a graph pseudo-action
	*
	* \param[in,out] graph Transition graph being executed
	* \param[in,out] pseudo Pseudo-action to execute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	execute_pseudo_action(pcmk__graph_t graph, pcmk__graph_action_t pseudo)
	{
	const char *task = crm_element_value(pseudo->xml, PCMK_XA_OPERATION);

	/* send to peers as well? */
	if (pcmk__str_eq(task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_casei)) {
	GHashTableIter iter;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *cmd = NULL;

	if (pcmk__str_eq(controld_globals.our_nodename, node->uname,
	pcmk__str_casei)) {
	continue;
	}

	cmd = create_request(task, pseudo->xml, node->uname,
	CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
	send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
	free_xml(cmd);
	}

	remote_ra_process_maintenance_nodes(pseudo->xml);
	} else {
	/* Check action for Pacemaker Remote node side effects */
	remote_ra_process_pseudo(pseudo->xml);
	}

	crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
	crm_element_value(pseudo->xml, PCMK__XA_OPERATION_KEY));
	te_action_confirmed(pseudo, graph);
	return pcmk_rc_ok;
	}

	static int
	get_target_rc(pcmk__graph_action_t *action)
	{
	int exit_status;

	pcmk__scan_min_int(crm_meta_value(action->params, PCMK__META_OP_TARGET_RC),
	&exit_status, 0);
	return exit_status;
	}

	/*!
	* \internal
	* \brief Execute a cluster action from a transition graph
	*
	* \param[in,out] graph Transition graph being executed
	* \param[in,out] action Cluster action to execute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	execute_cluster_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	char *counter = NULL;
	xmlNode *cmd = NULL;
	gboolean is_local = FALSE;

	const char *id = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;

	id = ID(action->xml);
	CRM_CHECK(!pcmk__str_empty(id), return EPROTO);

	task = crm_element_value(action->xml, PCMK_XA_OPERATION);
	CRM_CHECK(!pcmk__str_empty(task), return EPROTO);

	on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
	CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown);

	router_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
	if (router_node == NULL) {
	router_node = on_node;
	if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
	const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);

	if (pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) {
	router_node = controld_globals.our_nodename;
	}
	}
	}

	if (pcmk__str_eq(router_node, controld_globals.our_nodename,
	pcmk__str_casei)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_info("Handling controller request '%s' (%s on %s)%s%s",
	id, task, on_node, (is_local? " locally" : ""),
	(no_wait? " without waiting" : ""));

	if (is_local
	&& pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
	/* defer until everything else completes */
	crm_info("Controller request '%s' is a local shutdown", id);
	graph->completion_action = pcmk__graph_shutdown;
	graph->abort_reason = "local shutdown";
	te_action_confirmed(action, graph);
	return pcmk_rc_ok;

	} else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
	crm_node_t *peer = pcmk__get_node(0, router_node, NULL,
	pcmk__node_search_cluster);

	pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
	}

	cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);

	counter = pcmk__transition_key(controld_globals.transition_graph->id,
	action->id, get_target_rc(action),
	controld_globals.te_uuid);
	crm_xml_add(cmd, PCMK__XA_TRANSITION_KEY, counter);

	rc = send_cluster_message(pcmk__get_node(0, router_node, NULL,
	pcmk__node_search_cluster),
	crm_msg_crmd, cmd, TRUE);
	free(counter);
	free_xml(cmd);

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return ECOMM;

	} else if (no_wait) {
	te_action_confirmed(action, graph);

	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %ums instead",
	action->id, task, on_node, action->timeout, graph->network_delay);
	action->timeout = (int) graph->network_delay;
	}
	te_start_action_timer(graph, action);
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Synthesize an executor event for a resource action timeout
	*
	* \param[in] action Resource action that timed out
	* \param[in] target_rc Expected result of action that timed out
	*
	* Synthesize an executor event for a resource action timeout. (If the executor
	* gets a timeout while waiting for a resource action to complete, that will be
	* reported via the usual callback. This timeout means we didn't hear from the
	* executor itself or the controller that relayed the action to the executor.)
	*
	* \return Newly created executor event for result of \p action
	* \note The caller is responsible for freeing the return value using
	* lrmd_free_event().
	*/
	static lrmd_event_data_t *
	synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc)
	{
	lrmd_event_data_t *op = NULL;
	const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
	const char *reason = NULL;
	char *dynamic_reason = NULL;

	if (pcmk__str_eq(target, get_local_node_name(), pcmk__str_casei)) {
	reason = "Local executor did not return result in time";
	} else {
	const char *router_node = NULL;

	router_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
	if (router_node == NULL) {
	router_node = target;
	}
	dynamic_reason = crm_strdup_printf("Controller on %s did not return "
	"result in time", router_node);
	reason = dynamic_reason;
	}

	op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT,
	PCMK_OCF_UNKNOWN_ERROR, reason);
	op->call_id = -1;
	op->user_data = pcmk__transition_key(controld_globals.transition_graph->id,
	action->id, target_rc,
	controld_globals.te_uuid);
	free(dynamic_reason);
	return op;
	}

	static void
	controld_record_action_event(pcmk__graph_action_t *action,
	lrmd_event_data_t *op)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	xmlNode *state = NULL;
	xmlNode *rsc = NULL;
	xmlNode *action_rsc = NULL;

	int rc = pcmk_ok;

	const char *rsc_id = NULL;
	const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
	const char *task_uuid = crm_element_value(action->xml,
	PCMK__XA_OPERATION_KEY);
	const char *target_uuid = crm_element_value(action->xml,
	PCMK__META_ON_NODE_UUID);

	int target_rc = get_target_rc(action);

	action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
	if (action_rsc == NULL) {
	return;
	}

	rsc_id = ID(action_rsc);
	CRM_CHECK(rsc_id != NULL,
	crm_log_xml_err(action->xml, "Bad:action"); return);

	/*
	update the CIB

	<node_state id="hadev">
	<lrm>
	<lrm_resources>
	<lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
	*/

	- state = create_xml_node(NULL, XML_CIB_TAG_STATE);
	+ state = create_xml_node(NULL, PCMK__XE_NODE_STATE);

	crm_xml_add(state, PCMK_XA_ID, target_uuid);
	crm_xml_add(state, PCMK_XA_UNAME, target);

	rsc = create_xml_node(state, XML_CIB_TAG_LRM);
	crm_xml_add(rsc, PCMK_XA_ID, target_uuid);

	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
	crm_xml_add(rsc, PCMK_XA_ID, rsc_id);


	crm_copy_xml_element(action_rsc, rsc, PCMK_XA_TYPE);
	crm_copy_xml_element(action_rsc, rsc, PCMK_XA_CLASS);
	crm_copy_xml_element(action_rsc, rsc, PCMK_XA_PROVIDER);

	pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target,
	__func__);

	rc = cib_conn->cmds->modify(cib_conn, PCMK_XE_STATUS, state,
	cib_scope_local);
	fsa_register_cib_callback(rc, NULL, cib_action_updated);
	free_xml(state);

	crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)",
	rc, action->id, task_uuid, target);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update);
	}

	void
	controld_record_action_timeout(pcmk__graph_action_t *action)
	{
	lrmd_event_data_t *op = NULL;

	const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
	const char *task_uuid = crm_element_value(action->xml,
	PCMK__XA_OPERATION_KEY);

	int target_rc = get_target_rc(action);

	crm_warn("%s %d: %s on %s timed out",
	action->xml->name, action->id, task_uuid, target);

	op = synthesize_timeout_event(action, target_rc);
	controld_record_action_event(action, op);
	lrmd_free_event(op);
	}

	/*!
	* \internal
	* \brief Execute a resource action from a transition graph
	*
	* \param[in,out] graph Transition graph being executed
	* \param[in,out] action Resource action to execute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	execute_rsc_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	/* never overwrite stop actions in the CIB with
	* anything other than completed results
	*
	* Writing pending stops makes it look like the
	* resource is running again
	*/
	xmlNode *cmd = NULL;
	xmlNode *rsc_op = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;
	gboolean is_local = FALSE;

	char *counter = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;
	const char *task_uuid = NULL;

	CRM_ASSERT(action != NULL);
	CRM_ASSERT(action->xml != NULL);

	pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed);
	on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);

	CRM_CHECK(!pcmk__str_empty(on_node),
	crm_err("Corrupted command(id=%s) %s: no node",
	ID(action->xml), pcmk__s(task, "without task"));
	return pcmk_rc_node_unknown);

	rsc_op = action->xml;
	task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY);
	router_node = crm_element_value(rsc_op, PCMK__XA_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	}

	counter = pcmk__transition_key(controld_globals.transition_graph->id,
	action->id, get_target_rc(action),
	controld_globals.te_uuid);
	crm_xml_add(rsc_op, PCMK__XA_TRANSITION_KEY, counter);

	if (pcmk__str_eq(router_node, controld_globals.our_nodename,
	pcmk__str_casei)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
	task, task_uuid, (is_local? " locally" : ""), on_node,
	(no_wait? " without waiting" : ""), action->id);

	cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
	CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);

	if (is_local) {
	/* shortcut local resource commands */
	ha_msg_input_t data = {
	.msg = cmd,
	.xml = rsc_op,
	};

	fsa_data_t msg = {
	.id = 0,
	.data = &data,
	.data_type = fsa_dt_ha_msg,
	.fsa_input = I_NULL,
	.fsa_cause = C_FSA_INTERNAL,
	.actions = A_LRM_INVOKE,
	.origin = __func__,
	};

	do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state,
	I_NULL, &msg);

	} else {
	rc = send_cluster_message(pcmk__get_node(0, router_node, NULL,
	pcmk__node_search_cluster),
	crm_msg_lrmd, cmd, TRUE);
	}

	free(counter);
	free_xml(cmd);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return ECOMM;

	} else if (no_wait) {
	/* Just mark confirmed. Don't bump the job count only to immediately
	* decrement it.
	*/
	crm_info("Action %d confirmed - no wait", action->id);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(controld_globals.transition_graph, action);
	trigger_graph();

	} else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
	crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
	action->id, task, task_uuid, on_node, action->timeout);
	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %ums instead",
	action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
	action->timeout = (int) graph->network_delay;
	}
	te_update_job_count(action, 1);
	te_start_action_timer(graph, action);
	}

	return pcmk_rc_ok;
	}

	struct te_peer_s
	{
	char *name;
	int jobs;
	int migrate_jobs;
	};

	static void te_peer_free(gpointer p)
	{
	struct te_peer_s *peer = p;

	free(peer->name);
	free(peer);
	}

	void te_reset_job_counts(void)
	{
	GHashTableIter iter;
	struct te_peer_s *peer = NULL;

	if(te_targets == NULL) {
	te_targets = pcmk__strkey_table(NULL, te_peer_free);
	}

	g_hash_table_iter_init(&iter, te_targets);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
	peer->jobs = 0;
	peer->migrate_jobs = 0;
	}
	}

	static void
	te_update_job_count_on(const char *target, int offset, bool migrate)
	{
	struct te_peer_s *r = NULL;

	if(target == NULL \|\| te_targets == NULL) {
	return;
	}

	r = g_hash_table_lookup(te_targets, target);
	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	r->jobs += offset;
	if(migrate) {
	r->migrate_jobs += offset;
	}
	crm_trace("jobs[%s] = %d", target, r->jobs);
	}

	static void
	te_update_job_count(pcmk__graph_action_t *action, int offset)
	{
	const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
	const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);

	if ((action->type != pcmk__rsc_graph_action) \|\| (target == NULL)) {
	/* No limit on these */
	return;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all actions occurring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);

	if ((target == NULL)
	&& pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {

	const char *t1 = crm_meta_value(action->params,
	PCMK__META_MIGRATE_SOURCE);
	const char *t2 = crm_meta_value(action->params,
	PCMK__META_MIGRATE_TARGET);

	te_update_job_count_on(t1, offset, TRUE);
	te_update_job_count_on(t2, offset, TRUE);
	return;
	} else if (target == NULL) {
	target = crm_element_value(action->xml, PCMK__META_ON_NODE);
	}

	te_update_job_count_on(target, offset, FALSE);
	}

	/*!
	* \internal
	* \brief Check whether a graph action is allowed to be executed on a node
	*
	* \param[in] graph Transition graph being executed
	* \param[in] action Graph action being executed
	* \param[in] target Name of node where action should be executed
	*
	* \return true if action is allowed, otherwise false
	*/
	static bool
	allowed_on_node(const pcmk__graph_t graph, const pcmk__graph_action_t action,
	const char *target)
	{
	int limit = 0;
	struct te_peer_s *r = NULL;
	const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
	const char *id = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY);

	if(target == NULL) {
	/* No limit on these */
	return true;

	} else if(te_targets == NULL) {
	return false;
	}

	r = g_hash_table_lookup(te_targets, target);
	limit = throttle_get_job_limit(target);

	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	if(limit <= r->jobs) {
	crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
	target, limit, r->jobs, id);
	return false;

	} else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
	if (pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
	target, graph->migration_limit, r->migrate_jobs, id);
	return false;
	}
	}

	crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);

	return true;
	}

	/*!
	* \internal
	* \brief Check whether a graph action is allowed to be executed
	*
	* \param[in] graph Transition graph being executed
	* \param[in] action Graph action being executed
	*
	* \return true if action is allowed, otherwise false
	*/
	static bool
	graph_action_allowed(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *target = NULL;
	const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);

	if (action->type != pcmk__rsc_graph_action) {
	/* No limit on these */
	return true;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all actions occurring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);

	if ((target == NULL)
	&& pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	target = crm_meta_value(action->params, PCMK__META_MIGRATE_SOURCE);
	if (!allowed_on_node(graph, action, target)) {
	return false;
	}

	target = crm_meta_value(action->params, PCMK__META_MIGRATE_TARGET);

	} else if (target == NULL) {
	target = crm_element_value(action->xml, PCMK__META_ON_NODE);
	}

	return allowed_on_node(graph, action, target);
	}

	/*!
	* \brief Confirm a graph action (and optionally update graph)
	*
	* \param[in,out] action Action to confirm
	* \param[in,out] graph Update and trigger this graph (if non-NULL)
	*/
	void
	te_action_confirmed(pcmk__graph_action_t action, pcmk__graph_t graph)
	{
	if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
	if ((action->type == pcmk__rsc_graph_action)
	&& (crm_element_value(action->xml, PCMK__META_ON_NODE) != NULL)) {
	te_update_job_count(action, -1);
	}
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	}
	if (graph) {
	pcmk__update_graph(graph, action);
	trigger_graph();
	}
	}

	static pcmk__graph_functions_t te_graph_fns = {
	execute_pseudo_action,
	execute_rsc_action,
	execute_cluster_action,
	controld_execute_fence_action,
	graph_action_allowed,
	};

	/*
	* \internal
	* \brief Register the transitioner's graph functions with \p libpacemaker
	*/
	void
	controld_register_graph_functions(void)
	{
	pcmk__set_graph_functions(&te_graph_fns);
	}

	void
	notify_crmd(pcmk__graph_t *graph)
	{
	const char *type = "unknown";
	enum crmd_fsa_input event = I_NULL;

	crm_debug("Processing transition completion in state %s",
	fsa_state2string(controld_globals.fsa_state));

	CRM_CHECK(graph->complete, graph->complete = true);

	switch (graph->completion_action) {
	case pcmk__graph_wait:
	type = "stop";
	if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;
	case pcmk__graph_done:
	type = "done";
	if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;

	case pcmk__graph_restart:
	type = "restart";
	if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
	if (controld_get_period_transition_timer() > 0) {
	controld_stop_transition_timer();
	controld_start_transition_timer();
	} else {
	event = I_PE_CALC;
	}

	} else if (controld_globals.fsa_state == S_POLICY_ENGINE) {
	controld_set_fsa_action_flags(A_PE_INVOKE);
	controld_trigger_fsa();
	}
	break;

	case pcmk__graph_shutdown:
	type = "shutdown";
	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	event = I_STOP;

	} else {
	crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
	event = I_TERMINATE;
	}
	}

	crm_debug("Transition %d status: %s - %s", graph->id, type,
	pcmk__s(graph->abort_reason, "unspecified reason"));

	graph->abort_reason = NULL;
	graph->completion_action = pcmk__graph_done;

	if (event != I_NULL) {
	register_fsa_input(C_FSA_INTERNAL, event, NULL);
	} else {
	controld_trigger_fsa();
	}
	}
	diff --git a/daemons/controld/controld_te_callbacks.c b/daemons/controld/controld_te_callbacks.c
	index b2b7a4c404..98bcaacfdf 100644
	--- a/daemons/controld/controld_te_callbacks.c
	+++ b/daemons/controld/controld_te_callbacks.c
	@@ -1,697 +1,697 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/msg_xml.h>
	#include <crm/cluster.h> /* For ONLINESTATUS etc */

	#include <pacemaker-controld.h>

	void te_update_confirm(const char event, xmlNode msg);

	#define RSC_OP_PREFIX "//" PCMK__XE_DIFF_ADDED "//" PCMK_XE_CIB \
	"//" XML_LRM_TAG_RSC_OP "[@" PCMK_XA_ID "='"

	// An explicit PCMK_OPT_SHUTDOWN_LOCK of 0 means the lock has been cleared
	static bool
	shutdown_lock_cleared(xmlNode *lrm_resource)
	{
	time_t shutdown_lock = 0;

	return (crm_element_value_epoch(lrm_resource, PCMK_OPT_SHUTDOWN_LOCK,
	&shutdown_lock) == pcmk_ok)
	&& (shutdown_lock == 0);
	}

	static void
	te_update_diff_v1(const char event, xmlNode diff)
	{
	int lpc, max;
	xmlXPathObject *xpathObj = NULL;
	GString *rsc_op_xpath = NULL;

	CRM_CHECK(diff != NULL, return);

	pcmk__output_set_log_level(controld_globals.logger_out, LOG_TRACE);
	controld_globals.logger_out->message(controld_globals.logger_out,
	"xml-patchset", diff);

	if (cib__config_changed_v1(NULL, NULL, &diff)) {
	abort_transition(INFINITY, pcmk__graph_restart, "Non-status change",
	diff);
	goto bail; /* configuration changed */
	}

	/* Tickets Attributes - Added/Updated */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT
	"//" PCMK__XE_DIFF_ADDED
	"//" XML_CIB_TAG_TICKETS);
	if (numXpathResults(xpathObj) > 0) {
	xmlNode *aborted = getXpathResult(xpathObj, 0);

	abort_transition(INFINITY, pcmk__graph_restart,
	"Ticket attribute: update", aborted);
	goto bail;

	}
	freeXpathObject(xpathObj);

	/* Tickets Attributes - Removed */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT
	"//" PCMK__XE_DIFF_REMOVED
	"//" XML_CIB_TAG_TICKETS);
	if (numXpathResults(xpathObj) > 0) {
	xmlNode *aborted = getXpathResult(xpathObj, 0);

	abort_transition(INFINITY, pcmk__graph_restart,
	"Ticket attribute: removal", aborted);
	goto bail;
	}
	freeXpathObject(xpathObj);

	/* Transient Attributes - Removed */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT
	"//" PCMK__XE_DIFF_REMOVED
	"//" XML_TAG_TRANSIENT_NODEATTRS);
	if (numXpathResults(xpathObj) > 0) {
	xmlNode *aborted = getXpathResult(xpathObj, 0);

	abort_transition(INFINITY, pcmk__graph_restart,
	"Transient attribute: removal", aborted);
	goto bail;

	}
	freeXpathObject(xpathObj);

	// Check for lrm_resource entries
	xpathObj = xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT
	"//" PCMK__XE_DIFF_ADDED
	"//" XML_LRM_TAG_RESOURCE);
	max = numXpathResults(xpathObj);

	/*
	* Updates by, or in response to, graph actions will never affect more than
	* one resource at a time, so such updates indicate an LRM refresh. In that
	* case, start a new transition rather than check each result individually,
	* which can result in _huge_ speedups in large clusters.
	*
	* Unfortunately, we can only do so when there are no pending actions.
	* Otherwise, we could mistakenly throw away those results here, and
	* the cluster will stall waiting for them and time out the operation.
	*/
	if ((controld_globals.transition_graph->pending == 0) && (max > 1)) {
	crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
	max);
	crm_log_xml_trace(diff, "lrm-refresh");
	abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
	NULL);
	goto bail;
	}

	if (max == 1) {
	xmlNode *lrm_resource = getXpathResult(xpathObj, 0);

	if (shutdown_lock_cleared(lrm_resource)) {
	// @TODO would be more efficient to abort once after transition done
	abort_transition(INFINITY, pcmk__graph_restart,
	"Shutdown lock cleared", lrm_resource);
	// Still process results, so we stop timers and update failcounts
	}
	}
	freeXpathObject(xpathObj);

	/* Process operation updates */
	xpathObj =
	xpath_search(diff,
	"//" F_CIB_UPDATE_RESULT
	"//" PCMK__XE_DIFF_ADDED
	"//" XML_LRM_TAG_RSC_OP);
	max = numXpathResults(xpathObj);
	if (max > 0) {
	int lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
	const char *node = get_node_id(rsc_op);

	process_graph_event(rsc_op, node);
	}
	}
	freeXpathObject(xpathObj);

	/* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
	xpathObj = xpath_search(diff,
	"//" PCMK__XE_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
	max = numXpathResults(xpathObj);
	for (lpc = 0; lpc < max; lpc++) {
	const char *op_id = NULL;
	xmlXPathObject *op_match = NULL;
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match == NULL) { continue; };

	op_id = ID(match);

	if (rsc_op_xpath == NULL) {
	rsc_op_xpath = g_string_new(RSC_OP_PREFIX);
	} else {
	g_string_truncate(rsc_op_xpath, sizeof(RSC_OP_PREFIX) - 1);
	}
	pcmk__g_strcat(rsc_op_xpath, op_id, "']", NULL);

	op_match = xpath_search(diff, (const char *) rsc_op_xpath->str);
	if (numXpathResults(op_match) == 0) {
	/* Prevent false positives by matching cancelations too */
	const char *node = get_node_id(match);
	pcmk__graph_action_t *cancelled = get_cancel_action(op_id, node);

	if (cancelled == NULL) {
	crm_debug("No match for deleted action %s (%s on %s)",
	(const char *) rsc_op_xpath->str, op_id, node);
	abort_transition(INFINITY, pcmk__graph_restart,
	"Resource op removal", match);
	freeXpathObject(op_match);
	goto bail;

	} else {
	crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
	op_id, node, cancelled->id);
	}
	}

	freeXpathObject(op_match);
	}

	bail:
	freeXpathObject(xpathObj);
	if (rsc_op_xpath != NULL) {
	g_string_free(rsc_op_xpath, TRUE);
	}
	}

	static void
	process_lrm_resource_diff(xmlNode lrm_resource, const char node)
	{
	for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
	rsc_op = pcmk__xml_next(rsc_op)) {
	process_graph_event(rsc_op, node);
	}
	if (shutdown_lock_cleared(lrm_resource)) {
	// @TODO would be more efficient to abort once after transition done
	abort_transition(INFINITY, pcmk__graph_restart, "Shutdown lock cleared",
	lrm_resource);
	}
	}

	static void
	process_resource_updates(const char node, xmlNode xml, xmlNode *change,
	const char op, const char xpath)
	{
	xmlNode *rsc = NULL;

	if (xml == NULL) {
	return;
	}

	if (pcmk__xe_is(xml, XML_CIB_TAG_LRM)) {
	xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
	CRM_CHECK(xml != NULL, return);
	}

	CRM_CHECK(pcmk__xe_is(xml, XML_LRM_TAG_RESOURCES), return);

	/*
	* Updates by, or in response to, TE actions will never contain updates
	* for more than one resource at a time, so such updates indicate an
	* LRM refresh.
	*
	* In that case, start a new transition rather than check each result
	* individually, which can result in _huge_ speedups in large clusters.
	*
	* Unfortunately, we can only do so when there are no pending actions.
	* Otherwise, we could mistakenly throw away those results here, and
	* the cluster will stall waiting for them and time out the operation.
	*/
	if ((controld_globals.transition_graph->pending == 0)
	&& (xml->children != NULL) && (xml->children->next != NULL)) {

	crm_log_xml_trace(change, "lrm-refresh");
	abort_transition(INFINITY, pcmk__graph_restart, "History refresh",
	NULL);
	return;
	}

	for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
	rsc = pcmk__xml_next(rsc)) {
	crm_trace("Processing %s", ID(rsc));
	process_lrm_resource_diff(rsc, node);
	}
	}

	static char extract_node_uuid(const char xpath)
	{
	char *mutable_path = strdup(xpath);
	char *node_uuid = NULL;
	char *search = NULL;
	char *match = NULL;

	- match = strstr(mutable_path, "node_state[@" PCMK_XA_ID "=\'");
	+ match = strstr(mutable_path, PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'");
	if (match == NULL) {
	free(mutable_path);
	return NULL;
	}
	- match += strlen("node_state[@" PCMK_XA_ID "=\'");
	+ match += strlen(PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'");

	search = strchr(match, '\'');
	if (search == NULL) {
	free(mutable_path);
	return NULL;
	}
	search[0] = 0;

	node_uuid = strdup(match);
	free(mutable_path);
	return node_uuid;
	}

	static void
	abort_unless_down(const char xpath, const char op, xmlNode *change,
	const char *reason)
	{
	char *node_uuid = NULL;
	pcmk__graph_action_t *down = NULL;

	if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
	abort_transition(INFINITY, pcmk__graph_restart, reason, change);
	return;
	}

	node_uuid = extract_node_uuid(xpath);
	if(node_uuid == NULL) {
	crm_err("Could not extract node ID from %s", xpath);
	abort_transition(INFINITY, pcmk__graph_restart, reason, change);
	return;
	}

	down = match_down_event(node_uuid);
	if (down == NULL) {
	crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
	abort_transition(INFINITY, pcmk__graph_restart, reason, change);
	} else {
	crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
	}
	free(node_uuid);
	}

	static void
	process_op_deletion(const char xpath, xmlNode change)
	{
	char *mutable_key = strdup(xpath);
	char *key;
	char *node_uuid;

	// Extract the part of xpath between last pair of single quotes
	key = strrchr(mutable_key, '\'');
	if (key != NULL) {
	*key = '\0';
	key = strrchr(mutable_key, '\'');
	}
	if (key == NULL) {
	crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
	xpath);
	free(mutable_key);
	return;
	}
	++key;

	node_uuid = extract_node_uuid(xpath);
	if (confirm_cancel_action(key, node_uuid) == FALSE) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Resource operation removal", change);
	}
	free(mutable_key);
	free(node_uuid);
	}

	static void
	process_delete_diff(const char xpath, const char op, xmlNode *change)
	{
	if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
	process_op_deletion(xpath, change);

	} else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
	abort_unless_down(xpath, op, change, "Resource state removal");

	- } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
	+ } else if (strstr(xpath, "/" PCMK__XE_NODE_STATE "[")) {
	abort_unless_down(xpath, op, change, "Node state removal");

	} else {
	crm_trace("Ignoring delete of %s", xpath);
	}
	}

	static void
	process_node_state_diff(xmlNode state, xmlNode change, const char *op,
	const char *xpath)
	{
	xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);

	process_resource_updates(ID(state), lrm, change, op, xpath);
	}

	static void
	process_status_diff(xmlNode status, xmlNode change, const char *op,
	const char *xpath)
	{
	for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
	state = pcmk__xml_next(state)) {
	process_node_state_diff(state, change, op, xpath);
	}
	}

	static void
	process_cib_diff(xmlNode cib, xmlNode change, const char *op,
	const char *xpath)
	{
	xmlNode *status = first_named_child(cib, PCMK_XE_STATUS);
	xmlNode *config = first_named_child(cib, PCMK_XE_CONFIGURATION);

	if (status) {
	process_status_diff(status, change, op, xpath);
	}
	if (config) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Non-status-only change", change);
	}
	}

	static void
	te_update_diff_v2(xmlNode *diff)
	{
	crm_log_xml_trace(diff, "Patch:Raw");

	for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
	change = pcmk__xml_next(change)) {

	xmlNode *match = NULL;
	const char *name = NULL;
	const char *xpath = crm_element_value(change, PCMK_XA_PATH);

	// Possible ops: create, modify, delete, move
	const char *op = crm_element_value(change, PCMK_XA_OPERATION);

	// Ignore uninteresting updates
	if (op == NULL) {
	continue;

	} else if (xpath == NULL) {
	crm_trace("Ignoring %s change for version field", op);
	continue;

	} else if ((strcmp(op, "move") == 0)
	&& (strstr(xpath,
	"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION
	"/" PCMK_XE_RESOURCES) == NULL)) {
	/* We still need to consider moves within the resources section,
	* since they affect placement order.
	*/
	crm_trace("Ignoring move change at %s", xpath);
	continue;
	}

	// Find the result of create/modify ops
	if (strcmp(op, "create") == 0) {
	match = change->children;

	} else if (strcmp(op, "modify") == 0) {
	match = first_named_child(change, XML_DIFF_RESULT);
	if(match) {
	match = match->children;
	}

	} else if (!pcmk__str_any_of(op, "delete", "move", NULL)) {
	crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
	op, xpath);
	continue;
	}

	if (match) {
	if (match->type == XML_COMMENT_NODE) {
	crm_trace("Ignoring %s operation for comment at %s", op, xpath);
	continue;
	}
	name = (const char *)match->name;
	}

	crm_trace("Handling %s operation for %s%s%s",
	op, (xpath? xpath : "CIB"),
	(name? " matched by " : ""), (name? name : ""));

	if (strstr(xpath, "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION)) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Configuration change", change);
	break; // Won't be packaged with operation results we may be waiting for

	} else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
	\|\| pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	abort_transition(INFINITY, pcmk__graph_restart,
	"Ticket attribute change", change);
	break; // Won't be packaged with operation results we may be waiting for

	} else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
	\|\| pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS,
	pcmk__str_none)) {
	abort_unless_down(xpath, op, change, "Transient attribute change");
	break; // Won't be packaged with operation results we may be waiting for

	} else if (strcmp(op, "delete") == 0) {
	process_delete_diff(xpath, op, change);

	} else if (name == NULL) {
	crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
	op, xpath);

	} else if (strcmp(name, PCMK_XE_CIB) == 0) {
	process_cib_diff(match, change, op, xpath);

	} else if (strcmp(name, PCMK_XE_STATUS) == 0) {
	process_status_diff(match, change, op, xpath);

	- } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
	+ } else if (strcmp(name, PCMK__XE_NODE_STATE) == 0) {
	process_node_state_diff(match, change, op, xpath);

	} else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
	process_resource_updates(ID(match), match, change, op, xpath);

	} else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	process_resource_updates(local_node, match, change, op, xpath);
	free(local_node);

	} else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	process_lrm_resource_diff(match, local_node);
	free(local_node);

	} else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	process_graph_event(match, local_node);
	free(local_node);

	} else {
	crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
	op, xpath, name);
	}
	}
	}

	void
	te_update_diff(const char event, xmlNode msg)
	{
	xmlNode *diff = NULL;
	const char *op = NULL;
	int rc = -EINVAL;
	int format = 1;
	int p_add[] = { 0, 0, 0 };
	int p_del[] = { 0, 0, 0 };

	CRM_CHECK(msg != NULL, return);
	crm_element_value_int(msg, F_CIB_RC, &rc);

	if (controld_globals.transition_graph == NULL) {
	crm_trace("No graph");
	return;

	} else if (rc < pcmk_ok) {
	crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
	return;

	} else if (controld_globals.transition_graph->complete
	&& (controld_globals.fsa_state != S_IDLE)
	&& (controld_globals.fsa_state != S_TRANSITION_ENGINE)
	&& (controld_globals.fsa_state != S_POLICY_ENGINE)) {
	crm_trace("Filter state=%s (complete)",
	fsa_state2string(controld_globals.fsa_state));
	return;
	}

	op = crm_element_value(msg, F_CIB_OPERATION);
	diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	xml_patch_versions(diff, p_add, p_del);
	crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
	p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
	fsa_state2string(controld_globals.fsa_state));

	crm_element_value_int(diff, PCMK_XA_FORMAT, &format);
	switch (format) {
	case 1:
	te_update_diff_v1(event, diff);
	break;
	case 2:
	te_update_diff_v2(diff);
	break;
	default:
	crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
	format);
	}
	controld_remove_all_outside_events();
	}

	void
	process_te_message(xmlNode * msg, xmlNode * xml_data)
	{
	const char *value = NULL;
	xmlXPathObject *xpathObj = NULL;
	int nmatches = 0;

	CRM_CHECK(msg != NULL, return);

	// Transition requests must specify transition engine as subsystem
	value = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
	if (pcmk__str_empty(value)
	\|\| !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
	crm_info("Received invalid transition request: subsystem '%s' not '"
	CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
	return;
	}

	// Only the lrm_invoke command is supported as a transition request
	value = crm_element_value(msg, PCMK__XA_CRM_TASK);
	if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
	crm_info("Received invalid transition request: command '%s' not '"
	CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
	return;
	}

	// Transition requests must be marked as coming from the executor
	value = crm_element_value(msg, PCMK__XA_CRM_SYS_FROM);
	if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
	crm_info("Received invalid transition request: from '%s' not '"
	CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
	return;
	}

	crm_debug("Processing transition request with ref='%s' origin='%s'",
	pcmk__s(crm_element_value(msg, PCMK_XA_REFERENCE), ""),
	pcmk__s(crm_element_value(msg, PCMK__XA_SRC), ""));

	xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
	nmatches = numXpathResults(xpathObj);
	if (nmatches == 0) {
	crm_err("Received transition request with no results (bug?)");
	} else {
	for (int lpc = 0; lpc < nmatches; lpc++) {
	xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
	const char *node = get_node_id(rsc_op);

	process_graph_event(rsc_op, node);
	}
	}
	freeXpathObject(xpathObj);
	}

	void
	cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	if (rc < pcmk_ok) {
	crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
	}
	}

	/*!
	* \brief Handle a timeout in node-to-node communication
	*
	* \param[in,out] data Pointer to graph action
	*
	* \return FALSE (indicating that source should be not be re-added)
	*/
	gboolean
	action_timer_callback(gpointer data)
	{
	pcmk__graph_action_t action = (pcmk__graph_action_t ) data;
	const char *task = NULL;
	const char *on_node = NULL;
	const char *via_node = NULL;

	CRM_CHECK(data != NULL, return FALSE);

	stop_te_timer(action);

	task = crm_element_value(action->xml, PCMK_XA_OPERATION);
	on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
	via_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);

	if (controld_globals.transition_graph->complete) {
	crm_notice("Node %s did not send %s result (via %s) within %dms "
	"(ignoring because transition not in progress)",
	(on_node? on_node : ""), (task? task : "unknown action"),
	(via_node? via_node : "controller"), action->timeout);
	} else {
	/* fail the action */

	crm_err("Node %s did not send %s result (via %s) within %dms "
	"(action timeout plus " PCMK_OPT_CLUSTER_DELAY ")",
	(on_node? on_node : ""), (task? task : "unknown action"),
	(via_node? via_node : "controller"),
	(action->timeout
	+ controld_globals.transition_graph->network_delay));
	pcmk__log_graph_action(LOG_ERR, action);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);

	te_action_confirmed(action, controld_globals.transition_graph);
	abort_transition(INFINITY, pcmk__graph_restart, "Action lost", NULL);

	// Record timeout in the CIB if appropriate
	if ((action->type == pcmk__rsc_graph_action)
	&& controld_action_is_recordable(task)) {
	controld_record_action_timeout(action);
	}
	}

	return FALSE;
	}
	diff --git a/daemons/controld/controld_te_utils.c b/daemons/controld/controld_te_utils.c
	index d03230c5f5..eee3b16178 100644
	--- a/daemons/controld/controld_te_utils.c
	+++ b/daemons/controld/controld_te_utils.c
	@@ -1,510 +1,510 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	//! Triggers transition graph processing
	static crm_trigger_t *transition_trigger = NULL;

	static GHashTable *node_pending_timers = NULL;

	gboolean
	stop_te_timer(pcmk__graph_action_t *action)
	{
	if (action == NULL) {
	return FALSE;
	}
	if (action->timer != 0) {
	crm_trace("Stopping action timer");
	g_source_remove(action->timer);
	action->timer = 0;
	} else {
	crm_trace("Action timer was already stopped");
	return FALSE;
	}
	return TRUE;
	}

	static gboolean
	te_graph_trigger(gpointer user_data)
	{
	if (controld_globals.transition_graph == NULL) {
	crm_debug("Nothing to do");
	return TRUE;
	}

	crm_trace("Invoking graph %d in state %s",
	controld_globals.transition_graph->id,
	fsa_state2string(controld_globals.fsa_state));

	switch (controld_globals.fsa_state) {
	case S_STARTING:
	case S_PENDING:
	case S_NOT_DC:
	case S_HALT:
	case S_ILLEGAL:
	case S_STOPPING:
	case S_TERMINATE:
	return TRUE;
	default:
	break;
	}

	if (!controld_globals.transition_graph->complete) {
	enum pcmk__graph_status graph_rc;
	int orig_limit = controld_globals.transition_graph->batch_limit;
	int throttled_limit = throttle_get_total_job_limit(orig_limit);

	controld_globals.transition_graph->batch_limit = throttled_limit;
	graph_rc = pcmk__execute_graph(controld_globals.transition_graph);
	controld_globals.transition_graph->batch_limit = orig_limit;

	if (graph_rc == pcmk__graph_active) {
	crm_trace("Transition not yet complete");
	return TRUE;

	} else if (graph_rc == pcmk__graph_pending) {
	crm_trace("Transition not yet complete - no actions fired");
	return TRUE;
	}

	if (graph_rc != pcmk__graph_complete) {
	crm_warn("Transition failed: %s",
	pcmk__graph_status2text(graph_rc));
	pcmk__log_graph(LOG_NOTICE, controld_globals.transition_graph);
	}
	}

	crm_debug("Transition %d is now complete",
	controld_globals.transition_graph->id);
	controld_globals.transition_graph->complete = true;
	notify_crmd(controld_globals.transition_graph);

	return TRUE;
	}

	/*!
	* \internal
	* \brief Initialize transition trigger
	*/
	void
	controld_init_transition_trigger(void)
	{
	transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger,
	NULL);
	}

	/*!
	* \internal
	* \brief Destroy transition trigger
	*/
	void
	controld_destroy_transition_trigger(void)
	{
	mainloop_destroy_trigger(transition_trigger);
	transition_trigger = NULL;
	}

	void
	controld_trigger_graph_as(const char *fn, int line)
	{
	crm_trace("%s:%d - Triggered graph processing", fn, line);
	mainloop_set_trigger(transition_trigger);
	}

	static struct abort_timer_s {
	bool aborted;
	guint id;
	int priority;
	enum pcmk__graph_next action;
	const char *text;
	} abort_timer = { 0, };

	static gboolean
	abort_timer_popped(gpointer data)
	{
	struct abort_timer_s abort_timer = (struct abort_timer_s ) data;

	if (AM_I_DC && (abort_timer->aborted == FALSE)) {
	abort_transition(abort_timer->priority, abort_timer->action,
	abort_timer->text, NULL);
	}
	abort_timer->id = 0;
	return FALSE; // do not immediately reschedule timer
	}

	/*!
	* \internal
	* \brief Abort transition after delay, if not already aborted in that time
	*
	* \param[in] abort_text Must be literal string
	*/
	void
	abort_after_delay(int abort_priority, enum pcmk__graph_next abort_action,
	const char *abort_text, guint delay_ms)
	{
	if (abort_timer.id) {
	// Timer already in progress, stop and reschedule
	g_source_remove(abort_timer.id);
	}
	abort_timer.aborted = FALSE;
	abort_timer.priority = abort_priority;
	abort_timer.action = abort_action;
	abort_timer.text = abort_text;
	abort_timer.id = g_timeout_add(delay_ms, abort_timer_popped, &abort_timer);
	}

	static void
	free_node_pending_timer(gpointer data)
	{
	struct abort_timer_s node_pending_timer = (struct abort_timer_s ) data;

	if (node_pending_timer->id != 0) {
	g_source_remove(node_pending_timer->id);
	node_pending_timer->id = 0;
	}

	free(node_pending_timer);
	}

	static gboolean
	node_pending_timer_popped(gpointer key)
	{
	struct abort_timer_s *node_pending_timer = NULL;

	if (node_pending_timers == NULL) {
	return FALSE;
	}

	node_pending_timer = g_hash_table_lookup(node_pending_timers, key);
	if (node_pending_timer == NULL) {
	return FALSE;
	}

	crm_warn("Node with id '%s' pending timed out (%us) on joining the process "
	"group",
	(const char *) key, controld_globals.node_pending_timeout);

	if (controld_globals.node_pending_timeout > 0) {
	abort_timer_popped(node_pending_timer);
	}

	g_hash_table_remove(node_pending_timers, key);

	return FALSE; // do not reschedule timer
	}

	static void
	init_node_pending_timer(const crm_node_t *node, guint timeout)
	{
	struct abort_timer_s *node_pending_timer = NULL;
	char *key = NULL;

	if (node->uuid == NULL) {
	return;
	}

	if (node_pending_timers == NULL) {
	node_pending_timers = pcmk__strikey_table(free,
	free_node_pending_timer);

	// The timer is somehow already existing
	} else if (g_hash_table_lookup(node_pending_timers, node->uuid) != NULL) {
	return;
	}

	crm_notice("Waiting for pending %s with id '%s' to join the process "
	"group (timeout=%us)",
	node->uname ? node->uname : "node", node->uuid,
	controld_globals.node_pending_timeout);

	node_pending_timer = calloc(1, sizeof(struct abort_timer_s));
	CRM_ASSERT(node_pending_timer != NULL);

	node_pending_timer->aborted = FALSE;
	node_pending_timer->priority = INFINITY;
	node_pending_timer->action = pcmk__graph_restart;
	node_pending_timer->text = "Node pending timed out";

	key = strdup(node->uuid);
	CRM_ASSERT(key != NULL);

	g_hash_table_replace(node_pending_timers, key, node_pending_timer);

	node_pending_timer->id = g_timeout_add_seconds(timeout,
	node_pending_timer_popped,
	key);
	CRM_ASSERT(node_pending_timer->id != 0);
	}

	static void
	remove_node_pending_timer(const char *node_uuid)
	{
	if (node_pending_timers == NULL) {
	return;
	}

	g_hash_table_remove(node_pending_timers, node_uuid);
	}

	void
	controld_node_pending_timer(const crm_node_t *node)
	{
	long long remaining_timeout = 0;

	/* If the node is not an active cluster node, is leaving the cluster, or is
	* already part of CPG, or PCMK_OPT_NODE_PENDING_TIMEOUT is disabled, free
	* any node pending timer for it.
	*/
	if (pcmk_is_set(node->flags, crm_remote_node)
	\|\| (node->when_member <= 1) \|\| (node->when_online > 0)
	\|\| (controld_globals.node_pending_timeout == 0)) {
	remove_node_pending_timer(node->uuid);
	return;
	}

	// Node is a cluster member but offline in CPG

	remaining_timeout = node->when_member - time(NULL)
	+ controld_globals.node_pending_timeout;

	/* It already passed node pending timeout somehow.
	* Free any node pending timer of it.
	*/
	if (remaining_timeout <= 0) {
	remove_node_pending_timer(node->uuid);
	return;
	}

	init_node_pending_timer(node, remaining_timeout);
	}

	void
	controld_free_node_pending_timers(void)
	{
	if (node_pending_timers == NULL) {
	return;
	}

	g_hash_table_destroy(node_pending_timers);
	node_pending_timers = NULL;
	}

	static const char *
	abort2text(enum pcmk__graph_next abort_action)
	{
	switch (abort_action) {
	case pcmk__graph_done: return "done";
	case pcmk__graph_wait: return "stop";
	case pcmk__graph_restart: return "restart";
	case pcmk__graph_shutdown: return "shutdown";
	}
	return "unknown";
	}

	static bool
	update_abort_priority(pcmk__graph_t *graph, int priority,
	enum pcmk__graph_next action, const char *abort_reason)
	{
	bool change = FALSE;

	if (graph == NULL) {
	return change;
	}

	if (graph->abort_priority < priority) {
	crm_debug("Abort priority upgraded from %d to %d", graph->abort_priority, priority);
	graph->abort_priority = priority;
	if (graph->abort_reason != NULL) {
	crm_debug("'%s' abort superseded by %s", graph->abort_reason, abort_reason);
	}
	graph->abort_reason = abort_reason;
	change = TRUE;
	}

	if (graph->completion_action < action) {
	crm_debug("Abort action %s superseded by %s: %s",
	abort2text(graph->completion_action), abort2text(action), abort_reason);
	graph->completion_action = action;
	change = TRUE;
	}

	return change;
	}

	void
	abort_transition_graph(int abort_priority, enum pcmk__graph_next abort_action,
	const char abort_text, const xmlNode reason,
	const char *fn, int line)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };
	int level = LOG_INFO;
	const xmlNode *diff = NULL;
	const xmlNode *change = NULL;

	CRM_CHECK(controld_globals.transition_graph != NULL, return);

	switch (controld_globals.fsa_state) {
	case S_STARTING:
	case S_PENDING:
	case S_NOT_DC:
	case S_HALT:
	case S_ILLEGAL:
	case S_STOPPING:
	case S_TERMINATE:
	crm_info("Abort %s suppressed: state=%s (%scomplete)",
	abort_text, fsa_state2string(controld_globals.fsa_state),
	(controld_globals.transition_graph->complete? "" : "in"));
	return;
	default:
	break;
	}

	abort_timer.aborted = TRUE;
	controld_expect_sched_reply(NULL);

	if (!controld_globals.transition_graph->complete
	&& update_abort_priority(controld_globals.transition_graph,
	abort_priority, abort_action,
	abort_text)) {
	level = LOG_NOTICE;
	}

	if (reason != NULL) {
	const xmlNode *search = NULL;

	for(search = reason; search; search = search->parent) {
	if (pcmk__xe_is(search, XML_TAG_DIFF)) {
	diff = search;
	break;
	}
	}

	if(diff) {
	xml_patch_versions(diff, add, del);
	for(search = reason; search; search = search->parent) {
	if (pcmk__xe_is(search, XML_DIFF_CHANGE)) {
	change = search;
	break;
	}
	}
	}
	}

	if (reason == NULL) {
	do_crm_log(level,
	"Transition %d aborted: %s " CRM_XS " source=%s:%d "
	"complete=%s", controld_globals.transition_graph->id,
	abort_text, fn, line,
	pcmk__btoa(controld_globals.transition_graph->complete));

	} else if(change == NULL) {
	GString *local_path = pcmk__element_xpath(reason);
	CRM_ASSERT(local_path != NULL);

	do_crm_log(level, "Transition %d aborted by %s.%s: %s "
	CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
	controld_globals.transition_graph->id, reason->name,
	ID(reason), abort_text, add[0], add[1], add[2], fn, line,
	(const char *) local_path->str,
	pcmk__btoa(controld_globals.transition_graph->complete));
	g_string_free(local_path, TRUE);

	} else {
	const char *op = crm_element_value(change, PCMK_XA_OPERATION);
	const char *path = crm_element_value(change, PCMK_XA_PATH);

	if(change == reason) {
	if(strcmp(op, "create") == 0) {
	reason = reason->children;

	} else if(strcmp(op, "modify") == 0) {
	reason = first_named_child(reason, XML_DIFF_RESULT);
	if(reason) {
	reason = reason->children;
	}
	}
	CRM_CHECK(reason != NULL, goto done);
	}

	if(strcmp(op, "delete") == 0) {
	const char *shortpath = strrchr(path, '/');

	do_crm_log(level, "Transition %d aborted by deletion of %s: %s "
	CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
	controld_globals.transition_graph->id,
	(shortpath? (shortpath + 1) : path), abort_text,
	add[0], add[1], add[2], fn, line, path,
	pcmk__btoa(controld_globals.transition_graph->complete));

	} else if (pcmk__xe_is(reason, XML_CIB_TAG_NVPAIR)) {
	do_crm_log(level, "Transition %d aborted by %s doing %s %s=%s: %s "
	CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
	controld_globals.transition_graph->id,
	crm_element_value(reason, PCMK_XA_ID), op,
	crm_element_value(reason, PCMK_XA_NAME),
	crm_element_value(reason, PCMK_XA_VALUE),
	abort_text, add[0], add[1], add[2], fn, line, path,
	pcmk__btoa(controld_globals.transition_graph->complete));

	} else if (pcmk__xe_is(reason, XML_LRM_TAG_RSC_OP)) {
	const char *magic = crm_element_value(reason,
	PCMK__XA_TRANSITION_MAGIC);

	do_crm_log(level, "Transition %d aborted by operation %s '%s' on %s: %s "
	CRM_XS " magic=%s cib=%d.%d.%d source=%s:%d complete=%s",
	controld_globals.transition_graph->id,
	crm_element_value(reason, PCMK__XA_OPERATION_KEY), op,
	crm_element_value(reason, PCMK__META_ON_NODE),
	abort_text,
	magic, add[0], add[1], add[2], fn, line,
	pcmk__btoa(controld_globals.transition_graph->complete));

	} else if (pcmk__str_any_of((const char *) reason->name,
	- XML_CIB_TAG_STATE, XML_CIB_TAG_NODE, NULL)) {
	+ PCMK__XE_NODE_STATE, XML_CIB_TAG_NODE, NULL)) {
	const char *uname = crm_peer_uname(ID(reason));

	do_crm_log(level, "Transition %d aborted by %s '%s' on %s: %s "
	CRM_XS " cib=%d.%d.%d source=%s:%d complete=%s",
	controld_globals.transition_graph->id,
	reason->name, op, pcmk__s(uname, ID(reason)),
	abort_text, add[0], add[1], add[2], fn, line,
	pcmk__btoa(controld_globals.transition_graph->complete));

	} else {
	const char *id = ID(reason);

	do_crm_log(level, "Transition %d aborted by %s.%s '%s': %s "
	CRM_XS " cib=%d.%d.%d source=%s:%d path=%s complete=%s",
	controld_globals.transition_graph->id,
	reason->name, pcmk__s(id, ""), pcmk__s(op, "change"),
	abort_text, add[0], add[1], add[2], fn, line, path,
	pcmk__btoa(controld_globals.transition_graph->complete));
	}
	}

	done:
	if (controld_globals.transition_graph->complete) {
	if (controld_get_period_transition_timer() > 0) {
	controld_stop_transition_timer();
	controld_start_transition_timer();
	} else {
	register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL);
	}
	return;
	}

	trigger_graph();
	}
	diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c
	index d6b46ba5af..768ea9c8bd 100644
	--- a/daemons/controld/controld_utils.c
	+++ b/daemons/controld/controld_utils.c
	@@ -1,838 +1,838 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <stdint.h> // uint64_t

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	const char *
	fsa_input2string(enum crmd_fsa_input input)
	{
	const char *inputAsText = NULL;

	switch (input) {
	case I_NULL:
	inputAsText = "I_NULL";
	break;
	case I_CIB_OP:
	inputAsText = "I_CIB_OP (unused)";
	break;
	case I_CIB_UPDATE:
	inputAsText = "I_CIB_UPDATE";
	break;
	case I_DC_TIMEOUT:
	inputAsText = "I_DC_TIMEOUT";
	break;
	case I_ELECTION:
	inputAsText = "I_ELECTION";
	break;
	case I_PE_CALC:
	inputAsText = "I_PE_CALC";
	break;
	case I_RELEASE_DC:
	inputAsText = "I_RELEASE_DC";
	break;
	case I_ELECTION_DC:
	inputAsText = "I_ELECTION_DC";
	break;
	case I_ERROR:
	inputAsText = "I_ERROR";
	break;
	case I_FAIL:
	inputAsText = "I_FAIL";
	break;
	case I_INTEGRATED:
	inputAsText = "I_INTEGRATED";
	break;
	case I_FINALIZED:
	inputAsText = "I_FINALIZED";
	break;
	case I_NODE_JOIN:
	inputAsText = "I_NODE_JOIN";
	break;
	case I_JOIN_OFFER:
	inputAsText = "I_JOIN_OFFER";
	break;
	case I_JOIN_REQUEST:
	inputAsText = "I_JOIN_REQUEST";
	break;
	case I_JOIN_RESULT:
	inputAsText = "I_JOIN_RESULT";
	break;
	case I_NOT_DC:
	inputAsText = "I_NOT_DC";
	break;
	case I_RECOVERED:
	inputAsText = "I_RECOVERED";
	break;
	case I_RELEASE_FAIL:
	inputAsText = "I_RELEASE_FAIL";
	break;
	case I_RELEASE_SUCCESS:
	inputAsText = "I_RELEASE_SUCCESS";
	break;
	case I_RESTART:
	inputAsText = "I_RESTART";
	break;
	case I_PE_SUCCESS:
	inputAsText = "I_PE_SUCCESS";
	break;
	case I_ROUTER:
	inputAsText = "I_ROUTER";
	break;
	case I_SHUTDOWN:
	inputAsText = "I_SHUTDOWN";
	break;
	case I_STARTUP:
	inputAsText = "I_STARTUP";
	break;
	case I_TE_SUCCESS:
	inputAsText = "I_TE_SUCCESS";
	break;
	case I_STOP:
	inputAsText = "I_STOP";
	break;
	case I_DC_HEARTBEAT:
	inputAsText = "I_DC_HEARTBEAT";
	break;
	case I_WAIT_FOR_EVENT:
	inputAsText = "I_WAIT_FOR_EVENT";
	break;
	case I_LRM_EVENT:
	inputAsText = "I_LRM_EVENT";
	break;
	case I_PENDING:
	inputAsText = "I_PENDING";
	break;
	case I_HALT:
	inputAsText = "I_HALT";
	break;
	case I_TERMINATE:
	inputAsText = "I_TERMINATE";
	break;
	case I_ILLEGAL:
	inputAsText = "I_ILLEGAL";
	break;
	}

	if (inputAsText == NULL) {
	crm_err("Input %d is unknown", input);
	inputAsText = "<UNKNOWN_INPUT>";
	}

	return inputAsText;
	}

	const char *
	fsa_state2string(enum crmd_fsa_state state)
	{
	const char *stateAsText = NULL;

	switch (state) {
	case S_IDLE:
	stateAsText = "S_IDLE";
	break;
	case S_ELECTION:
	stateAsText = "S_ELECTION";
	break;
	case S_INTEGRATION:
	stateAsText = "S_INTEGRATION";
	break;
	case S_FINALIZE_JOIN:
	stateAsText = "S_FINALIZE_JOIN";
	break;
	case S_NOT_DC:
	stateAsText = "S_NOT_DC";
	break;
	case S_POLICY_ENGINE:
	stateAsText = "S_POLICY_ENGINE";
	break;
	case S_RECOVERY:
	stateAsText = "S_RECOVERY";
	break;
	case S_RELEASE_DC:
	stateAsText = "S_RELEASE_DC";
	break;
	case S_PENDING:
	stateAsText = "S_PENDING";
	break;
	case S_STOPPING:
	stateAsText = "S_STOPPING";
	break;
	case S_TERMINATE:
	stateAsText = "S_TERMINATE";
	break;
	case S_TRANSITION_ENGINE:
	stateAsText = "S_TRANSITION_ENGINE";
	break;
	case S_STARTING:
	stateAsText = "S_STARTING";
	break;
	case S_HALT:
	stateAsText = "S_HALT";
	break;
	case S_ILLEGAL:
	stateAsText = "S_ILLEGAL";
	break;
	}

	if (stateAsText == NULL) {
	crm_err("State %d is unknown", state);
	stateAsText = "<UNKNOWN_STATE>";
	}

	return stateAsText;
	}

	const char *
	fsa_cause2string(enum crmd_fsa_cause cause)
	{
	const char *causeAsText = NULL;

	switch (cause) {
	case C_UNKNOWN:
	causeAsText = "C_UNKNOWN";
	break;
	case C_STARTUP:
	causeAsText = "C_STARTUP";
	break;
	case C_IPC_MESSAGE:
	causeAsText = "C_IPC_MESSAGE";
	break;
	case C_HA_MESSAGE:
	causeAsText = "C_HA_MESSAGE";
	break;
	case C_TIMER_POPPED:
	causeAsText = "C_TIMER_POPPED";
	break;
	case C_SHUTDOWN:
	causeAsText = "C_SHUTDOWN";
	break;
	case C_LRM_OP_CALLBACK:
	causeAsText = "C_LRM_OP_CALLBACK";
	break;
	case C_CRMD_STATUS_CALLBACK:
	causeAsText = "C_CRMD_STATUS_CALLBACK";
	break;
	case C_FSA_INTERNAL:
	causeAsText = "C_FSA_INTERNAL";
	break;
	}

	if (causeAsText == NULL) {
	crm_err("Cause %d is unknown", cause);
	causeAsText = "<UNKNOWN_CAUSE>";
	}

	return causeAsText;
	}

	const char *
	fsa_action2string(long long action)
	{
	const char *actionAsText = NULL;

	switch (action) {

	case A_NOTHING:
	actionAsText = "A_NOTHING";
	break;
	case A_ELECTION_START:
	actionAsText = "A_ELECTION_START";
	break;
	case A_DC_JOIN_FINAL:
	actionAsText = "A_DC_JOIN_FINAL";
	break;
	case A_READCONFIG:
	actionAsText = "A_READCONFIG";
	break;
	case O_RELEASE:
	actionAsText = "O_RELEASE";
	break;
	case A_STARTUP:
	actionAsText = "A_STARTUP";
	break;
	case A_STARTED:
	actionAsText = "A_STARTED";
	break;
	case A_HA_CONNECT:
	actionAsText = "A_HA_CONNECT";
	break;
	case A_HA_DISCONNECT:
	actionAsText = "A_HA_DISCONNECT";
	break;
	case A_LRM_CONNECT:
	actionAsText = "A_LRM_CONNECT";
	break;
	case A_LRM_EVENT:
	actionAsText = "A_LRM_EVENT";
	break;
	case A_LRM_INVOKE:
	actionAsText = "A_LRM_INVOKE";
	break;
	case A_LRM_DISCONNECT:
	actionAsText = "A_LRM_DISCONNECT";
	break;
	case O_LRM_RECONNECT:
	actionAsText = "O_LRM_RECONNECT";
	break;
	case A_CL_JOIN_QUERY:
	actionAsText = "A_CL_JOIN_QUERY";
	break;
	case A_DC_TIMER_STOP:
	actionAsText = "A_DC_TIMER_STOP";
	break;
	case A_DC_TIMER_START:
	actionAsText = "A_DC_TIMER_START";
	break;
	case A_INTEGRATE_TIMER_START:
	actionAsText = "A_INTEGRATE_TIMER_START";
	break;
	case A_INTEGRATE_TIMER_STOP:
	actionAsText = "A_INTEGRATE_TIMER_STOP";
	break;
	case A_FINALIZE_TIMER_START:
	actionAsText = "A_FINALIZE_TIMER_START";
	break;
	case A_FINALIZE_TIMER_STOP:
	actionAsText = "A_FINALIZE_TIMER_STOP";
	break;
	case A_ELECTION_COUNT:
	actionAsText = "A_ELECTION_COUNT";
	break;
	case A_ELECTION_VOTE:
	actionAsText = "A_ELECTION_VOTE";
	break;
	case A_ELECTION_CHECK:
	actionAsText = "A_ELECTION_CHECK";
	break;
	case A_CL_JOIN_ANNOUNCE:
	actionAsText = "A_CL_JOIN_ANNOUNCE";
	break;
	case A_CL_JOIN_REQUEST:
	actionAsText = "A_CL_JOIN_REQUEST";
	break;
	case A_CL_JOIN_RESULT:
	actionAsText = "A_CL_JOIN_RESULT";
	break;
	case A_DC_JOIN_OFFER_ALL:
	actionAsText = "A_DC_JOIN_OFFER_ALL";
	break;
	case A_DC_JOIN_OFFER_ONE:
	actionAsText = "A_DC_JOIN_OFFER_ONE";
	break;
	case A_DC_JOIN_PROCESS_REQ:
	actionAsText = "A_DC_JOIN_PROCESS_REQ";
	break;
	case A_DC_JOIN_PROCESS_ACK:
	actionAsText = "A_DC_JOIN_PROCESS_ACK";
	break;
	case A_DC_JOIN_FINALIZE:
	actionAsText = "A_DC_JOIN_FINALIZE";
	break;
	case A_MSG_PROCESS:
	actionAsText = "A_MSG_PROCESS";
	break;
	case A_MSG_ROUTE:
	actionAsText = "A_MSG_ROUTE";
	break;
	case A_RECOVER:
	actionAsText = "A_RECOVER";
	break;
	case A_DC_RELEASE:
	actionAsText = "A_DC_RELEASE";
	break;
	case A_DC_RELEASED:
	actionAsText = "A_DC_RELEASED";
	break;
	case A_DC_TAKEOVER:
	actionAsText = "A_DC_TAKEOVER";
	break;
	case A_SHUTDOWN:
	actionAsText = "A_SHUTDOWN";
	break;
	case A_SHUTDOWN_REQ:
	actionAsText = "A_SHUTDOWN_REQ";
	break;
	case A_STOP:
	actionAsText = "A_STOP ";
	break;
	case A_EXIT_0:
	actionAsText = "A_EXIT_0";
	break;
	case A_EXIT_1:
	actionAsText = "A_EXIT_1";
	break;
	case O_CIB_RESTART:
	actionAsText = "O_CIB_RESTART";
	break;
	case A_CIB_START:
	actionAsText = "A_CIB_START";
	break;
	case A_CIB_STOP:
	actionAsText = "A_CIB_STOP";
	break;
	case A_TE_INVOKE:
	actionAsText = "A_TE_INVOKE";
	break;
	case O_TE_RESTART:
	actionAsText = "O_TE_RESTART";
	break;
	case A_TE_START:
	actionAsText = "A_TE_START";
	break;
	case A_TE_STOP:
	actionAsText = "A_TE_STOP";
	break;
	case A_TE_HALT:
	actionAsText = "A_TE_HALT";
	break;
	case A_TE_CANCEL:
	actionAsText = "A_TE_CANCEL";
	break;
	case A_PE_INVOKE:
	actionAsText = "A_PE_INVOKE";
	break;
	case O_PE_RESTART:
	actionAsText = "O_PE_RESTART";
	break;
	case A_PE_START:
	actionAsText = "A_PE_START";
	break;
	case A_PE_STOP:
	actionAsText = "A_PE_STOP";
	break;
	case A_NODE_BLOCK:
	actionAsText = "A_NODE_BLOCK";
	break;
	case A_UPDATE_NODESTATUS:
	actionAsText = "A_UPDATE_NODESTATUS";
	break;
	case A_LOG:
	actionAsText = "A_LOG ";
	break;
	case A_ERROR:
	actionAsText = "A_ERROR ";
	break;
	case A_WARN:
	actionAsText = "A_WARN ";
	break;
	/* Composite actions */
	case A_DC_TIMER_START \| A_CL_JOIN_QUERY:
	actionAsText = "A_DC_TIMER_START\|A_CL_JOIN_QUERY";
	break;
	}

	if (actionAsText == NULL) {
	crm_err("Action %.16llx is unknown", action);
	actionAsText = "<UNKNOWN_ACTION>";
	}

	return actionAsText;
	}

	void
	fsa_dump_inputs(int log_level, const char *text, long long input_register)
	{
	if (input_register == A_NOTHING) {
	return;
	}
	if (text == NULL) {
	text = "Input register contents:";
	}

	if (pcmk_is_set(input_register, R_THE_DC)) {
	crm_trace("%s %.16llx (R_THE_DC)", text, R_THE_DC);
	}
	if (pcmk_is_set(input_register, R_STARTING)) {
	crm_trace("%s %.16llx (R_STARTING)", text, R_STARTING);
	}
	if (pcmk_is_set(input_register, R_SHUTDOWN)) {
	crm_trace("%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN);
	}
	if (pcmk_is_set(input_register, R_STAYDOWN)) {
	crm_trace("%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN);
	}
	if (pcmk_is_set(input_register, R_JOIN_OK)) {
	crm_trace("%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK);
	}
	if (pcmk_is_set(input_register, R_READ_CONFIG)) {
	crm_trace("%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG);
	}
	if (pcmk_is_set(input_register, R_INVOKE_PE)) {
	crm_trace("%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE);
	}
	if (pcmk_is_set(input_register, R_CIB_CONNECTED)) {
	crm_trace("%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_PE_CONNECTED)) {
	crm_trace("%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_TE_CONNECTED)) {
	crm_trace("%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_LRM_CONNECTED)) {
	crm_trace("%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED);
	}
	if (pcmk_is_set(input_register, R_CIB_REQUIRED)) {
	crm_trace("%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED);
	}
	if (pcmk_is_set(input_register, R_PE_REQUIRED)) {
	crm_trace("%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED);
	}
	if (pcmk_is_set(input_register, R_TE_REQUIRED)) {
	crm_trace("%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED);
	}
	if (pcmk_is_set(input_register, R_REQ_PEND)) {
	crm_trace("%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND);
	}
	if (pcmk_is_set(input_register, R_PE_PEND)) {
	crm_trace("%s %.16llx (R_PE_PEND)", text, R_PE_PEND);
	}
	if (pcmk_is_set(input_register, R_TE_PEND)) {
	crm_trace("%s %.16llx (R_TE_PEND)", text, R_TE_PEND);
	}
	if (pcmk_is_set(input_register, R_RESP_PEND)) {
	crm_trace("%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND);
	}
	if (pcmk_is_set(input_register, R_CIB_DONE)) {
	crm_trace("%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE);
	}
	if (pcmk_is_set(input_register, R_HAVE_CIB)) {
	crm_trace("%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB);
	}
	if (pcmk_is_set(input_register, R_MEMBERSHIP)) {
	crm_trace("%s %.16llx (R_MEMBERSHIP)", text, R_MEMBERSHIP);
	}
	if (pcmk_is_set(input_register, R_PEER_DATA)) {
	crm_trace("%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA);
	}
	if (pcmk_is_set(input_register, R_IN_RECOVERY)) {
	crm_trace("%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY);
	}
	}

	void
	fsa_dump_actions(uint64_t action, const char *text)
	{
	if (pcmk_is_set(action, A_READCONFIG)) {
	crm_trace("Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text);
	}
	if (pcmk_is_set(action, A_STARTUP)) {
	crm_trace("Action %.16llx (A_STARTUP) %s", A_STARTUP, text);
	}
	if (pcmk_is_set(action, A_STARTED)) {
	crm_trace("Action %.16llx (A_STARTED) %s", A_STARTED, text);
	}
	if (pcmk_is_set(action, A_HA_CONNECT)) {
	crm_trace("Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text);
	}
	if (pcmk_is_set(action, A_HA_DISCONNECT)) {
	crm_trace("Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text);
	}
	if (pcmk_is_set(action, A_LRM_CONNECT)) {
	crm_trace("Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text);
	}
	if (pcmk_is_set(action, A_LRM_EVENT)) {
	crm_trace("Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text);
	}
	if (pcmk_is_set(action, A_LRM_INVOKE)) {
	crm_trace("Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text);
	}
	if (pcmk_is_set(action, A_LRM_DISCONNECT)) {
	crm_trace("Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text);
	}
	if (pcmk_is_set(action, A_DC_TIMER_STOP)) {
	crm_trace("Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text);
	}
	if (pcmk_is_set(action, A_DC_TIMER_START)) {
	crm_trace("Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text);
	}
	if (pcmk_is_set(action, A_INTEGRATE_TIMER_START)) {
	crm_trace("Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text);
	}
	if (pcmk_is_set(action, A_INTEGRATE_TIMER_STOP)) {
	crm_trace("Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text);
	}
	if (pcmk_is_set(action, A_FINALIZE_TIMER_START)) {
	crm_trace("Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text);
	}
	if (pcmk_is_set(action, A_FINALIZE_TIMER_STOP)) {
	crm_trace("Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text);
	}
	if (pcmk_is_set(action, A_ELECTION_COUNT)) {
	crm_trace("Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text);
	}
	if (pcmk_is_set(action, A_ELECTION_VOTE)) {
	crm_trace("Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text);
	}
	if (pcmk_is_set(action, A_ELECTION_CHECK)) {
	crm_trace("Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text);
	}
	if (pcmk_is_set(action, A_CL_JOIN_ANNOUNCE)) {
	crm_trace("Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text);
	}
	if (pcmk_is_set(action, A_CL_JOIN_REQUEST)) {
	crm_trace("Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text);
	}
	if (pcmk_is_set(action, A_CL_JOIN_RESULT)) {
	crm_trace("Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_OFFER_ALL)) {
	crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_OFFER_ONE)) {
	crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_PROCESS_REQ)) {
	crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_PROCESS_ACK)) {
	crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text);
	}
	if (pcmk_is_set(action, A_DC_JOIN_FINALIZE)) {
	crm_trace("Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text);
	}
	if (pcmk_is_set(action, A_MSG_PROCESS)) {
	crm_trace("Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text);
	}
	if (pcmk_is_set(action, A_MSG_ROUTE)) {
	crm_trace("Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text);
	}
	if (pcmk_is_set(action, A_RECOVER)) {
	crm_trace("Action %.16llx (A_RECOVER) %s", A_RECOVER, text);
	}
	if (pcmk_is_set(action, A_DC_RELEASE)) {
	crm_trace("Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text);
	}
	if (pcmk_is_set(action, A_DC_RELEASED)) {
	crm_trace("Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text);
	}
	if (pcmk_is_set(action, A_DC_TAKEOVER)) {
	crm_trace("Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text);
	}
	if (pcmk_is_set(action, A_SHUTDOWN)) {
	crm_trace("Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text);
	}
	if (pcmk_is_set(action, A_SHUTDOWN_REQ)) {
	crm_trace("Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text);
	}
	if (pcmk_is_set(action, A_STOP)) {
	crm_trace("Action %.16llx (A_STOP ) %s", A_STOP, text);
	}
	if (pcmk_is_set(action, A_EXIT_0)) {
	crm_trace("Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text);
	}
	if (pcmk_is_set(action, A_EXIT_1)) {
	crm_trace("Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text);
	}
	if (pcmk_is_set(action, A_CIB_START)) {
	crm_trace("Action %.16llx (A_CIB_START) %s", A_CIB_START, text);
	}
	if (pcmk_is_set(action, A_CIB_STOP)) {
	crm_trace("Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text);
	}
	if (pcmk_is_set(action, A_TE_INVOKE)) {
	crm_trace("Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text);
	}
	if (pcmk_is_set(action, A_TE_START)) {
	crm_trace("Action %.16llx (A_TE_START) %s", A_TE_START, text);
	}
	if (pcmk_is_set(action, A_TE_STOP)) {
	crm_trace("Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text);
	}
	if (pcmk_is_set(action, A_TE_CANCEL)) {
	crm_trace("Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text);
	}
	if (pcmk_is_set(action, A_PE_INVOKE)) {
	crm_trace("Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text);
	}
	if (pcmk_is_set(action, A_PE_START)) {
	crm_trace("Action %.16llx (A_PE_START) %s", A_PE_START, text);
	}
	if (pcmk_is_set(action, A_PE_STOP)) {
	crm_trace("Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text);
	}
	if (pcmk_is_set(action, A_NODE_BLOCK)) {
	crm_trace("Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text);
	}
	if (pcmk_is_set(action, A_UPDATE_NODESTATUS)) {
	crm_trace("Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text);
	}
	if (pcmk_is_set(action, A_LOG)) {
	crm_trace("Action %.16llx (A_LOG ) %s", A_LOG, text);
	}
	if (pcmk_is_set(action, A_ERROR)) {
	crm_trace("Action %.16llx (A_ERROR ) %s", A_ERROR, text);
	}
	if (pcmk_is_set(action, A_WARN)) {
	crm_trace("Action %.16llx (A_WARN ) %s", A_WARN, text);
	}
	}

	gboolean
	update_dc(xmlNode * msg)
	{
	char *last_dc = controld_globals.dc_name;
	const char *dc_version = NULL;
	const char *welcome_from = NULL;

	if (msg != NULL) {
	gboolean invalid = FALSE;

	dc_version = crm_element_value(msg, PCMK_XA_VERSION);
	welcome_from = crm_element_value(msg, PCMK__XA_SRC);

	CRM_CHECK(dc_version != NULL, return FALSE);
	CRM_CHECK(welcome_from != NULL, return FALSE);

	if (AM_I_DC
	&& !pcmk__str_eq(welcome_from, controld_globals.our_nodename,
	pcmk__str_casei)) {
	invalid = TRUE;

	} else if ((controld_globals.dc_name != NULL)
	&& !pcmk__str_eq(welcome_from, controld_globals.dc_name,
	pcmk__str_casei)) {
	invalid = TRUE;
	}

	if (invalid) {
	if (AM_I_DC) {
	crm_err("Not updating DC to %s (%s): we are also a DC",
	welcome_from, dc_version);
	} else {
	crm_warn("New DC %s is not %s",
	welcome_from, controld_globals.dc_name);
	}

	controld_set_fsa_action_flags(A_CL_JOIN_QUERY \| A_DC_TIMER_START);
	controld_trigger_fsa();
	return FALSE;
	}
	}

	controld_globals.dc_name = NULL; // freed as last_dc
	pcmk__str_update(&(controld_globals.dc_name), welcome_from);
	pcmk__str_update(&(controld_globals.dc_version), dc_version);

	if (pcmk__str_eq(controld_globals.dc_name, last_dc, pcmk__str_casei)) {
	/* do nothing */

	} else if (controld_globals.dc_name != NULL) {
	crm_node_t *dc_node = pcmk__get_node(0, controld_globals.dc_name, NULL,
	pcmk__node_search_cluster);

	crm_info("Set DC to %s (%s)",
	controld_globals.dc_name,
	pcmk__s(controld_globals.dc_version, "unknown version"));
	pcmk__update_peer_expected(__func__, dc_node, CRMD_JOINSTATE_MEMBER);

	} else if (last_dc != NULL) {
	crm_info("Unset DC (was %s)", last_dc);
	}

	free(last_dc);
	return TRUE;
	}

	void crmd_peer_down(crm_node_t *peer, bool full)
	{
	if(full && peer->state == NULL) {
	pcmk__update_peer_state(__func__, peer, CRM_NODE_LOST, 0);
	crm_update_peer_proc(__func__, peer, crm_proc_none, NULL);
	}
	crm_update_peer_join(__func__, peer, crm_join_none);
	pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
	}

	/*!
	* \internal
	* \brief Check feature set compatibility of DC and joining node
	*
	* Return true if a joining node's CRM feature set is compatible with the
	* current DC's. The feature sets are compatible if they have the same major
	* version number, and the DC's minor version number is the same or older than
	* the joining node's. The minor-minor version is intended solely to allow
	* resource agents to detect feature support, and so is ignored.
	*
	* \param[in] dc_version DC's feature set
	* \param[in] join_version Joining node's version
	*/
	bool
	feature_set_compatible(const char dc_version, const char join_version)
	{
	char *dc_minor = NULL;
	char *join_minor = NULL;
	long dc_v = 0;
	long join_v = 0;

	// Get DC's major version
	errno = 0;
	dc_v = strtol(dc_version, &dc_minor, 10);
	if (errno) {
	return FALSE;
	}

	// Get joining node's major version
	errno = 0;
	join_v = strtol(join_version, &join_minor, 10);
	if (errno) {
	return FALSE;
	}

	// Major version component must be identical
	if (dc_v != join_v) {
	return FALSE;
	}

	// Get DC's minor version
	if (*dc_minor == '.') {
	++dc_minor;
	}
	errno = 0;
	dc_v = strtol(dc_minor, NULL, 10);
	if (errno) {
	return FALSE;
	}

	// Get joining node's minor version
	if (*join_minor == '.') {
	++join_minor;
	}
	errno = 0;
	join_v = strtol(join_minor, NULL, 10);
	if (errno) {
	return FALSE;
	}

	// DC's minor version must be the same or older
	return dc_v <= join_v;
	}

	const char *
	get_node_id(xmlNode *lrm_rsc_op)
	{
	xmlNode *node = lrm_rsc_op;

	- while ((node != NULL) && !pcmk__xe_is(node, XML_CIB_TAG_STATE)) {
	+ while ((node != NULL) && !pcmk__xe_is(node, PCMK__XE_NODE_STATE)) {
	node = node->parent;
	}

	CRM_CHECK(node != NULL, return NULL);
	return ID(node);
	}
	diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h
	index 659af5c93d..5ccf62c6a0 100644
	--- a/include/crm/common/xml_internal.h
	+++ b/include/crm/common/xml_internal.h
	@@ -1,453 +1,453 @@
	/*
	* Copyright 2017-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__XML_INTERNAL__H
	# define PCMK__XML_INTERNAL__H

	/*
	* Internal-only wrappers for and extensions to libxml2 (libxslt)
	*/

	# include <stdlib.h>
	# include <stdio.h>
	# include <string.h>

	# include <crm/crm.h> /* transitively imports qblog.h */
	# include <crm/common/output_internal.h>

	# include <libxml/relaxng.h>

	/*!
	* \brief Base for directing lib{xml2,xslt} log into standard libqb backend
	*
	* This macro implements the core of what can be needed for directing
	* libxml2 or libxslt error messaging into standard, preconfigured
	* libqb-backed log stream.
	*
	* It's a bit unfortunate that libxml2 (and more sparsely, also libxslt)
	* emits a single message by chunks (location is emitted separatedly from
	* the message itself), so we have to take the effort to combine these
	* chunks back to single message. Whether to do this or not is driven
	* with \p dechunk toggle.
	*
	* The form of a macro was chosen for implicit deriving of __FILE__, etc.
	* and also because static dechunking buffer should be differentiated per
	* library (here we assume different functions referring to this macro
	* will not ever be using both at once), preferably also per-library
	* context of use to avoid clashes altogether.
	*
	* Note that we cannot use qb_logt, because callsite data have to be known
	* at the moment of compilation, which it is not always the case -- xml_log
	* (and unfortunately there's no clear explanation of the fail to compile).
	*
	* Also note that there's no explicit guard against said libraries producing
	* never-newline-terminated chunks (which would just keep consuming memory),
	* as it's quite improbable. Termination of the program in between the
	* same-message chunks will raise a flag with valgrind and the likes, though.
	*
	* And lastly, regarding how dechunking combines with other non-message
	* parameters -- for \p priority, most important running specification
	* wins (possibly elevated to LOG_ERR in case of nonconformance with the
	* newline-termination "protocol"), \p dechunk is expected to always be
	* on once it was at the start, and the rest (\p postemit and \p prefix)
	* are picked directly from the last chunk entry finalizing the message
	* (also reasonable to always have it the same with all related entries).
	*
	* \param[in] priority Syslog priority for the message to be logged
	* \param[in] dechunk Whether to dechunk new-line terminated message
	* \param[in] postemit Code to be executed once message is sent out
	* \param[in] prefix How to prefix the message or NULL for raw passing
	* \param[in] fmt Format string as with printf-like functions
	* \param[in] ap Variable argument list to supplement \p fmt format string
	*/
	#define PCMK__XML_LOG_BASE(priority, dechunk, postemit, prefix, fmt, ap) \
	do { \
	if (!(dechunk) && (prefix) == NULL) { /* quick pass */ \
	qb_log_from_external_source_va(__func__, __FILE__, (fmt), \
	(priority), __LINE__, 0, (ap)); \
	(void) (postemit); \
	} else { \
	int CXLB_len = 0; \
	char *CXLB_buf = NULL; \
	static int CXLB_buffer_len = 0; \
	static char *CXLB_buffer = NULL; \
	static uint8_t CXLB_priority = 0; \
	\
	CXLB_len = vasprintf(&CXLB_buf, (fmt), (ap)); \
	\
	if (CXLB_len <= 0 \|\| CXLB_buf[CXLB_len - 1] == '\n' \|\| !(dechunk)) { \
	if (CXLB_len < 0) { \
	CXLB_buf = (char ) "LOG CORRUPTION HAZARD"; /we don't modify*/\
	CXLB_priority = QB_MIN(CXLB_priority, LOG_ERR); \
	} else if (CXLB_len > 0 /* && (dechunk) */ \
	&& CXLB_buf[CXLB_len - 1] == '\n') { \
	CXLB_buf[CXLB_len - 1] = '\0'; \
	} \
	if (CXLB_buffer) { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s%s", \
	CXLB_priority, __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buffer, CXLB_buf); \
	free(CXLB_buffer); \
	} else { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s", \
	(priority), __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buf); \
	} \
	if (CXLB_len < 0) { \
	CXLB_buf = NULL; /* restore temporary override */ \
	} \
	CXLB_buffer = NULL; \
	CXLB_buffer_len = 0; \
	(void) (postemit); \
	\
	} else if (CXLB_buffer == NULL) { \
	CXLB_buffer_len = CXLB_len; \
	CXLB_buffer = CXLB_buf; \
	CXLB_buf = NULL; \
	CXLB_priority = (priority); /* remember as a running severest */ \
	\
	} else { \
	CXLB_buffer = realloc(CXLB_buffer, 1 + CXLB_buffer_len + CXLB_len); \
	memcpy(CXLB_buffer + CXLB_buffer_len, CXLB_buf, CXLB_len); \
	CXLB_buffer_len += CXLB_len; \
	CXLB_buffer[CXLB_buffer_len] = '\0'; \
	CXLB_priority = QB_MIN(CXLB_priority, (priority)); /* severest? */ \
	} \
	free(CXLB_buf); \
	} \
	} while (0)

	/*
	* \enum pcmk__xml_fmt_options
	* \brief Bit flags to control format in XML logs and dumps
	*/
	enum pcmk__xml_fmt_options {
	//! Exclude certain XML attributes (for calculating digests)
	pcmk__xml_fmt_filtered = (1 << 0),

	//! Include indentation and newlines
	pcmk__xml_fmt_pretty = (1 << 1),

	//! Include the opening tag of an XML element, and include XML comments
	pcmk__xml_fmt_open = (1 << 3),

	//! Include the children of an XML element
	pcmk__xml_fmt_children = (1 << 4),

	//! Include the closing tag of an XML element
	pcmk__xml_fmt_close = (1 << 5),

	//! Include XML text nodes
	pcmk__xml_fmt_text = (1 << 6),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a created XML subtree
	pcmk__xml_fmt_diff_plus = (1 << 7),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a removed XML subtree
	pcmk__xml_fmt_diff_minus = (1 << 8),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a minimal version of an XML diff (only showing the changes)
	pcmk__xml_fmt_diff_short = (1 << 9),
	};

	int pcmk__xml_show(pcmk__output_t out, const char prefix, const xmlNode *data,
	int depth, uint32_t options);
	int pcmk__xml_show_changes(pcmk__output_t out, const xmlNode xml);

	/* XML search strings for guest, remote and pacemaker_remote nodes */

	/* search string to find CIB resources entries for cluster nodes */
	#define PCMK__XP_MEMBER_NODE_CONFIG \
	"//" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_NODES \
	"/" XML_CIB_TAG_NODE "[not(@type) or @type='member']"

	/* search string to find CIB resources entries for guest nodes */
	#define PCMK__XP_GUEST_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" XML_CIB_TAG_RESOURCE \
	"//" XML_TAG_META_SETS "//" XML_CIB_TAG_NVPAIR \
	"[@name='" PCMK_META_REMOTE_NODE "']"

	/* search string to find CIB resources entries for remote nodes */
	#define PCMK__XP_REMOTE_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" XML_CIB_TAG_RESOURCE \
	"[@type='remote'][@provider='pacemaker']"

	/* search string to find CIB node status entries for pacemaker_remote nodes */
	-#define PCMK__XP_REMOTE_NODE_STATUS \
	- "//" PCMK_XE_CIB "//" PCMK_XE_STATUS "//" XML_CIB_TAG_STATE \
	+#define PCMK__XP_REMOTE_NODE_STATUS \
	+ "//" PCMK_XE_CIB "//" PCMK_XE_STATUS "//" PCMK__XE_NODE_STATE \
	"[@" XML_NODE_IS_REMOTE "='true']"
	/*!
	* \internal
	* \brief Serialize XML (using libxml) into provided descriptor
	*
	* \param[in] fd File descriptor to (piece-wise) write to
	* \param[in] cur XML subtree to proceed
	*
	* \return a standard Pacemaker return code
	*/
	int pcmk__xml2fd(int fd, xmlNode *cur);

	enum pcmk__xml_artefact_ns {
	pcmk__xml_artefact_ns_legacy_rng = 1,
	pcmk__xml_artefact_ns_legacy_xslt,
	pcmk__xml_artefact_ns_base_rng,
	pcmk__xml_artefact_ns_base_xslt,
	};

	void pcmk__strip_xml_text(xmlNode *xml);
	const char pcmk__xe_add_last_written(xmlNode xe);

	xmlNode pcmk__xe_match(const xmlNode parent, const char *node_name,
	const char attr_n, const char attr_v);

	void pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data);

	GString pcmk__element_xpath(const xmlNode xml);

	/*!
	* \internal
	* \brief Get the root directory to scan XML artefacts of given kind for
	*
	* \param[in] ns governs the hierarchy nesting against the inherent root dir
	*
	* \return root directory to scan XML artefacts of given kind for
	*/
	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns);

	/*!
	* \internal
	* \brief Get the fully unwrapped path to particular XML artifact (RNG/XSLT)
	*
	* \param[in] ns denotes path forming details (parent dir, suffix)
	* \param[in] filespec symbolic file specification to be combined with
	* #artefact_ns to form the final path
	* \return unwrapped path to particular XML artifact (RNG/XSLT)
	*/
	char *pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns,
	const char *filespec);

	/*!
	* \internal
	* \brief Check whether an XML element is of a particular type
	*
	* \param[in] xml XML element to compare
	* \param[in] name XML element name to compare
	*
	* \return \c true if \p xml is of type \p name, otherwise \c false
	*/
	static inline bool
	pcmk__xe_is(const xmlNode xml, const char name)
	{
	return (xml != NULL) && (xml->name != NULL) && (name != NULL)
	&& (strcmp((const char *) xml->name, name) == 0);
	}

	/*!
	* \internal
	* \brief Return first non-text child node of an XML node
	*
	* \param[in] parent XML node to check
	*
	* \return First non-text child node of \p parent (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_first_child(const xmlNode *parent)
	{
	xmlNode *child = (parent? parent->children : NULL);

	while (child && (child->type == XML_TEXT_NODE)) {
	child = child->next;
	}
	return child;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling node of an XML node
	*
	* \param[in] child XML node to check
	*
	* \return Next non-text sibling of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_next(const xmlNode *child)
	{
	xmlNode *next = (child? child->next : NULL);

	while (next && (next->type == XML_TEXT_NODE)) {
	next = next->next;
	}
	return next;
	}

	/*!
	* \internal
	* \brief Return first non-text child element of an XML node
	*
	* \param[in] parent XML node to check
	*
	* \return First child element of \p parent (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xe_first_child(const xmlNode *parent)
	{
	xmlNode *child = (parent? parent->children : NULL);

	while (child && (child->type != XML_ELEMENT_NODE)) {
	child = child->next;
	}
	return child;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling element of an XML element
	*
	* \param[in] child XML element to check
	*
	* \return Next sibling element of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xe_next(const xmlNode *child)
	{
	xmlNode *next = child? child->next : NULL;

	while (next && (next->type != XML_ELEMENT_NODE)) {
	next = next->next;
	}
	return next;
	}

	/*!
	* \internal
	* \brief Like pcmk__xe_set_props, but takes a va_list instead of
	* arguments directly.
	*
	* \param[in,out] node XML to add attributes to
	* \param[in] pairs NULL-terminated list of name/value pairs to add
	*/
	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs);

	/*!
	* \internal
	* \brief Add a NULL-terminated list of name/value pairs to the given
	* XML node as properties.
	*
	* \param[in,out] node XML node to add properties to
	* \param[in] ... NULL-terminated list of name/value pairs
	*
	* \note A NULL name terminates the arguments; a NULL value will be skipped.
	*/
	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	G_GNUC_NULL_TERMINATED;

	/*!
	* \internal
	* \brief Get first attribute of an XML element
	*
	* \param[in] xe XML element to check
	*
	* \return First attribute of \p xe (or NULL if \p xe is NULL or has none)
	*/
	static inline xmlAttr *
	pcmk__xe_first_attr(const xmlNode *xe)
	{
	return (xe == NULL)? NULL : xe->properties;
	}

	/*!
	* \internal
	* \brief Extract the ID attribute from an XML element
	*
	* \param[in] xpath String to search
	* \param[in] node Node to get the ID for
	*
	* \return ID attribute of \p node in xpath string \p xpath
	*/
	char *
	pcmk__xpath_node_id(const char xpath, const char node);

	/* internal XML-related utilities */

	enum xml_private_flags {
	pcmk__xf_none = 0x0000,
	pcmk__xf_dirty = 0x0001,
	pcmk__xf_deleted = 0x0002,
	pcmk__xf_created = 0x0004,
	pcmk__xf_modified = 0x0008,

	pcmk__xf_tracking = 0x0010,
	pcmk__xf_processed = 0x0020,
	pcmk__xf_skip = 0x0040,
	pcmk__xf_moved = 0x0080,

	pcmk__xf_acl_enabled = 0x0100,
	pcmk__xf_acl_read = 0x0200,
	pcmk__xf_acl_write = 0x0400,
	pcmk__xf_acl_deny = 0x0800,

	pcmk__xf_acl_create = 0x1000,
	pcmk__xf_acl_denied = 0x2000,
	pcmk__xf_lazy = 0x4000,
	};

	void pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag);

	/*!
	* \internal
	* \brief Iterate over child elements of \p xml
	*
	* This function iterates over the children of \p xml, performing the
	* callback function \p handler on each node. If the callback returns
	* a value other than pcmk_rc_ok, the iteration stops and the value is
	* returned. It is therefore possible that not all children will be
	* visited.
	*
	* \param[in,out] xml The starting XML node. Can be NULL.
	* \param[in] child_element_name The name that the node must match in order
	* for \p handler to be run. If NULL, all
	* child elements will match.
	* \param[in] handler The callback function.
	* \param[in,out] userdata User data to pass to the callback function.
	* Can be NULL.
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__xe_foreach_child(xmlNode xml, const char child_element_name,
	int (handler)(xmlNode xml, void *userdata),
	void *userdata);

	static inline const char *
	pcmk__xml_attr_value(const xmlAttr *attr)
	{
	return ((attr == NULL) \|\| (attr->children == NULL))? NULL
	: (const char *) attr->children->content;
	}


	gboolean pcmk__validate_xml(xmlNode xml_blob, const char validation,
	xmlRelaxNGValidityErrorFunc error_handler,
	void *error_handler_context);

	void pcmk__log_known_schemas(void);
	const char *pcmk__remote_schema_dir(void);
	void pcmk__sort_schemas(void);

	#endif // PCMK__XML_INTERNAL__H
	diff --git a/include/crm_internal.h b/include/crm_internal.h
	index 79873b113c..9b06a38093 100644
	--- a/include/crm_internal.h
	+++ b/include/crm_internal.h
	@@ -1,208 +1,209 @@
	/*
	* Copyright 2006-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CRM_INTERNAL__H
	# define CRM_INTERNAL__H

	# ifndef PCMK__CONFIG_H
	# define PCMK__CONFIG_H
	# include <config.h>
	# endif

	# include <portability.h>

	/* Our minimum glib dependency is 2.42. Define that as both the minimum and
	* maximum glib APIs that are allowed (i.e. APIs that were already deprecated
	* in 2.42, and APIs introduced after 2.42, cannot be used by Pacemaker code).
	*/
	#define GLIB_VERSION_MIN_REQUIRED GLIB_VERSION_2_42
	#define GLIB_VERSION_MAX_ALLOWED GLIB_VERSION_2_42

	# include <glib.h>
	# include <stdbool.h>
	# include <libxml/tree.h>

	/* Public API headers can guard including deprecated API headers with this
	* symbol, thus preventing internal code (which includes this header) from using
	* deprecated APIs, while still allowing external code to use them by default.
	*/
	#define PCMK_ALLOW_DEPRECATED 0

	# include <crm/lrmd.h>
	# include <crm/common/logging.h>
	# include <crm/common/logging_internal.h>
	# include <crm/common/ipc_internal.h>
	# include <crm/common/options_internal.h>
	# include <crm/common/output_internal.h>
	# include <crm/common/xml_internal.h>
	# include <crm/common/internal.h>
	# include <locale.h>
	# include <gettext.h>

	#define N_(String) (String)

	#ifdef ENABLE_NLS
	# define _(String) gettext(String)
	#else
	# define _(String) (String)
	#endif


	/*
	* XML element names used only by internal code
	*/

	+#define PCMK__XE_NODE_STATE "node_state"
	#define PCMK__XE_PING_RESPONSE "ping_response"

	// @COMPAT Deprecated since 2.1.7
	#define PCMK__XE_DIFF_ADDED "diff-added"

	// @COMPAT Deprecated since 2.1.7
	#define PCMK__XE_DIFF_REMOVED "diff-removed"

	/* @COMPAT Deprecated since 2.0.0; alias for <clone> with PCMK_META_PROMOTABLE
	* set to "true"
	*/
	#define PCMK__XE_PROMOTABLE_LEGACY "master"


	/*
	* XML attribute names used only by internal code
	*/

	#define PCMK__XA_ATTR_DAMPENING "attr_dampening"
	#define PCMK__XA_ATTR_FORCE "attrd_is_force_write"
	#define PCMK__XA_ATTR_INTERVAL "attr_clear_interval"
	#define PCMK__XA_ATTR_IS_PRIVATE "attr_is_private"
	#define PCMK__XA_ATTR_IS_REMOTE "attr_is_remote"
	#define PCMK__XA_ATTR_NAME "attr_name"
	#define PCMK__XA_ATTR_NODE_ID "attr_host_id"
	#define PCMK__XA_ATTR_NODE_NAME "attr_host"
	#define PCMK__XA_ATTR_OPERATION "attr_clear_operation"
	#define PCMK__XA_ATTR_PATTERN "attr_regex"
	#define PCMK__XA_ATTR_RESOURCE "attr_resource"
	#define PCMK__XA_ATTR_SECTION "attr_section"
	#define PCMK__XA_ATTR_SET "attr_set"
	#define PCMK__XA_ATTR_SET_TYPE "attr_set_type"
	#define PCMK__XA_ATTR_SYNC_POINT "attr_sync_point"
	#define PCMK__XA_ATTR_USER "attr_user"
	#define PCMK__XA_ATTR_UUID "attr_key"
	#define PCMK__XA_ATTR_VALUE "attr_value"
	#define PCMK__XA_ATTR_VERSION "attr_version"
	#define PCMK__XA_ATTR_WRITER "attr_writer"
	#define PCMK__XA_CALL_ID "call-id"
	#define PCMK__XA_CONFIG_ERRORS "config-errors"
	#define PCMK__XA_CONFIG_WARNINGS "config-warnings"
	#define PCMK__XA_CONFIRM "confirm"
	#define PCMK__XA_CONN_HOST "connection_host"
	#define PCMK__XA_CRMD "crmd"
	#define PCMK__XA_CRMD_STATE "crmd_state"
	#define PCMK__XA_CRM_HOST_TO "crm_host_to"
	#define PCMK__XA_CRM_LIMIT_MAX "crm-limit-max"
	#define PCMK__XA_CRM_LIMIT_MODE "crm-limit-mode"
	#define PCMK__XA_CRM_SUBSYSTEM "crm_subsystem"
	#define PCMK__XA_CRM_SYS_FROM "crm_sys_from"
	#define PCMK__XA_CRM_SYS_TO "crm_sys_to"
	#define PCMK__XA_CRM_TASK "crm_task"
	#define PCMK__XA_CRM_TGRAPH_IN "crm-tgraph-in"
	#define PCMK__XA_CRM_USER "crm_user"
	#define PCMK__XA_DC_LEAVING "dc-leaving"
	#define PCMK__XA_DIGEST "digest"
	#define PCMK__XA_ELECTION_AGE_SEC "election-age-sec"
	#define PCMK__XA_ELECTION_AGE_NANO_SEC "election-age-nano-sec"
	#define PCMK__XA_ELECTION_ID "election-id"
	#define PCMK__XA_ELECTION_OWNER "election-owner"
	#define PCMK__XA_EXPECTED "expected"
	#define PCMK__XA_FILE "file"
	#define PCMK__XA_GRAPH_ERRORS "graph-errors"
	#define PCMK__XA_GRAPH_WARNINGS "graph-warnings"
	#define PCMK__XA_IN_CCM "in_ccm"
	#define PCMK__XA_JOIN "join"
	#define PCMK__XA_JOIN_ID "join_id"
	#define PCMK__XA_LONG_ID "long-id"
	#define PCMK__XA_MODE "mode"
	#define PCMK__XA_NODE_START_STATE "node_start_state"
	#define PCMK__XA_OPERATION_KEY "operation_key"
	#define PCMK__XA_OP_DIGEST "op-digest"
	#define PCMK__XA_OP_FORCE_RESTART "op-force-restart"
	#define PCMK__XA_OP_RESTART_DIGEST "op-restart-digest"
	#define PCMK__XA_OP_SECURE_DIGEST "op-secure-digest"
	#define PCMK__XA_OP_SECURE_PARAMS "op-secure-params"
	#define PCMK__XA_OP_STATUS "op-status"
	#define PCMK__XA_PACEMAKERD_STATE "pacemakerd_state"
	#define PCMK__XA_PRIORITY "priority"
	#define PCMK__XA_RC_CODE "rc-code"
	#define PCMK__XA_REAP "reap"

	/* Actions to be executed on Pacemaker Remote nodes are routed through the
	* controller on the cluster node hosting the remote connection. That cluster
	* node is considered the router node for the action.
	*/
	#define PCMK__XA_ROUTER_NODE "router_node"

	#define PCMK__XA_RSC_ID "rsc-id"
	#define PCMK__XA_SCHEMA "schema"
	#define PCMK__XA_SCHEMAS "schemas"
	#define PCMK__XA_SRC "src"
	#define PCMK__XA_SUBT "subt" // subtype
	#define PCMK__XA_T "t" // type
	#define PCMK__XA_TASK "task"
	#define PCMK__XA_TRANSITION_KEY "transition-key"
	#define PCMK__XA_TRANSITION_MAGIC "transition-magic"
	#define PCMK__XA_UPTIME "uptime"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_FIRST_INSTANCE "first-instance"

	// @COMPAT Deprecated since 1.1.12
	#define PCMK__XA_REF "ref"

	// @COMPAT Deprecated since 2.1.6
	#define PCMK__XA_REPLACE "replace"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_RSC_INSTANCE "rsc-instance"

	// @COMPAT Deprecated since 1.1.12
	#define PCMK__XA_TAG "tag"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_THEN_INSTANCE "then-instance"

	// @COMPAT Deprecated since 2.1.5
	#define PCMK__XA_WITH_RSC_INSTANCE "with-rsc-instance"


	/*
	* IPC service names that are only used internally
	*/

	# define PCMK__SERVER_BASED_RO "cib_ro"
	# define PCMK__SERVER_BASED_RW "cib_rw"
	# define PCMK__SERVER_BASED_SHM "cib_shm"

	/*
	* IPC commands that can be sent to Pacemaker daemons
	*/

	#define PCMK__ATTRD_CMD_PEER_REMOVE "peer-remove"
	#define PCMK__ATTRD_CMD_UPDATE "update"
	#define PCMK__ATTRD_CMD_UPDATE_BOTH "update-both"
	#define PCMK__ATTRD_CMD_UPDATE_DELAY "update-delay"
	#define PCMK__ATTRD_CMD_QUERY "query"
	#define PCMK__ATTRD_CMD_REFRESH "refresh"
	#define PCMK__ATTRD_CMD_FLUSH "flush"
	#define PCMK__ATTRD_CMD_SYNC "sync"
	#define PCMK__ATTRD_CMD_SYNC_RESPONSE "sync-response"
	#define PCMK__ATTRD_CMD_CLEAR_FAILURE "clear-failure"
	#define PCMK__ATTRD_CMD_CONFIRM "confirm"

	#define PCMK__CONTROLD_CMD_NODES "list-nodes"

	#endif /* CRM_INTERNAL__H */
	diff --git a/lib/cib/cib_attrs.c b/lib/cib/cib_attrs.c
	index ce0dbbc8e1..4ae59768ed 100644
	--- a/lib/cib/cib_attrs.c
	+++ b/lib/cib/cib_attrs.c
	@@ -1,744 +1,744 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/cib/internal.h>

	static pcmk__output_t *
	new_output_object(const char *ty)
	{
	int rc = pcmk_rc_ok;
	pcmk__output_t *out = NULL;
	const char* argv[] = { "", NULL };
	pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_LOG,
	PCMK__SUPPORTED_FORMAT_TEXT,
	{ NULL, NULL, NULL }
	};

	pcmk__register_formats(NULL, formats);
	rc = pcmk__output_new(&out, ty, NULL, (char**)argv);
	if ((rc != pcmk_rc_ok) \|\| (out == NULL)) {
	crm_err("Can't out due to internal error: %s", pcmk_rc_str(rc));
	return NULL;
	}

	return out;
	}

	static int
	find_attr(cib_t cib, const char section, const char *node_uuid,
	const char attr_set_type, const char set_name, const char *attr_id,
	const char attr_name, const char user_name, xmlNode **result)
	{
	int rc = pcmk_rc_ok;

	const char *xpath_base = NULL;
	GString *xpath = NULL;
	xmlNode *xml_search = NULL;
	const char *set_type = NULL;
	const char *node_type = NULL;

	if (attr_set_type) {
	set_type = attr_set_type;
	} else {
	set_type = XML_TAG_ATTR_SETS;
	}

	if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) {
	node_uuid = NULL;
	set_type = XML_CIB_TAG_PROPSET;

	} else if (pcmk__strcase_any_of(section,
	PCMK_XE_OP_DEFAULTS, PCMK_XE_RSC_DEFAULTS,
	NULL)) {
	node_uuid = NULL;
	set_type = XML_TAG_META_SETS;

	} else if (pcmk__str_eq(section, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
	node_uuid = NULL;
	section = PCMK_XE_STATUS;
	node_type = XML_CIB_TAG_TICKETS;

	} else if (node_uuid == NULL) {
	return EINVAL;
	}

	xpath_base = pcmk_cib_xpath_for(section);
	if (xpath_base == NULL) {
	crm_warn("%s CIB section not known", section);
	return ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	g_string_append(xpath, xpath_base);

	if (pcmk__str_eq(node_type, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
	pcmk__g_strcat(xpath, "//", node_type, NULL);

	} else if (node_uuid) {
	const char *node_type = XML_CIB_TAG_NODE;

	if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) {
	- node_type = XML_CIB_TAG_STATE;
	+ node_type = PCMK__XE_NODE_STATE;
	set_type = XML_TAG_TRANSIENT_NODEATTRS;
	}
	pcmk__g_strcat(xpath,
	"//", node_type, "[@" PCMK_XA_ID "='", node_uuid, "']",
	NULL);
	}

	pcmk__g_strcat(xpath, "//", set_type, NULL);
	if (set_name) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", set_name, "']", NULL);
	}

	g_string_append(xpath, "//nvpair");

	if (attr_id && attr_name) {
	pcmk__g_strcat(xpath,
	"[@" PCMK_XA_ID "='", attr_id, "' "
	"and @" PCMK_XA_NAME "='", attr_name, "']", NULL);

	} else if (attr_id) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "']", NULL);

	} else if (attr_name) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_NAME "='", attr_name, "']", NULL);
	}

	rc = cib_internal_op(cib, PCMK__CIB_REQUEST_QUERY, NULL,
	(const char *) xpath->str, NULL, &xml_search,
	cib_sync_call\|cib_scope_local\|cib_xpath, user_name);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_trace("Query failed for attribute %s (section=%s, node=%s, set=%s, xpath=%s): %s",
	attr_name, section, pcmk__s(node_uuid, "<null>"),
	pcmk__s(set_name, "<null>"), (const char *) xpath->str,
	pcmk_rc_str(rc));
	} else {
	rc = pcmk_rc_ok;
	crm_log_xml_debug(xml_search, "Match");
	}

	g_string_free(xpath, TRUE);
	*result = xml_search;
	return rc;
	}

	static int
	handle_multiples(pcmk__output_t out, xmlNode search, const char *attr_name)
	{
	if ((search != NULL) && (search->children != NULL)) {
	xmlNode *child = NULL;

	out->info(out, "Multiple attributes match name=%s", attr_name);
	for (child = pcmk__xml_first_child(search); child != NULL;
	child = pcmk__xml_next(child)) {
	out->info(out, " Value: %s \t(id=%s)",
	crm_element_value(child, PCMK_XA_VALUE), ID(child));
	}
	return ENOTUNIQ;

	} else {
	return pcmk_rc_ok;
	}
	}

	int
	cib__update_node_attr(pcmk__output_t out, cib_t cib, int call_options, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *attr_value,
	const char user_name, const char node_type)
	{
	const char *tag = NULL;
	int rc = pcmk_rc_ok;
	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_search = NULL;

	char *local_attr_id = NULL;
	char *local_set_name = NULL;

	CRM_CHECK((out != NULL) && (cib != NULL) && (section != NULL)
	&& ((attr_id != NULL) \|\| (attr_name != NULL))
	&& (attr_value != NULL), return EINVAL);

	rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id,
	attr_name, user_name, &xml_search);

	if (rc == pcmk_rc_ok) {
	if (handle_multiples(out, xml_search, attr_name) == ENOTUNIQ) {
	free_xml(xml_search);
	return ENOTUNIQ;
	} else {
	pcmk__str_update(&local_attr_id,
	crm_element_value(xml_search, PCMK_XA_ID));
	attr_id = local_attr_id;
	free_xml(xml_search);
	goto do_modify;
	}

	} else if (rc != ENXIO) {
	free_xml(xml_search);
	return rc;

	/* } else if(attr_id == NULL) { */
	/* return EINVAL; */

	} else {
	free_xml(xml_search);
	crm_trace("%s does not exist, create it", attr_name);
	if (pcmk__str_eq(section, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
	node_uuid = NULL;
	section = PCMK_XE_STATUS;
	node_type = XML_CIB_TAG_TICKETS;

	xml_top = create_xml_node(xml_obj, PCMK_XE_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);

	} else if (pcmk__str_eq(section, PCMK_XE_NODES, pcmk__str_casei)) {

	if (node_uuid == NULL) {
	return EINVAL;
	}

	if (pcmk__str_eq(node_type, "remote", pcmk__str_casei)) {
	xml_top = create_xml_node(xml_obj, PCMK_XE_NODES);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_NODE);
	crm_xml_add(xml_obj, PCMK_XA_TYPE, "remote");
	crm_xml_add(xml_obj, PCMK_XA_ID, node_uuid);
	crm_xml_add(xml_obj, PCMK_XA_UNAME, node_uuid);
	} else {
	tag = XML_CIB_TAG_NODE;
	}

	} else if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) {
	tag = XML_TAG_TRANSIENT_NODEATTRS;
	if (node_uuid == NULL) {
	return EINVAL;
	}

	- xml_top = create_xml_node(xml_obj, XML_CIB_TAG_STATE);
	+ xml_top = create_xml_node(xml_obj, PCMK__XE_NODE_STATE);
	crm_xml_add(xml_top, PCMK_XA_ID, node_uuid);
	xml_obj = xml_top;

	} else {
	tag = section;
	node_uuid = NULL;
	}

	if (set_name == NULL) {
	if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) {
	local_set_name = strdup(CIB_OPTIONS_FIRST);

	} else if (pcmk__str_eq(node_type, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
	local_set_name = crm_strdup_printf("%s-%s", section,
	XML_CIB_TAG_TICKETS);

	} else if (node_uuid) {
	local_set_name = crm_strdup_printf("%s-%s", section, node_uuid);

	if (set_type) {
	char *tmp_set_name = local_set_name;

	local_set_name = crm_strdup_printf("%s-%s", tmp_set_name,
	set_type);
	free(tmp_set_name);
	}
	} else {
	local_set_name = crm_strdup_printf("%s-options", section);
	}
	set_name = local_set_name;
	}

	if (attr_id == NULL) {
	local_attr_id = crm_strdup_printf("%s-%s", set_name, attr_name);
	crm_xml_sanitize_id(local_attr_id);
	attr_id = local_attr_id;

	} else if (attr_name == NULL) {
	attr_name = attr_id;
	}

	crm_trace("Creating %s/%s", section, tag);
	if (tag != NULL) {
	xml_obj = create_xml_node(xml_obj, tag);
	crm_xml_add(xml_obj, PCMK_XA_ID, node_uuid);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}
	}

	if (node_uuid == NULL && !pcmk__str_eq(node_type, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
	if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) {
	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_PROPSET);
	} else {
	xml_obj = create_xml_node(xml_obj, XML_TAG_META_SETS);
	}

	} else if (set_type) {
	xml_obj = create_xml_node(xml_obj, set_type);

	} else {
	xml_obj = create_xml_node(xml_obj, XML_TAG_ATTR_SETS);
	}
	crm_xml_add(xml_obj, PCMK_XA_ID, set_name);

	if (xml_top == NULL) {
	xml_top = xml_obj;
	}
	}

	do_modify:
	xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_trace(xml_top, "update_attr");
	rc = cib_internal_op(cib, PCMK__CIB_REQUEST_MODIFY, NULL, section, xml_top,
	NULL, call_options, user_name);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);

	out->err(out, "Error setting %s=%s (section=%s, set=%s): %s",
	attr_name, attr_value, section, pcmk__s(set_name, "<null>"),
	pcmk_rc_str(rc));
	crm_log_xml_info(xml_top, "Update");
	} else {
	rc = pcmk_rc_ok;
	}

	free(local_set_name);
	free(local_attr_id);
	free_xml(xml_top);

	return rc;
	}

	int
	cib__get_node_attrs(pcmk__output_t out, cib_t cib, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *user_name,
	xmlNode **result)
	{
	int rc = pcmk_rc_ok;

	CRM_ASSERT(result != NULL);
	CRM_CHECK(section != NULL, return EINVAL);

	*result = NULL;

	rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id, attr_name,
	user_name, result);

	if (rc != pcmk_rc_ok) {
	crm_trace("Query failed for attribute %s (section=%s node=%s set=%s): %s",
	pcmk__s(attr_name, "with unspecified name"),
	section, pcmk__s(set_name, "<null>"),
	pcmk__s(node_uuid, "<null>"), pcmk_rc_str(rc));
	}

	return rc;
	}

	int
	cib__delete_node_attr(pcmk__output_t out, cib_t cib, int options, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *attr_value,
	const char *user_name)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_search = NULL;
	char *local_attr_id = NULL;

	CRM_CHECK(section != NULL, return EINVAL);
	CRM_CHECK(attr_name != NULL \|\| attr_id != NULL, return EINVAL);

	if (attr_id == NULL) {
	rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id,
	attr_name, user_name, &xml_search);

	if (rc != pcmk_rc_ok \|\| handle_multiples(out, xml_search, attr_name) == ENOTUNIQ) {
	free_xml(xml_search);
	return rc;
	} else {
	pcmk__str_update(&local_attr_id,
	crm_element_value(xml_search, PCMK_XA_ID));
	attr_id = local_attr_id;
	free_xml(xml_search);
	}
	}

	xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, attr_value);

	rc = cib_internal_op(cib, PCMK__CIB_REQUEST_DELETE, NULL, section, xml_obj,
	NULL, options, user_name);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	} else {
	rc = pcmk_rc_ok;
	out->info(out, "Deleted %s %s: id=%s%s%s%s%s",
	section, node_uuid ? "attribute" : "option", local_attr_id,
	set_name ? " set=" : "", set_name ? set_name : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}

	free(local_attr_id);
	free_xml(xml_obj);
	return rc;
	}

	int
	find_nvpair_attr_delegate(cib_t cib, const char attr, const char *section,
	const char node_uuid, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, gboolean to_console,
	char *value, const char user_name)
	{
	pcmk__output_t *out = NULL;
	xmlNode *xml_search = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = find_attr(cib, section, node_uuid, attr_set_type, set_name, attr_id,
	attr_name, user_name, &xml_search);

	if (rc == pcmk_rc_ok) {
	rc = handle_multiples(out, xml_search, attr_name);

	if (rc == pcmk_rc_ok) {
	pcmk__str_update(value, crm_element_value(xml_search, attr));
	}
	}

	out->finish(out, CRM_EX_OK, true, NULL);
	free_xml(xml_search);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	int
	update_attr_delegate(cib_t cib, int call_options, const char section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *attr_value,
	gboolean to_console, const char user_name, const char node_type)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = cib__update_node_attr(out, cib, call_options, section, node_uuid, set_type,
	set_name, attr_id, attr_name, attr_value, user_name,
	node_type);

	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	int
	read_attr_delegate(cib_t cib, const char section, const char *node_uuid,
	const char set_type, const char set_name, const char *attr_id,
	const char attr_name, char *attr_value, gboolean to_console,
	const char *user_name)
	{
	pcmk__output_t *out = NULL;
	xmlNode *result = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = cib__get_node_attrs(out, cib, section, node_uuid, set_type, set_name,
	attr_id, attr_name, user_name, &result);

	if (rc == pcmk_rc_ok) {
	if (result->children == NULL) {
	pcmk__str_update(attr_value,
	crm_element_value(result, PCMK_XA_VALUE));
	} else {
	rc = ENOTUNIQ;
	}
	}

	out->finish(out, CRM_EX_OK, true, NULL);
	free_xml(result);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	int
	delete_attr_delegate(cib_t cib, int options, const char section, const char *node_uuid,
	const char set_type, const char set_name, const char *attr_id,
	const char attr_name, const char attr_value, gboolean to_console,
	const char *user_name)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = cib__delete_node_attr(out, cib, options, section, node_uuid, set_type,
	set_name, attr_id, attr_name, attr_value, user_name);

	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	/*!
	* \internal
	* \brief Parse node UUID from search result
	*
	* \param[in] result XML search result
	* \param[out] uuid If non-NULL, where to store parsed UUID
	* \param[out] is_remote If non-NULL, set TRUE if result is remote node
	*
	* \return pcmk_ok if UUID was successfully parsed, -ENXIO otherwise
	*/
	static int
	get_uuid_from_result(const xmlNode result, char uuid, int is_remote)
	{
	int rc = -ENXIO;
	const char *tag;
	const char *parsed_uuid = NULL;
	int parsed_is_remote = FALSE;

	if (result == NULL) {
	return rc;
	}

	/* If there are multiple results, the first is sufficient */
	tag = (const char *) (result->name);
	if (pcmk__str_eq(tag, "xpath-query", pcmk__str_casei)) {
	result = pcmk__xml_first_child(result);
	CRM_CHECK(result != NULL, return rc);
	tag = (const char *) (result->name);
	}

	if (pcmk__str_eq(tag, XML_CIB_TAG_NODE, pcmk__str_casei)) {
	/* Result is <node> tag from <nodes> section */

	if (pcmk__str_eq(crm_element_value(result, PCMK_XA_TYPE), "remote",
	pcmk__str_casei)) {
	parsed_uuid = crm_element_value(result, PCMK_XA_UNAME);
	parsed_is_remote = TRUE;
	} else {
	parsed_uuid = ID(result);
	parsed_is_remote = FALSE;
	}

	} else if (pcmk__str_eq(tag, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
	/* Result is <primitive> for ocf:pacemaker:remote resource */

	parsed_uuid = ID(result);
	parsed_is_remote = TRUE;

	} else if (pcmk__str_eq(tag, XML_CIB_TAG_NVPAIR, pcmk__str_casei)) {
	/* Result is PCMK_META_REMOTE_NODE parameter of <primitive> for guest
	* node
	*/

	parsed_uuid = crm_element_value(result, PCMK_XA_VALUE);
	parsed_is_remote = TRUE;

	- } else if (pcmk__str_eq(tag, XML_CIB_TAG_STATE, pcmk__str_casei)) {
	- /* Result is <node_state> tag from <status> section */
	+ } else if (pcmk__str_eq(tag, PCMK__XE_NODE_STATE, pcmk__str_casei)) {
	+ // Result is PCMK__XE_NODE_STATE tag from PCMK_XE_STATUS section

	parsed_uuid = crm_element_value(result, PCMK_XA_UNAME);
	if (pcmk__xe_attr_is_true(result, XML_NODE_IS_REMOTE)) {
	parsed_is_remote = TRUE;
	}
	}

	if (parsed_uuid) {
	if (uuid) {
	*uuid = strdup(parsed_uuid);
	}
	if (is_remote) {
	*is_remote = parsed_is_remote;
	}
	rc = pcmk_ok;
	}

	return rc;
	}

	/* Search string to find a node by name, as:
	* - cluster or remote node in nodes section
	* - remote node in resources section
	* - guest node in resources section
	* - orphaned remote node or bundle guest node in status section
	*/
	#define XPATH_UPPER_TRANS "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
	#define XPATH_LOWER_TRANS "abcdefghijklmnopqrstuvwxyz"
	#define XPATH_NODE \
	"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_NODES \
	"/" XML_CIB_TAG_NODE "[translate(@" PCMK_XA_UNAME ",'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \
	"\|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \
	"/" XML_CIB_TAG_RESOURCE \
	"[@class='ocf'][@provider='pacemaker'][@type='remote'][translate(@id,'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \
	"\|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \
	"/" XML_CIB_TAG_RESOURCE "/" XML_TAG_META_SETS "/" XML_CIB_TAG_NVPAIR \
	"[@name='" PCMK_META_REMOTE_NODE "'][translate(@value,'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \
	- "\|/" PCMK_XE_CIB "/" PCMK_XE_STATUS "/" XML_CIB_TAG_STATE \
	+ "\|/" PCMK_XE_CIB "/" PCMK_XE_STATUS "/" PCMK__XE_NODE_STATE \
	"[@" XML_NODE_IS_REMOTE "='true'][translate(@" PCMK_XA_ID ",'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']"

	int
	query_node_uuid(cib_t * the_cib, const char uname, char uuid, int is_remote_node)
	{
	int rc = pcmk_ok;
	char *xpath_string;
	xmlNode *xml_search = NULL;
	char *host_lowercase = NULL;

	CRM_ASSERT(uname != NULL);

	host_lowercase = g_ascii_strdown(uname, -1);

	if (uuid) {
	*uuid = NULL;
	}
	if (is_remote_node) {
	*is_remote_node = FALSE;
	}

	xpath_string = crm_strdup_printf(XPATH_NODE, host_lowercase, host_lowercase, host_lowercase, host_lowercase);
	if (cib_internal_op(the_cib, PCMK__CIB_REQUEST_QUERY, NULL, xpath_string,
	NULL, &xml_search,
	cib_sync_call\|cib_scope_local\|cib_xpath,
	NULL) == pcmk_ok) {
	rc = get_uuid_from_result(xml_search, uuid, is_remote_node);
	} else {
	rc = -ENXIO;
	}
	free(xpath_string);
	free_xml(xml_search);
	g_free(host_lowercase);

	if (rc != pcmk_ok) {
	crm_debug("Could not map node name '%s' to a UUID: %s",
	uname, pcmk_strerror(rc));
	} else {
	crm_info("Mapped node name '%s' to UUID %s", uname, (uuid? *uuid : ""));
	}
	return rc;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/cib/util_compat.h>

	int
	query_node_uname(cib_t * the_cib, const char uuid, char *uname)
	{
	int rc = pcmk_ok;
	xmlNode *a_child = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *fragment = NULL;
	const char *child_name = NULL;

	CRM_ASSERT(uname != NULL);
	CRM_ASSERT(uuid != NULL);

	rc = the_cib->cmds->query(the_cib, PCMK_XE_NODES, &fragment,
	cib_sync_call \| cib_scope_local);
	if (rc != pcmk_ok) {
	return rc;
	}

	xml_obj = fragment;
	CRM_CHECK(pcmk__xe_is(xml_obj, PCMK_XE_NODES), return -ENOMSG);
	crm_log_xml_trace(xml_obj, "Result section");

	rc = -ENXIO;
	*uname = NULL;

	for (a_child = pcmk__xml_first_child(xml_obj); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {

	if (pcmk__str_eq((const char *)a_child->name, XML_CIB_TAG_NODE,
	pcmk__str_none)) {
	child_name = ID(a_child);
	if (pcmk__str_eq(uuid, child_name, pcmk__str_casei)) {
	child_name = crm_element_value(a_child, PCMK_XA_UNAME);
	if (child_name != NULL) {
	*uname = strdup(child_name);
	rc = pcmk_ok;
	}
	break;
	}
	}
	}

	free_xml(fragment);
	return rc;
	}

	int
	set_standby(cib_t * the_cib, const char uuid, const char scope, const char *standby_value)
	{
	int rc = pcmk_ok;
	char *attr_id = NULL;

	CRM_CHECK(uuid != NULL, return -EINVAL);
	CRM_CHECK(standby_value != NULL, return -EINVAL);

	if (pcmk__strcase_any_of(scope, "reboot", PCMK_XE_STATUS, NULL)) {
	scope = PCMK_XE_STATUS;
	attr_id = crm_strdup_printf("transient-standby-%.256s", uuid);

	} else {
	scope = PCMK_XE_NODES;
	attr_id = crm_strdup_printf("standby-%.256s", uuid);
	}

	rc = update_attr_delegate(the_cib, cib_sync_call, scope, uuid, NULL, NULL,
	attr_id, "standby", standby_value, TRUE, NULL, NULL);

	free(attr_id);
	return rc;
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c
	index 9a01cf0769..fa80f6e59b 100644
	--- a/lib/cluster/membership.c
	+++ b/lib/cluster/membership.c
	@@ -1,1385 +1,1385 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/param.h>
	#include <sys/types.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <glib.h>
	#include <crm/common/ipc.h>
	#include <crm/common/xml_internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/stonith-ng.h>
	#include "crmcluster_private.h"

	/* The peer cache remembers cluster nodes that have been seen.
	* This is managed mostly automatically by libcluster, based on
	* cluster membership events.
	*
	* Because cluster nodes can have conflicting names or UUIDs,
	* the hash table key is a uniquely generated ID.
	*/
	GHashTable *crm_peer_cache = NULL;

	/*
	* The remote peer cache tracks pacemaker_remote nodes. While the
	* value has the same type as the peer cache's, it is tracked separately for
	* three reasons: pacemaker_remote nodes can't have conflicting names or UUIDs,
	* so the name (which is also the UUID) is used as the hash table key; there
	* is no equivalent of membership events, so management is not automatic; and
	* most users of the peer cache need to exclude pacemaker_remote nodes.
	*
	* That said, using a single cache would be more logical and less error-prone,
	* so it would be a good idea to merge them one day.
	*
	* libcluster provides two avenues for populating the cache:
	* crm_remote_peer_get() and crm_remote_peer_cache_remove() directly manage it,
	* while crm_remote_peer_cache_refresh() populates it via the CIB.
	*/
	GHashTable *crm_remote_peer_cache = NULL;

	/*
	* The known node cache tracks cluster and remote nodes that have been seen in
	* the CIB. It is useful mainly when a caller needs to know about a node that
	* may no longer be in the membership, but doesn't want to add the node to the
	* main peer cache tables.
	*/
	static GHashTable *known_node_cache = NULL;

	unsigned long long crm_peer_seq = 0;
	gboolean crm_have_quorum = FALSE;
	static gboolean crm_autoreap = TRUE;

	// Flag setting and clearing for crm_node_t:flags

	#define set_peer_flags(peer, flags_to_set) do { \
	(peer)->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Peer", (peer)->uname, \
	(peer)->flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define clear_peer_flags(peer, flags_to_clear) do { \
	(peer)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Peer", (peer)->uname, \
	(peer)->flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	static void update_peer_uname(crm_node_t node, const char uname);
	static crm_node_t find_known_node(const char id, const char *uname);

	int
	crm_remote_peer_cache_size(void)
	{
	if (crm_remote_peer_cache == NULL) {
	return 0;
	}
	return g_hash_table_size(crm_remote_peer_cache);
	}

	/*!
	* \brief Get a remote node peer cache entry, creating it if necessary
	*
	* \param[in] node_name Name of remote node
	*
	* \return Cache entry for node on success, NULL (and set errno) otherwise
	*
	* \note When creating a new entry, this will leave the node state undetermined,
	* so the caller should also call pcmk__update_peer_state() if the state
	* is known.
	* \note Because this can add and remove cache entries, callers should not
	* assume any previously obtained cache entry pointers remain valid.
	*/
	crm_node_t *
	crm_remote_peer_get(const char *node_name)
	{
	crm_node_t *node;
	char *node_name_copy = NULL;

	if (node_name == NULL) {
	errno = EINVAL;
	return NULL;
	}

	/* It's theoretically possible that the node was added to the cluster peer
	* cache before it was known to be a Pacemaker Remote node. Remove that
	* entry unless it has a node ID, which means the name actually is
	* associated with a cluster node. (@TODO return an error in that case?)
	*/
	node = pcmk__search_node_caches(0, node_name, pcmk__node_search_cluster);
	if ((node != NULL) && (node->uuid == NULL)) {
	/* node_name could be a pointer into the cache entry being removed, so
	* reassign it to a copy before the original gets freed
	*/
	node_name_copy = strdup(node_name);
	if (node_name_copy == NULL) {
	errno = ENOMEM;
	return NULL;
	}
	node_name = node_name_copy;
	reap_crm_member(0, node_name);
	}

	/* Return existing cache entry if one exists */
	node = g_hash_table_lookup(crm_remote_peer_cache, node_name);
	if (node) {
	free(node_name_copy);
	return node;
	}

	/* Allocate a new entry */
	node = calloc(1, sizeof(crm_node_t));
	if (node == NULL) {
	free(node_name_copy);
	return NULL;
	}

	/* Populate the essential information */
	set_peer_flags(node, crm_remote_node);
	node->uuid = strdup(node_name);
	if (node->uuid == NULL) {
	free(node);
	errno = ENOMEM;
	free(node_name_copy);
	return NULL;
	}

	/* Add the new entry to the cache */
	g_hash_table_replace(crm_remote_peer_cache, node->uuid, node);
	crm_trace("added %s to remote cache", node_name);

	/* Update the entry's uname, ensuring peer status callbacks are called */
	update_peer_uname(node, node_name);
	free(node_name_copy);
	return node;
	}

	/*!
	* \brief Remove a node from the Pacemaker Remote node cache
	*
	* \param[in] node_name Name of node to remove from cache
	*
	* \note The caller must be careful not to use \p node_name after calling this
	* function if it might be a pointer into the cache entry being removed.
	*/
	void
	crm_remote_peer_cache_remove(const char *node_name)
	{
	/* Do a lookup first, because node_name could be a pointer within the entry
	* being removed -- we can't log it after removing it.
	*/
	if (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL) {
	crm_trace("Removing %s from Pacemaker Remote node cache", node_name);
	g_hash_table_remove(crm_remote_peer_cache, node_name);
	}
	}

	/*!
	* \internal
	* \brief Return node status based on a CIB status entry
	*
	* \param[in] node_state XML of node state
	*
	- * \return CRM_NODE_LOST if PCMK__XA_IN_CCM is false in node_state,
	- * CRM_NODE_MEMBER otherwise
	+ * \return \c CRM_NODE_LOST if \c PCMK__XA_IN_CCM is false in
	+ * \c PCMK__XE_NODE_STATE, \c CRM_NODE_MEMBER otherwise
	* \note Unlike most boolean XML attributes, this one defaults to true, for
	* backward compatibility with older controllers that don't set it.
	*/
	static const char *
	remote_state_from_cib(const xmlNode *node_state)
	{
	bool status = false;

	if ((pcmk__xe_get_bool_attr(node_state, PCMK__XA_IN_CCM,
	&status) == pcmk_rc_ok) && !status) {
	return CRM_NODE_LOST;
	} else {
	return CRM_NODE_MEMBER;
	}
	}

	/* user data for looping through remote node xpath searches */
	struct refresh_data {
	const char field; / XML attribute to check for node name */
	gboolean has_state; /* whether to update node state based on XML */
	};

	/*!
	* \internal
	* \brief Process one pacemaker_remote node xpath search result
	*
	* \param[in] result XML search result
	* \param[in] user_data what to look for in the XML
	*/
	static void
	remote_cache_refresh_helper(xmlNode result, void user_data)
	{
	const struct refresh_data *data = user_data;
	const char *remote = crm_element_value(result, data->field);
	const char *state = NULL;
	crm_node_t *node;

	CRM_CHECK(remote != NULL, return);

	/* Determine node's state, if the result has it */
	if (data->has_state) {
	state = remote_state_from_cib(result);
	}

	/* Check whether cache already has entry for node */
	node = g_hash_table_lookup(crm_remote_peer_cache, remote);

	if (node == NULL) {
	/* Node is not in cache, so add a new entry for it */
	node = crm_remote_peer_get(remote);
	CRM_ASSERT(node);
	if (state) {
	pcmk__update_peer_state(__func__, node, state, 0);
	}

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	/* Node is in cache and hasn't been updated already, so mark it clean */
	clear_peer_flags(node, crm_node_dirty);
	if (state) {
	pcmk__update_peer_state(__func__, node, state, 0);
	}
	}
	}

	static void
	mark_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	set_peer_flags((crm_node_t *) value, crm_node_dirty);
	}

	static gboolean
	is_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	return pcmk_is_set(((crm_node_t*)value)->flags, crm_node_dirty);
	}

	/*!
	* \brief Repopulate the remote peer cache based on CIB XML
	*
	* \param[in] xmlNode CIB XML to parse
	*/
	void
	crm_remote_peer_cache_refresh(xmlNode *cib)
	{
	struct refresh_data data;

	crm_peer_init();

	/* First, we mark all existing cache entries as dirty,
	* so that later we can remove any that weren't in the CIB.
	* We don't empty the cache, because we need to detect changes in state.
	*/
	g_hash_table_foreach(crm_remote_peer_cache, mark_dirty, NULL);

	/* Look for guest nodes and remote nodes in the status section */
	data.field = PCMK_XA_ID;
	data.has_state = TRUE;
	crm_foreach_xpath_result(cib, PCMK__XP_REMOTE_NODE_STATUS,
	remote_cache_refresh_helper, &data);

	/* Look for guest nodes and remote nodes in the configuration section,
	* because they may have just been added and not have a status entry yet.
	* In that case, the cached node state will be left NULL, so that the
	* peer status callback isn't called until we're sure the node started
	* successfully.
	*/
	data.field = PCMK_XA_VALUE;
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, PCMK__XP_GUEST_NODE_CONFIG,
	remote_cache_refresh_helper, &data);
	data.field = PCMK_XA_ID;
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, PCMK__XP_REMOTE_NODE_CONFIG,
	remote_cache_refresh_helper, &data);

	/* Remove all old cache entries that weren't seen in the CIB */
	g_hash_table_foreach_remove(crm_remote_peer_cache, is_dirty, NULL);
	}

	gboolean
	crm_is_peer_active(const crm_node_t * node)
	{
	if(node == NULL) {
	return FALSE;
	}

	if (pcmk_is_set(node->flags, crm_remote_node)) {
	/* remote nodes are never considered active members. This
	* guarantees they will never be considered for DC membership.*/
	return FALSE;
	}
	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	return crm_is_corosync_peer_active(node);
	}
	#endif
	crm_err("Unhandled cluster type: %s", name_for_cluster_type(get_cluster_type()));
	return FALSE;
	}

	static gboolean
	crm_reap_dead_member(gpointer key, gpointer value, gpointer user_data)
	{
	crm_node_t *node = value;
	crm_node_t *search = user_data;

	if (search == NULL) {
	return FALSE;

	} else if (search->id && node->id != search->id) {
	return FALSE;

	} else if (search->id == 0 && !pcmk__str_eq(node->uname, search->uname, pcmk__str_casei)) {
	return FALSE;

	} else if (crm_is_peer_active(value) == FALSE) {
	crm_info("Removing node with name %s and id %u from membership cache",
	(node->uname? node->uname : "unknown"), node->id);
	return TRUE;
	}
	return FALSE;
	}

	/*!
	* \brief Remove all peer cache entries matching a node ID and/or uname
	*
	* \param[in] id ID of node to remove (or 0 to ignore)
	* \param[in] name Uname of node to remove (or NULL to ignore)
	*
	* \return Number of cache entries removed
	*
	* \note The caller must be careful not to use \p name after calling this
	* function if it might be a pointer into the cache entry being removed.
	*/
	guint
	reap_crm_member(uint32_t id, const char *name)
	{
	int matches = 0;
	crm_node_t search = { 0, };

	if (crm_peer_cache == NULL) {
	crm_trace("Membership cache not initialized, ignoring purge request");
	return 0;
	}

	search.id = id;
	pcmk__str_update(&search.uname, name);
	matches = g_hash_table_foreach_remove(crm_peer_cache, crm_reap_dead_member, &search);
	if(matches) {
	crm_notice("Purged %d peer%s with id=%u%s%s from the membership cache",
	matches, pcmk__plural_s(matches), search.id,
	(search.uname? " and/or uname=" : ""),
	(search.uname? search.uname : ""));

	} else {
	crm_info("No peers with id=%u%s%s to purge from the membership cache",
	search.id, (search.uname? " and/or uname=" : ""),
	(search.uname? search.uname : ""));
	}

	free(search.uname);
	return matches;
	}

	static void
	count_peer(gpointer key, gpointer value, gpointer user_data)
	{
	guint *count = user_data;
	crm_node_t *node = value;

	if (crm_is_peer_active(node)) {
	count = count + 1;
	}
	}

	guint
	crm_active_peers(void)
	{
	guint count = 0;

	if (crm_peer_cache) {
	g_hash_table_foreach(crm_peer_cache, count_peer, &count);
	}
	return count;
	}

	static void
	destroy_crm_node(gpointer data)
	{
	crm_node_t *node = data;

	crm_trace("Destroying entry for node %u: %s", node->id, node->uname);

	free(node->uname);
	free(node->state);
	free(node->uuid);
	free(node->expected);
	free(node->conn_host);
	free(node);
	}

	void
	crm_peer_init(void)
	{
	if (crm_peer_cache == NULL) {
	crm_peer_cache = pcmk__strikey_table(free, destroy_crm_node);
	}

	if (crm_remote_peer_cache == NULL) {
	crm_remote_peer_cache = pcmk__strikey_table(NULL, destroy_crm_node);
	}

	if (known_node_cache == NULL) {
	known_node_cache = pcmk__strikey_table(free, destroy_crm_node);
	}
	}

	void
	crm_peer_destroy(void)
	{
	if (crm_peer_cache != NULL) {
	crm_trace("Destroying peer cache with %d members", g_hash_table_size(crm_peer_cache));
	g_hash_table_destroy(crm_peer_cache);
	crm_peer_cache = NULL;
	}

	if (crm_remote_peer_cache != NULL) {
	crm_trace("Destroying remote peer cache with %d members", g_hash_table_size(crm_remote_peer_cache));
	g_hash_table_destroy(crm_remote_peer_cache);
	crm_remote_peer_cache = NULL;
	}

	if (known_node_cache != NULL) {
	crm_trace("Destroying known node cache with %d members",
	g_hash_table_size(known_node_cache));
	g_hash_table_destroy(known_node_cache);
	known_node_cache = NULL;
	}

	}

	static void (peer_status_callback)(enum crm_status_type, crm_node_t ,
	const void *) = NULL;

	/*!
	* \brief Set a client function that will be called after peer status changes
	*
	* \param[in] dispatch Pointer to function to use as callback
	*
	* \note Previously, client callbacks were responsible for peer cache
	* management. This is no longer the case, and client callbacks should do
	* only client-specific handling. Callbacks MUST NOT add or remove entries
	* in the peer caches.
	*/
	void
	crm_set_status_callback(void (dispatch) (enum crm_status_type, crm_node_t , const void *))
	{
	peer_status_callback = dispatch;
	}

	/*!
	* \brief Tell the library whether to automatically reap lost nodes
	*
	* If TRUE (the default), calling crm_update_peer_proc() will also update the
	* peer state to CRM_NODE_MEMBER or CRM_NODE_LOST, and pcmk__update_peer_state()
	* will reap peers whose state changes to anything other than CRM_NODE_MEMBER.
	* Callers should leave this enabled unless they plan to manage the cache
	* separately on their own.
	*
	* \param[in] autoreap TRUE to enable automatic reaping, FALSE to disable
	*/
	void
	crm_set_autoreap(gboolean autoreap)
	{
	crm_autoreap = autoreap;
	}

	static void
	dump_peer_hash(int level, const char *caller)
	{
	GHashTableIter iter;
	const char *id = NULL;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, (gpointer ) &id, (gpointer ) &node)) {
	do_crm_log(level, "%s: Node %u/%s = %p - %s", caller, node->id, node->uname, node, id);
	}
	}

	static gboolean
	hash_find_by_data(gpointer key, gpointer value, gpointer user_data)
	{
	return value == user_data;
	}

	/*!
	* \internal
	* \brief Search caches for a node (cluster or Pacemaker Remote)
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] flags Group of enum pcmk__node_search_flags
	*
	* \return Node cache entry if found, otherwise NULL
	*/
	crm_node_t *
	pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags)
	{
	crm_node_t *node = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	crm_peer_init();

	if ((uname != NULL) && pcmk_is_set(flags, pcmk__node_search_remote)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	}

	if ((node == NULL) && pcmk_is_set(flags, pcmk__node_search_cluster)) {
	node = pcmk__search_cluster_node_cache(id, uname, NULL);
	}

	if ((node == NULL) && pcmk_is_set(flags, pcmk__node_search_known)) {
	char *id_str = (id == 0)? NULL : crm_strdup_printf("%u", id);

	node = find_known_node(id_str, uname);
	free(id_str);
	}

	return node;
	}

	/*!
	* \internal
	* \brief Purge a node from cache (both cluster and Pacemaker Remote)
	*
	* \param[in] node_name If not NULL, purge only nodes with this name
	* \param[in] node_id If not 0, purge cluster nodes only if they have this ID
	*
	* \note If \p node_name is NULL and \p node_id is 0, no nodes will be purged.
	* If \p node_name is not NULL and \p node_id is not 0, Pacemaker Remote
	* nodes that match \p node_name will be purged, and cluster nodes that
	* match both \p node_name and \p node_id will be purged.
	* \note The caller must be careful not to use \p node_name after calling this
	* function if it might be a pointer into a cache entry being removed.
	*/
	void
	pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id)
	{
	char *node_name_copy = NULL;

	if ((node_name == NULL) && (node_id == 0U)) {
	return;
	}

	// Purge from Pacemaker Remote node cache
	if ((node_name != NULL)
	&& (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL)) {
	/* node_name could be a pointer into the cache entry being purged,
	* so reassign it to a copy before the original gets freed
	*/
	node_name_copy = strdup(node_name);
	CRM_ASSERT(node_name_copy != NULL);
	node_name = node_name_copy;

	crm_trace("Purging %s from Pacemaker Remote node cache", node_name);
	g_hash_table_remove(crm_remote_peer_cache, node_name);
	}

	reap_crm_member(node_id, node_name);
	free(node_name_copy);
	}

	/*!
	* \internal
	* \brief Search cluster node cache
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] uuid If not NULL while id is 0, node UUID instead of cluster
	* node ID to search for
	*
	* \return Cluster node cache entry if found, otherwise NULL
	*/
	crm_node_t *
	pcmk__search_cluster_node_cache(unsigned int id, const char *uname,
	const char *uuid)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	crm_node_t *by_id = NULL;
	crm_node_t *by_name = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	crm_peer_init();

	if (uname != NULL) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id > 0) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node->id == id) {
	crm_trace("ID match: %u = %p", node->id, node);
	by_id = node;
	break;
	}
	}

	} else if (uuid != NULL) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (pcmk__str_eq(node->uuid, uuid, pcmk__str_casei)) {
	crm_trace("UUID match: %s = %p", node->uuid, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if(by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %u/%s", by_id, id, uname);

	} else if(by_id == NULL && by_name) {
	crm_trace("Only one: %p for %u/%s", by_name, id, uname);

	if(id && by_name->id) {
	dump_peer_hash(LOG_WARNING, __func__);
	crm_crit("Node %u and %u share the same name '%s'",
	id, by_name->id, uname);
	node = NULL; /* Create a new one */

	} else {
	node = by_name;
	}

	} else if(by_name == NULL && by_id) {
	crm_trace("Only one: %p for %u/%s", by_id, id, uname);

	if(uname && by_id->uname) {
	dump_peer_hash(LOG_WARNING, __func__);
	crm_crit("Node '%s' and '%s' share the same cluster nodeid %u: assuming '%s' is correct",
	uname, by_id->uname, id, uname);
	}

	} else if(uname && by_id->uname) {
	if(pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	crm_notice("Node '%s' has changed its ID from %u to %u", by_id->uname, by_name->id, by_id->id);
	g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);

	} else {
	crm_warn("Node '%s' and '%s' share the same cluster nodeid: %u %s", by_id->uname, by_name->uname, id, uname);
	dump_peer_hash(LOG_INFO, __func__);
	crm_abort(__FILE__, __func__, __LINE__, "member weirdness", TRUE,
	TRUE);
	}

	} else if(id && by_name->id) {
	crm_warn("Node %u and %u share the same name: '%s'", by_id->id, by_name->id, uname);

	} else {
	/* Simple merge */

	/* Only corosync-based clusters use node IDs. The functions that call
	* pcmk__update_peer_state() and crm_update_peer_proc() only know
	* nodeid, so 'by_id' is authoritative when merging.
	*/
	dump_peer_hash(LOG_DEBUG, __func__);

	crm_info("Merging %p into %p", by_name, by_id);
	g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);
	}

	return node;
	}

	#if SUPPORT_COROSYNC
	static guint
	remove_conflicting_peer(crm_node_t *node)
	{
	int matches = 0;
	GHashTableIter iter;
	crm_node_t *existing_node = NULL;

	if (node->id == 0 \|\| node->uname == NULL) {
	return 0;
	}

	if (!pcmk__corosync_has_nodelist()) {
	return 0;
	}

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &existing_node)) {
	if (existing_node->id > 0
	&& existing_node->id != node->id
	&& existing_node->uname != NULL
	&& strcasecmp(existing_node->uname, node->uname) == 0) {

	if (crm_is_peer_active(existing_node)) {
	continue;
	}

	crm_warn("Removing cached offline node %u/%s which has conflicting uname with %u",
	existing_node->id, existing_node->uname, node->id);

	g_hash_table_iter_remove(&iter);
	matches++;
	}
	}

	return matches;
	}
	#endif

	/*!
	* \brief Get a cluster node cache entry
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] uuid If not NULL while id is 0, node UUID instead of cluster
	* node ID to search for
	* \param[in] flags Group of enum pcmk__node_search_flags
	*
	* \return (Possibly newly created) cluster node cache entry
	*/
	/* coverity[-alloc] Memory is referenced in one or both hashtables */
	crm_node_t *
	pcmk__get_node(unsigned int id, const char uname, const char uuid,
	uint32_t flags)
	{
	crm_node_t *node = NULL;
	char *uname_lookup = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	crm_peer_init();

	// Check the Pacemaker Remote node cache first
	if (pcmk_is_set(flags, pcmk__node_search_remote)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	if (node != NULL) {
	return node;
	}
	}

	if (!pcmk_is_set(flags, pcmk__node_search_cluster)) {
	return NULL;
	}

	node = pcmk__search_cluster_node_cache(id, uname, uuid);

	/* if uname wasn't provided, and find_peer did not turn up a uname based on id.
	* we need to do a lookup of the node name using the id in the cluster membership. */
	if ((node == NULL \|\| node->uname == NULL) && (uname == NULL)) {
	uname_lookup = get_node_name(id);
	}

	if (uname_lookup) {
	uname = uname_lookup;
	crm_trace("Inferred a name of '%s' for node %u", uname, id);

	/* try to turn up the node one more time now that we know the uname. */
	if (node == NULL) {
	node = pcmk__search_cluster_node_cache(id, uname, uuid);
	}
	}

	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = calloc(1, sizeof(crm_node_t));
	CRM_ASSERT(node);

	crm_info("Created entry %s/%p for node %s/%u (%d total)",
	uniqueid, node, uname, id, 1 + g_hash_table_size(crm_peer_cache));
	g_hash_table_replace(crm_peer_cache, uniqueid, node);
	}

	if(id > 0 && uname && (node->id == 0 \|\| node->uname == NULL)) {
	crm_info("Node %u is now known as %s", id, uname);
	}

	if(id > 0 && node->id == 0) {
	node->id = id;
	}

	if (uname && (node->uname == NULL)) {
	update_peer_uname(node, uname);
	}

	if(node->uuid == NULL) {
	if (uuid == NULL) {
	uuid = crm_peer_uuid(node);
	}

	if (uuid) {
	crm_info("Node %u has uuid %s", id, uuid);

	} else {
	crm_info("Cannot obtain a UUID for node %u/%s", id, node->uname);
	}
	}

	free(uname_lookup);

	return node;
	}

	/*!
	* \internal
	* \brief Update a node's uname
	*
	* \param[in,out] node Node object to update
	* \param[in] uname New name to set
	*
	* \note This function should not be called within a peer cache iteration,
	* because in some cases it can remove conflicting cache entries,
	* which would invalidate the iterator.
	*/
	static void
	update_peer_uname(crm_node_t node, const char uname)
	{
	CRM_CHECK(uname != NULL,
	crm_err("Bug: can't update node name without name"); return);
	CRM_CHECK(node != NULL,
	crm_err("Bug: can't update node name to %s without node", uname);
	return);

	if (pcmk__str_eq(uname, node->uname, pcmk__str_casei)) {
	crm_debug("Node uname '%s' did not change", uname);
	return;
	}

	for (const char c = uname; c; ++c) {
	if ((c >= 'A') && (c <= 'Z')) {
	crm_warn("Node names with capitals are discouraged, consider changing '%s'",
	uname);
	break;
	}
	}

	pcmk__str_update(&node->uname, uname);

	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_uname, node, NULL);
	}

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster() && !pcmk_is_set(node->flags, crm_remote_node)) {
	remove_conflicting_peer(node);
	}
	#endif
	}

	/*!
	* \internal
	* \brief Get log-friendly string equivalent of a process flag
	*
	* \param[in] proc Process flag
	*
	* \return Log-friendly string equivalent of \p proc
	*/
	static inline const char *
	proc2text(enum crm_proc_flag proc)
	{
	const char *text = "unknown";

	switch (proc) {
	case crm_proc_none:
	text = "none";
	break;
	case crm_proc_based:
	text = "pacemaker-based";
	break;
	case crm_proc_controld:
	text = "pacemaker-controld";
	break;
	case crm_proc_schedulerd:
	text = "pacemaker-schedulerd";
	break;
	case crm_proc_execd:
	text = "pacemaker-execd";
	break;
	case crm_proc_attrd:
	text = "pacemaker-attrd";
	break;
	case crm_proc_fenced:
	text = "pacemaker-fenced";
	break;
	case crm_proc_cpg:
	text = "corosync-cpg";
	break;
	}
	return text;
	}

	/*!
	* \internal
	* \brief Update a node's process information (and potentially state)
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] flag Bitmask of new process information
	* \param[in] status node status (online, offline, etc.)
	*
	* \return NULL if any node was reaped from peer caches, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible, otherwise
	* reaping could invalidate the iterator.
	*/
	crm_node_t *
	crm_update_peer_proc(const char source, crm_node_t node, uint32_t flag, const char *status)
	{
	uint32_t last = 0;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set %s to %s for NULL",
	source, proc2text(flag), status);
	return NULL);

	/* Pacemaker doesn't spawn processes on remote nodes */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return node;
	}

	last = node->processes;
	if (status == NULL) {
	node->processes = flag;
	if (node->processes != last) {
	changed = TRUE;
	}

	} else if (pcmk__str_eq(status, ONLINESTATUS, pcmk__str_casei)) {
	if ((node->processes & flag) != flag) {
	node->processes = pcmk__set_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	} else if (node->processes & flag) {
	node->processes = pcmk__clear_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	if (changed) {
	if (status == NULL && flag <= crm_proc_none) {
	crm_info("%s: Node %s[%u] - all processes are now offline", source, node->uname,
	node->id);
	} else {
	crm_info("%s: Node %s[%u] - %s is now %s", source, node->uname, node->id,
	proc2text(flag), status);
	}

	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	node->when_online = time(NULL);

	} else {
	node->when_online = 0;
	}

	/* Call the client callback first, then update the peer state,
	* in case the node will be reaped
	*/
	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_processes, node, &last);
	}

	/* The client callback shouldn't touch the peer caches,
	* but as a safety net, bail if the peer cache was destroyed.
	*/
	if (crm_peer_cache == NULL) {
	return NULL;
	}

	if (crm_autoreap) {
	const char *peer_state = NULL;

	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	peer_state = CRM_NODE_MEMBER;
	} else {
	peer_state = CRM_NODE_LOST;
	}
	node = pcmk__update_peer_state(__func__, node, peer_state, 0);
	}
	} else {
	crm_trace("%s: Node %s[%u] - %s is unchanged (%s)", source, node->uname, node->id,
	proc2text(flag), status);
	}
	return node;
	}

	/*!
	* \internal
	* \brief Update a cluster node cache entry's expected join state
	*
	* \param[in] source Caller's function name (for logging)
	* \param[in,out] node Node to update
	* \param[in] expected Node's new join state
	*/
	void
	pcmk__update_peer_expected(const char source, crm_node_t node,
	const char *expected)
	{
	char *last = NULL;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set 'expected' to %s", source, expected);
	return);

	/* Remote nodes don't participate in joins */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return;
	}

	last = node->expected;
	if (expected != NULL && !pcmk__str_eq(node->expected, expected, pcmk__str_casei)) {
	node->expected = strdup(expected);
	changed = TRUE;
	}

	if (changed) {
	crm_info("%s: Node %s[%u] - expected state is now %s (was %s)", source, node->uname, node->id,
	expected, last);
	free(last);
	} else {
	crm_trace("%s: Node %s[%u] - expected state is unchanged (%s)", source, node->uname,
	node->id, expected);
	}
	}

	/*!
	* \internal
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	* \param[in,out] iter If not NULL, pointer to node's peer cache iterator
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function may be called from
	* within a peer cache iteration if the iterator is supplied.
	*/
	static crm_node_t *
	update_peer_state_iter(const char source, crm_node_t node, const char *state,
	uint64_t membership, GHashTableIter *iter)
	{
	gboolean is_member;

	CRM_CHECK(node != NULL,
	crm_err("Could not set state for unknown host to %s"
	CRM_XS " source=%s", state, source);
	return NULL);

	is_member = pcmk__str_eq(state, CRM_NODE_MEMBER, pcmk__str_casei);
	if (is_member) {
	node->when_lost = 0;
	if (membership) {
	node->last_seen = membership;
	}
	}

	if (state && !pcmk__str_eq(node->state, state, pcmk__str_casei)) {
	char *last = node->state;

	if (is_member) {
	node->when_member = time(NULL);

	} else {
	node->when_member = 0;
	}

	node->state = strdup(state);
	crm_notice("Node %s state is now %s " CRM_XS
	" nodeid=%u previous=%s source=%s", node->uname, state,
	node->id, (last? last : "unknown"), source);
	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_nstate, node, last);
	}
	free(last);

	if (crm_autoreap && !is_member
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/* We only autoreap from the peer cache, not the remote peer cache,
	* because the latter should be managed only by
	* crm_remote_peer_cache_refresh().
	*/
	if(iter) {
	crm_notice("Purged 1 peer with id=%u and/or uname=%s from the membership cache", node->id, node->uname);
	g_hash_table_iter_remove(iter);

	} else {
	reap_crm_member(node->id, node->uname);
	}
	node = NULL;
	}

	} else {
	crm_trace("Node %s state is unchanged (%s) " CRM_XS
	" nodeid=%u source=%s", node->uname, state, node->id, source);
	}
	return node;
	}

	/*!
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible,
	* otherwise reaping could invalidate the iterator.
	*/
	crm_node_t *
	pcmk__update_peer_state(const char source, crm_node_t node,
	const char *state, uint64_t membership)
	{
	return update_peer_state_iter(source, node, state, membership, NULL);
	}

	/*!
	* \internal
	* \brief Reap all nodes from cache whose membership information does not match
	*
	* \param[in] membership Membership ID of nodes to keep
	*/
	void
	pcmk__reap_unseen_nodes(uint64_t membership)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;

	crm_trace("Reaping unseen nodes...");
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&node)) {
	if (node->last_seen != membership) {
	if (node->state) {
	/*
	* Calling update_peer_state_iter() allows us to
	* remove the node from crm_peer_cache without
	* invalidating our iterator
	*/
	update_peer_state_iter(__func__, node, CRM_NODE_LOST,
	membership, &iter);

	} else {
	crm_info("State of node %s[%u] is still unknown",
	node->uname, node->id);
	}
	}
	}
	}

	static crm_node_t *
	find_known_node(const char id, const char uname)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	crm_node_t *by_id = NULL;
	crm_node_t *by_name = NULL;

	if (uname) {
	g_hash_table_iter_init(&iter, known_node_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id) {
	g_hash_table_iter_init(&iter, known_node_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(strcasecmp(node->uuid, id) == 0) {
	crm_trace("ID match: %s= %p", id, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if (by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %s/%s", by_id, id, uname);

	} else if (by_id == NULL && by_name) {
	crm_trace("Only one: %p for %s/%s", by_name, id, uname);

	if (id) {
	node = NULL;

	} else {
	node = by_name;
	}

	} else if (by_name == NULL && by_id) {
	crm_trace("Only one: %p for %s/%s", by_id, id, uname);

	if (uname) {
	node = NULL;
	}

	} else if (uname && by_id->uname
	&& pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	/* Multiple nodes have the same uname in the CIB.
	* Return by_id. */

	} else if (id && by_name->uuid
	&& pcmk__str_eq(id, by_name->uuid, pcmk__str_casei)) {
	/* Multiple nodes have the same id in the CIB.
	* Return by_name. */
	node = by_name;

	} else {
	node = NULL;
	}

	if (node == NULL) {
	crm_debug("Couldn't find node%s%s%s%s",
	id? " " : "",
	id? id : "",
	uname? " with name " : "",
	uname? uname : "");
	}

	return node;
	}

	static void
	known_node_cache_refresh_helper(xmlNode xml_node, void user_data)
	{
	const char *id = crm_element_value(xml_node, PCMK_XA_ID);
	const char *uname = crm_element_value(xml_node, PCMK_XA_UNAME);
	crm_node_t * node = NULL;

	CRM_CHECK(id != NULL && uname !=NULL, return);
	node = find_known_node(id, uname);

	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = calloc(1, sizeof(crm_node_t));
	CRM_ASSERT(node != NULL);

	node->uname = strdup(uname);
	CRM_ASSERT(node->uname != NULL);

	node->uuid = strdup(id);
	CRM_ASSERT(node->uuid != NULL);

	g_hash_table_replace(known_node_cache, uniqueid, node);

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	pcmk__str_update(&node->uname, uname);

	/* Node is in cache and hasn't been updated already, so mark it clean */
	clear_peer_flags(node, crm_node_dirty);
	}

	}

	static void
	refresh_known_node_cache(xmlNode *cib)
	{
	crm_peer_init();

	g_hash_table_foreach(known_node_cache, mark_dirty, NULL);

	crm_foreach_xpath_result(cib, PCMK__XP_MEMBER_NODE_CONFIG,
	known_node_cache_refresh_helper, NULL);

	/* Remove all old cache entries that weren't seen in the CIB */
	g_hash_table_foreach_remove(known_node_cache, is_dirty, NULL);
	}

	void
	pcmk__refresh_node_caches_from_cib(xmlNode *cib)
	{
	crm_remote_peer_cache_refresh(cib);
	refresh_known_node_cache(cib);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/cluster/compat.h>

	int
	crm_terminate_member(int nodeid, const char uname, void unused)
	{
	return stonith_api_kick(nodeid, uname, 120, TRUE);
	}

	int
	crm_terminate_member_no_mainloop(int nodeid, const char uname, int connection)
	{
	return stonith_api_kick(nodeid, uname, 120, TRUE);
	}

	crm_node_t *
	crm_get_peer(unsigned int id, const char *uname)
	{
	return pcmk__get_node(id, uname, NULL, pcmk__node_search_cluster);
	}

	crm_node_t *
	crm_get_peer_full(unsigned int id, const char *uname, int flags)
	{
	return pcmk__get_node(id, uname, NULL, flags);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/pacemaker/pcmk_injections.c b/lib/pacemaker/pcmk_injections.c
	index ebaaca7744..3d3f40fc84 100644
	--- a/lib/pacemaker/pcmk_injections.c
	+++ b/lib/pacemaker/pcmk_injections.c
	@@ -1,788 +1,795 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>

	#include <sys/stat.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/util.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/xml_internal.h>
	#include <crm/lrmd_events.h> // lrmd_event_data_t, etc.
	#include <crm/lrmd_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include "libpacemaker_private.h"

	bool pcmk__simulate_node_config = false;

	#define XPATH_NODE_CONFIG "//" XML_CIB_TAG_NODE "[@" PCMK_XA_UNAME "='%s']"
	-#define XPATH_NODE_STATE "//" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']"
	-#define XPATH_NODE_STATE_BY_ID "//" XML_CIB_TAG_STATE "[@" PCMK_XA_ID "='%s']"
	+#define XPATH_NODE_STATE "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']"
	+#define XPATH_NODE_STATE_BY_ID "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "='%s']"
	#define XPATH_RSC_HISTORY XPATH_NODE_STATE \
	"//" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']"


	/*!
	* \internal
	* \brief Inject a fictitious transient node attribute into scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	- * \param[in,out] cib_node node_state XML to inject attribute into
	+ * \param[in,out] cib_node \c PCMK__XE_NODE_STATE XML to inject attribute into
	* \param[in] name Transient node attribute name to inject
	* \param[in] value Transient node attribute value to inject
	*/
	static void
	inject_transient_attr(pcmk__output_t out, xmlNode cib_node,
	const char name, const char value)
	{
	xmlNode *attrs = NULL;
	xmlNode *instance_attrs = NULL;
	const char *node_uuid = ID(cib_node);

	out->message(out, "inject-attr", name, value, cib_node);

	attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	if (attrs == NULL) {
	attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
	crm_xml_add(attrs, PCMK_XA_ID, node_uuid);
	}

	instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
	if (instance_attrs == NULL) {
	instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
	crm_xml_add(instance_attrs, PCMK_XA_ID, node_uuid);
	}

	crm_create_nvpair_xml(instance_attrs, NULL, name, value);
	}

	/*!
	* \internal
	* \brief Inject a fictitious fail count into a scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node Node state XML to inject into
	* \param[in] resource ID of resource for fail count to inject
	* \param[in] task Action name for fail count to inject
	* \param[in] interval_ms Action interval (in milliseconds) for fail count
	* \param[in] rc Action result for fail count to inject (if 0, or
	* 7 when interval_ms is 0, inject nothing)
	*/
	void
	pcmk__inject_failcount(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char task,
	guint interval_ms, int rc)
	{
	if (rc == 0) {
	return;

	} else if ((rc == 7) && (interval_ms == 0)) {
	return;

	} else {
	char *name = NULL;
	char *now = pcmk__ttoa(time(NULL));

	name = pcmk__failcount_name(resource, task, interval_ms);
	inject_transient_attr(out, cib_node, name, "value++");
	free(name);

	name = pcmk__lastfailure_name(resource, task, interval_ms);
	inject_transient_attr(out, cib_node, name, now);
	free(name);

	free(now);
	}
	}

	/*!
	* \internal
	* \brief Create a CIB configuration entry for a fictitious node
	*
	* \param[in,out] cib_conn CIB object to use
	* \param[in] node Node name to use
	*/
	static void
	create_node_entry(cib_t cib_conn, const char node)
	{
	int rc = pcmk_ok;
	char *xpath = crm_strdup_printf(XPATH_NODE_CONFIG, node);

	rc = cib_conn->cmds->query(cib_conn, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if (rc == -ENXIO) { // Only add if not already existing
	xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);

	crm_xml_add(cib_object, PCMK_XA_ID, node); // Use node name as ID
	crm_xml_add(cib_object, PCMK_XA_UNAME, node);
	cib_conn->cmds->create(cib_conn, PCMK_XE_NODES, cib_object,
	cib_sync_call\|cib_scope_local);
	/* Not bothering with subsequent query to see if it exists,
	we'll bomb out later in the call to query_node_uuid()... */

	free_xml(cib_object);
	}

	free(xpath);
	}

	/*!
	* \internal
	* \brief Synthesize a fake executor event for an action
	*
	* \param[in] cib_resource XML for any existing resource action history
	* \param[in] task Name of action to synthesize
	* \param[in] interval_ms Interval of action to synthesize
	* \param[in] outcome Result of action to synthesize
	*
	* \return Newly allocated executor event
	* \note It is the caller's responsibility to free the result with
	* lrmd_free_event().
	*/
	static lrmd_event_data_t *
	create_op(const xmlNode cib_resource, const char task, guint interval_ms,
	int outcome)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *xop = NULL;

	op = lrmd_new_event(ID(cib_resource), task, interval_ms);
	lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
	op->params = NULL; // Not needed for simulation purposes
	op->t_run = (unsigned int) time(NULL);
	op->t_rcchange = op->t_run;

	// Use a call ID higher than any existing history entries
	op->call_id = 0;
	for (xop = pcmk__xe_first_child(cib_resource); xop != NULL;
	xop = pcmk__xe_next(xop)) {

	int tmp = 0;

	crm_element_value_int(xop, PCMK__XA_CALL_ID, &tmp);
	if (tmp > op->call_id) {
	op->call_id = tmp;
	}
	}
	op->call_id++;

	return op;
	}

	/*!
	* \internal
	* \brief Inject a fictitious resource history entry into a scheduler input
	*
	* \param[in,out] cib_resource Resource history XML to inject entry into
	* \param[in,out] op Action result to inject
	* \param[in] target_rc Expected result for action to inject
	*
	* \return XML of injected resource history entry
	*/
	xmlNode *
	pcmk__inject_action_result(xmlNode cib_resource, lrmd_event_data_t op,
	int target_rc)
	{
	return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
	target_rc, NULL, crm_system_name);
	}

	/*!
	* \internal
	* \brief Inject a fictitious node into a scheduler input
	*
	* \param[in,out] cib_conn Scheduler input CIB to inject node into
	* \param[in] node Name of node to inject
	* \param[in] uuid UUID of node to inject
	*
	- * \return XML of node_state entry for new node
	+ * \return XML of \c PCMK__XE_NODE_STATE entry for new node
	* \note If the global pcmk__simulate_node_config has been set to true, a
	* node entry in the configuration section will be added, as well as a
	* node state entry in the status section.
	*/
	xmlNode *
	pcmk__inject_node(cib_t cib_conn, const char node, const char *uuid)
	{
	int rc = pcmk_ok;
	xmlNode *cib_object = NULL;
	char *xpath = crm_strdup_printf(XPATH_NODE_STATE, node);
	bool duplicate = false;
	char *found_uuid = NULL;

	if (pcmk__simulate_node_config) {
	create_node_entry(cib_conn, node);
	}

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
	- crm_err("Detected multiple node_state entries for xpath=%s, bailing",
	+ crm_err("Detected multiple " PCMK__XE_NODE_STATE " entries for "
	+ "xpath=%s, bailing",
	xpath);
	duplicate = true;
	goto done;
	}

	if (rc == -ENXIO) {
	if (uuid == NULL) {
	query_node_uuid(cib_conn, node, &found_uuid, NULL);
	} else {
	found_uuid = strdup(uuid);
	}

	if (found_uuid) {
	char *xpath_by_uuid = crm_strdup_printf(XPATH_NODE_STATE_BY_ID,
	found_uuid);

	- // It's possible that a node_state entry doesn't have an uname yet.
	+ /* It's possible that a PCMK__XE_NODE_STATE entry doesn't have a
	+ * PCMK_XA_UNAME yet
	+ */
	rc = cib_conn->cmds->query(cib_conn, xpath_by_uuid, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
	crm_err("Can't inject node state for %s because multiple "
	"state entries found for ID %s", node, found_uuid);
	duplicate = true;
	free(xpath_by_uuid);
	goto done;

	} else if (cib_object != NULL) {
	crm_xml_add(cib_object, PCMK_XA_UNAME, node);

	rc = cib_conn->cmds->modify(cib_conn, PCMK_XE_STATUS,
	cib_object,
	cib_sync_call\|cib_scope_local);
	}

	free(xpath_by_uuid);
	}
	}

	if (rc == -ENXIO) {
	- cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
	+ cib_object = create_xml_node(NULL, PCMK__XE_NODE_STATE);
	crm_xml_add(cib_object, PCMK_XA_ID, found_uuid);
	crm_xml_add(cib_object, PCMK_XA_UNAME, node);
	cib_conn->cmds->create(cib_conn, PCMK_XE_STATUS, cib_object,
	cib_sync_call\|cib_scope_local);
	free_xml(cib_object);

	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	crm_trace("Injecting node state for %s (rc=%d)", node, rc);
	}

	done:
	free(found_uuid);
	free(xpath);

	if (duplicate) {
	crm_log_xml_warn(cib_object, "Duplicates");
	crm_exit(CRM_EX_SOFTWARE);
	return NULL; // not reached, but makes static analysis happy
	}

	CRM_ASSERT(rc == pcmk_ok);
	return cib_object;
	}

	/*!
	* \internal
	* \brief Inject a fictitious node state change into a scheduler input
	*
	* \param[in,out] cib_conn Scheduler input CIB to inject into
	* \param[in] node Name of node to inject change for
	* \param[in] up If true, change state to online, otherwise offline
	*
	* \return XML of changed (or added) node state entry
	*/
	xmlNode *
	pcmk__inject_node_state_change(cib_t cib_conn, const char node, bool up)
	{
	xmlNode *cib_node = pcmk__inject_node(cib_conn, node, NULL);

	if (up) {
	pcmk__xe_set_props(cib_node,
	PCMK__XA_IN_CCM, PCMK_VALUE_TRUE,
	PCMK__XA_CRMD, ONLINESTATUS,
	PCMK__XA_JOIN, CRMD_JOINSTATE_MEMBER,
	PCMK__XA_EXPECTED, CRMD_JOINSTATE_MEMBER,
	NULL);
	} else {
	pcmk__xe_set_props(cib_node,
	PCMK__XA_IN_CCM, PCMK_VALUE_FALSE,
	PCMK__XA_CRMD, OFFLINESTATUS,
	PCMK__XA_JOIN, CRMD_JOINSTATE_DOWN,
	PCMK__XA_EXPECTED, CRMD_JOINSTATE_DOWN,
	NULL);
	}
	crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, crm_system_name);
	return cib_node;
	}

	/*!
	* \internal
	* \brief Check whether a node has history for a given resource
	*
	* \param[in,out] cib_node Node state XML to check
	* \param[in] resource Resource name to check for
	*
	* \return Resource's lrm_resource XML entry beneath \p cib_node if found,
	* otherwise NULL
	*/
	static xmlNode *
	find_resource_xml(xmlNode cib_node, const char resource)
	{
	const char *node = crm_element_value(cib_node, PCMK_XA_UNAME);
	char *xpath = crm_strdup_printf(XPATH_RSC_HISTORY, node, resource);
	xmlNode *match = get_xpath_object(xpath, cib_node, LOG_TRACE);

	free(xpath);
	return match;
	}

	/*!
	* \internal
	* \brief Inject a resource history element into a scheduler input
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in,out] cib_node Node state XML to inject resource history entry into
	* \param[in] resource ID (in configuration) of resource to inject
	* \param[in] lrm_name ID as used in history (could be clone instance)
	* \param[in] rclass Resource agent class of resource to inject
	* \param[in] rtype Resource agent type of resource to inject
	* \param[in] rprovider Resource agent provider of resource to inject
	*
	* \return XML of injected resource history element
	* \note If a history element already exists under either \p resource or
	* \p lrm_name, this will return it rather than injecting a new one.
	*/
	xmlNode *
	pcmk__inject_resource_history(pcmk__output_t out, xmlNode cib_node,
	const char resource, const char lrm_name,
	const char rclass, const char rtype,
	const char *rprovider)
	{
	xmlNode *lrm = NULL;
	xmlNode *container = NULL;
	xmlNode *cib_resource = NULL;

	cib_resource = find_resource_xml(cib_node, resource);
	if (cib_resource != NULL) {
	/* If an existing LRM history entry uses the resource name,
	* continue using it, even if lrm_name is different.
	*/
	return cib_resource;
	}

	// Check for history entry under preferred name
	if (strcmp(resource, lrm_name) != 0) {
	cib_resource = find_resource_xml(cib_node, lrm_name);
	if (cib_resource != NULL) {
	return cib_resource;
	}
	}

	if ((rclass == NULL) \|\| (rtype == NULL)) {
	// @TODO query configuration for class, provider, type
	out->err(out,
	"Resource %s not found in the status section of %s "
	"(supply class and type to continue)",
	resource, ID(cib_node));
	return NULL;

	} else if (!pcmk__strcase_any_of(rclass,
	PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_STONITH,
	PCMK_RESOURCE_CLASS_SERVICE,
	PCMK_RESOURCE_CLASS_UPSTART,
	PCMK_RESOURCE_CLASS_SYSTEMD,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	out->err(out, "Invalid class for %s: %s", resource, rclass);
	return NULL;

	} else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
	&& (rprovider == NULL)) {
	// @TODO query configuration for provider
	out->err(out, "Please specify the provider for resource %s", resource);
	return NULL;
	}

	crm_info("Injecting new resource %s into node state '%s'",
	lrm_name, ID(cib_node));

	lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
	if (lrm == NULL) {
	const char *node_uuid = ID(cib_node);

	lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
	crm_xml_add(lrm, PCMK_XA_ID, node_uuid);
	}

	container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
	if (container == NULL) {
	container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
	}

	cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);

	// If we're creating a new entry, use the preferred name
	crm_xml_add(cib_resource, PCMK_XA_ID, lrm_name);

	crm_xml_add(cib_resource, PCMK_XA_CLASS, rclass);
	crm_xml_add(cib_resource, PCMK_XA_PROVIDER, rprovider);
	crm_xml_add(cib_resource, PCMK_XA_TYPE, rtype);

	return cib_resource;
	}

	static int
	find_ticket_state(pcmk__output_t out, cib_t the_cib, const char *ticket_id,
	xmlNode **ticket_state_xml)
	{
	int rc = pcmk_ok;
	xmlNode *xml_search = NULL;

	GString *xpath = g_string_sized_new(256);

	CRM_ASSERT(ticket_state_xml != NULL);
	*ticket_state_xml = NULL;

	g_string_append(xpath,
	"/" PCMK_XE_CIB "/" PCMK_XE_STATUS
	"/" XML_CIB_TAG_TICKETS);

	if (ticket_id) {
	pcmk__g_strcat(xpath,
	"/" XML_CIB_TAG_TICKET_STATE
	"[@" PCMK_XA_ID "=\"", ticket_id, "\"]", NULL);
	}
	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call\|cib_scope_local\|cib_xpath);
	g_string_free(xpath, TRUE);

	if (rc != pcmk_ok) {
	return rc;
	}

	crm_log_xml_debug(xml_search, "Match");
	if ((xml_search->children != NULL) && (ticket_id != NULL)) {
	out->err(out, "Multiple ticket_states match ticket_id=%s", ticket_id);
	}
	*ticket_state_xml = xml_search;

	return rc;
	}

	/*!
	* \internal
	* \brief Inject a ticket attribute into ticket state
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in] ticket_id Ticket whose state should be changed
	* \param[in] attr_name Ticket attribute name to inject
	* \param[in] attr_value Boolean value of ticket attribute to inject
	* \param[in,out] cib CIB object to use
	*
	* \return Standard Pacemaker return code
	*/
	static int
	set_ticket_state_attr(pcmk__output_t out, const char ticket_id,
	const char attr_name, bool attr_value, cib_t cib)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_top = NULL;
	xmlNode *ticket_state_xml = NULL;

	// Check for an existing ticket state entry
	rc = find_ticket_state(out, cib, ticket_id, &ticket_state_xml);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) { // Ticket state found, use it
	crm_debug("Injecting attribute into existing ticket state %s",
	ticket_id);
	xml_top = ticket_state_xml;

	} else if (rc == ENXIO) { // No ticket state, create it
	xmlNode *xml_obj = NULL;

	xml_top = create_xml_node(NULL, PCMK_XE_STATUS);
	xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
	ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
	crm_xml_add(ticket_state_xml, PCMK_XA_ID, ticket_id);

	} else { // Error
	return rc;
	}

	// Add the attribute to the ticket state
	pcmk__xe_set_bool_attr(ticket_state_xml, attr_name, attr_value);
	crm_log_xml_debug(xml_top, "Update");

	// Commit the change to the CIB
	rc = cib->cmds->modify(cib, PCMK_XE_STATUS, xml_top,
	cib_sync_call\|cib_scope_local);
	rc = pcmk_legacy2rc(rc);

	free_xml(xml_top);
	return rc;
	}

	/*!
	* \internal
	* \brief Inject a fictitious action into the cluster
	*
	* \param[in,out] out Output object for displaying error messages
	* \param[in] spec Action specification to inject
	* \param[in,out] cib CIB object for scheduler input
	* \param[in] scheduler Scheduler data
	*/
	static void
	inject_action(pcmk__output_t out, const char spec, cib_t *cib,
	const pcmk_scheduler_t *scheduler)
	{
	int rc;
	int outcome = PCMK_OCF_OK;
	guint interval_ms = 0;

	char *key = NULL;
	char *node = NULL;
	char *task = NULL;
	char *resource = NULL;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *rprovider = NULL;

	xmlNode *cib_op = NULL;
	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	const pcmk_resource_t *rsc = NULL;
	lrmd_event_data_t *op = NULL;

	out->message(out, "inject-spec", spec);

	key = calloc(1, strlen(spec) + 1);
	node = calloc(1, strlen(spec) + 1);
	rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
	if (rc != 3) {
	out->err(out, "Invalid operation spec: %s. Only found %d fields",
	spec, rc);
	goto done;
	}

	parse_op_key(key, &resource, &task, &interval_ms);

	rsc = pe_find_resource(scheduler->resources, resource);
	if (rsc == NULL) {
	out->err(out, "Invalid resource name: %s", resource);
	goto done;
	}

	rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	rprovider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);

	cib_node = pcmk__inject_node(cib, node, NULL);
	CRM_ASSERT(cib_node != NULL);

	pcmk__inject_failcount(out, cib_node, resource, task, interval_ms, outcome);

	cib_resource = pcmk__inject_resource_history(out, cib_node,
	resource, resource,
	rclass, rtype, rprovider);
	CRM_ASSERT(cib_resource != NULL);

	op = create_op(cib_resource, task, interval_ms, outcome);
	CRM_ASSERT(op != NULL);

	cib_op = pcmk__inject_action_result(cib_resource, op, 0);
	CRM_ASSERT(cib_op != NULL);
	lrmd_free_event(op);

	rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(task);
	free(node);
	free(key);
	}

	/*!
	* \internal
	* \brief Inject fictitious scheduler inputs
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] cib CIB object for scheduler input to modify
	* \param[in] injections Injections to apply
	*/
	void
	pcmk__inject_scheduler_input(pcmk_scheduler_t scheduler, cib_t cib,
	const pcmk_injections_t *injections)
	{
	int rc = pcmk_ok;
	const GList *iter = NULL;
	xmlNode *cib_node = NULL;
	pcmk__output_t *out = scheduler->priv;

	out->message(out, "inject-modify-config", injections->quorum,
	injections->watchdog);
	if (injections->quorum != NULL) {
	xmlNode *top = create_xml_node(NULL, PCMK_XE_CIB);

	/* crm_xml_add(top, PCMK_XA_DC_UUID, dc_uuid); */
	crm_xml_add(top, PCMK_XA_HAVE_QUORUM, injections->quorum);

	rc = cib->cmds->modify(cib, NULL, top, cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	}

	if (injections->watchdog != NULL) {
	rc = cib__update_node_attr(out, cib, cib_sync_call\|cib_scope_local,
	PCMK_XE_CRM_CONFIG, NULL, NULL, NULL,
	NULL, PCMK_OPT_HAVE_WATCHDOG,
	injections->watchdog, NULL, NULL);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->node_up; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;

	out->message(out, "inject-modify-node", "Online", node);

	cib_node = pcmk__inject_node_state_change(cib, node, true);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (iter = injections->node_down; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;
	char *xpath = NULL;

	out->message(out, "inject-modify-node", "Offline", node);

	cib_node = pcmk__inject_node_state_change(cib, node, false);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);

	- xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
	- node, XML_CIB_TAG_LRM);
	+ xpath = crm_strdup_printf("//" PCMK__XE_NODE_STATE
	+ "[@" PCMK_XA_UNAME "='%s']"
	+ "/" XML_CIB_TAG_LRM,
	+ node);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	free(xpath);

	- xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
	- node, XML_TAG_TRANSIENT_NODEATTRS);
	+ xpath = crm_strdup_printf("//" PCMK__XE_NODE_STATE
	+ "[@" PCMK_XA_UNAME "='%s']"
	+ "/" XML_TAG_TRANSIENT_NODEATTRS,
	+ node);
	cib->cmds->remove(cib, xpath, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);
	free(xpath);
	}

	for (iter = injections->node_fail; iter != NULL; iter = iter->next) {
	const char node = (const char ) iter->data;

	out->message(out, "inject-modify-node", "Failing", node);

	cib_node = pcmk__inject_node_state_change(cib, node, true);
	crm_xml_add(cib_node, PCMK__XA_IN_CCM, PCMK_VALUE_FALSE);
	CRM_ASSERT(cib_node != NULL);

	rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);
	free_xml(cib_node);
	}

	for (iter = injections->ticket_grant; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Granting", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "granted", true, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_revoke; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Revoking", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "granted", false, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_standby; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Standby", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "standby", true, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->ticket_activate; iter != NULL; iter = iter->next) {
	const char ticket_id = (const char ) iter->data;

	out->message(out, "inject-modify-ticket", "Activating", ticket_id);

	rc = set_ticket_state_attr(out, ticket_id, "standby", false, cib);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	for (iter = injections->op_inject; iter != NULL; iter = iter->next) {
	inject_action(out, (const char *) iter->data, cib, scheduler);
	}

	if (!out->is_quiet(out)) {
	out->end_list(out);
	}
	}

	void
	pcmk_free_injections(pcmk_injections_t *injections)
	{
	if (injections == NULL) {
	return;
	}

	g_list_free_full(injections->node_up, g_free);
	g_list_free_full(injections->node_down, g_free);
	g_list_free_full(injections->node_fail, g_free);
	g_list_free_full(injections->op_fail, g_free);
	g_list_free_full(injections->op_inject, g_free);
	g_list_free_full(injections->ticket_grant, g_free);
	g_list_free_full(injections->ticket_revoke, g_free);
	g_list_free_full(injections->ticket_standby, g_free);
	g_list_free_full(injections->ticket_activate, g_free);
	free(injections->quorum);
	free(injections->watchdog);

	free(injections);
	}
	diff --git a/lib/pacemaker/pcmk_resource.c b/lib/pacemaker/pcmk_resource.c
	index 50244d31a7..758672a562 100644
	--- a/lib/pacemaker/pcmk_resource.c
	+++ b/lib/pacemaker/pcmk_resource.c
	@@ -1,248 +1,249 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <errno.h>
	#include <glib.h>
	#include <libxml/tree.h>

	#include <crm/cib/internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/results.h>
	#include <crm/common/output_internal.h>
	#include <crm/pengine/internal.h>

	#include <pacemaker.h>
	#include <pacemaker-internal.h>

	// Search path for resource operation history (takes node name and resource ID)
	-#define XPATH_OP_HISTORY "//" PCMK_XE_STATUS \
	- "/" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']" \
	- "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	+#define XPATH_OP_HISTORY "//" PCMK_XE_STATUS \
	+ "/" PCMK__XE_NODE_STATE \
	+ "[@" PCMK_XA_UNAME "='%s']" \
	+ "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']"

	static xmlNode *
	best_op(const pcmk_resource_t rsc, const pcmk_node_t node)
	{
	char *xpath = NULL;
	xmlNode *history = NULL;
	xmlNode *best = NULL;
	bool best_effective_op = false;
	guint best_interval = 0;
	bool best_failure = false;
	const char *best_digest = NULL;

	// Find node's resource history
	xpath = crm_strdup_printf(XPATH_OP_HISTORY, node->details->uname, rsc->id);
	history = get_xpath_object(xpath, rsc->cluster->input, LOG_NEVER);
	free(xpath);

	// Examine each history entry
	for (xmlNode *lrm_rsc_op = first_named_child(history, XML_LRM_TAG_RSC_OP);
	lrm_rsc_op != NULL; lrm_rsc_op = crm_next_same_xml(lrm_rsc_op)) {

	const char *digest = crm_element_value(lrm_rsc_op,
	PCMK__XA_OP_RESTART_DIGEST);
	guint interval_ms = 0;
	const char *task = crm_element_value(lrm_rsc_op, PCMK_XA_OPERATION);
	bool effective_op = false;
	bool failure = pcmk__ends_with(ID(lrm_rsc_op), "_last_failure_0");


	crm_element_value_ms(lrm_rsc_op, PCMK_META_INTERVAL, &interval_ms);
	effective_op = interval_ms == 0
	&& pcmk__strcase_any_of(task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE,
	PCMK_ACTION_MIGRATE_FROM, NULL);

	if (best == NULL) {
	goto is_best;
	}

	if (best_effective_op) {
	// Do not use an ineffective op if there's an effective one.
	if (!effective_op) {
	continue;
	}
	// Do not use an ineffective non-recurring op if there's a recurring one
	} else if (best_interval != 0
	&& !effective_op
	&& interval_ms == 0) {
	continue;
	}

	// Do not use last failure if there's a successful one.
	if (!best_failure && failure) {
	continue;
	}

	// Do not use an op without a restart digest if there's one with.
	if (best_digest != NULL && digest == NULL) {
	continue;
	}

	// Do not use an older op if there's a newer one.
	if (pe__is_newer_op(best, lrm_rsc_op, true) > 0) {
	continue;
	}

	is_best:
	best = lrm_rsc_op;
	best_effective_op = effective_op;
	best_interval = interval_ms;
	best_failure = failure;
	best_digest = digest;
	}
	return best;
	}

	/*!
	* \internal
	* \brief Remove a resource
	*
	* \param[in,out] cib An open connection to the CIB
	* \param[in] cib_opts Options to use in the CIB operation call
	* \param[in] rsc_id Resource to remove
	* \param[in] rsc_type Type of the resource ("primitive", "group", etc.)
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__resource_delete(cib_t cib, uint32_t cib_opts, const char rsc_id,
	const char *rsc_type)
	{
	int rc = pcmk_rc_ok;
	xmlNode *msg_data = NULL;

	if (cib == NULL) {
	return ENOTCONN;
	}

	if (rsc_id == NULL \|\| rsc_type == NULL) {
	return EINVAL;
	}

	msg_data = create_xml_node(NULL, rsc_type);
	crm_xml_add(msg_data, PCMK_XA_ID, rsc_id);

	rc = cib->cmds->remove(cib, PCMK_XE_RESOURCES, msg_data, cib_opts);
	rc = pcmk_legacy2rc(rc);

	free_xml(msg_data);
	return rc;
	}

	int
	pcmk_resource_delete(xmlNodePtr xml, const char rsc_id, const char *rsc_type)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;
	uint32_t cib_opts = cib_sync_call;
	cib_t *cib = NULL;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	cib = cib_new();
	if (cib == NULL) {
	rc = pcmk_rc_cib_corrupt;
	goto done;
	}

	rc = cib->cmds->signon(cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto done;
	}

	rc = pcmk__resource_delete(cib, cib_opts, rsc_id, rsc_type);

	done:
	if (cib != NULL) {
	cib__clean_up_connection(&cib);
	}

	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	return rc;
	}

	/*!
	* \internal
	* \brief Calculate and output resource operation digests
	*
	* \param[in,out] out Output object
	* \param[in,out] rsc Resource to calculate digests for
	* \param[in] node Node whose operation history should be used
	* \param[in] overrides Hash table of configuration parameters to override
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__resource_digests(pcmk__output_t out, pcmk_resource_t rsc,
	const pcmk_node_t node, GHashTable overrides)
	{
	const char *task = NULL;
	xmlNode *xml_op = NULL;
	pcmk__op_digest_t *digests = NULL;
	guint interval_ms = 0;
	int rc = pcmk_rc_ok;

	if ((out == NULL) \|\| (rsc == NULL) \|\| (node == NULL)) {
	return EINVAL;
	}
	if (rsc->variant != pcmk_rsc_variant_primitive) {
	// Only primitives get operation digests
	return EOPNOTSUPP;
	}

	// Find XML of operation history to use
	xml_op = best_op(rsc, node);

	// Generate an operation key
	if (xml_op != NULL) {
	task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);
	}
	if (task == NULL) { // Assume start if no history is available
	task = PCMK_ACTION_START;
	interval_ms = 0;
	}

	// Calculate and show digests
	digests = pe__calculate_digests(rsc, task, &interval_ms, node, xml_op,
	overrides, true, rsc->cluster);
	rc = out->message(out, "digests", rsc, node, task, interval_ms, digests);

	pe__free_digests(digests);
	return rc;
	}

	// @COMPAT The scheduler parameter is unused and can be removed at the next break
	int
	pcmk_resource_digests(xmlNodePtr xml, pcmk_resource_t rsc,
	const pcmk_node_t node, GHashTable overrides,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	pcmk__register_lib_messages(out);
	rc = pcmk__resource_digests(out, rsc, node, overrides);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	return rc;
	}
	diff --git a/lib/pacemaker/pcmk_sched_actions.c b/lib/pacemaker/pcmk_sched_actions.c
	index f3c604a27a..52d44b8cf5 100644
	--- a/lib/pacemaker/pcmk_sched_actions.c
	+++ b/lib/pacemaker/pcmk_sched_actions.c
	@@ -1,1932 +1,1933 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/param.h>
	#include <glib.h>

	#include <crm/lrmd_internal.h>
	#include <crm/common/scheduler_internal.h>
	#include <pacemaker-internal.h>
	#include "libpacemaker_private.h"

	/*!
	* \internal
	* \brief Get the action flags relevant to ordering constraints
	*
	* \param[in,out] action Action to check
	* \param[in] node Node that other action in the ordering is on
	* (used only for clone resource actions)
	*
	* \return Action flags that should be used for orderings
	*/
	static uint32_t
	action_flags_for_ordering(pcmk_action_t action, const pcmk_node_t node)
	{
	bool runnable = false;
	uint32_t flags;

	// For non-resource actions, return the action flags
	if (action->rsc == NULL) {
	return action->flags;
	}

	/* For non-clone resources, or a clone action not assigned to a node,
	* return the flags as determined by the resource method without a node
	* specified.
	*/
	flags = action->rsc->cmds->action_flags(action, NULL);
	if ((node == NULL) \|\| !pe_rsc_is_clone(action->rsc)) {
	return flags;
	}

	/* Otherwise (i.e., for clone resource actions on a specific node), first
	* remember whether the non-node-specific action is runnable.
	*/
	runnable = pcmk_is_set(flags, pcmk_action_runnable);

	// Then recheck the resource method with the node
	flags = action->rsc->cmds->action_flags(action, node);

	/* For clones in ordering constraints, the node-specific "runnable" doesn't
	* matter, just the non-node-specific setting (i.e., is the action runnable
	* anywhere).
	*
	* This applies only to runnable, and only for ordering constraints. This
	* function shouldn't be used for other types of constraints without
	* changes. Not very satisfying, but it's logical and appears to work well.
	*/
	if (runnable && !pcmk_is_set(flags, pcmk_action_runnable)) {
	pcmk__set_raw_action_flags(flags, action->rsc->id,
	pcmk_action_runnable);
	}
	return flags;
	}

	/*!
	* \internal
	* \brief Get action UUID that should be used with a resource ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the UUID and resource of the first action in an
	* ordering, this returns the UUID of the action that should actually be used
	* for ordering (for example, "CLONE_started_0" instead of "CLONE_start_0").
	*
	* \param[in] first_uuid UUID of first action in ordering
	* \param[in] first_rsc Resource of first action in ordering
	*
	* \return Newly allocated copy of UUID to use with ordering
	* \note It is the caller's responsibility to free the return value.
	*/
	static char *
	action_uuid_for_ordering(const char *first_uuid,
	const pcmk_resource_t *first_rsc)
	{
	guint interval_ms = 0;
	char *uuid = NULL;
	char *rid = NULL;
	char *first_task_str = NULL;
	enum action_tasks first_task = pcmk_action_unspecified;
	enum action_tasks remapped_task = pcmk_action_unspecified;

	// Only non-notify actions for collective resources need remapping
	if ((strstr(first_uuid, PCMK_ACTION_NOTIFY) != NULL)
	\|\| (first_rsc->variant < pcmk_rsc_variant_group)) {
	goto done;
	}

	// Only non-recurring actions need remapping
	CRM_ASSERT(parse_op_key(first_uuid, &rid, &first_task_str, &interval_ms));
	if (interval_ms > 0) {
	goto done;
	}

	first_task = text2task(first_task_str);
	switch (first_task) {
	case pcmk_action_stop:
	case pcmk_action_start:
	case pcmk_action_notify:
	case pcmk_action_promote:
	case pcmk_action_demote:
	remapped_task = first_task + 1;
	break;
	case pcmk_action_stopped:
	case pcmk_action_started:
	case pcmk_action_notified:
	case pcmk_action_promoted:
	case pcmk_action_demoted:
	remapped_task = first_task;
	break;
	case pcmk_action_monitor:
	case pcmk_action_shutdown:
	case pcmk_action_fence:
	break;
	default:
	crm_err("Unknown action '%s' in ordering", first_task_str);
	break;
	}

	if (remapped_task != pcmk_action_unspecified) {
	/* If a clone or bundle has notifications enabled, the ordering will be
	* relative to when notifications have been sent for the remapped task.
	*/
	if (pcmk_is_set(first_rsc->flags, pcmk_rsc_notify)
	&& (pe_rsc_is_clone(first_rsc) \|\| pe_rsc_is_bundled(first_rsc))) {
	uuid = pcmk__notify_key(rid, "confirmed-post",
	task2text(remapped_task));
	} else {
	uuid = pcmk__op_key(rid, task2text(remapped_task), 0);
	}
	pcmk__rsc_trace(first_rsc,
	"Remapped action UUID %s to %s for ordering purposes",
	first_uuid, uuid);
	}

	done:
	if (uuid == NULL) {
	uuid = strdup(first_uuid);
	CRM_ASSERT(uuid != NULL);
	}
	free(first_task_str);
	free(rid);
	return uuid;
	}

	/*!
	* \internal
	* \brief Get actual action that should be used with an ordering
	*
	* When an action is ordered relative to an action for a collective resource
	* (clone, group, or bundle), it actually needs to be ordered after all
	* instances of the collective have completed the relevant action (for example,
	* given "start CLONE then start RSC", RSC must wait until all instances of
	* CLONE have started). Given the first action in an ordering, this returns the
	* the action that should actually be used for ordering (for example, the
	* started action instead of the start action).
	*
	* \param[in] action First action in an ordering
	*
	* \return Actual action that should be used for the ordering
	*/
	static pcmk_action_t *
	action_for_ordering(pcmk_action_t *action)
	{
	pcmk_action_t *result = action;
	pcmk_resource_t *rsc = action->rsc;

	if ((rsc != NULL) && (rsc->variant >= pcmk_rsc_variant_group)
	&& (action->uuid != NULL)) {
	char *uuid = action_uuid_for_ordering(action->uuid, rsc);

	result = find_first_action(rsc->actions, uuid, NULL, NULL);
	if (result == NULL) {
	crm_warn("Not remapping %s to %s because %s does not have "
	"remapped action", action->uuid, uuid, rsc->id);
	result = action;
	}
	free(uuid);
	}
	return result;
	}

	/*!
	* \internal
	* \brief Wrapper for update_ordered_actions() method for readability
	*
	* \param[in,out] rsc Resource to call method for
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this
	* node (only used when interleaving instances)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates
	* (may include pcmk_action_optional to affect only
	* mandatory actions, and pe_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	static inline uint32_t
	update(pcmk_resource_t rsc, pcmk_action_t first, pcmk_action_t *then,
	const pcmk_node_t *node, uint32_t flags, uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler)
	{
	return rsc->cmds->update_ordered_actions(first, then, node, flags, filter,
	type, scheduler);
	}

	/*!
	* \internal
	* \brief Update flags for ordering's actions appropriately for ordering's flags
	*
	* \param[in,out] first First action in an ordering
	* \param[in,out] then Then action in an ordering
	* \param[in] first_flags Action flags for \p first for ordering purposes
	* \param[in] then_flags Action flags for \p then for ordering purposes
	* \param[in,out] order Action wrapper for \p first in ordering
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags
	*/
	static uint32_t
	update_action_for_ordering_flags(pcmk_action_t first, pcmk_action_t then,
	uint32_t first_flags, uint32_t then_flags,
	pcmk__related_action_t *order,
	pcmk_scheduler_t *scheduler)
	{
	uint32_t changed = pcmk__updated_none;

	/* The node will only be used for clones. If interleaved, node will be NULL,
	* otherwise the ordering scope will be limited to the node. Normally, the
	* whole 'then' clone should restart if 'first' is restarted, so then->node
	* is needed.
	*/
	pcmk_node_t *node = then->node;

	if (pcmk_is_set(order->type, pcmk__ar_first_implies_same_node_then)) {
	/* For unfencing, only instances of 'then' on the same node as 'first'
	* (the unfencing operation) should restart, so reset node to
	* first->node, at which point this case is handled like a normal
	* pcmk__ar_first_implies_then.
	*/
	pcmk__clear_relation_flags(order->type,
	pcmk__ar_first_implies_same_node_then);
	pcmk__set_relation_flags(order->type, pcmk__ar_first_implies_then);
	node = first->node;
	pcmk__rsc_trace(then->rsc,
	"%s then %s: mapped "
	"pcmk__ar_first_implies_same_node_then to "
	"pcmk__ar_first_implies_then on %s",
	first->uuid, then->uuid, pcmk__node_name(node));
	}

	if (pcmk_is_set(order->type, pcmk__ar_first_implies_then)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pcmk_action_optional,
	pcmk_action_optional, pcmk__ar_first_implies_then,
	scheduler);
	} else if (!pcmk_is_set(first_flags, pcmk_action_optional)
	&& pcmk_is_set(then->flags, pcmk_action_optional)) {
	pcmk__clear_action_flags(then, pcmk_action_optional);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_first_implies_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_intermediate_stop)
	&& (then->rsc != NULL)) {
	enum pe_action_flags restart = pcmk_action_optional
	\|pcmk_action_runnable;

	changed \|= update(then->rsc, first, then, node, first_flags, restart,
	pcmk__ar_intermediate_stop, scheduler);
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_intermediate_stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_then_implies_first)) {
	if (first->rsc != NULL) {
	changed \|= update(first->rsc, first, then, node, first_flags,
	pcmk_action_optional, pcmk__ar_then_implies_first,
	scheduler);
	} else if (!pcmk_is_set(first_flags, pcmk_action_optional)
	&& pcmk_is_set(first->flags, pcmk_action_runnable)) {
	pcmk__clear_action_flags(first, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_then_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_promoted_then_implies_first)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node,
	first_flags & pcmk_action_optional,
	pcmk_action_optional,
	pcmk__ar_promoted_then_implies_first, scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after "
	"pcmk__ar_promoted_then_implies_first",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_min_runnable)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pcmk__ar_min_runnable,
	scheduler);

	} else if (pcmk_is_set(first_flags, pcmk_action_runnable)) {
	// We have another runnable instance of "first"
	then->runnable_before++;

	/* Mark "then" as runnable if it requires a certain number of
	* "before" instances to be runnable, and they now are.
	*/
	if ((then->runnable_before >= then->required_runnable_before)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)) {

	pcmk__set_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	}
	pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_min_runnable",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_nested_remote_probe)
	&& (then->rsc != NULL)) {

	if (!pcmk_is_set(first_flags, pcmk_action_runnable)
	&& (first->rsc != NULL) && (first->rsc->running_on != NULL)) {

	pcmk__rsc_trace(then->rsc,
	"%s then %s: ignoring because first is stopping",
	first->uuid, then->uuid);
	order->type = (enum pe_ordering) pcmk__ar_none;
	} else {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable,
	pcmk__ar_unrunnable_first_blocks, scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_nested_remote_probe",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_unrunnable_first_blocks)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable,
	pcmk__ar_unrunnable_first_blocks, scheduler);

	} else if (!pcmk_is_set(first_flags, pcmk_action_runnable)
	&& pcmk_is_set(then->flags, pcmk_action_runnable)) {

	pcmk__clear_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_unrunnable_first_blocks",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_unmigratable_then_blocks)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_optional,
	pcmk__ar_unmigratable_then_blocks, scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after "
	"pcmk__ar_unmigratable_then_blocks",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_first_else_then)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_optional, pcmk__ar_first_else_then,
	scheduler);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after pcmk__ar_first_else_then",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_ordered)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pcmk__ar_ordered,
	scheduler);
	}
	pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_ordered",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(order->type, pcmk__ar_asymmetric)) {
	if (then->rsc != NULL) {
	changed \|= update(then->rsc, first, then, node, first_flags,
	pcmk_action_runnable, pcmk__ar_asymmetric,
	scheduler);
	}
	pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_asymmetric",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	if (pcmk_is_set(first->flags, pcmk_action_runnable)
	&& pcmk_is_set(order->type, pcmk__ar_first_implies_then_graphed)
	&& !pcmk_is_set(first_flags, pcmk_action_optional)) {

	pcmk__rsc_trace(then->rsc, "%s will be in graph because %s is required",
	then->uuid, first->uuid);
	pcmk__set_action_flags(then, pcmk_action_always_in_graph);
	// Don't bother marking 'then' as changed just for this
	}

	if (pcmk_is_set(order->type, pcmk__ar_then_implies_first_graphed)
	&& !pcmk_is_set(then_flags, pcmk_action_optional)) {

	pcmk__rsc_trace(then->rsc, "%s will be in graph because %s is required",
	first->uuid, then->uuid);
	pcmk__set_action_flags(first, pcmk_action_always_in_graph);
	// Don't bother marking 'first' as changed just for this
	}

	if (pcmk_any_flags_set(order->type, pcmk__ar_first_implies_then
	\|pcmk__ar_then_implies_first
	\|pcmk__ar_intermediate_stop)
	&& (first->rsc != NULL)
	&& !pcmk_is_set(first->rsc->flags, pcmk_rsc_managed)
	&& pcmk_is_set(first->rsc->flags, pcmk_rsc_blocked)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)
	&& pcmk__str_eq(first->task, PCMK_ACTION_STOP, pcmk__str_none)) {

	if (pcmk_is_set(then->flags, pcmk_action_runnable)) {
	pcmk__clear_action_flags(then, pcmk_action_runnable);
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	}
	pcmk__rsc_trace(then->rsc,
	"%s then %s: %s after checking whether first "
	"is blocked, unmanaged, unrunnable stop",
	first->uuid, then->uuid,
	(changed? "changed" : "unchanged"));
	}

	return changed;
	}

	// Convenience macros for logging action properties

	#define action_type_str(flags) \
	(pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action")

	#define action_optional_str(flags) \
	(pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required")

	#define action_runnable_str(flags) \
	(pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable")

	#define action_node_str(a) \
	(((a)->node == NULL)? "no node" : (a)->node->details->uname)

	/*!
	* \internal
	* \brief Update an action's flags for all orderings where it is "then"
	*
	* \param[in,out] then Action to update
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__update_action_for_orderings(pcmk_action_t *then,
	pcmk_scheduler_t *scheduler)
	{
	GList *lpc = NULL;
	uint32_t changed = pcmk__updated_none;
	int last_flags = then->flags;

	pcmk__rsc_trace(then->rsc, "Updating %s %s (%s %s) on %s",
	action_type_str(then->flags), then->uuid,
	action_optional_str(then->flags),
	action_runnable_str(then->flags), action_node_str(then));

	if (pcmk_is_set(then->flags, pcmk_action_min_runnable)) {
	/* Initialize current known "runnable before" actions. As
	* update_action_for_ordering_flags() is called for each of then's
	* before actions, this number will increment as runnable 'first'
	* actions are encountered.
	*/
	then->runnable_before = 0;

	if (then->required_runnable_before == 0) {
	/* @COMPAT This ordering constraint uses the deprecated
	* "require-all=false" attribute. Treat it like
	* PCMK_META_CLONE_MIN=1.
	*/
	then->required_runnable_before = 1;
	}

	/* The pcmk__ar_min_runnable clause of
	* update_action_for_ordering_flags() (called below)
	* will reset runnable if appropriate.
	*/
	pcmk__clear_action_flags(then, pcmk_action_runnable);
	}

	for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
	pcmk__related_action_t *other = lpc->data;
	pcmk_action_t *first = other->action;

	pcmk_node_t *then_node = then->node;
	pcmk_node_t *first_node = first->node;

	if ((first->rsc != NULL)
	&& (first->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(first->task, PCMK_ACTION_START, pcmk__str_none)) {

	first_node = first->rsc->fns->location(first->rsc, NULL, FALSE);
	if (first_node != NULL) {
	pcmk__rsc_trace(first->rsc, "Found %s for 'first' %s",
	pcmk__node_name(first_node), first->uuid);
	}
	}

	if ((then->rsc != NULL)
	&& (then->rsc->variant == pcmk_rsc_variant_group)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)) {

	then_node = then->rsc->fns->location(then->rsc, NULL, FALSE);
	if (then_node != NULL) {
	pcmk__rsc_trace(then->rsc, "Found %s for 'then' %s",
	pcmk__node_name(then_node), then->uuid);
	}
	}

	// Disable constraint if it only applies when on same node, but isn't
	if (pcmk_is_set(other->type, pcmk__ar_if_on_same_node)
	&& (first_node != NULL) && (then_node != NULL)
	&& !pcmk__same_node(first_node, then_node)) {

	pcmk__rsc_trace(then->rsc,
	"Disabled ordering %s on %s then %s on %s: "
	"not same node",
	other->action->uuid, pcmk__node_name(first_node),
	then->uuid, pcmk__node_name(then_node));
	other->type = (enum pe_ordering) pcmk__ar_none;
	continue;
	}

	pcmk__clear_updated_flags(changed, then, pcmk__updated_first);

	if ((first->rsc != NULL)
	&& pcmk_is_set(other->type, pcmk__ar_then_cancels_first)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {

	/* 'then' is required, so we must abandon 'first'
	* (e.g. a required stop cancels any agent reload).
	*/
	pcmk__set_action_flags(other->action, pcmk_action_optional);
	if (!strcmp(first->task, PCMK_ACTION_RELOAD_AGENT)) {
	pcmk__clear_rsc_flags(first->rsc, pcmk_rsc_reload);
	}
	}

	if ((first->rsc != NULL) && (then->rsc != NULL)
	&& (first->rsc != then->rsc) && !is_parent(then->rsc, first->rsc)) {
	first = action_for_ordering(first);
	}
	if (first != other->action) {
	pcmk__rsc_trace(then->rsc, "Ordering %s after %s instead of %s",
	then->uuid, first->uuid, other->action->uuid);
	}

	pcmk__rsc_trace(then->rsc,
	"%s (%#.6x) then %s (%#.6x): type=%#.6x node=%s",
	first->uuid, first->flags, then->uuid, then->flags,
	other->type, action_node_str(first));

	if (first == other->action) {
	/* 'first' was not remapped (e.g. from 'start' to 'running'), which
	* could mean it is a non-resource action, a primitive resource
	* action, or already expanded.
	*/
	uint32_t first_flags, then_flags;

	first_flags = action_flags_for_ordering(first, then_node);
	then_flags = action_flags_for_ordering(then, first_node);

	changed \|= update_action_for_ordering_flags(first, then,
	first_flags, then_flags,
	other, scheduler);

	/* 'first' was for a complex resource (clone, group, etc),
	* create a new dependency if necessary
	*/
	} else if (order_actions(first, then, other->type)) {
	/* This was the first time 'first' and 'then' were associated,
	* start again to get the new actions_before list
	*/
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	pcmk__rsc_trace(then->rsc,
	"Disabled ordering %s then %s in favor of %s "
	"then %s",
	other->action->uuid, then->uuid, first->uuid,
	then->uuid);
	other->type = (enum pe_ordering) pcmk__ar_none;
	}


	if (pcmk_is_set(changed, pcmk__updated_first)) {
	crm_trace("Re-processing %s and its 'after' actions "
	"because it changed", first->uuid);
	for (GList *lpc2 = first->actions_after; lpc2 != NULL;
	lpc2 = lpc2->next) {
	pcmk__related_action_t *other = lpc2->data;

	pcmk__update_action_for_orderings(other->action, scheduler);
	}
	pcmk__update_action_for_orderings(first, scheduler);
	}
	}

	if (pcmk_is_set(then->flags, pcmk_action_min_runnable)) {
	if (last_flags == then->flags) {
	pcmk__clear_updated_flags(changed, then, pcmk__updated_then);
	} else {
	pcmk__set_updated_flags(changed, then, pcmk__updated_then);
	}
	}

	if (pcmk_is_set(changed, pcmk__updated_then)) {
	crm_trace("Re-processing %s and its 'after' actions because it changed",
	then->uuid);
	if (pcmk_is_set(last_flags, pcmk_action_runnable)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)) {
	pcmk__block_colocation_dependents(then);
	}
	pcmk__update_action_for_orderings(then, scheduler);
	for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
	pcmk__related_action_t *other = lpc->data;

	pcmk__update_action_for_orderings(other->action, scheduler);
	}
	}
	}

	static inline bool
	is_primitive_action(const pcmk_action_t *action)
	{
	return (action != NULL) && (action->rsc != NULL)
	&& (action->rsc->variant == pcmk_rsc_variant_primitive);
	}

	/*!
	* \internal
	* \brief Clear a single action flag and set reason text
	*
	* \param[in,out] action Action whose flag should be cleared
	* \param[in] flag Action flag that should be cleared
	* \param[in] reason Action that is the reason why flag is being cleared
	*/
	#define clear_action_flag_because(action, flag, reason) do { \
	if (pcmk_is_set((action)->flags, (flag))) { \
	pcmk__clear_action_flags(action, flag); \
	if ((action)->rsc != (reason)->rsc) { \
	char *reason_text = pe__action2reason((reason), (flag)); \
	pe_action_set_reason((action), reason_text, false); \
	free(reason_text); \
	} \
	} \
	} while (0)

	/*!
	* \internal
	* \brief Update actions in an asymmetric ordering
	*
	* If the "first" action in an asymmetric ordering is unrunnable, make the
	* "second" action unrunnable as well, if appropriate.
	*
	* \param[in] first 'First' action in an asymmetric ordering
	* \param[in,out] then 'Then' action in an asymmetric ordering
	*/
	static void
	handle_asymmetric_ordering(const pcmk_action_t first, pcmk_action_t then)
	{
	/* Only resource actions after an unrunnable 'first' action need updates for
	* asymmetric ordering.
	*/
	if ((then->rsc == NULL)
	\|\| pcmk_is_set(first->flags, pcmk_action_runnable)) {
	return;
	}

	// Certain optional 'then' actions are unaffected by unrunnable 'first'
	if (pcmk_is_set(then->flags, pcmk_action_optional)) {
	enum rsc_role_e then_rsc_role = then->rsc->fns->state(then->rsc, TRUE);

	if ((then_rsc_role == pcmk_role_stopped)
	&& pcmk__str_eq(then->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	/* If 'then' should stop after 'first' but is already stopped, the
	* ordering is irrelevant.
	*/
	return;
	} else if ((then_rsc_role >= pcmk_role_started)
	&& pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)
	&& pe__rsc_running_on_only(then->rsc, then->node)) {
	/* Similarly if 'then' should start after 'first' but is already
	* started on a single node.
	*/
	return;
	}
	}

	// 'First' can't run, so 'then' can't either
	clear_action_flag_because(then, pcmk_action_optional, first);
	clear_action_flag_because(then, pcmk_action_runnable, first);
	}

	/*!
	* \internal
	* \brief Set action bits appropriately when pe_restart_order is used
	*
	* \param[in,out] first 'First' action in an ordering with pe_restart_order
	* \param[in,out] then 'Then' action in an ordering with pe_restart_order
	* \param[in] filter What action flags to care about
	*
	* \note pe_restart_order is set for "stop resource before starting it" and
	* "stop later group member before stopping earlier group member"
	*/
	static void
	handle_restart_ordering(pcmk_action_t first, pcmk_action_t then,
	uint32_t filter)
	{
	const char *reason = NULL;

	CRM_ASSERT(is_primitive_action(first));
	CRM_ASSERT(is_primitive_action(then));

	// We need to update the action in two cases:

	// ... if 'then' is required
	if (pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {
	reason = "restart";
	}

	/* ... if 'then' is unrunnable action on same resource (if a resource
	* should restart but can't start, we still want to stop)
	*/
	if (pcmk_is_set(filter, pcmk_action_runnable)
	&& !pcmk_is_set(then->flags, pcmk_action_runnable)
	&& pcmk_is_set(then->rsc->flags, pcmk_rsc_managed)
	&& (first->rsc == then->rsc)) {
	reason = "stop";
	}

	if (reason == NULL) {
	return;
	}

	pcmk__rsc_trace(first->rsc, "Handling %s -> %s for %s",
	first->uuid, then->uuid, reason);

	// Make 'first' required if it is runnable
	if (pcmk_is_set(first->flags, pcmk_action_runnable)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	// Make 'first' required if 'then' is required
	if (!pcmk_is_set(then->flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	// Make 'first' unmigratable if 'then' is unmigratable
	if (!pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}

	// Make 'then' unrunnable if 'first' is required but unrunnable
	if (!pcmk_is_set(first->flags, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)) {
	clear_action_flag_because(then, pcmk_action_runnable, first);
	}
	}

	/*!
	* \internal
	* \brief Update two actions according to an ordering between them
	*
	* Given information about an ordering of two actions, update the actions' flags
	* (and runnable_before members if appropriate) as appropriate for the ordering.
	* Effects may cascade to other orderings involving the actions as well.
	*
	* \param[in,out] first 'First' action in an ordering
	* \param[in,out] then 'Then' action in an ordering
	* \param[in] node If not NULL, limit scope of ordering to this node
	* (ignored)
	* \param[in] flags Action flags for \p first for ordering purposes
	* \param[in] filter Action flags to limit scope of certain updates (may
	* include pcmk_action_optional to affect only
	* mandatory actions, and pcmk_action_runnable to
	* affect only runnable actions)
	* \param[in] type Group of enum pcmk__action_relation_flags to apply
	* \param[in,out] scheduler Scheduler data
	*
	* \return Group of enum pcmk__updated flags indicating what was updated
	*/
	uint32_t
	pcmk__update_ordered_actions(pcmk_action_t first, pcmk_action_t then,
	const pcmk_node_t *node, uint32_t flags,
	uint32_t filter, uint32_t type,
	pcmk_scheduler_t *scheduler)
	{
	uint32_t changed = pcmk__updated_none;
	uint32_t then_flags = 0U;
	uint32_t first_flags = 0U;

	CRM_ASSERT((first != NULL) && (then != NULL) && (scheduler != NULL));

	then_flags = then->flags;
	first_flags = first->flags;
	if (pcmk_is_set(type, pcmk__ar_asymmetric)) {
	handle_asymmetric_ordering(first, then);
	}

	if (pcmk_is_set(type, pcmk__ar_then_implies_first)
	&& !pcmk_is_set(then_flags, pcmk_action_optional)) {
	// Then is required, and implies first should be, too

	if (pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(flags, pcmk_action_optional)
	&& pcmk_is_set(first_flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}

	if (pcmk_is_set(flags, pcmk_action_migratable)
	&& !pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}
	}

	if (pcmk_is_set(type, pcmk__ar_promoted_then_implies_first)
	&& (then->rsc != NULL) && (then->rsc->role == pcmk_role_promoted)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(then->flags, pcmk_action_optional)) {

	clear_action_flag_because(first, pcmk_action_optional, then);

	if (pcmk_is_set(first->flags, pcmk_action_migratable)
	&& !pcmk_is_set(then->flags, pcmk_action_migratable)) {
	clear_action_flag_because(first, pcmk_action_migratable, then);
	}
	}

	if (pcmk_is_set(type, pcmk__ar_unmigratable_then_blocks)
	&& pcmk_is_set(filter, pcmk_action_optional)) {

	if (!pcmk_all_flags_set(then->flags, pcmk_action_migratable
	\|pcmk_action_runnable)) {
	clear_action_flag_because(first, pcmk_action_runnable, then);
	}

	if (!pcmk_is_set(then->flags, pcmk_action_optional)) {
	clear_action_flag_because(first, pcmk_action_optional, then);
	}
	}

	if (pcmk_is_set(type, pcmk__ar_first_else_then)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_runnable)) {

	clear_action_flag_because(then, pcmk_action_migratable, first);
	pcmk__clear_action_flags(then, pcmk_action_pseudo);
	}

	if (pcmk_is_set(type, pcmk__ar_unrunnable_first_blocks)
	&& pcmk_is_set(filter, pcmk_action_runnable)
	&& pcmk_is_set(then->flags, pcmk_action_runnable)
	&& !pcmk_is_set(flags, pcmk_action_runnable)) {

	clear_action_flag_because(then, pcmk_action_runnable, first);
	clear_action_flag_because(then, pcmk_action_migratable, first);
	}

	if (pcmk_is_set(type, pcmk__ar_first_implies_then)
	&& pcmk_is_set(filter, pcmk_action_optional)
	&& pcmk_is_set(then->flags, pcmk_action_optional)
	&& !pcmk_is_set(flags, pcmk_action_optional)
	&& !pcmk_is_set(first->flags, pcmk_action_migratable)) {

	clear_action_flag_because(then, pcmk_action_optional, first);
	}

	if (pcmk_is_set(type, pcmk__ar_intermediate_stop)) {
	handle_restart_ordering(first, then, filter);
	}

	if (then_flags != then->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_then);
	pcmk__rsc_trace(then->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'first' %s (%#.6x)",
	then->uuid, pcmk__node_name(then->node),
	then->flags, then_flags, first->uuid, first->flags);

	if ((then->rsc != NULL) && (then->rsc->parent != NULL)) {
	// Required to handle "X_stop then X_start" for cloned groups
	pcmk__update_action_for_orderings(then, scheduler);
	}
	}

	if (first_flags != first->flags) {
	pcmk__set_updated_flags(changed, first, pcmk__updated_first);
	pcmk__rsc_trace(first->rsc,
	"%s on %s: flags are now %#.6x (was %#.6x) "
	"because of 'then' %s (%#.6x)",
	first->uuid, pcmk__node_name(first->node),
	first->flags, first_flags, then->uuid, then->flags);
	}

	return changed;
	}

	/*!
	* \internal
	* \brief Trace-log an action (optionally with its dependent actions)
	*
	* \param[in] pre_text If not NULL, prefix the log with this plus ": "
	* \param[in] action Action to log
	* \param[in] details If true, recursively log dependent actions
	*/
	void
	pcmk__log_action(const char pre_text, const pcmk_action_t action,
	bool details)
	{
	const char *node_uname = NULL;
	const char *node_uuid = NULL;
	const char *desc = NULL;

	CRM_CHECK(action != NULL, return);

	if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	if (action->node != NULL) {
	node_uname = action->node->details->uname;
	node_uuid = action->node->details->id;
	} else {
	node_uname = "<none>";
	}
	}

	switch (text2task(action->task)) {
	case pcmk_action_fence:
	case pcmk_action_shutdown:
	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	desc = "Pseudo ";
	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	desc = "Optional ";
	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	default:
	if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	desc = "Optional ";
	} else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	desc = "Pseudo ";
	} else if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
	desc = "!!Non-Startable!! ";
	} else {
	desc = "(Provisional) ";
	}
	crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
	((pre_text == NULL)? "" : pre_text),
	((pre_text == NULL)? "" : ": "),
	desc, action->id, action->uuid,
	(action->rsc? action->rsc->id : "<none>"),
	(node_uname? "\ton " : ""), (node_uname? node_uname : ""),
	(node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
	(node_uuid? ")" : ""));
	break;
	}

	if (details) {
	const GList *iter = NULL;
	const pcmk__related_action_t *other = NULL;

	crm_trace("\t\t====== Preceding Actions");
	for (iter = action->actions_before; iter != NULL; iter = iter->next) {
	other = (const pcmk__related_action_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== Subsequent Actions");
	for (iter = action->actions_after; iter != NULL; iter = iter->next) {
	other = (const pcmk__related_action_t *) iter->data;
	pcmk__log_action("\t\t", other->action, false);
	}
	crm_trace("\t\t====== End");

	} else {
	crm_trace("\t\t(before=%d, after=%d)",
	g_list_length(action->actions_before),
	g_list_length(action->actions_after));
	}
	}

	/*!
	* \internal
	* \brief Create a new shutdown action for a node
	*
	* \param[in,out] node Node being shut down
	*
	* \return Newly created shutdown action for \p node
	*/
	pcmk_action_t *
	pcmk__new_shutdown_action(pcmk_node_t *node)
	{
	char *shutdown_id = NULL;
	pcmk_action_t *shutdown_op = NULL;

	CRM_ASSERT(node != NULL);

	shutdown_id = crm_strdup_printf("%s-%s", PCMK_ACTION_DO_SHUTDOWN,
	node->details->uname);

	shutdown_op = custom_action(NULL, shutdown_id, PCMK_ACTION_DO_SHUTDOWN,
	node, FALSE, node->details->data_set);

	pcmk__order_stops_before_shutdown(node, shutdown_op);
	add_hash_param(shutdown_op->meta, PCMK__META_OP_NO_WAIT, PCMK_VALUE_TRUE);
	return shutdown_op;
	}

	/*!
	* \internal
	* \brief Calculate and add an operation digest to XML
	*
	* Calculate an operation digest, which enables us to later determine when a
	* restart is needed due to the resource's parameters being changed, and add it
	* to given XML.
	*
	* \param[in] op Operation result from executor
	* \param[in,out] update XML to add digest to
	*/
	static void
	add_op_digest_to_xml(const lrmd_event_data_t op, xmlNode update)
	{
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}
	args_xml = create_xml_node(NULL, PCMK_XE_PARAMETERS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	pcmk__filter_op_for_digest(args_xml);
	digest = calculate_operation_digest(args_xml, NULL);
	crm_xml_add(update, PCMK__XA_OP_DIGEST, digest);
	free_xml(args_xml);
	free(digest);
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"

	/*!
	* \internal
	* \brief Create XML for resource operation history update
	*
	* \param[in,out] parent Parent XML node to add to
	* \param[in,out] op Operation event data
	* \param[in] caller_version DC feature set
	* \param[in] target_rc Expected result of operation
	* \param[in] node Name of node on which operation was performed
	* \param[in] origin Arbitrary description of update source
	*
	* \return Newly created XML node for history update
	*/
	xmlNode *
	pcmk__create_history_xml(xmlNode parent, lrmd_event_data_t op,
	const char *caller_version, int target_rc,
	const char node, const char origin)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;

	CRM_CHECK(op != NULL, return NULL);
	crm_trace("Creating history XML for %s-interval %s action for %s on %s "
	"(DC version: %s, origin: %s)",
	pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
	((node == NULL)? "no node" : node), caller_version, origin);

	task = op->op_type;

	/* Record a successful agent reload as a start, and a failed one as a
	* monitor, to make life easier for the scheduler when determining the
	* current state.
	*
	* @COMPAT We should check "reload" here only if the operation was for a
	* pre-OCF-1.1 resource agent, but we don't know that here, and we should
	* only ever get results for actions scheduled by us, so we can reasonably
	* assume any "reload" is actually a pre-1.1 agent reload.
	*/
	if (pcmk__str_any_of(task, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT,
	NULL)) {
	if (op->op_status == PCMK_EXEC_DONE) {
	task = PCMK_ACTION_START;
	} else {
	task = PCMK_ACTION_MONITOR;
	}
	}

	key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);

	if (op->op_status != PCMK_EXEC_PENDING) {
	/* Ignore notify errors.
	*
	* @TODO It might be better to keep the correct result here, and
	* ignore it in process_graph_event().
	*/
	lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
	}

	/* Migration history is preserved separately, which usually matters for
	* multiple nodes and is important for future cluster transitions.
	*/
	} else if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	op_id = strdup(key);

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
	if (op->interval_ms == 0) {
	/* Ensure 'last' gets updated, in case PCMK_META_RECORD_PENDING is
	* true
	*/
	op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval_ms > 0) {
	op_id = strdup(key);

	} else {
	op_id = pcmk__op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, PCMK_XA_ID, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for: " PCMK__OP_FMT
	" %d from %s", op->rsc_id, op->op_type, op->interval_ms,
	op->call_id, origin);
	local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
	FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if (magic == NULL) {
	magic = crm_strdup_printf("%d:%d;%s", op->op_status, op->rc,
	(const char *) op->user_data);
	}

	crm_xml_add(xml_op, PCMK_XA_ID, op_id);
	crm_xml_add(xml_op, PCMK__XA_OPERATION_KEY, key);
	crm_xml_add(xml_op, PCMK_XA_OPERATION, task);
	crm_xml_add(xml_op, PCMK_XA_CRM_DEBUG_ORIGIN, origin);
	crm_xml_add(xml_op, PCMK_XA_CRM_FEATURE_SET, caller_version);
	crm_xml_add(xml_op, PCMK__XA_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, PCMK__XA_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, PCMK_XA_EXIT_REASON, pcmk__s(exit_reason, ""));
	crm_xml_add(xml_op, PCMK__META_ON_NODE, node); // For context during triage

	crm_xml_add_int(xml_op, PCMK__XA_CALL_ID, op->call_id);
	crm_xml_add_int(xml_op, PCMK__XA_RC_CODE, op->rc);
	crm_xml_add_int(xml_op, PCMK__XA_OP_STATUS, op->op_status);
	crm_xml_add_ms(xml_op, PCMK_META_INTERVAL, op->interval_ms);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (" PCMK__OP_FMT
	"): last=%u change=%u exec=%u queue=%u",
	op->rsc_id, op->op_type, op->interval_ms,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
	// Recurring ops may have changed rc after initial run
	crm_xml_add_ll(xml_op, PCMK_XA_LAST_RC_CHANGE,
	(long long) op->t_rcchange);
	} else {
	crm_xml_add_ll(xml_op, PCMK_XA_LAST_RC_CHANGE,
	(long long) op->t_run);
	}

	crm_xml_add_int(xml_op, PCMK_XA_EXEC_TIME, op->exec_time);
	crm_xml_add_int(xml_op, PCMK_XA_QUEUE_TIME, op->queue_time);
	}
	}

	if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* Record PCMK__META_MIGRATE_SOURCE and PCMK__META_MIGRATE_TARGET always
	* for migrate ops.
	*/
	const char *name = PCMK__META_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = PCMK__META_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	add_op_digest_to_xml(op, xml_op);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	/*!
	* \internal
	* \brief Check whether an action shutdown-locks a resource to a node
	*
	* If the PCMK_OPT_SHUTDOWN_LOCK cluster property is set, resources will not be
	* recovered on a different node if cleanly stopped, and may start only on that
	* same node. This function checks whether that applies to a given action, so
	* that the transition graph can be marked appropriately.
	*
	* \param[in] action Action to check
	*
	* \return true if \p action locks its resource to the action's node,
	* otherwise false
	*/
	bool
	pcmk__action_locks_rsc_to_node(const pcmk_action_t *action)
	{
	// Only resource actions taking place on resource's lock node are locked
	if ((action == NULL) \|\| (action->rsc == NULL)
	\|\| !pcmk__same_node(action->node, action->rsc->lock_node)) {
	return false;
	}

	/* During shutdown, only stops are locked (otherwise, another action such as
	* a demote would cause the controller to clear the lock)
	*/
	if (action->node->details->shutdown && (action->task != NULL)
	&& (strcmp(action->task, PCMK_ACTION_STOP) != 0)) {
	return false;
	}

	return true;
	}

	/* lowest to highest */
	static gint
	sort_action_id(gconstpointer a, gconstpointer b)
	{
	const pcmk__related_action_t *action_wrapper2 = a;
	const pcmk__related_action_t *action_wrapper1 = b;

	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}
	if (action_wrapper1->action->id < action_wrapper2->action->id) {
	return 1;
	}
	if (action_wrapper1->action->id > action_wrapper2->action->id) {
	return -1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Remove any duplicate action inputs, merging action flags
	*
	* \param[in,out] action Action whose inputs should be checked
	*/
	void
	pcmk__deduplicate_action_inputs(pcmk_action_t *action)
	{
	GList *item = NULL;
	GList *next = NULL;
	pcmk__related_action_t *last_input = NULL;

	action->actions_before = g_list_sort(action->actions_before,
	sort_action_id);
	for (item = action->actions_before; item != NULL; item = next) {
	pcmk__related_action_t *input = item->data;

	next = item->next;
	if ((last_input != NULL)
	&& (input->action->id == last_input->action->id)) {
	crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
	input->action->uuid, input->action->id,
	action->uuid, action->id);

	/* For the purposes of scheduling, the ordering flags no longer
	* matter, but crm_simulate looks at certain ones when creating a
	* dot graph. Combining the flags is sufficient for that purpose.
	*/
	last_input->type \|= input->type;
	if (input->state == pe_link_dumped) {
	last_input->state = pe_link_dumped;
	}

	free(item->data);
	action->actions_before = g_list_delete_link(action->actions_before,
	item);
	} else {
	last_input = input;
	input->state = pe_link_not_dumped;
	}
	}
	}

	/*!
	* \internal
	* \brief Output all scheduled actions
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__output_actions(pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;

	// Output node (non-resource) actions
	for (GList *iter = scheduler->actions; iter != NULL; iter = iter->next) {
	char *node_name = NULL;
	char *task = NULL;
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	if (action->rsc != NULL) {
	continue; // Resource actions will be output later

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	continue; // This action was not scheduled
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	task = strdup("Shutdown");

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta,
	"stonith_action");

	task = crm_strdup_printf("Fence (%s)", op);

	} else {
	continue; // Don't display other node action types
	}

	if (pe__is_guest_node(action->node)) {
	const pcmk_resource_t *remote = action->node->details->remote_rsc;

	node_name = crm_strdup_printf("%s (resource: %s)",
	pcmk__node_name(action->node),
	remote->container->id);
	} else if (action->node != NULL) {
	node_name = crm_strdup_printf("%s", pcmk__node_name(action->node));
	}

	out->message(out, "node-action", task, node_name, action->reason);

	free(node_name);
	free(task);
	}

	// Output resource actions
	for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	rsc->cmds->output_actions(rsc);
	}
	}

	/*!
	* \internal
	* \brief Get action name needed to compare digest for configuration changes
	*
	* \param[in] task Action name from history
	* \param[in] interval_ms Action interval (in milliseconds)
	*
	* \return Action name whose digest should be compared
	*/
	static const char *
	task_for_digest(const char *task, guint interval_ms)
	{
	/* Certain actions need to be compared against the parameters used to start
	* the resource.
	*/
	if ((interval_ms == 0)
	&& pcmk__str_any_of(task, PCMK_ACTION_MONITOR, PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_PROMOTE, NULL)) {
	task = PCMK_ACTION_START;
	}
	return task;
	}

	/*!
	* \internal
	* \brief Check whether only sanitized parameters to an action changed
	*
	* When collecting CIB files for troubleshooting, crm_report will mask
	* sensitive resource parameters. If simulations were run using that, affected
	* resources would appear to need a restart, which would complicate
	* troubleshooting. To avoid that, we save a "secure digest" of non-sensitive
	* parameters. This function used that digest to check whether only masked
	* parameters are different.
	*
	* \param[in] xml_op Resource history entry with secure digest
	* \param[in] digest_data Operation digest information being compared
	* \param[in] scheduler Scheduler data
	*
	* \return true if only sanitized parameters changed, otherwise false
	*/
	static bool
	only_sanitized_changed(const xmlNode *xml_op,
	const pcmk__op_digest_t *digest_data,
	const pcmk_scheduler_t *scheduler)
	{
	const char *digest_secure = NULL;

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_sanitized)) {
	// The scheduler is not being run as a simulation
	return false;
	}

	digest_secure = crm_element_value(xml_op, PCMK__XA_OP_SECURE_DIGEST);

	return (digest_data->rc != pcmk__digest_match) && (digest_secure != NULL)
	&& (digest_data->digest_secure_calc != NULL)
	&& (strcmp(digest_data->digest_secure_calc, digest_secure) == 0);
	}

	/*!
	* \internal
	* \brief Force a restart due to a configuration change
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in] task Name of action whose configuration changed
	* \param[in] interval_ms Action interval (in milliseconds)
	* \param[in,out] node Node where resource should be restarted
	*/
	static void
	force_restart(pcmk_resource_t rsc, const char task, guint interval_ms,
	pcmk_node_t *node)
	{
	char *key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk_action_t *required = custom_action(rsc, key, task, NULL, FALSE,
	rsc->cluster);

	pe_action_set_reason(required, "resource definition change", true);
	trigger_unfencing(rsc, node, "Device parameters changed", NULL,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Schedule a reload of a resource on a node
	*
	* \param[in,out] data Resource to reload
	* \param[in] user_data Where resource should be reloaded
	*/
	static void
	schedule_reload(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *rsc = data;
	const pcmk_node_t *node = user_data;
	pcmk_action_t *reload = NULL;

	// For collective resources, just call recursively for children
	if (rsc->variant > pcmk_rsc_variant_primitive) {
	g_list_foreach(rsc->children, schedule_reload, user_data);
	return;
	}

	// Skip the reload in certain situations
	if ((node == NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_managed)
	\|\| pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	pcmk__rsc_trace(rsc, "Skip reload of %s:%s%s %s",
	rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " unmanaged",
	pcmk_is_set(rsc->flags, pcmk_rsc_failed)? " failed" : "",
	(node == NULL)? "inactive" : node->details->uname);
	return;
	}

	/* If a resource's configuration changed while a start was pending,
	* force a full restart instead of a reload.
	*/
	if (pcmk_is_set(rsc->flags, pcmk_rsc_start_pending)) {
	pcmk__rsc_trace(rsc,
	"%s: preventing agent reload because start pending",
	rsc->id);
	custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
	rsc->cluster);
	return;
	}

	// Schedule the reload
	pcmk__set_rsc_flags(rsc, pcmk_rsc_reload);
	reload = custom_action(rsc, reload_key(rsc), PCMK_ACTION_RELOAD_AGENT, node,
	FALSE, rsc->cluster);
	pe_action_set_reason(reload, "resource definition change", FALSE);

	// Set orderings so that a required stop or demote cancels the reload
	pcmk__new_ordering(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
	pcmk__ar_ordered\|pcmk__ar_then_cancels_first,
	rsc->cluster);
	pcmk__new_ordering(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
	pcmk__ar_ordered\|pcmk__ar_then_cancels_first,
	rsc->cluster);
	}

	/*!
	* \internal
	* \brief Handle any configuration change for an action
	*
	* Given an action from resource history, if the resource's configuration
	* changed since the action was done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, etc.).
	*
	* \param[in,out] rsc Resource that action is for
	* \param[in,out] node Node that action was on
	* \param[in] xml_op Action XML from resource history
	*
	* \return true if action configuration changed, otherwise false
	*/
	bool
	pcmk__check_action_config(pcmk_resource_t rsc, pcmk_node_t node,
	const xmlNode *xml_op)
	{
	guint interval_ms = 0;
	const char *task = NULL;
	const pcmk__op_digest_t *digest_data = NULL;

	CRM_CHECK((rsc != NULL) && (node != NULL) && (xml_op != NULL),
	return false);

	task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	CRM_CHECK(task != NULL, return false);

	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);

	// If this is a recurring action, check whether it has been orphaned
	if (interval_ms > 0) {
	if (pcmk__find_action_config(rsc, task, interval_ms, false) != NULL) {
	pcmk__rsc_trace(rsc,
	"%s-interval %s for %s on %s is in configuration",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node));
	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_cancel_removed_actions)) {
	pcmk__schedule_cancel(rsc,
	crm_element_value(xml_op, PCMK__XA_CALL_ID),
	task, interval_ms, node, "orphan");
	return true;
	} else {
	pcmk__rsc_debug(rsc, "%s-interval %s for %s on %s is orphaned",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node));
	return true;
	}
	}

	crm_trace("Checking %s-interval %s for %s on %s for configuration changes",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node));
	task = task_for_digest(task, interval_ms);
	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, rsc->cluster);

	if (only_sanitized_changed(xml_op, digest_data, rsc->cluster)) {
	if (!pcmk__is_daemon && (rsc->cluster->priv != NULL)) {
	pcmk__output_t *out = rsc->cluster->priv;

	out->info(out,
	"Only 'private' parameters to %s-interval %s for %s "
	"on %s changed: %s",
	pcmk__readable_interval(interval_ms), task, rsc->id,
	pcmk__node_name(node),
	crm_element_value(xml_op, PCMK__XA_TRANSITION_MAGIC));
	}
	return false;
	}

	switch (digest_data->rc) {
	case pcmk__digest_restart:
	crm_log_xml_debug(digest_data->params_restart, "params:restart");
	force_restart(rsc, task, interval_ms, node);
	return true;

	case pcmk__digest_unknown:
	case pcmk__digest_mismatch:
	// Changes that can potentially be handled by an agent reload

	if (interval_ms > 0) {
	/* Recurring actions aren't reloaded per se, they are just
	* re-scheduled so the next run uses the new parameters.
	* The old instance will be cancelled automatically.
	*/
	crm_log_xml_debug(digest_data->params_all, "params:reschedule");
	pcmk__reschedule_recurring(rsc, task, interval_ms, node);

	} else if (crm_element_value(xml_op,
	PCMK__XA_OP_RESTART_DIGEST) != NULL) {
	// Agent supports reload, so use it
	trigger_unfencing(rsc, node,
	"Device parameters changed (reload)", NULL,
	rsc->cluster);
	crm_log_xml_debug(digest_data->params_all, "params:reload");
	schedule_reload((gpointer) rsc, (gpointer) node);

	} else {
	pcmk__rsc_trace(rsc,
	"Restarting %s "
	"because agent doesn't support reload",
	rsc->id);
	crm_log_xml_debug(digest_data->params_restart,
	"params:restart");
	force_restart(rsc, task, interval_ms, node);
	}
	return true;

	default:
	break;
	}
	return false;
	}

	/*!
	* \internal
	* \brief Create a list of resource's action history entries, sorted by call ID
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[out] start_index Where to store index of start-like action, if any
	* \param[out] stop_index Where to store index of stop action, if any
	*/
	static GList *
	rsc_history_as_list(const xmlNode rsc_entry, int start_index, int *stop_index)
	{
	GList *ops = NULL;

	for (xmlNode *rsc_op = first_named_child(rsc_entry, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {
	ops = g_list_prepend(ops, rsc_op);
	}
	ops = g_list_sort(ops, sort_op_by_callid);
	calculate_active_ops(ops, start_index, stop_index);
	return ops;
	}

	/*!
	* \internal
	* \brief Process a resource's action history from the CIB status
	*
	* Given a resource's action history, if the resource's configuration
	* changed since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in] rsc_entry Resource's <lrm_rsc_op> status XML
	* \param[in,out] rsc Resource whose history is being processed
	* \param[in,out] node Node whose history is being processed
	*/
	static void
	process_rsc_history(const xmlNode rsc_entry, pcmk_resource_t rsc,
	pcmk_node_t *node)
	{
	int offset = -1;
	int stop_index = 0;
	int start_index = 0;
	GList *sorted_op_list = NULL;

	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pe_rsc_is_anon_clone(pe__const_top_resource(rsc, false))) {
	pcmk__rsc_trace(rsc,
	"Skipping configuration check "
	"for orphaned clone instance %s",
	rsc->id);
	} else {
	pcmk__rsc_trace(rsc,
	"Skipping configuration check and scheduling "
	"clean-up for orphaned resource %s", rsc->id);
	pcmk__schedule_cleanup(rsc, node, false);
	}
	return;
	}

	if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, false)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}
	pcmk__rsc_trace(rsc,
	"Skipping configuration check for %s "
	"because no longer active on %s",
	rsc->id, pcmk__node_name(node));
	return;
	}

	pcmk__rsc_trace(rsc, "Checking for configuration changes for %s on %s",
	rsc->id, pcmk__node_name(node));

	if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, true)) {
	pcmk__schedule_cleanup(rsc, node, false);
	}

	sorted_op_list = rsc_history_as_list(rsc_entry, &start_index, &stop_index);
	if (start_index < stop_index) {
	return; // Resource is stopped
	}

	for (GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	xmlNode rsc_op = (xmlNode ) iter->data;
	const char *task = NULL;
	guint interval_ms = 0;

	if (++offset < start_index) {
	// Skip actions that happened before a start
	continue;
	}

	task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	crm_element_value_ms(rsc_op, PCMK_META_INTERVAL, &interval_ms);

	if ((interval_ms > 0)
	&& (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)
	\|\| node->details->maintenance)) {
	// Maintenance mode cancels recurring operations
	pcmk__schedule_cancel(rsc,
	crm_element_value(rsc_op, PCMK__XA_CALL_ID),
	task, interval_ms, node, "maintenance mode");

	} else if ((interval_ms > 0)
	\|\| pcmk__strcase_any_of(task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START,
	PCMK_ACTION_PROMOTE,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	/* If a resource operation failed, and the operation's definition
	* has changed, clear any fail count so they can be retried fresh.
	*/

	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't assigned resources to nodes yet, so if the
	* REMOTE_CONTAINER_HACK is used, we may calculate the digest
	* based on the literal "#uname" value rather than the properly
	* substituted value. That would mistakenly make the action
	* definition appear to have been changed. Defer the check until
	* later in this case.
	*/
	pe__add_param_check(rsc_op, rsc, node, pcmk__check_active,
	rsc->cluster);

	} else if (pcmk__check_action_config(rsc, node, rsc_op)
	&& (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
	NULL) != 0)) {
	pe__clear_failcount(rsc, node, "action definition changed",
	rsc->cluster);
	}
	}
	}
	g_list_free(sorted_op_list);
	}

	/*!
	* \internal
	* \brief Process a node's action history from the CIB status
	*
	* Given a node's resource history, if the resource's configuration changed
	* since the actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] node Node whose history is being processed
	* \param[in] lrm_rscs Node's <lrm_resources> from CIB status XML
	*/
	static void
	process_node_history(pcmk_node_t node, const xmlNode lrm_rscs)
	{
	crm_trace("Processing node history for %s", pcmk__node_name(node));
	for (const xmlNode *rsc_entry = first_named_child(lrm_rscs,
	XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	if (rsc_entry->children != NULL) {
	GList *result = pcmk__rscs_matching_id(ID(rsc_entry),
	node->details->data_set);

	for (GList *iter = result; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->variant == pcmk_rsc_variant_primitive) {
	process_rsc_history(rsc_entry, rsc, node);
	}
	}
	g_list_free(result);
	}
	}
	}

	// XPath to find a node's resource history
	-#define XPATH_NODE_HISTORY "/" PCMK_XE_CIB "/" PCMK_XE_STATUS \
	- "/" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']" \
	+#define XPATH_NODE_HISTORY "/" PCMK_XE_CIB "/" PCMK_XE_STATUS \
	+ "/" PCMK__XE_NODE_STATE \
	+ "[@" PCMK_XA_UNAME "='%s']" \
	"/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES

	/*!
	* \internal
	* \brief Process any resource configuration changes in the CIB status
	*
	* Go through all nodes' resource history, and if a resource's configuration
	* changed since its actions were done, schedule any actions needed (restart,
	* reload, unfencing, rescheduling recurring actions, clean-up, etc.).
	* (This also cancels recurring actions for maintenance mode, which is not
	* entirely related but convenient to do here.)
	*
	* \param[in,out] scheduler Scheduler data
	*/
	void
	pcmk__handle_rsc_config_changes(pcmk_scheduler_t *scheduler)
	{
	crm_trace("Check resource and action configuration for changes");

	/* Rather than iterate through the status section, iterate through the nodes
	* and search for the appropriate status subsection for each. This skips
	* orphaned nodes and lets us eliminate some cases before searching the XML.
	*/
	for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	/* Don't bother checking actions for a node that can't run actions ...
	* unless it's in maintenance mode, in which case we still need to
	* cancel any existing recurring monitors.
	*/
	if (node->details->maintenance
	\|\| pcmk__node_available(node, false, false)) {

	char *xpath = NULL;
	xmlNode *history = NULL;

	xpath = crm_strdup_printf(XPATH_NODE_HISTORY, node->details->uname);
	history = get_xpath_object(xpath, scheduler->input, LOG_NEVER);
	free(xpath);

	process_node_history(node, history);
	}
	}
	}
	diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c
	index 6580545112..356e626a92 100644
	--- a/lib/pacemaker/pcmk_simulate.c
	+++ b/lib/pacemaker/pcmk_simulate.c
	@@ -1,1005 +1,1005 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/common/scheduler.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#include <stdint.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include "libpacemaker_private.h"

	static pcmk__output_t *out = NULL;
	static cib_t *fake_cib = NULL;
	static GList *fake_resource_list = NULL;
	static const GList *fake_op_fail_list = NULL;

	static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
	const char *use_date);

	/*!
	* \internal
	* \brief Create an action name for use in a dot graph
	*
	* \param[in] action Action to create name for
	* \param[in] verbose If true, add action ID to name
	*
	* \return Newly allocated string with action name
	* \note It is the caller's responsibility to free the result.
	*/
	static char *
	create_action_name(const pcmk_action_t *action, bool verbose)
	{
	char *action_name = NULL;
	const char *prefix = "";
	const char *action_host = NULL;
	const char *clone_name = NULL;
	const char *task = action->task;

	if (action->node != NULL) {
	action_host = action->node->details->uname;
	} else if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	action_host = "<none>";
	}

	if (pcmk__str_eq(action->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
	prefix = "Cancel ";
	task = action->cancel_task;
	}

	if (action->rsc != NULL) {
	clone_name = action->rsc->clone_name;
	}

	if (clone_name != NULL) {
	char *key = NULL;
	guint interval_ms = 0;

	if (pcmk__guint_from_hash(action->meta, PCMK_META_INTERVAL, 0,
	&interval_ms) != pcmk_rc_ok) {
	interval_ms = 0;
	}

	if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_NOTIFIED, NULL)) {
	const char *n_type = g_hash_table_lookup(action->meta,
	"notify_key_type");
	const char *n_task = g_hash_table_lookup(action->meta,
	"notify_key_operation");

	CRM_ASSERT(n_type != NULL);
	CRM_ASSERT(n_task != NULL);
	key = pcmk__notify_key(clone_name, n_type, n_task);
	} else {
	key = pcmk__op_key(clone_name, task, interval_ms);
	}

	if (action_host != NULL) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, key, action_host);
	} else {
	action_name = crm_strdup_printf("%s%s", prefix, key);
	}
	free(key);

	} else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
	pcmk__str_none)) {
	const char *op = g_hash_table_lookup(action->meta, "stonith_action");

	action_name = crm_strdup_printf("%s%s '%s' %s",
	prefix, action->task, op, action_host);

	} else if (action->rsc && action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->uuid, action_host);

	} else if (action_host) {
	action_name = crm_strdup_printf("%s%s %s",
	prefix, action->task, action_host);

	} else {
	action_name = crm_strdup_printf("%s", action->uuid);
	}

	if (verbose) {
	char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id);

	free(action_name);
	action_name = with_id;
	}
	return action_name;
	}

	/*!
	* \internal
	* \brief Display the status of a cluster
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] show_opts How to modify display (as pcmk_show_opt_e flags)
	* \param[in] section_opts Sections to display (as pcmk_section_e flags)
	* \param[in] title What to use as list title
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_cluster_status(pcmk_scheduler_t *scheduler, uint32_t show_opts,
	uint32_t section_opts, const char *title,
	bool print_spacer)
	{
	pcmk__output_t *out = scheduler->priv;
	GList *all = NULL;
	crm_exit_t stonith_rc = 0;
	enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;

	section_opts \|= pcmk_section_nodes \| pcmk_section_resources;
	show_opts \|= pcmk_show_inactive_rscs \| pcmk_show_failed_detail;

	all = g_list_prepend(all, (gpointer) "*");

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "%s", title);
	out->message(out, "cluster-status",
	scheduler, state, stonith_rc, NULL,
	false, section_opts, show_opts, NULL, all, all);
	out->end_list(out);

	g_list_free(all);
	}

	/*!
	* \internal
	* \brief Display a summary of all actions scheduled in a transition
	*
	* \param[in,out] scheduler Scheduler data (fully scheduled)
	* \param[in] print_spacer Whether to display a spacer first
	*/
	static void
	print_transition_summary(pcmk_scheduler_t *scheduler, bool print_spacer)
	{
	pcmk__output_t *out = scheduler->priv;

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	out->begin_list(out, NULL, NULL, "Transition Summary");
	pcmk__output_actions(scheduler);
	out->end_list(out);
	}

	/*!
	* \internal
	* \brief Reset scheduler input, output, date, and flags
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] input What to set as cluster input
	* \param[in] out What to set as cluster output object
	* \param[in] use_date What to set as cluster's current timestamp
	* \param[in] flags Group of enum pcmk_scheduler_flags to set
	*/
	static void
	reset(pcmk_scheduler_t scheduler, xmlNodePtr input, pcmk__output_t out,
	const char *use_date, unsigned int flags)
	{
	scheduler->input = input;
	scheduler->priv = out;
	set_effective_date(scheduler, true, use_date);
	if (pcmk_is_set(flags, pcmk_sim_sanitized)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_sanitized);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
	}
	}

	/*!
	* \brief Write out a file in dot(1) format describing the actions that will
	* be taken by the scheduler in response to an input CIB file.
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] dot_file The filename to write
	* \param[in] all_actions Write all actions, even those that are optional
	* or are on unmanaged resources
	* \param[in] verbose Add extra information, such as action IDs, to the
	* output
	*
	* \return Standard Pacemaker return code
	*/
	static int
	write_sim_dotfile(pcmk_scheduler_t scheduler, const char dot_file,
	bool all_actions, bool verbose)
	{
	GList *iter = NULL;
	FILE *dot_strm = fopen(dot_file, "w");

	if (dot_strm == NULL) {
	return errno;
	}

	fprintf(dot_strm, " digraph \"g\" {\n");
	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;
	const char *style = "dashed";
	const char *font = "black";
	const char *color = "black";
	char *action_name = create_action_name(action, verbose);

	if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
	font = "orange";
	}

	if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)) {
	style = "bold";
	color = "green";

	} else if ((action->rsc != NULL)
	&& !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)) {
	color = "red";
	font = "purple";
	if (!all_actions) {
	goto do_not_write;
	}

	} else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
	color = "blue";
	if (!all_actions) {
	goto do_not_write;
	}

	} else {
	color = "red";
	CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pcmk_action_runnable));
	}

	pcmk__set_action_flags(action, pcmk_action_added_to_graph);
	fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
	action_name, style, color, font);
	do_not_write:
	free(action_name);
	}

	for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
	pcmk_action_t action = (pcmk_action_t ) iter->data;

	for (GList *before_iter = action->actions_before;
	before_iter != NULL; before_iter = before_iter->next) {

	pcmk__related_action_t *before = before_iter->data;

	char *before_name = NULL;
	char *after_name = NULL;
	const char *style = "dashed";
	bool optional = true;

	if (before->state == pe_link_dumped) {
	optional = false;
	style = "bold";
	} else if ((uint32_t) before->type == pcmk__ar_none) {
	continue;
	} else if (pcmk_is_set(before->action->flags,
	pcmk_action_added_to_graph)
	&& pcmk_is_set(action->flags, pcmk_action_added_to_graph)
	&& (uint32_t) before->type != pcmk__ar_if_on_same_node_or_target) {
	optional = false;
	}

	if (all_actions \|\| !optional) {
	before_name = create_action_name(before->action, verbose);
	after_name = create_action_name(action, verbose);
	fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
	before_name, after_name, style);
	free(before_name);
	free(after_name);
	}
	}
	}

	fprintf(dot_strm, "}\n");
	fflush(dot_strm);
	fclose(dot_strm);
	return pcmk_rc_ok;
	}

	/*!
	* \brief Profile the configuration updates and scheduler actions in a single
	* CIB file, printing the profiling timings.
	*
	* \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in] xml_file The CIB file to profile
	* \param[in] repeat Number of times to run
	* \param[in,out] scheduler Scheduler data
	* \param[in] use_date The date to set the cluster's time to (may be NULL)
	*/
	static void
	profile_file(const char *xml_file, long long repeat,
	pcmk_scheduler_t scheduler, const char use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	xmlNode *cib_object = NULL;
	clock_t start = 0;
	clock_t end;
	unsigned long long scheduler_flags = pcmk_sched_no_compat;

	CRM_ASSERT(out != NULL);

	cib_object = filename2xml(xml_file);
	start = clock();

	if (pcmk_find_cib_element(cib_object, PCMK_XE_STATUS) == NULL) {
	create_xml_node(cib_object, PCMK_XE_STATUS);
	}

	if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	free_xml(cib_object);
	return;
	}

	if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
	free_xml(cib_object);
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	scheduler_flags \|= pcmk_sched_output_scores;
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	scheduler_flags \|= pcmk_sched_show_utilization;
	}

	for (int i = 0; i < repeat; ++i) {
	xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object);

	scheduler->input = input;
	set_effective_date(scheduler, false, use_date);
	pcmk__schedule_actions(input, scheduler_flags, scheduler);
	pe_reset_working_set(scheduler);
	}

	end = clock();
	out->message(out, "profile", xml_file, start, end);
	}

	void
	pcmk__profile_dir(const char *dir, long long repeat,
	pcmk_scheduler_t scheduler, const char use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	struct dirent **namelist;

	int file_num = scandir(dir, &namelist, 0, alphasort);

	CRM_ASSERT(out != NULL);

	if (file_num > 0) {
	struct stat prop;
	char buffer[FILENAME_MAX];

	out->begin_list(out, NULL, NULL, "Timings");

	while (file_num--) {
	if ('.' == namelist[file_num]->d_name[0]) {
	free(namelist[file_num]);
	continue;

	} else if (!pcmk__ends_with_ext(namelist[file_num]->d_name,
	".xml")) {
	free(namelist[file_num]);
	continue;
	}
	snprintf(buffer, sizeof(buffer), "%s/%s",
	dir, namelist[file_num]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	profile_file(buffer, repeat, scheduler, use_date);
	}
	free(namelist[file_num]);
	}
	free(namelist);

	out->end_list(out);
	}
	}

	/*!
	* \brief Set the date of the cluster, either to the value given by
	* \p use_date, or to the "execution-date" value in the CIB.
	*
	* \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
	* object before this function is called.
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] print_original If \p true, the "execution-date" should
	* also be printed
	* \param[in] use_date The date to set the cluster's time to
	* (may be NULL)
	*/
	static void
	set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
	const char *use_date)
	{
	pcmk__output_t *out = scheduler->priv;
	time_t original_date = 0;

	CRM_ASSERT(out != NULL);

	crm_element_value_epoch(scheduler->input, "execution-date", &original_date);

	if (use_date) {
	scheduler->now = crm_time_new(use_date);
	out->info(out, "Setting effective cluster time: %s", use_date);
	crm_time_log(LOG_NOTICE, "Pretending 'now' is", scheduler->now,
	crm_time_log_date \| crm_time_log_timeofday);

	} else if (original_date != 0) {
	scheduler->now = pcmk__copy_timet(original_date);

	if (print_original) {
	char *when = crm_time_as_string(scheduler->now,
	crm_time_log_date\|crm_time_log_timeofday);

	out->info(out, "Using the original execution date of: %s", when);
	free(when);
	}
	}
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a pseudo-action in a graph
	*
	* \param[in,out] graph Graph to update with pseudo-action result
	* \param[in,out] action Pseudo-action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_pseudo_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, PCMK__META_ON_NODE);
	const char *task = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-pseudo-action", node, task);

	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate executing a resource action in a graph
	*
	* \param[in,out] graph Graph to update with resource action result
	* \param[in,out] action Resource action to simulate executing
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_resource_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	int rc;
	lrmd_event_data_t *op = NULL;
	int target_outcome = PCMK_OCF_OK;

	const char *rtype = NULL;
	const char *rclass = NULL;
	const char *resource = NULL;
	const char *rprovider = NULL;
	const char *resource_config_name = NULL;
	const char *operation = crm_element_value(action->xml, PCMK_XA_OPERATION);
	const char *target_rc_s = crm_meta_value(action->params,
	PCMK__META_OP_TARGET_RC);

	xmlNode *cib_node = NULL;
	xmlNode *cib_resource = NULL;
	xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	char *node = crm_element_value_copy(action->xml, PCMK__META_ON_NODE);
	char *uuid = NULL;
	const char *router_node = crm_element_value(action->xml,
	PCMK__XA_ROUTER_NODE);

	// Certain actions don't need to be displayed or history entries
	if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
	crm_debug("No history injection for %s op on %s", operation, node);
	goto done; // Confirm action and update graph
	}

	if (action_rsc == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "Bad");
	free(node);
	return EPROTO;
	}

	/* A resource might be known by different names in the configuration and in
	* the action (for example, a clone instance). Grab the configuration name
	* (which is preferred when writing history), and if necessary, the instance
	* name.
	*/
	resource_config_name = crm_element_value(action_rsc, PCMK_XA_ID);
	if (resource_config_name == NULL) { // Shouldn't be possible
	crm_log_xml_err(action->xml, "No ID");
	free(node);
	return EPROTO;
	}
	resource = resource_config_name;
	if (pe_find_resource(fake_resource_list, resource) == NULL) {
	const char *longname = crm_element_value(action_rsc, PCMK__XA_LONG_ID);

	if ((longname != NULL)
	&& (pe_find_resource(fake_resource_list, longname) != NULL)) {
	resource = longname;
	}
	}

	// Certain actions need to be displayed but don't need history entries
	if (pcmk__strcase_any_of(operation, PCMK_ACTION_DELETE,
	PCMK_ACTION_META_DATA, NULL)) {
	out->message(out, "inject-rsc-action", resource, operation, node,
	(guint) 0);
	goto done; // Confirm action and update graph
	}

	rclass = crm_element_value(action_rsc, PCMK_XA_CLASS);
	rtype = crm_element_value(action_rsc, PCMK_XA_TYPE);
	rprovider = crm_element_value(action_rsc, PCMK_XA_PROVIDER);

	pcmk__scan_min_int(target_rc_s, &target_outcome, 0);

	CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL,
	cib_sync_call\|cib_scope_local) == pcmk_ok);

	// Ensure the action node is in the CIB
	uuid = crm_element_value_copy(action->xml, PCMK__META_ON_NODE_UUID);
	cib_node = pcmk__inject_node(fake_cib, node,
	((router_node == NULL)? uuid: node));
	free(uuid);
	CRM_ASSERT(cib_node != NULL);

	// Add a history entry for the action
	cib_resource = pcmk__inject_resource_history(out, cib_node, resource,
	resource_config_name,
	rclass, rtype, rprovider);
	if (cib_resource == NULL) {
	crm_err("Could not simulate action %d history for resource %s",
	action->id, resource);
	free(node);
	free_xml(cib_node);
	return EINVAL;
	}

	// Simulate and display an executor event for the action result
	op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
	target_outcome, "User-injected result");
	out->message(out, "inject-rsc-action", resource, op->op_type, node,
	op->interval_ms);

	// Check whether action is in a list of desired simulated failures
	for (const GList *iter = fake_op_fail_list;
	iter != NULL; iter = iter->next) {
	const char spec = (const char ) iter->data;
	char *key = NULL;
	const char *match_name = NULL;

	// Allow user to specify anonymous clone with or without instance number
	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
	op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource;
	}
	free(key);

	// If not found, try the resource's name in the configuration
	if ((match_name == NULL)
	&& (strcmp(resource, resource_config_name) != 0)) {

	key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name,
	op->op_type, op->interval_ms, node);
	if (strncasecmp(key, spec, strlen(key)) == 0) {
	match_name = resource_config_name;
	}
	free(key);
	}

	if (match_name == NULL) {
	continue; // This failed action entry doesn't match
	}

	// ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
	rc = sscanf(spec, "%[^=]=%d", (int ) &op->rc);
	if (rc != 1) {
	out->err(out, "Invalid failed operation '%s' "
	"(result code must be integer)", spec);
	continue; // Keep checking other list entries
	}

	out->info(out, "Pretending action %d failed with rc=%d",
	action->id, op->rc);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
	graph->abort_priority = INFINITY;
	pcmk__inject_failcount(out, cib_node, match_name, op->op_type,
	op->interval_ms, op->rc);
	break;
	}

	pcmk__inject_action_result(cib_resource, op, target_outcome);
	lrmd_free_event(op);
	rc = fake_cib->cmds->modify(fake_cib, PCMK_XE_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	done:
	free(node);
	free_xml(cib_node);
	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a cluster action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Cluster action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_cluster_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *node = crm_element_value(action->xml, PCMK__META_ON_NODE);
	const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
	xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	out->message(out, "inject-cluster-action", node, task, rsc);
	pcmk__update_graph(graph, action);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Simulate successfully executing a fencing action
	*
	* \param[in,out] graph Graph to update with action result
	* \param[in,out] action Fencing action to simulate
	*
	* \return Standard Pacemaker return code
	*/
	static int
	simulate_fencing_action(pcmk__graph_t graph, pcmk__graph_action_t action)
	{
	const char *op = crm_meta_value(action->params, "stonith_action");
	char *target = crm_element_value_copy(action->xml, PCMK__META_ON_NODE);

	out->message(out, "inject-fencing-action", target, op);

	if (!pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
	int rc = pcmk_ok;
	GString *xpath = g_string_sized_new(512);

	// Set node state to offline
	xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target,
	false);

	CRM_ASSERT(cib_node != NULL);
	crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);
	rc = fake_cib->cmds->replace(fake_cib, PCMK_XE_STATUS, cib_node,
	cib_sync_call\|cib_scope_local);
	CRM_ASSERT(rc == pcmk_ok);

	// Simulate controller clearing node's resource history and attributes
	pcmk__g_strcat(xpath,
	- "//" XML_CIB_TAG_STATE
	+ "//" PCMK__XE_NODE_STATE
	"[@" PCMK_XA_UNAME "='", target, "']/" XML_CIB_TAG_LRM,
	NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	g_string_truncate(xpath, 0);
	pcmk__g_strcat(xpath,
	- "//" XML_CIB_TAG_STATE
	+ "//" PCMK__XE_NODE_STATE
	"[@" PCMK_XA_UNAME "='", target, "']"
	"/" XML_TAG_TRANSIENT_NODEATTRS, NULL);
	fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
	cib_xpath\|cib_sync_call\|cib_scope_local);

	free_xml(cib_node);
	g_string_free(xpath, TRUE);
	}

	pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
	pcmk__update_graph(graph, action);
	free(target);
	return pcmk_rc_ok;
	}

	enum pcmk__graph_status
	pcmk__simulate_transition(pcmk_scheduler_t scheduler, cib_t cib,
	const GList *op_fail_list)
	{
	pcmk__graph_t *transition = NULL;
	enum pcmk__graph_status graph_rc;

	pcmk__graph_functions_t simulation_fns = {
	simulate_pseudo_action,
	simulate_resource_action,
	simulate_cluster_action,
	simulate_fencing_action,
	};

	out = scheduler->priv;

	fake_cib = cib;
	fake_op_fail_list = op_fail_list;

	if (!out->is_quiet(out)) {
	out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
	}

	pcmk__set_graph_functions(&simulation_fns);
	transition = pcmk__unpack_graph(scheduler->graph, crm_system_name);
	pcmk__log_graph(LOG_DEBUG, transition);

	fake_resource_list = scheduler->resources;
	do {
	graph_rc = pcmk__execute_graph(transition);
	} while (graph_rc == pcmk__graph_active);
	fake_resource_list = NULL;

	if (graph_rc != pcmk__graph_complete) {
	out->err(out, "Transition failed: %s",
	pcmk__graph_status2text(graph_rc));
	pcmk__log_graph(LOG_ERR, transition);
	out->err(out, "An invalid transition was produced");
	}
	pcmk__free_graph(transition);

	if (!out->is_quiet(out)) {
	// If not quiet, we'll need the resulting CIB for later display
	xmlNode *cib_object = NULL;
	int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object,
	cib_sync_call\|cib_scope_local);

	CRM_ASSERT(rc == pcmk_ok);
	pe_reset_working_set(scheduler);
	scheduler->input = cib_object;
	out->end_list(out);
	}
	return graph_rc;
	}

	int
	pcmk__simulate(pcmk_scheduler_t scheduler, pcmk__output_t out,
	const pcmk_injections_t *injections, unsigned int flags,
	uint32_t section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	int printed = pcmk_rc_no_output;
	int rc = pcmk_rc_ok;
	xmlNodePtr input = NULL;
	cib_t *cib = NULL;

	rc = cib__signon_query(out, &cib, &input);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}

	reset(scheduler, input, out, use_date, flags);
	cluster_status(scheduler);

	if ((cib->variant == cib_native)
	&& pcmk_is_set(section_opts, pcmk_section_times)) {
	if (pcmk__our_nodename == NULL) {
	// Currently used only in the times section
	pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0);
	}
	scheduler->localhost = pcmk__our_nodename;
	}

	if (!out->is_quiet(out)) {
	const bool show_pending = pcmk_is_set(flags, pcmk_sim_show_pending);

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	printed = out->message(out, "maint-mode", scheduler->flags);
	}

	if (scheduler->disabled_resources \|\| scheduler->blocked_resources) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	printed = out->info(out,
	"%d of %d resource instances DISABLED and "
	"%d BLOCKED from further action due to failure",
	scheduler->disabled_resources,
	scheduler->ninstances,
	scheduler->blocked_resources);
	}

	/* Most formatted output headers use caps for each word, but this one
	* only has the first word capitalized for compatibility with pcs.
	*/
	print_cluster_status(scheduler, (show_pending? pcmk_show_pending : 0),
	section_opts, "Current cluster status",
	(printed == pcmk_rc_ok));
	printed = pcmk_rc_ok;
	}

	// If the user requested any injections, handle them
	if ((injections->node_down != NULL)
	\|\| (injections->node_fail != NULL)
	\|\| (injections->node_up != NULL)
	\|\| (injections->op_inject != NULL)
	\|\| (injections->ticket_activate != NULL)
	\|\| (injections->ticket_grant != NULL)
	\|\| (injections->ticket_revoke != NULL)
	\|\| (injections->ticket_standby != NULL)
	\|\| (injections->watchdog != NULL)) {

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	pcmk__inject_scheduler_input(scheduler, cib, injections);
	printed = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &input, cib_sync_call);
	if (rc != pcmk_rc_ok) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}

	cleanup_calculations(scheduler);
	reset(scheduler, input, out, use_date, flags);
	cluster_status(scheduler);
	}

	if (input_file != NULL) {
	rc = write_xml_file(input, input_file, FALSE);
	if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	goto simulate_done;
	}
	}

	if (pcmk_any_flags_set(flags, pcmk_sim_process \| pcmk_sim_simulate)) {
	pcmk__output_t *logger_out = NULL;
	unsigned long long scheduler_flags = pcmk_sched_no_compat;

	if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	scheduler_flags \|= pcmk_sched_output_scores;
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	scheduler_flags \|= pcmk_sched_show_utilization;
	}

	if (pcmk_all_flags_set(scheduler->flags,
	pcmk_sched_output_scores
	\|pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL,
	"Assignment Scores and Utilization Information");
	printed = pcmk_rc_ok;

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Assignment Scores");
	printed = pcmk_rc_ok;

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	out->begin_list(out, NULL, NULL, "Utilization Information");
	printed = pcmk_rc_ok;

	} else {
	rc = pcmk__log_output_new(&logger_out);
	if (rc != pcmk_rc_ok) {
	goto simulate_done;
	}
	pe__register_messages(logger_out);
	pcmk__register_lib_messages(logger_out);
	scheduler->priv = logger_out;
	}

	pcmk__schedule_actions(input, scheduler_flags, scheduler);

	if (logger_out == NULL) {
	out->end_list(out);
	} else {
	logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
	pcmk__output_free(logger_out);
	scheduler->priv = out;
	}

	input = NULL; /* Don't try and free it twice */

	if (graph_file != NULL) {
	rc = write_xml_file(scheduler->graph, graph_file, FALSE);
	if (rc < 0) {
	rc = pcmk_rc_graph_error;
	goto simulate_done;
	}
	}

	if (dot_file != NULL) {
	rc = write_sim_dotfile(scheduler, dot_file,
	pcmk_is_set(flags, pcmk_sim_all_actions),
	pcmk_is_set(flags, pcmk_sim_verbose));
	if (rc != pcmk_rc_ok) {
	rc = pcmk_rc_dot_error;
	goto simulate_done;
	}
	}

	if (!out->is_quiet(out)) {
	print_transition_summary(scheduler, printed == pcmk_rc_ok);
	}
	}

	rc = pcmk_rc_ok;

	if (!pcmk_is_set(flags, pcmk_sim_simulate)) {
	goto simulate_done;
	}

	PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
	if (pcmk__simulate_transition(scheduler, cib, injections->op_fail)
	!= pcmk__graph_complete) {
	rc = pcmk_rc_invalid_transition;
	}

	if (out->is_quiet(out)) {
	goto simulate_done;
	}

	set_effective_date(scheduler, true, use_date);

	if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
	}
	if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
	}

	cluster_status(scheduler);
	print_cluster_status(scheduler, 0, section_opts, "Revised Cluster Status",
	true);

	simulate_done:
	cib__clean_up_connection(&cib);
	return rc;
	}

	int
	pcmk_simulate(xmlNodePtr xml, pcmk_scheduler_t scheduler,
	const pcmk_injections_t *injections, unsigned int flags,
	unsigned int section_opts, const char *use_date,
	const char input_file, const char graph_file,
	const char *dot_file)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	rc = pcmk__simulate(scheduler, out, injections, flags, section_opts,
	use_date, input_file, graph_file, dot_file);
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	return rc;
	}
	diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
	index a8ebaea59c..064dba8e33 100644
	--- a/lib/pengine/failcounts.c
	+++ b/lib/pengine/failcounts.c
	@@ -1,472 +1,472 @@
	/*
	* Copyright 2008-2024 the Pacemaker project contributors
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/types.h>
	#include <regex.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/pengine/internal.h>

	static gboolean
	is_matched_failure(const char rsc_id, const xmlNode conf_op_xml,
	const xmlNode *lrm_op_xml)
	{
	gboolean matched = FALSE;
	const char *conf_op_name = NULL;
	const char *lrm_op_task = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	guint lrm_op_interval_ms = 0;
	const char *lrm_op_id = NULL;
	char *last_failure_key = NULL;

	if (rsc_id == NULL \|\| conf_op_xml == NULL \|\| lrm_op_xml == NULL) {
	return FALSE;
	}

	// Get name and interval from configured op
	conf_op_name = crm_element_value(conf_op_xml, PCMK_XA_NAME);
	conf_op_interval_spec = crm_element_value(conf_op_xml, PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(conf_op_interval_spec, &conf_op_interval_ms);

	// Get name and interval from op history entry
	lrm_op_task = crm_element_value(lrm_op_xml, PCMK_XA_OPERATION);
	crm_element_value_ms(lrm_op_xml, PCMK_META_INTERVAL, &lrm_op_interval_ms);

	if ((conf_op_interval_ms != lrm_op_interval_ms)
	\|\| !pcmk__str_eq(conf_op_name, lrm_op_task, pcmk__str_casei)) {
	return FALSE;
	}

	lrm_op_id = ID(lrm_op_xml);
	last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);

	if (pcmk__str_eq(last_failure_key, lrm_op_id, pcmk__str_casei)) {
	matched = TRUE;

	} else {
	char *expected_op_key = pcmk__op_key(rsc_id, conf_op_name,
	conf_op_interval_ms);

	if (pcmk__str_eq(expected_op_key, lrm_op_id, pcmk__str_casei)) {
	int rc = 0;
	int target_rc = pe__target_rc_from_xml(lrm_op_xml);

	crm_element_value_int(lrm_op_xml, PCMK__XA_RC_CODE, &rc);
	if (rc != target_rc) {
	matched = TRUE;
	}
	}
	free(expected_op_key);
	}

	free(last_failure_key);
	return matched;
	}

	static gboolean
	block_failure(const pcmk_node_t node, pcmk_resource_t rsc,
	const xmlNode *xml_op)
	{
	char *xml_name = clone_strip(rsc->id);

	/* @TODO This xpath search occurs after template expansion, but it is unable
	* to properly detect on-fail in id-ref, operation meta-attributes, or
	* op_defaults, or evaluate rules.
	*
	* Also, on-fail defaults to block (in unpack_operation()) for stop actions
	* when stonith is disabled.
	*
	* Ideally, we'd unpack the operation before this point, and pass in a
	* meta-attributes table that takes all that into consideration.
	*/
	char *xpath = crm_strdup_printf("//" XML_CIB_TAG_RESOURCE
	"[@" PCMK_XA_ID "='%s']"
	"//" PCMK_XE_OP
	"[@" PCMK_META_ON_FAIL "='block']",
	xml_name);

	xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
	gboolean should_block = FALSE;

	free(xpath);

	if (xpathObj) {
	int max = numXpathResults(xpathObj);
	int lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *pref = getXpathResult(xpathObj, lpc);

	if (xml_op) {
	should_block = is_matched_failure(xml_name, pref, xml_op);
	if (should_block) {
	break;
	}

	} else {
	const char *conf_op_name = NULL;
	const char *conf_op_interval_spec = NULL;
	guint conf_op_interval_ms = 0;
	char *lrm_op_xpath = NULL;
	xmlXPathObject *lrm_op_xpathObj = NULL;

	// Get name and interval from configured op
	conf_op_name = crm_element_value(pref, PCMK_XA_NAME);
	conf_op_interval_spec = crm_element_value(pref,
	PCMK_META_INTERVAL);
	pcmk_parse_interval_spec(conf_op_interval_spec,
	&conf_op_interval_ms);

	-#define XPATH_FMT "//" XML_CIB_TAG_STATE "[@" PCMK_XA_UNAME "='%s']" \
	+#define XPATH_FMT "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']" \
	"//" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']" \
	"/" XML_LRM_TAG_RSC_OP "[@" PCMK_XA_OPERATION "='%s']" \
	"[@" PCMK_META_INTERVAL "='%u']"

	lrm_op_xpath = crm_strdup_printf(XPATH_FMT,
	node->details->uname, xml_name,
	conf_op_name,
	conf_op_interval_ms);
	lrm_op_xpathObj = xpath_search(rsc->cluster->input, lrm_op_xpath);

	free(lrm_op_xpath);

	if (lrm_op_xpathObj) {
	int max2 = numXpathResults(lrm_op_xpathObj);
	int lpc2 = 0;

	for (lpc2 = 0; lpc2 < max2; lpc2++) {
	xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
	lpc2);

	should_block = is_matched_failure(xml_name, pref,
	lrm_op_xml);
	if (should_block) {
	break;
	}
	}
	}
	freeXpathObject(lrm_op_xpathObj);

	if (should_block) {
	break;
	}
	}
	}
	}

	free(xml_name);
	freeXpathObject(xpathObj);

	return should_block;
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(const pcmk_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	return pcmk_is_set(rsc->flags, pcmk_rsc_unique)? strdup(name) : clone_strip(name);
	}

	/*!
	* \internal
	* \brief Compile regular expression to match a failure-related node attribute
	*
	* \param[in] prefix Attribute prefix to match
	* \param[in] rsc_name Resource name to match as used in failure attributes
	* \param[in] is_legacy Whether DC uses per-resource fail counts
	* \param[in] is_unique Whether the resource is a globally unique clone
	* \param[out] re Where to store resulting regular expression
	*
	* \return Standard Pacemaker return code
	* \note Fail attributes are named like PREFIX-RESOURCE#OP_INTERVAL.
	* The caller is responsible for freeing re with regfree().
	*/
	static int
	generate_fail_regex(const char prefix, const char rsc_name,
	gboolean is_legacy, gboolean is_unique, regex_t *re)
	{
	char *pattern;

	/* @COMPAT DC < 1.1.17: Fail counts used to be per-resource rather than
	* per-operation.
	*/
	const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");

	/* Ignore instance numbers for anything other than globally unique clones.
	* Anonymous clone fail counts could contain an instance number if the
	* clone was initially unique, failed, then was converted to anonymous.
	* @COMPAT Also, before 1.1.8, anonymous clone fail counts always contained
	* clone instance numbers.
	*/
	const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");

	pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
	instance_pattern, op_pattern);
	if (regcomp(re, pattern, REG_EXTENDED\|REG_NOSUB) != 0) {
	free(pattern);
	return EINVAL;
	}

	free(pattern);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Compile regular expressions to match failure-related node attributes
	*
	* \param[in] rsc Resource being checked for failures
	* \param[out] failcount_re Storage for regular expression for fail count
	* \param[out] lastfailure_re Storage for regular expression for last failure
	*
	* \return Standard Pacemaker return code
	* \note On success, the caller is responsible for freeing the expressions with
	* regfree().
	*/
	static int
	generate_fail_regexes(const pcmk_resource_t *rsc,
	regex_t failcount_re, regex_t lastfailure_re)
	{
	int rc = pcmk_rc_ok;
	char *rsc_name = rsc_fail_name(rsc);
	const char *version = crm_element_value(rsc->cluster->input,
	PCMK_XA_CRM_FEATURE_SET);

	// @COMPAT Pacemaker <= 1.1.16 used a single fail count per resource
	gboolean is_legacy = (compare_version(version, "3.0.13") < 0);

	if (generate_fail_regex(PCMK__FAIL_COUNT_PREFIX, rsc_name, is_legacy,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique),
	failcount_re) != pcmk_rc_ok) {
	rc = EINVAL;

	} else if (generate_fail_regex(PCMK__LAST_FAILURE_PREFIX, rsc_name,
	is_legacy,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique),
	lastfailure_re) != pcmk_rc_ok) {
	rc = EINVAL;
	regfree(failcount_re);
	}

	free(rsc_name);
	return rc;
	}

	// Data for fail-count-related iterators
	struct failcount_data {
	const pcmk_node_t *node;// Node to check for fail count
	pcmk_resource_t *rsc; // Resource to check for fail count
	uint32_t flags; // Fail count flags
	const xmlNode *xml_op; // History entry for expiration purposes (or NULL)
	regex_t failcount_re; // Fail count regular expression to match
	regex_t lastfailure_re; // Last failure regular expression to match
	int failcount; // Fail count so far
	time_t last_failure; // Time of most recent failure so far
	};

	/*!
	* \internal
	* \brief Update fail count and last failure appropriately for a node attribute
	*
	* \param[in] key Node attribute name
	* \param[in] value Node attribute value
	* \param[in] user_data Fail count data to update
	*/
	static void
	update_failcount_for_attr(gpointer key, gpointer value, gpointer user_data)
	{
	struct failcount_data *fc_data = user_data;

	// If this is a matching fail count attribute, update fail count
	if (regexec(&(fc_data->failcount_re), (const char *) key, 0, NULL, 0) == 0) {
	fc_data->failcount = pcmk__add_scores(fc_data->failcount,
	char2score(value));
	pcmk__rsc_trace(fc_data->rsc, "Added %s (%s) to %s fail count (now %s)",
	(const char ) key, (const char ) value,
	fc_data->rsc->id,
	pcmk_readable_score(fc_data->failcount));
	return;
	}

	// If this is a matching last failure attribute, update last failure
	if (regexec(&(fc_data->lastfailure_re), (const char *) key, 0, NULL,
	0) == 0) {
	long long last_ll;

	if (pcmk__scan_ll(value, &last_ll, 0LL) == pcmk_rc_ok) {
	fc_data->last_failure = (time_t) QB_MAX(fc_data->last_failure,
	last_ll);
	}
	}
	}

	/*!
	* \internal
	* \brief Update fail count and last failure appropriately for a filler resource
	*
	* \param[in] data Filler resource
	* \param[in] user_data Fail count data to update
	*/
	static void
	update_failcount_for_filler(gpointer data, gpointer user_data)
	{
	pcmk_resource_t *filler = data;
	struct failcount_data *fc_data = user_data;
	time_t filler_last_failure = 0;

	fc_data->failcount += pe_get_failcount(fc_data->node, filler,
	&filler_last_failure, fc_data->flags,
	fc_data->xml_op);
	fc_data->last_failure = QB_MAX(fc_data->last_failure, filler_last_failure);
	}

	/*!
	* \internal
	* \brief Get a resource's fail count on a node
	*
	* \param[in] node Node to check
	* \param[in,out] rsc Resource to check
	* \param[out] last_failure If not NULL, where to set time of most recent
	* failure of \p rsc on \p node
	* \param[in] flags Group of enum pcmk__fc_flags
	* \param[in] xml_op If not NULL, consider only the action in this
	* history entry when determining whether on-fail
	* is configured as "blocked", otherwise consider
	* all actions configured for \p rsc
	*
	* \return Fail count for \p rsc on \p node according to \p flags
	*/
	int
	pe_get_failcount(const pcmk_node_t node, pcmk_resource_t rsc,
	time_t last_failure, uint32_t flags, const xmlNode xml_op)
	{
	struct failcount_data fc_data = {
	.node = node,
	.rsc = rsc,
	.flags = flags,
	.xml_op = xml_op,
	.failcount = 0,
	.last_failure = (time_t) 0,
	};

	// Calculate resource failcount as sum of all matching operation failcounts
	CRM_CHECK(generate_fail_regexes(rsc, &fc_data.failcount_re,
	&fc_data.lastfailure_re) == pcmk_rc_ok,
	return 0);
	g_hash_table_foreach(node->details->attrs, update_failcount_for_attr,
	&fc_data);
	regfree(&(fc_data.failcount_re));
	regfree(&(fc_data.lastfailure_re));

	// If failure blocks the resource, disregard any failure timeout
	if ((fc_data.failcount > 0) && (rsc->failure_timeout > 0)
	&& block_failure(node, rsc, xml_op)) {

	pcmk__config_warn("Ignoring failure timeout %d for %s "
	"because it conflicts with on-fail=block",
	rsc->failure_timeout, rsc->id);
	rsc->failure_timeout = 0;
	}

	// If all failures have expired, ignore fail count
	if (pcmk_is_set(flags, pcmk__fc_effective) && (fc_data.failcount > 0)
	&& (fc_data.last_failure > 0) && (rsc->failure_timeout != 0)) {

	time_t now = get_effective_time(rsc->cluster);

	if (now > (fc_data.last_failure + rsc->failure_timeout)) {
	pcmk__rsc_debug(rsc, "Failcount for %s on %s expired after %ds",
	rsc->id, pcmk__node_name(node),
	rsc->failure_timeout);
	fc_data.failcount = 0;
	}
	}

	/* Add the fail count of any filler resources, except that we never want the
	* fail counts of a bundle container's fillers to count towards the
	* container's fail count.
	*
	* Most importantly, a Pacemaker Remote connection to a bundle container
	* is a filler of the container, but can reside on a different node than the
	* container itself. Counting its fail count on its node towards the
	* container's fail count on that node could lead to attempting to stop the
	* container on the wrong node.
	*/
	if (pcmk_is_set(flags, pcmk__fc_fillers) && (rsc->fillers != NULL)
	&& !pe_rsc_is_bundled(rsc)) {

	g_list_foreach(rsc->fillers, update_failcount_for_filler, &fc_data);
	if (fc_data.failcount > 0) {
	pcmk__rsc_info(rsc,
	"Container %s and the resources within it "
	"have failed %s time%s on %s",
	rsc->id, pcmk_readable_score(fc_data.failcount),
	pcmk__plural_s(fc_data.failcount),
	pcmk__node_name(node));
	}

	} else if (fc_data.failcount > 0) {
	pcmk__rsc_info(rsc, "%s has failed %s time%s on %s",
	rsc->id, pcmk_readable_score(fc_data.failcount),
	pcmk__plural_s(fc_data.failcount),
	pcmk__node_name(node));
	}

	if (last_failure != NULL) {
	if ((fc_data.failcount > 0) && (fc_data.last_failure > 0)) {
	*last_failure = fc_data.last_failure;
	} else {
	*last_failure = 0;
	}
	}
	return fc_data.failcount;
	}

	/*!
	* \brief Schedule a controller operation to clear a fail count
	*
	* \param[in,out] rsc Resource with failure
	* \param[in] node Node failure occurred on
	* \param[in] reason Readable description why needed (for logging)
	* \param[in,out] scheduler Scheduler data cluster
	*
	* \return Scheduled action
	*/
	pcmk_action_t *
	pe__clear_failcount(pcmk_resource_t rsc, const pcmk_node_t node,
	const char reason, pcmk_scheduler_t scheduler)
	{
	char *key = NULL;
	pcmk_action_t *clear = NULL;

	CRM_CHECK(rsc && node && reason && scheduler, return NULL);

	key = pcmk__op_key(rsc->id, PCMK_ACTION_CLEAR_FAILCOUNT, 0);
	clear = custom_action(rsc, key, PCMK_ACTION_CLEAR_FAILCOUNT, node, FALSE,
	scheduler);
	add_hash_param(clear->meta, PCMK__META_OP_NO_WAIT, PCMK_VALUE_TRUE);
	crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
	rsc->id, pcmk__node_name(node), reason, clear->uuid);
	return clear;
	}
	diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
	index 531011ba08..da71685687 100644
	--- a/lib/pengine/pe_output.c
	+++ b/lib/pengine/pe_output.c
	@@ -1,3208 +1,3208 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm/common/xml_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/cib/util.h>
	#include <crm/msg_xml.h>
	#include <crm/pengine/internal.h>

	const char *
	pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
	{
	const char * desc = NULL;
	// User-supplied description
	if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only\|pcmk_show_description)) {
	desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);
	}
	return desc;
	}

	/* Never display node attributes whose name starts with one of these prefixes */
	#define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX, \
	"shutdown", PCMK_NODE_ATTR_TERMINATE, "standby", "#", \
	NULL }

	static int
	compare_attribute(gconstpointer a, gconstpointer b)
	{
	int rc;

	rc = strcmp((const char )a, (const char )b);

	return rc;
	}

	/*!
	* \internal
	* \brief Determine whether extended information about an attribute should be added.
	*
	* \param[in] node Node that ran this resource
	* \param[in,out] rsc_list List of resources for this node
	* \param[in,out] scheduler Scheduler data
	* \param[in] attrname Attribute to find
	* \param[out] expected_score Expected value for this attribute
	*
	* \return true if extended information should be printed, false otherwise
	* \note Currently, extended information is only supported for ping/pingd
	* resources, for which a message will be printed if connectivity is lost
	* or degraded.
	*/
	static bool
	add_extra_info(const pcmk_node_t node, GList rsc_list,
	pcmk_scheduler_t scheduler, const char attrname,
	int *expected_score)
	{
	GList *gIter = NULL;

	for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;
	const char *type = g_hash_table_lookup(rsc->meta, PCMK_XA_TYPE);
	const char *name = NULL;
	GHashTable *params = NULL;

	if (rsc->children != NULL) {
	if (add_extra_info(node, rsc->children, scheduler, attrname,
	expected_score)) {
	return true;
	}
	}

	if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
	continue;
	}

	params = pe_rsc_params(rsc, node, scheduler);
	name = g_hash_table_lookup(params, PCMK_XA_NAME);

	if (name == NULL) {
	name = "pingd";
	}

	/* To identify the resource with the attribute name. */
	if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
	int host_list_num = 0;
	const char *hosts = g_hash_table_lookup(params, "host_list");
	const char *multiplier = g_hash_table_lookup(params, "multiplier");
	int multiplier_i;

	if (hosts) {
	char **host_list = g_strsplit(hosts, " ", 0);
	host_list_num = g_strv_length(host_list);
	g_strfreev(host_list);
	}

	if ((multiplier == NULL)
	\|\| (pcmk__scan_min_int(multiplier, &multiplier_i,
	INT_MIN) != pcmk_rc_ok)) {
	/* The ocf:pacemaker:ping resource agent defaults multiplier to
	* 1. The agent currently does not handle invalid text, but it
	* should, and this would be a reasonable choice ...
	*/
	multiplier_i = 1;
	}
	expected_score = host_list_num multiplier_i;

	return true;
	}
	}
	return false;
	}

	static GList *
	filter_attr_list(GList attr_list, char name)
	{
	int i;
	const char *filt_str[] = FILTER_STR;

	CRM_CHECK(name != NULL, return attr_list);

	/* filtering automatic attributes */
	for (i = 0; filt_str[i] != NULL; i++) {
	if (g_str_has_prefix(name, filt_str[i])) {
	return attr_list;
	}
	}

	return g_list_insert_sorted(attr_list, name, compare_attribute);
	}

	static GList *
	get_operation_list(xmlNode *rsc_entry) {
	GList *op_list = NULL;
	xmlNode *rsc_op = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
	rsc_op = pcmk__xe_next(rsc_op)) {
	const char *task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	const char *interval_ms_s = crm_element_value(rsc_op,
	PCMK_META_INTERVAL);
	const char *op_rc = crm_element_value(rsc_op, PCMK__XA_RC_CODE);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* Ignore notifies and some probes */
	if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)
	\|\| (pcmk__str_eq(task, "probe", pcmk__str_none)
	&& (op_rc_i == CRM_EX_NOT_RUNNING))) {
	continue;
	}

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
	op_list = g_list_append(op_list, rsc_op);
	}
	}

	op_list = g_list_sort(op_list, sort_op_by_callid);
	return op_list;
	}

	static void
	add_dump_node(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr node = user_data;
	pcmk_create_xml_text_node(node, (const char ) key, (const char ) value);
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	char *new_text = crm_strdup_printf("%s %s=%s",
	dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	#define XPATH_STACK "//" XML_CIB_TAG_NVPAIR \
	"[@" PCMK_XA_NAME "='" \
	PCMK_OPT_CLUSTER_INFRASTRUCTURE "']"

	static const char *
	get_cluster_stack(pcmk_scheduler_t *scheduler)
	{
	xmlNode *stack = get_xpath_object(XPATH_STACK, scheduler->input, LOG_DEBUG);
	return (stack != NULL)? crm_element_value(stack, PCMK_XA_VALUE) : "unknown";
	}

	static char *
	last_changed_string(const char last_written, const char user,
	const char client, const char origin) {
	if (last_written != NULL \|\| user != NULL \|\| client != NULL \|\| origin != NULL) {
	return crm_strdup_printf("%s%s%s%s%s%s%s",
	last_written ? last_written : "",
	user ? " by " : "",
	user ? user : "",
	client ? " via " : "",
	client ? client : "",
	origin ? " on " : "",
	origin ? origin : "");
	} else {
	return strdup("");
	}
	}

	static char *
	op_history_string(xmlNode xml_op, const char task, const char *interval_ms_s,
	int rc, bool print_timing) {
	const char *call = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	char *interval_str = NULL;
	char *buf = NULL;

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *pair = pcmk__format_nvpair("interval", interval_ms_s, "ms");
	interval_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	if (print_timing) {
	char *last_change_str = NULL;
	char *exec_str = NULL;
	char *queue_str = NULL;

	const char *value = NULL;

	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok)
	&& (epoch > 0)) {
	char *epoch_str = pcmk__epoch2str(&epoch, 0);

	last_change_str = crm_strdup_printf(" %s=\"%s\"",
	PCMK_XA_LAST_RC_CHANGE,
	pcmk__s(epoch_str, ""));
	free(epoch_str);
	}

	value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	if (value) {
	char *pair = pcmk__format_nvpair(PCMK_XA_EXEC_TIME, value, "ms");
	exec_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	if (value) {
	char *pair = pcmk__format_nvpair(PCMK_XA_QUEUE_TIME, value, "ms");
	queue_str = crm_strdup_printf(" %s", pair);
	free(pair);
	}

	buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
	interval_str ? interval_str : "",
	last_change_str ? last_change_str : "",
	exec_str ? exec_str : "",
	queue_str ? queue_str : "",
	rc, services_ocf_exitcode_str(rc));

	if (last_change_str) {
	free(last_change_str);
	}

	if (exec_str) {
	free(exec_str);
	}

	if (queue_str) {
	free(queue_str);
	}
	} else {
	buf = crm_strdup_printf("(%s) %s%s%s", call, task,
	interval_str ? ":" : "",
	interval_str ? interval_str : "");
	}

	if (interval_str) {
	free(interval_str);
	}

	return buf;
	}

	static char *
	resource_history_string(pcmk_resource_t rsc, const char rsc_id, bool all,
	int failcount, time_t last_failure) {
	char *buf = NULL;

	if (rsc == NULL) {
	buf = crm_strdup_printf("%s: orphan", rsc_id);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *failcount_s = NULL;
	char *lastfail_s = NULL;

	if (failcount > 0) {
	failcount_s = crm_strdup_printf(" %s=%d", PCMK__FAIL_COUNT_PREFIX,
	failcount);
	} else {
	failcount_s = strdup("");
	}
	if (last_failure > 0) {
	buf = pcmk__epoch2str(&last_failure, 0);
	lastfail_s = crm_strdup_printf(" %s='%s'",
	PCMK__LAST_FAILURE_PREFIX, buf);
	free(buf);
	}

	buf = crm_strdup_printf("%s: " PCMK_META_MIGRATION_THRESHOLD "=%d%s%s",
	rsc_id, rsc->migration_threshold, failcount_s,
	lastfail_s? lastfail_s : "");
	free(failcount_s);
	free(lastfail_s);
	} else {
	buf = crm_strdup_printf("%s:", rsc_id);
	}

	return buf;
	}

	/*!
	* \internal
	* \brief Get a node's feature set for status display purposes
	*
	* \param[in] node Node to check
	*
	* \return String representation of feature set if the node is fully up (using
	* "<3.15.1" for older nodes that don't set the #feature-set attribute),
	* otherwise NULL
	*/
	static const char *
	get_node_feature_set(const pcmk_node_t *node)
	{
	if (node->details->online && node->details->expected_up
	&& !pe__is_guest_or_remote_node(node)) {

	const char *feature_set = g_hash_table_lookup(node->details->attrs,
	CRM_ATTR_FEATURE_SET);

	/* The feature set attribute is present since 3.15.1. If it is missing,
	* then the node must be running an earlier version.
	*/
	return pcmk__s(feature_set, "<3.15.1");
	}
	return NULL;
	}

	static bool
	is_mixed_version(pcmk_scheduler_t *scheduler)
	{
	const char *feature_set = NULL;
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;
	const char *node_feature_set = get_node_feature_set(node);
	if (node_feature_set != NULL) {
	if (feature_set == NULL) {
	feature_set = node_feature_set;
	} else if (strcmp(feature_set, node_feature_set) != 0) {
	return true;
	}
	}
	}
	return false;
	}

	static char *
	formatted_xml_buf(const pcmk_resource_t *rsc, bool raw)
	{
	if (raw) {
	return dump_xml_formatted(rsc->orig_xml ? rsc->orig_xml : rsc->xml);
	} else {
	return dump_xml_formatted(rsc->xml);
	}
	}

	#define XPATH_DC_VERSION "//" XML_CIB_TAG_NVPAIR \
	"[@" PCMK_XA_NAME "='" PCMK_OPT_DC_VERSION "']"

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	if (pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
	"enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
	static int
	cluster_summary_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	int rc = pcmk_rc_no_output;
	const char *stack_s = get_cluster_stack(scheduler);

	if (pcmk_is_set(section_opts, pcmk_section_stack)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-stack", stack_s, pcmkd_state);
	}

	/* Always print DC if none, even if not requested */
	if ((scheduler->dc_node == NULL)
	\|\| pcmk_is_set(section_opts, pcmk_section_dc)) {
	xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
	scheduler->input, LOG_DEBUG);
	const char *dc_version_s = dc_version?
	crm_element_value(dc_version, PCMK_XA_VALUE)
	: NULL;
	const char *quorum = crm_element_value(scheduler->input,
	PCMK_XA_HAVE_QUORUM);
	char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
	bool mixed_version = is_mixed_version(scheduler);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-dc", scheduler->dc_node, quorum,
	dc_version_s, dc_name, mixed_version);
	free(dc_name);
	}

	if (pcmk_is_set(section_opts, pcmk_section_times)) {
	const char *last_written = crm_element_value(scheduler->input,
	PCMK_XA_CIB_LAST_WRITTEN);
	const char *user = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_USER);
	const char *client = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_CLIENT);
	const char *origin = crm_element_value(scheduler->input,
	PCMK_XA_UPDATE_ORIGIN);

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-times",
	scheduler->localhost, last_written, user, client, origin);
	}

	if (pcmk_is_set(section_opts, pcmk_section_counts)) {
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
	out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
	scheduler->ninstances, scheduler->disabled_resources,
	scheduler->blocked_resources);
	}

	if (pcmk_is_set(section_opts, pcmk_section_options)) {
	/* Kind of a hack - close the list we may have opened earlier in this
	* function so we can put all the options into their own list. We
	* only want to do this on HTML output, though.
	*/
	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	out->begin_list(out, NULL, NULL, "Config Options");
	out->message(out, "cluster-options", scheduler);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);

	if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
	if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	return rc;
	}

	char *
	pe__node_display_name(pcmk_node_t *node, bool print_detail)
	{
	char *node_name;
	const char *node_host = NULL;
	const char *node_id = NULL;
	int name_len;

	CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));

	/* Host is displayed only if this is a guest node and detail is requested */
	if (print_detail && pe__is_guest_node(node)) {
	const pcmk_resource_t *container = node->details->remote_rsc->container;
	const pcmk_node_t *host_node = pcmk__current_node(container);

	if (host_node && host_node->details) {
	node_host = host_node->details->uname;
	}
	if (node_host == NULL) {
	node_host = ""; /* so we at least get "uname@" to indicate guest */
	}
	}

	/* Node ID is displayed if different from uname and detail is requested */
	if (print_detail && !pcmk__str_eq(node->details->uname, node->details->id, pcmk__str_casei)) {
	node_id = node->details->id;
	}

	/* Determine name length */
	name_len = strlen(node->details->uname) + 1;
	if (node_host) {
	name_len += strlen(node_host) + 1; /* "@node_host" */
	}
	if (node_id) {
	name_len += strlen(node_id) + 3; /* + " (node_id)" */
	}

	/* Allocate and populate display name */
	node_name = malloc(name_len);
	CRM_ASSERT(node_name != NULL);
	strcpy(node_name, node->details->uname);
	if (node_host) {
	strcat(node_name, "@");
	strcat(node_name, node_host);
	}
	if (node_id) {
	strcat(node_name, " (");
	strcat(node_name, node_id);
	strcat(node_name, ")");
	}
	return node_name;
	}

	int
	pe__name_and_nvpairs_xml(pcmk__output_t out, bool is_list, const char tag_name
	, size_t pairs_count, ...)
	{
	xmlNodePtr xml_node = NULL;
	va_list args;

	CRM_ASSERT(tag_name != NULL);

	xml_node = pcmk__output_xml_peek_parent(out);
	CRM_ASSERT(xml_node != NULL);
	xml_node = create_xml_node(xml_node, tag_name);

	va_start(args, pairs_count);
	while(pairs_count--) {
	const char param_name = va_arg(args, const char );
	const char param_value = va_arg(args, const char );
	if (param_name && param_value) {
	crm_xml_add(xml_node, param_name, param_value);
	}
	};
	va_end(args);

	if (is_list) {
	pcmk__output_xml_push_parent(out, xml_node);
	}
	return pcmk_rc_ok;
	}

	static const char *
	role_desc(enum rsc_role_e role)
	{
	if (role == pcmk_role_promoted) {
	#ifdef PCMK__COMPAT_2_0
	return "as " PCMK__ROLE_PROMOTED_LEGACY " ";
	#else
	return "in " PCMK__ROLE_PROMOTED " role ";
	#endif
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	pcmk__output_create_html_node(out, "li", NULL, NULL, buf);

	free(node_name);
	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts = va_arg(args, uint32_t);

	char *node_name = pe__node_display_name(pe_node,
	pcmk_is_set(show_opts, pcmk_show_node_id));
	out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
	location->id, location->rsc->id,
	role_desc(location->role_filter), node_name);

	free(node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban", "pcmk_node_t ", "pcmk__location_t ", "uint32_t")
	static int
	ban_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t pe_node = va_arg(args, pcmk_node_t );
	pcmk__location_t location = va_arg(args, pcmk__location_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
	char *weight_s = pcmk__itoa(pe_node->weight);

	pcmk__output_create_xml_node(out, "ban",
	PCMK_XA_ID, location->id,
	"resource", location->rsc->id,
	"node", pe_node->details->uname,
	"weight", weight_s,
	"promoted-only", promoted_only,
	/* This is a deprecated alias for
	* promoted_only. Removing it will break
	* backward compatibility of the API schema,
	* which will require an API schema major
	* version bump.
	*/
	"master_only", promoted_only,
	NULL);

	free(weight_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t ", "const char ", "GList *",
	"uint32_t", "bool")
	static int
	ban_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	const char prefix = va_arg(args, const char );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	GList gIter, gIter2;
	int rc = pcmk_rc_no_output;

	/* Print each ban */
	for (gIter = scheduler->placement_constraints;
	gIter != NULL; gIter = gIter->next) {
	pcmk__location_t *location = gIter->data;
	const pcmk_resource_t *rsc = location->rsc;

	if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
	only_rsc, pcmk__str_star_matches)) {
	continue;
	}

	for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter2->data;

	if (node->weight < 0) {
	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
	out->message(out, "ban", node, location, show_opts);
	}
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_html(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *nnodes_str = crm_strdup_printf("%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	pcmk_create_html_node(nodes_node, "span", NULL, NULL, nnodes_str);
	free(nnodes_str);

	if (ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");

	s = crm_strdup_printf(", %d ", nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else if (ndisabled && !nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	ndisabled);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL, ")");
	} else if (!ndisabled && nblocked) {
	char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
	nresources, pcmk__plural_s(nresources),
	nblocked);
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);

	pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
	pcmk_create_html_node(resources_node, "span", NULL, NULL,
	" from further action due to failure)");
	} else {
	char *s = crm_strdup_printf("%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
	free(s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_text(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	out->list_item(out, NULL, "%d node%s configured",
	nnodes, pcmk__plural_s(nnodes));

	if (ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED, %d BLOCKED from "
	"further action due to failure)",
	nresources, pcmk__plural_s(nresources), ndisabled,
	nblocked);
	} else if (ndisabled && !nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d DISABLED)",
	nresources, pcmk__plural_s(nresources), ndisabled);
	} else if (!ndisabled && nblocked) {
	out->list_item(out, NULL, "%d resource instance%s configured "
	"(%d BLOCKED from further action "
	"due to failure)",
	nresources, pcmk__plural_s(nresources), nblocked);
	} else {
	out->list_item(out, NULL, "%d resource instance%s configured",
	nresources, pcmk__plural_s(nresources));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
	static int
	cluster_counts_xml(pcmk__output_t *out, va_list args) {
	unsigned int nnodes = va_arg(args, unsigned int);
	int nresources = va_arg(args, int);
	int ndisabled = va_arg(args, int);
	int nblocked = va_arg(args, int);

	xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "nodes_configured", NULL);
	xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "resources_configured", NULL);

	char *s = pcmk__itoa(nnodes);
	crm_xml_add(nodes_node, "number", s);
	free(s);

	s = pcmk__itoa(nresources);
	crm_xml_add(resources_node, "number", s);
	free(s);

	s = pcmk__itoa(ndisabled);
	crm_xml_add(resources_node, "disabled", s);
	free(s);

	s = pcmk__itoa(nblocked);
	crm_xml_add(resources_node, "blocked", s);
	free(s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Current DC: ");

	if (dc) {
	char *buf = crm_strdup_printf("%s (version %s) -", dc_name,
	dc_version_s ? dc_version_s : "unknown");
	pcmk_create_html_node(node, "span", NULL, NULL, buf);
	free(buf);

	if (mixed_version) {
	pcmk_create_html_node(node, "span", NULL, "warning",
	" MIXED-VERSION");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " partition");
	if (crm_is_true(quorum)) {
	pcmk_create_html_node(node, "span", NULL, NULL, " with");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", " WITHOUT");
	}
	pcmk_create_html_node(node, "span", NULL, NULL, " quorum");
	} else {
	pcmk_create_html_node(node, "span", NULL, "warning", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	out->list_item(out, "Current DC",
	"%s (version %s) - %spartition %s quorum",
	dc_name, dc_version_s ? dc_version_s : "unknown",
	mixed_version ? "MIXED-VERSION " : "",
	crm_is_true(quorum) ? "with" : "WITHOUT");
	} else {
	out->list_item(out, "Current DC", "NONE");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t ", "const char ", "const char *",
	"char *", "int")
	static int
	cluster_dc_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t dc = va_arg(args, pcmk_node_t );
	const char quorum = va_arg(args, const char );
	const char dc_version_s = va_arg(args, const char );
	char dc_name G_GNUC_UNUSED = va_arg(args, char );
	bool mixed_version = va_arg(args, int);

	if (dc) {
	pcmk__output_create_xml_node(out, "current_dc",
	"present", PCMK_VALUE_TRUE,
	PCMK_XA_VERSION, pcmk__s(dc_version_s, ""),
	PCMK_XA_NAME, dc->details->uname,
	PCMK_XA_ID, dc->details->id,
	"with_quorum", pcmk__btoa(crm_is_true(quorum)),
	"mixed_version", pcmk__btoa(mixed_version),
	NULL);
	} else {
	pcmk__output_create_xml_node(out, "current_dc",
	"present", PCMK_VALUE_FALSE,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
	static int
	cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
	unsigned long long flags = va_arg(args, unsigned long long);

	if (pcmk_is_set(flags, pcmk_sched_in_maintenance)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will not attempt to start, stop or recover services\n");
	return pcmk_rc_ok;
	} else if (pcmk_is_set(flags, pcmk_sched_stop_all)) {
	pcmk__formatted_printf(out, "\n * Resource management is DISABLED *\n");
	pcmk__formatted_printf(out, " The cluster will keep all resources stopped\n");
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_html(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "DISABLED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will not attempt to start, stop, or recover services)");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
	pcmk_create_html_node(node, "span", NULL, "bold", "STOPPED");
	pcmk_create_html_node(node, "span", NULL, NULL,
	" (the cluster will keep all resources stopped)");
	} else {
	out->list_item(out, NULL, "Resource management: enabled");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_log(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
	return out->info(out, "Resource management is DISABLED. The cluster will not attempt to start, stop or recover services.");
	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
	return out->info(out, "Resource management is DISABLED. The cluster has stopped all resources.");
	} else {
	return pcmk_rc_no_output;
	}
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_text(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	out->list_item(out, NULL, "STONITH of failed nodes enabled");
	} else {
	out->list_item(out, NULL, "STONITH of failed nodes disabled");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	out->list_item(out, NULL, "Cluster is symmetric");
	} else {
	out->list_item(out, NULL, "Cluster is asymmetric");
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	out->list_item(out, NULL, "No quorum policy: Freeze resources");
	break;

	case pcmk_no_quorum_stop:
	out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
	break;

	case pcmk_no_quorum_demote:
	out->list_item(out, NULL, "No quorum policy: Demote promotable "
	"resources and stop all other resources");
	break;

	case pcmk_no_quorum_ignore:
	out->list_item(out, NULL, "No quorum policy: Ignore");
	break;

	case pcmk_no_quorum_fence:
	out->list_item(out, NULL, "No quorum policy: Suicide");
	break;
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
	static int
	cluster_options_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );

	const char *no_quorum_policy = NULL;
	char *stonith_timeout_str = pcmk__itoa(scheduler->stonith_timeout);
	char priority_fencing_delay_str = pcmk__itoa(scheduler->priority_fencing_delay 1000);

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	no_quorum_policy = "freeze";
	break;

	case pcmk_no_quorum_stop:
	no_quorum_policy = "stop";
	break;

	case pcmk_no_quorum_demote:
	no_quorum_policy = "demote";
	break;

	case pcmk_no_quorum_ignore:
	no_quorum_policy = "ignore";
	break;

	case pcmk_no_quorum_fence:
	no_quorum_policy = "suicide";
	break;
	}

	pcmk__output_create_xml_node(out, "cluster_options",
	PCMK_OPT_STONITH_ENABLED,
	pcmk__flag_text(scheduler->flags,
	pcmk_sched_fencing_enabled),

	PCMK_OPT_SYMMETRIC_CLUSTER,
	pcmk__flag_text(scheduler->flags,
	pcmk_sched_symmetric_cluster),

	PCMK_OPT_NO_QUORUM_POLICY, no_quorum_policy,

	PCMK_OPT_MAINTENANCE_MODE,
	pcmk__flag_text(scheduler->flags,
	pcmk_sched_in_maintenance),

	PCMK_OPT_STOP_ALL_RESOURCES,
	pcmk__flag_text(scheduler->flags,
	pcmk_sched_stop_all),

	PCMK_OPT_STONITH_TIMEOUT "-ms",
	stonith_timeout_str,

	PCMK_OPT_PRIORITY_FENCING_DELAY "-ms",
	priority_fencing_delay_str,

	NULL);
	free(stonith_timeout_str);
	free(priority_fencing_delay_str);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_html(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);

	pcmk_create_html_node(node, "span", NULL, "bold", "Stack: ");
	pcmk_create_html_node(node, "span", NULL, NULL, stack_s);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	pcmk_create_html_node(node, "span", NULL, NULL, " (");
	pcmk_create_html_node(node, "span", NULL, NULL,
	pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	pcmk_create_html_node(node, "span", NULL, NULL, ")");
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_text(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	out->list_item(out, "Stack", "%s (%s)",
	stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
	} else {
	out->list_item(out, "Stack", "%s", stack_s);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
	static int
	cluster_stack_xml(pcmk__output_t *out, va_list args) {
	const char stack_s = va_arg(args, const char );
	enum pcmk_pacemakerd_state pcmkd_state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	const char *state_s = NULL;

	if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
	}

	pcmk__output_create_xml_node(out, "stack",
	PCMK_XA_TYPE, stack_s,
	"pacemakerd-state", state_s,
	NULL);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_html(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
	xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk_create_html_node(updated_node, "span", NULL, "bold", "Last updated: ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, time_s);

	if (our_nodename != NULL) {
	pcmk_create_html_node(updated_node, "span", NULL, NULL, " on ");
	pcmk_create_html_node(updated_node, "span", NULL, NULL, our_nodename);
	}

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	pcmk_create_html_node(changed_node, "span", NULL, "bold", "Last change: ");
	pcmk_create_html_node(changed_node, "span", NULL, NULL, time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_xml(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	pcmk__output_create_xml_node(out, "last_update",
	"time", time_s,
	PCMK_XA_ORIGIN, our_nodename,
	NULL);

	pcmk__output_create_xml_node(out, "last_change",
	"time", last_written ? last_written : "",
	"user", user ? user : "",
	"client", client ? client : "",
	PCMK_XA_ORIGIN, pcmk__s(origin, ""),
	NULL);

	free(time_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("cluster-times", "const char ", "const char ",
	"const char ", "const char ", "const char *")
	static int
	cluster_times_text(pcmk__output_t *out, va_list args) {
	const char our_nodename = va_arg(args, const char );
	const char last_written = va_arg(args, const char );
	const char user = va_arg(args, const char );
	const char client = va_arg(args, const char );
	const char origin = va_arg(args, const char );

	char *time_s = pcmk__epoch2str(NULL, 0);

	out->list_item(out, "Last updated", "%s%s%s",
	time_s, (our_nodename != NULL)? " on " : "",
	pcmk__s(our_nodename, ""));

	free(time_s);
	time_s = last_changed_string(last_written, user, client, origin);

	out->list_item(out, "Last change", " %s", time_s);

	free(time_s);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Display a failed action in less-technical natural language
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_friendly(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	char *rsc_id = NULL;
	char *task = NULL;
	guint interval_ms = 0;
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)
	\|\| !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {
	rsc_id = strdup("unknown resource");
	task = strdup("unknown action");
	interval_ms = 0;
	}
	CRM_ASSERT((rsc_id != NULL) && (task != NULL));

	str = g_string_sized_new(256); // Should be sufficient for most messages

	pcmk__g_strcat(str, rsc_id, " ", NULL);

	if (interval_ms != 0) {
	pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
	NULL);
	}
	pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
	node_name, NULL);

	if (status == PCMK_EXEC_DONE) {
	pcmk__g_strcat(str, " returned '", services_ocf_exitcode_str(rc), "'",
	NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
	}

	} else {
	pcmk__g_strcat(str, " could not be executed (",
	pcmk_exec_status_str(status), NULL);
	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ": ", exit_reason, NULL);
	}
	g_string_append_c(str, ')');
	}


	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *s = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str, " at ", s, NULL);
	free(s);
	}
	if (!pcmk__str_empty(exec_time)) {
	int exec_time_ms = 0;

	if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
	&& (exec_time_ms > 0)) {

	pcmk__g_strcat(str, " after ",
	pcmk__readable_interval(exec_time_ms), NULL);
	}
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	free(rsc_id);
	free(task);
	}

	/*!
	* \internal
	* \brief Display a failed action with technical details
	*
	* \param[in,out] out Output object to use for display
	* \param[in] xml_op XML containing failed action
	* \param[in] op_key Operation key of failed action
	* \param[in] node_name Where failed action occurred
	* \param[in] rc OCF exit code of failed action
	* \param[in] status Execution status of failed action
	* \param[in] exit_reason Exit reason given for failed action
	* \param[in] exec_time String containing execution time in milliseconds
	*/
	static void
	failed_action_technical(pcmk__output_t out, const xmlNode xml_op,
	const char op_key, const char node_name, int rc,
	int status, const char *exit_reason,
	const char *exec_time)
	{
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	const char *exit_status = services_ocf_exitcode_str(rc);
	const char *lrm_status = pcmk_exec_status_str(status);
	time_t last_change_epoch = 0;
	GString *str = NULL;

	if (pcmk__str_empty(op_key)) {
	op_key = "unknown operation";
	}
	if (pcmk__str_empty(exit_status)) {
	exit_status = "unknown exit status";
	}
	if (pcmk__str_empty(call_id)) {
	call_id = "unknown";
	}

	str = g_string_sized_new(256);

	g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
	op_key, node_name, exit_status, rc, call_id,
	lrm_status);

	if (!pcmk__str_empty(exit_reason)) {
	pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change_epoch) == pcmk_ok) {
	char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);

	pcmk__g_strcat(str,
	", " PCMK_XA_LAST_RC_CHANGE "="
	"'", last_change_str, "'", NULL);
	free(last_change_str);
	}
	if (!pcmk__str_empty(queue_time)) {
	pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
	}
	if (!pcmk__str_empty(exec_time)) {
	pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
	}

	out->list_item(out, NULL, "%s", str->str);
	g_string_free(str, TRUE);
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_default(pcmk__output_t *out, va_list args)
	{
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *op_key = pcmk__xe_history_key(xml_op);
	const char *node_name = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *exit_reason = crm_element_value(xml_op, PCMK_XA_EXIT_REASON);
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	int rc;
	int status;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	0);

	if (pcmk__str_empty(node_name)) {
	node_name = "unknown node";
	}

	if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
	failed_action_technical(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	} else {
	failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
	exit_reason, exec_time);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
	static int
	failed_action_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);

	const char *op_key = pcmk__xe_history_key(xml_op);
	const char *op_key_name = "op_key";
	int rc;
	int status;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *exit_reason = crm_element_value(xml_op, PCMK_XA_EXIT_REASON);

	time_t epoch = 0;
	char *rc_s = NULL;
	char *reason_s = crm_xml_escape(exit_reason ? exit_reason : "none");
	xmlNodePtr node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);
	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	0);

	rc_s = pcmk__itoa(rc);
	if (crm_element_value(xml_op, PCMK__XA_OPERATION_KEY) == NULL) {
	op_key_name = PCMK_XA_ID;
	}
	node = pcmk__output_create_xml_node(out, "failure",
	op_key_name, op_key,
	"node", uname,
	"exitstatus", services_ocf_exitcode_str(rc),
	"exitreason", pcmk__s(reason_s, ""),
	"exitcode", rc_s,
	"call", call_id,
	"status", pcmk_exec_status_str(status),
	NULL);
	free(rc_s);

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {

	const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	guint interval_ms = 0;
	char *interval_ms_s = NULL;
	char *rc_change = pcmk__epoch2str(&epoch,
	crm_time_log_date
	\|crm_time_log_timeofday
	\|crm_time_log_with_timezone);

	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);

	pcmk__xe_set_props(node,
	PCMK_XA_LAST_RC_CHANGE, rc_change,
	"queued", queue_time,
	"exec", crm_element_value(xml_op, PCMK_XA_EXEC_TIME),
	"interval", interval_ms_s,
	"task", crm_element_value(xml_op, PCMK_XA_OPERATION),
	NULL);

	free(interval_ms_s);
	free(rc_change);
	}

	free(reason_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t ", "GList ",
	"GList *", "uint32_t", "bool")
	static int
	failed_action_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *xml_op = NULL;
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(scheduler->failed) == 0) {
	return rc;
	}

	for (xml_op = pcmk__xml_first_child(scheduler->failed); xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {
	char *rsc = NULL;

	if (!pcmk__str_in_list(crm_element_value(xml_op, PCMK_XA_UNAME),
	only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	if (pcmk_xe_mask_probe_failure(xml_op)) {
	continue;
	}

	if (!parse_op_key(pcmk__xe_history_key(xml_op), &rsc, NULL, NULL)) {
	continue;
	}

	if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
	free(rsc);
	continue;
	}

	free(rsc);

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
	out->message(out, "failed-action", xml_op, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
	{
	int health = pe__node_health(node);

	// Cluster membership
	if (node->details->online) {
	pcmk_create_html_node(parent, "span", NULL, "online", " online");
	} else {
	pcmk_create_html_node(parent, "span", NULL, "offline", " OFFLINE");
	}

	// Standby mode
	if (node->details->standby_onfail && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail,"
	" with active resources)");
	} else if (node->details->standby_onfail) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby due to on-fail)");
	} else if (node->details->standby && (node->details->running_rsc != NULL)) {
	pcmk_create_html_node(parent, "span", NULL, "standby",
	" (in standby, with active resources)");
	} else if (node->details->standby) {
	pcmk_create_html_node(parent, "span", NULL, "standby", " (in standby)");
	}

	// Maintenance mode
	if (node->details->maintenance) {
	pcmk_create_html_node(parent, "span", NULL, "maint",
	" (in maintenance mode)");
	}

	// Node health
	if (health < 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_red",
	" (health is RED)");
	} else if (health == 0) {
	pcmk_create_html_node(parent, "span", NULL, "health_yellow",
	" (health is YELLOW)");
	}

	// Feature set
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	char *buf = crm_strdup_printf(", feature set %s", feature_set);
	pcmk_create_html_node(parent, "span", NULL, NULL, buf);
	free(buf);
	}
	}
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
	"GList ", "GList ")
	static int
	node_html(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (full) {
	xmlNodePtr item_node;

	if (pcmk_all_flags_set(show_opts, pcmk_show_brief \| pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "Resources");
	pe__rscs_brief_output(out, rscs, new_show_opts);
	out->end_list(out);
	}

	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc2 = NULL;
	int rc = pcmk_rc_no_output;

	out->begin_list(out, NULL, NULL, "%s:", node_name);
	item_node = pcmk__output_xml_create_parent(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
	status_node(node, item_node, show_opts);

	for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc2->data;
	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	pcmk__output_xml_pop_parent(out);
	out->end_list(out);

	} else {
	char *buf = crm_strdup_printf("%s:", node_name);

	item_node = pcmk__output_create_xml_node(out, "li", NULL);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	status_node(node, item_node, show_opts);

	free(buf);
	}
	} else {
	out->begin_list(out, NULL, NULL, "%s:", node_name);
	}

	free(node_name);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Get a human-friendly textual description of a node's status
	*
	* \param[in] node Node to check
	*
	* \return String representation of node's status
	*/
	static const char *
	node_text_status(const pcmk_node_t *node)
	{
	if (node->details->unclean) {
	if (node->details->online) {
	return "UNCLEAN (online)";

	} else if (node->details->pending) {
	return "UNCLEAN (pending)";

	} else {
	return "UNCLEAN (offline)";
	}

	} else if (node->details->pending) {
	return "pending";

	} else if (node->details->standby_onfail && node->details->online) {
	return "standby (on-fail)";

	} else if (node->details->standby) {
	if (node->details->online) {
	if (node->details->running_rsc) {
	return "standby (with active resources)";
	} else {
	return "standby";
	}
	} else {
	return "OFFLINE (standby)";
	}

	} else if (node->details->maintenance) {
	if (node->details->online) {
	return "maintenance";
	} else {
	return "OFFLINE (maintenance)";
	}

	} else if (node->details->online) {
	return "online";
	}

	return "OFFLINE";
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_text(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	GString *str = g_string_sized_new(64);
	int health = pe__node_health(node);

	// Create a summary line with node type, name, and status
	if (pe__is_guest_node(node)) {
	g_string_append(str, "GuestNode");
	} else if (pe__is_remote_node(node)) {
	g_string_append(str, "RemoteNode");
	} else {
	g_string_append(str, "Node");
	}
	pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);

	if (health < 0) {
	g_string_append(str, " (health is RED)");
	} else if (health == 0) {
	g_string_append(str, " (health is YELLOW)");
	}
	if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
	const char *feature_set = get_node_feature_set(node);
	if (feature_set != NULL) {
	pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
	}
	}

	/* If we're grouping by node, print its resources */
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (pcmk_is_set(show_opts, pcmk_show_brief)) {
	GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);

	if (rscs != NULL) {
	uint32_t new_show_opts = (show_opts \| pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	pe__rscs_brief_output(out, rscs, new_show_opts);

	out->end_list(out);
	out->end_list(out);

	g_list_free(rscs);
	}

	} else {
	GList *gIter2 = NULL;

	out->begin_list(out, NULL, NULL, "%s", str->str);
	out->begin_list(out, NULL, NULL, "Resources");

	for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter2->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}

	out->end_list(out);
	out->end_list(out);
	}
	} else {
	out->list_item(out, NULL, "%s", str->str);
	}

	g_string_free(str, TRUE);
	free(node_name);
	} else {
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
	out->begin_list(out, NULL, NULL, "Node: %s", node_name);
	free(node_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node", "pcmk_node_t ", "uint32_t", "bool", "GList ",
	"GList *")
	static int
	node_xml(pcmk__output_t *out, va_list args) {
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
	bool full = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	if (full) {
	const char *node_type = "unknown";
	char *length_s = pcmk__itoa(g_list_length(node->details->running_rsc));
	int health = pe__node_health(node);
	const char *health_s = NULL;
	const char *feature_set;

	switch (node->details->type) {
	case pcmk_node_variant_cluster:
	node_type = "member";
	break;
	case pcmk_node_variant_remote:
	node_type = "remote";
	break;
	case node_ping:
	node_type = "ping";
	break;
	}

	if (health < 0) {
	health_s = "red";
	} else if (health == 0) {
	health_s = "yellow";
	} else {
	health_s = "green";
	}

	feature_set = get_node_feature_set(node);

	pe__name_and_nvpairs_xml(out, true, "node", 15,
	PCMK_XA_NAME, node->details->uname,
	PCMK_XA_ID, node->details->id,
	"online", pcmk__btoa(node->details->online),
	"standby", pcmk__btoa(node->details->standby),
	"standby_onfail", pcmk__btoa(node->details->standby_onfail),
	"maintenance", pcmk__btoa(node->details->maintenance),
	"pending", pcmk__btoa(node->details->pending),
	"unclean", pcmk__btoa(node->details->unclean),
	"health", health_s,
	"feature_set", feature_set,
	"shutdown", pcmk__btoa(node->details->shutdown),
	"expected_up", pcmk__btoa(node->details->expected_up),
	"is_dc", pcmk__btoa(node->details->is_dc),
	"resources_running", length_s,
	PCMK_XA_TYPE, node_type);

	if (pe__is_guest_node(node)) {
	xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
	crm_xml_add(xml_node, "id_as_resource", node->details->remote_rsc->container->id);
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *lpc = NULL;

	for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	show_opts \|= pcmk_show_rsc_only;
	out->message(out, crm_map_element_name(rsc->xml), show_opts,
	rsc, only_node, only_rsc);
	}
	}

	free(length_s);

	out->end_list(out);
	} else {
	pcmk__output_xml_create_parent(out, "node",
	PCMK_XA_NAME, node->details->uname,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_text(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}
	if (v <= 0) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
	} else if (v < expected_score) {
	out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}
	} else {
	out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_html(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	if (add_extra) {
	int v;
	char *s = crm_strdup_printf("%s: %s", name, value);
	xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);

	if (value == NULL) {
	v = 0;
	} else {
	pcmk__scan_min_int(value, &v, INT_MIN);
	}

	pcmk_create_html_node(item_node, "span", NULL, NULL, s);
	free(s);

	if (v <= 0) {
	pcmk_create_html_node(item_node, "span", NULL, "bold", "(connectivity is lost)");
	} else if (v < expected_score) {
	char *buf = crm_strdup_printf("(connectivity is degraded -- expected %d", expected_score);
	pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
	free(buf);
	}
	} else {
	out->list_item(out, NULL, "%s: %s", name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	gchar *node_str = NULL;
	char *last_change_str = NULL;

	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
	PCMK_EXEC_UNKNOWN);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const pcmk_node_t *node = pcmk__current_node(rsc);
	const char *target_role = g_hash_table_lookup(rsc->meta,
	PCMK_META_TARGET_ROLE);
	uint32_t show_opts = pcmk_show_rsc_only \| pcmk_show_pending;

	if (node == NULL) {
	node = rsc->pending_node;
	}

	node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
	show_opts, target_role, false);
	} else {
	node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change) == pcmk_ok) {
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
	PCMK_XA_LAST_RC_CHANGE,
	pcmk__trim(ctime(&last_change)),
	exec_time);
	}

	out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
	node_str, pcmk__xe_history_key(xml_op),
	crm_element_value(xml_op, PCMK_XA_UNAME),
	crm_element_value(xml_op, PCMK__XA_CALL_ID),
	crm_element_value(xml_op, PCMK__XA_RC_CODE),
	last_change_str ? last_change_str : "",
	pcmk_exec_status_str(status));

	g_free(node_str);
	free(last_change_str);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
	static int
	node_and_op_xml(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);

	pcmk_resource_t *rsc = NULL;
	const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	const char *rc_s = crm_element_value(xml_op, PCMK__XA_RC_CODE);
	const char *op_rsc = crm_element_value(xml_op, "resource");
	int status;
	time_t last_change = 0;
	xmlNode *node = NULL;

	pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS),
	&status, PCMK_EXEC_UNKNOWN);
	node = pcmk__output_create_xml_node(out, PCMK_XE_OPERATION,
	PCMK_XA_OP, pcmk__xe_history_key(xml_op),
	"node", uname,
	"call", call_id,
	"rc", rc_s,
	"status", pcmk_exec_status_str(status),
	NULL);

	rsc = pe_find_resource(scheduler->resources, op_rsc);

	if (rsc) {
	const char *class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	const char *provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	bool has_provider = pcmk_is_set(pcmk_get_ra_caps(class),
	pcmk_ra_cap_provider);

	char *agent_tuple = crm_strdup_printf("%s:%s:%s",
	class,
	(has_provider? provider : ""),
	kind);

	pcmk__xe_set_props(node,
	PCMK_XA_RSC, rsc_printable_id(rsc),
	"agent", agent_tuple,
	NULL);
	free(agent_tuple);
	}

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&last_change) == pcmk_ok) {
	const char *last_rc_change = pcmk__trim(ctime(&last_change));
	const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);

	pcmk__xe_set_props(node,
	PCMK_XA_LAST_RC_CHANGE, last_rc_change,
	PCMK_XA_EXEC_TIME, exec_time,
	NULL);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute", "const char ", "const char ", "bool", "int")
	static int
	node_attribute_xml(pcmk__output_t *out, va_list args) {
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );
	bool add_extra = va_arg(args, int);
	int expected_score = va_arg(args, int);

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_ATTRIBUTE,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (add_extra) {
	char *buf = pcmk__itoa(expected_score);
	crm_xml_add(node, "expected", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
	"bool", "GList ", "GList ")
	static int
	node_attribute_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	int rc = pcmk_rc_no_output;

	/* Display each node's attributes */
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *node = gIter->data;

	GList *attr_list = NULL;
	GHashTableIter iter;
	gpointer key;

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	g_hash_table_iter_init(&iter, node->details->attrs);
	while (g_hash_table_iter_next (&iter, &key, NULL)) {
	attr_list = filter_attr_list(attr_list, key);
	}

	if (attr_list == NULL) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node, pcmk__str_star_matches\|pcmk__str_casei)) {
	g_list_free(attr_list);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");

	out->message(out, "node", node, show_opts, false, only_node, only_rsc);

	for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
	const char *name = aIter->data;
	const char *value = NULL;
	int expected_score = 0;
	bool add_extra = false;

	value = pe_node_attribute_raw(node, name);

	add_extra = add_extra_info(node, node->details->running_rsc,
	scheduler, name, &expected_score);

	/* Print attribute name and value */
	out->message(out, "node-attribute", name, value, add_extra,
	expected_score);
	}

	g_list_free(attr_list);
	out->end_list(out);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s capacity:",
	comment, pcmk__node_name(node));

	g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t ", "const char ")
	static int
	node_capacity_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char comment = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "capacity",
	"node", node->details->uname,
	"comment", comment,
	NULL);
	g_hash_table_foreach(node->details->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t ", "pcmk_node_t ",
	"xmlNodePtr", "GList ", "GList ", "uint32_t", "uint32_t")
	static int
	node_history_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	xmlNode node_state = va_arg(args, xmlNode );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);

	xmlNode *lrm_rsc = NULL;
	xmlNode *rsc_entry = NULL;
	int rc = pcmk_rc_no_output;

	lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);

	/* Print history of each of the node's resources */
	for (rsc_entry = first_named_child(lrm_rsc, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {
	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	/* We can't use is_filtered here to filter group resources. For is_filtered,
	* we have to decide whether to check the parent or not. If we check the
	* parent, all elements of a group will always be printed because that's how
	* is_filtered works for groups. If we do not check the parent, sometimes
	* this will filter everything out.
	*
	* For other resource types, is_filtered is okay.
	*/
	if (parent->variant == pcmk_rsc_variant_group) {
	if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
	pcmk__str_star_matches)
	&& !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
	pcmk__str_star_matches)) {
	continue;
	}
	} else {
	if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
	continue;
	}
	}

	if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	if (failcount <= 0) {
	continue;
	}

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-history", rsc, rsc_id, false,
	failcount, last_failure, false);
	} else {
	GList *op_list = get_operation_list(rsc_entry);
	pcmk_resource_t *rsc = NULL;

	if (op_list == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources,
	crm_element_value(rsc_entry, PCMK_XA_ID));

	if (rc == pcmk_rc_no_output) {
	rc = pcmk_rc_ok;
	out->message(out, "node", node, show_opts, false, only_node,
	only_rsc);
	}

	out->message(out, "resource-operation-list", scheduler, rsc, node,
	op_list, show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_html(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_text(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	/* space-separated lists of node names */
	GString *online_nodes = NULL;
	GString *online_remote_nodes = NULL;
	GString *online_guest_nodes = NULL;
	GString *offline_nodes = NULL;
	GString *offline_remote_nodes = NULL;

	int rc = pcmk_rc_no_output;

	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;
	char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	free(node_name);
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");

	// Determine whether to display node individually or in a list
	if (node->details->unclean \|\| node->details->pending
	\|\| (node->details->standby_onfail && node->details->online)
	\|\| node->details->standby \|\| node->details->maintenance
	\|\| pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
	\|\| pcmk_is_set(show_opts, pcmk_show_feature_set)
	\|\| (pe__node_health(node) <= 0)) {
	// Display node individually

	} else if (node->details->online) {
	// Display online node in a list
	if (pe__is_guest_node(node)) {
	pcmk__add_word(&online_guest_nodes, 1024, node_name);

	} else if (pe__is_remote_node(node)) {
	pcmk__add_word(&online_remote_nodes, 1024, node_name);

	} else {
	pcmk__add_word(&online_nodes, 1024, node_name);
	}
	free(node_name);
	continue;

	} else {
	// Display offline node in a list
	if (pe__is_remote_node(node)) {
	pcmk__add_word(&offline_remote_nodes, 1024, node_name);

	} else if (pe__is_guest_node(node)) {
	/* ignore offline guest nodes */

	} else {
	pcmk__add_word(&offline_nodes, 1024, node_name);
	}
	free(node_name);
	continue;
	}

	/* If we get here, node is in bad state, or we're grouping by node */
	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	free(node_name);
	}

	/* If we're not grouping by node, summarize nodes by status */
	if (online_nodes != NULL) {
	out->list_item(out, "Online", "[ %s ]",
	(const char *) online_nodes->str);
	g_string_free(online_nodes, TRUE);
	}
	if (offline_nodes != NULL) {
	out->list_item(out, "OFFLINE", "[ %s ]",
	(const char *) offline_nodes->str);
	g_string_free(offline_nodes, TRUE);
	}
	if (online_remote_nodes) {
	out->list_item(out, "RemoteOnline", "[ %s ]",
	(const char *) online_remote_nodes->str);
	g_string_free(online_remote_nodes, TRUE);
	}
	if (offline_remote_nodes) {
	out->list_item(out, "RemoteOFFLINE", "[ %s ]",
	(const char *) offline_remote_nodes->str);
	g_string_free(offline_remote_nodes, TRUE);
	}
	if (online_guest_nodes != NULL) {
	out->list_item(out, "GuestOnline", "[ %s ]",
	(const char *) online_guest_nodes->str);
	g_string_free(online_guest_nodes, TRUE);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList ", "GList ", "GList *", "uint32_t", "bool")
	static int
	node_list_xml(pcmk__output_t *out, va_list args) {
	GList nodes = va_arg(args, GList );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer G_GNUC_UNUSED = va_arg(args, int);

	/* PCMK_XE_NODES acts as the list's element name for CLI tools that force
	* --xml-simple-list. Otherwise PCMK_XE_NODES is the value of the list's
	* PCMK_XA_NAME attribute.
	*/
	out->begin_list(out, NULL, NULL, PCMK_XE_NODES);
	for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t node = (pcmk_node_t ) gIter->data;

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	out->message(out, "node", node, show_opts, true, only_node, only_rsc);
	}
	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t ", "GList ", "GList *",
	"uint32_t", "uint32_t", "bool")
	static int
	node_summary(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	uint32_t section_opts = va_arg(args, uint32_t);
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_spacer = va_arg(args, int);

	xmlNode *node_state = NULL;
	xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
	PCMK_XE_STATUS);
	int rc = pcmk_rc_no_output;

	if (xmlChildElementCount(cib_status) == 0) {
	return rc;
	}

	- for (node_state = first_named_child(cib_status, XML_CIB_TAG_STATE);
	+ for (node_state = first_named_child(cib_status, PCMK__XE_NODE_STATE);
	node_state != NULL; node_state = crm_next_same_xml(node_state)) {
	pcmk_node_t *node = pe_find_node_id(scheduler->nodes, ID(node_state));

	if (!node \|\| !node->details \|\| !node->details->online) {
	continue;
	}

	if (!pcmk__str_in_list(node->details->uname, only_node,
	pcmk__str_star_matches\|pcmk__str_casei)) {
	continue;
	}

	PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
	pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");

	out->message(out, "node-history-list", scheduler, node, node_state,
	only_node, only_rsc, section_opts, show_opts);
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	if (rsc) {
	out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
	prefix, rsc->id, uname, score);
	} else {
	out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t ", "const char ",
	"const char ", "const char ")
	static int
	node_weight_xml(pcmk__output_t *out, va_list args)
	{
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	const char prefix = va_arg(args, const char );
	const char uname = va_arg(args, const char );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "node_weight",
	"function", prefix,
	"node", uname,
	PCMK_XA_SCORE, score,
	NULL);

	if (rsc) {
	crm_xml_add(node, PCMK_XA_ID, rsc->id);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_text(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
	pcmk_is_set(show_opts, pcmk_show_timing));

	out->list_item(out, NULL, "%s", buf);

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char ", "const char ", "int", "uint32_t")
	static int
	op_history_xml(pcmk__output_t *out, va_list args) {
	xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
	const char task = va_arg(args, const char );
	const char interval_ms_s = va_arg(args, const char );
	int rc = va_arg(args, int);
	uint32_t show_opts = va_arg(args, uint32_t);

	const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
	char *rc_s = pcmk__itoa(rc);
	xmlNodePtr node = pcmk__output_create_xml_node(out, "operation_history",
	"call", call_id,
	"task", task,
	"rc", rc_s,
	"rc_text", services_ocf_exitcode_str(rc),
	NULL);
	free(rc_s);

	if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
	char *s = crm_strdup_printf("%sms", interval_ms_s);
	crm_xml_add(node, "interval", s);
	free(s);
	}

	if (pcmk_is_set(show_opts, pcmk_show_timing)) {
	const char *value = NULL;
	time_t epoch = 0;

	if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&epoch) == pcmk_ok) && (epoch > 0)) {
	char *s = pcmk__epoch2str(&epoch, 0);
	crm_xml_add(node, PCMK_XA_LAST_RC_CHANGE, s);
	free(s);
	}

	value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, PCMK_XA_EXEC_TIME, s);
	free(s);
	}
	value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
	if (value) {
	char *s = crm_strdup_printf("%sms", value);
	crm_xml_add(node, PCMK_XA_QUEUE_TIME, s);
	free(s);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	out->list_item(out, NULL, "%s promotion score on %s: %s",
	child_rsc->id,
	chosen? chosen->details->uname : "none",
	score);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	promotion_score_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t child_rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t chosen = va_arg(args, pcmk_node_t );
	const char score = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, "promotion_score",
	PCMK_XA_ID, child_rsc->id,
	PCMK_XA_SCORE, score,
	NULL);

	if (chosen) {
	crm_xml_add(node, "node", chosen->details->uname);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
	static int
	resource_config_text(pcmk__output_t *out, va_list args) {
	const pcmk_resource_t rsc = va_arg(args, const pcmk_resource_t );
	bool raw = va_arg(args, int);

	char *rsc_xml = formatted_xml_buf(rsc, raw);

	pcmk__formatted_printf(out, "Resource XML:\n");
	out->output_xml(out, "xml", rsc_xml);

	free(rsc_xml);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);

	if (as_header) {
	out->begin_list(out, NULL, NULL, "%s", buf);
	} else {
	out->list_item(out, NULL, "%s", buf);
	}

	free(buf);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t ", "const char ",
	"bool", "int", "time_t", "bool")
	static int
	resource_history_xml(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char rsc_id = va_arg(args, const char );
	bool all = va_arg(args, int);
	int failcount = va_arg(args, int);
	time_t last_failure = va_arg(args, time_t);
	bool as_header = va_arg(args, int);

	xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource_history",
	PCMK_XA_ID, rsc_id,
	NULL);

	if (rsc == NULL) {
	pcmk__xe_set_bool_attr(node, "orphan", true);
	} else if (all \|\| failcount \|\| last_failure > 0) {
	char *migration_s = pcmk__itoa(rsc->migration_threshold);

	pcmk__xe_set_props(node, "orphan", PCMK_VALUE_FALSE,
	PCMK_META_MIGRATION_THRESHOLD, migration_s,
	NULL);
	free(migration_s);

	if (failcount > 0) {
	char *s = pcmk__itoa(failcount);

	crm_xml_add(node, PCMK__FAIL_COUNT_PREFIX, s);
	free(s);
	}

	if (last_failure > 0) {
	char *s = pcmk__epoch2str(&last_failure, 0);

	crm_xml_add(node, PCMK__LAST_FAILURE_PREFIX, s);
	free(s);
	}
	}

	if (!as_header) {
	pcmk__output_xml_pop_parent(out);
	}

	return pcmk_rc_ok;
	}

	static void
	print_resource_header(pcmk__output_t *out, uint32_t show_opts)
	{
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	/* Active resources have already been printed by node */
	out->begin_list(out, NULL, NULL, "Inactive Resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->begin_list(out, NULL, NULL, "Full List of Resources");
	} else {
	out->begin_list(out, NULL, NULL, "Active Resources");
	}
	}


	PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
	"GList ", "GList ", "bool")
	static int
	resource_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	uint32_t show_opts = va_arg(args, uint32_t);
	bool print_summary = va_arg(args, int);
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );
	bool print_spacer = va_arg(args, int);

	GList *rsc_iter;
	int rc = pcmk_rc_no_output;
	bool printed_header = false;

	/* If we already showed active resources by node, and
	* we're not showing inactive resources, we have nothing to do
	*/
	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
	!pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	return rc;
	}

	/* If we haven't already printed resources grouped by node,
	* and brief output was requested, print resource summary */
	if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	GList *rscs = pe__filter_rsc_list(scheduler->resources, only_rsc);

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;

	rc = pe__rscs_brief_output(out, rscs, show_opts);
	g_list_free(rscs);
	}

	/* For each resource, display it if appropriate */
	for (rsc_iter = scheduler->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rsc_iter->data;
	int x;

	/* Complex resources may have some sub-resources active and some inactive */
	gboolean is_active = rsc->fns->active(rsc, TRUE);
	gboolean partially_active = rsc->fns->active(rsc, FALSE);

	/* Skip inactive orphans (deleted but still in CIB) */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed) && !is_active) {
	continue;

	/* Skip active resources if we already displayed them by node */
	} else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	if (is_active) {
	continue;
	}

	/* Skip primitives already counted in a brief summary */
	} else if (pcmk_is_set(show_opts, pcmk_show_brief)
	&& (rsc->variant == pcmk_rsc_variant_primitive)) {
	continue;

	/* Skip resources that aren't at least partially active,
	* unless we're displaying inactive resources
	*/
	} else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	continue;

	} else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
	continue;
	}

	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	/* Print this resource */
	x = out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc,
	only_node, only_rsc);
	if (x == pcmk_rc_ok) {
	rc = pcmk_rc_ok;
	}
	}

	if (print_summary && rc != pcmk_rc_ok) {
	if (!printed_header) {
	PCMK__OUTPUT_SPACER_IF(out, print_spacer);
	print_resource_header(out, show_opts);
	printed_header = true;
	}

	if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
	out->list_item(out, NULL, "No inactive resources");
	} else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
	out->list_item(out, NULL, "No resources");
	} else {
	out->list_item(out, NULL, "No active resources");
	}
	}

	if (printed_header) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
	"pcmk_resource_t ", "pcmk_node_t ", "GList *", "uint32_t")
	static int
	resource_operation_list(pcmk__output_t *out, va_list args)
	{
	pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
	pcmk_scheduler_t *);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	GList op_list = va_arg(args, GList );
	uint32_t show_opts = va_arg(args, uint32_t);

	GList *gIter = NULL;
	int rc = pcmk_rc_no_output;

	/* Print each operation */
	for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode xml_op = (xmlNode ) gIter->data;
	const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	const char *interval_ms_s = crm_element_value(xml_op,
	PCMK_META_INTERVAL);
	const char *op_rc = crm_element_value(xml_op, PCMK__XA_RC_CODE);
	int op_rc_i;

	pcmk__scan_min_int(op_rc, &op_rc_i, 0);

	/* Display 0-interval monitors as "probe" */
	if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
	&& pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches \| pcmk__str_casei)) {
	task = "probe";
	}

	/* If this is the first printed operation, print heading for resource */
	if (rc == pcmk_rc_no_output) {
	time_t last_failure = 0;
	int failcount = pe_get_failcount(node, rsc, &last_failure,
	pcmk__fc_default, NULL);

	out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
	failcount, last_failure, true);
	rc = pcmk_rc_ok;
	}

	/* Print the operation */
	out->message(out, "op-history", xml_op, task, interval_ms_s,
	op_rc_i, show_opts);
	}

	/* Free the list we created (no need to free the individual items) */
	g_list_free(op_list);

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
	fn, rsc->id, pcmk__node_name(node));

	g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
	out->list_item(out, NULL, "%s", dump_text);
	free(dump_text);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t ", "pcmk_node_t ",
	"const char *")
	static int
	resource_util_xml(pcmk__output_t *out, va_list args)
	{
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );
	const char fn = va_arg(args, const char );

	xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "utilization",
	"resource", rsc->id,
	"node", node->details->uname,
	"function", fn,
	NULL);
	g_hash_table_foreach(rsc->utilization, add_dump_node, xml_node);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_html(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, NULL, "%s:\t%s%s %s=\"%s\"", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, NULL, "%s:\t%s%s", ticket->id,
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_text(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	if (ticket->last_granted > -1) {
	char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);

	out->list_item(out, ticket->id, "%s%s %s=\"%s\"",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "",
	"last-granted", pcmk__s(epoch_str, ""));
	free(epoch_str);
	} else {
	out->list_item(out, ticket->id, "%s%s",
	ticket->granted ? "granted" : "revoked",
	ticket->standby ? " [standby]" : "");
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
	static int
	ticket_xml(pcmk__output_t *out, va_list args) {
	pcmk_ticket_t ticket = va_arg(args, pcmk_ticket_t );

	xmlNodePtr node = NULL;

	node = pcmk__output_create_xml_node(out, "ticket",
	PCMK_XA_ID, ticket->id,
	"status", ticket->granted ? "granted" : "revoked",
	"standby", pcmk__btoa(ticket->standby),
	NULL);

	if (ticket->last_granted > -1) {
	char *buf = pcmk__epoch2str(&ticket->last_granted, 0);

	crm_xml_add(node, "last-granted", buf);
	free(buf);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("ticket-list", "pcmk_scheduler_t *", "bool")
	static int
	ticket_list(pcmk__output_t *out, va_list args) {
	pcmk_scheduler_t scheduler = va_arg(args, pcmk_scheduler_t );
	bool print_spacer = va_arg(args, int);

	GHashTableIter iter;
	gpointer key, value;

	if (g_hash_table_size(scheduler->tickets) == 0) {
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_SPACER_IF(out, print_spacer);

	/* Print section heading */
	out->begin_list(out, NULL, NULL, "Tickets");

	/* Print each ticket */
	g_hash_table_iter_init(&iter, scheduler->tickets);
	while (g_hash_table_iter_next(&iter, &key, &value)) {
	pcmk_ticket_t ticket = (pcmk_ticket_t ) value;
	out->message(out, "ticket", ticket);
	}

	/* Close section */
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "ban", "default", ban_text },
	{ "ban", "html", ban_html },
	{ "ban", "xml", ban_xml },
	{ "ban-list", "default", ban_list },
	{ "bundle", "default", pe__bundle_text },
	{ "bundle", "xml", pe__bundle_xml },
	{ "bundle", "html", pe__bundle_html },
	{ "clone", "default", pe__clone_default },
	{ "clone", "xml", pe__clone_xml },
	{ "cluster-counts", "default", cluster_counts_text },
	{ "cluster-counts", "html", cluster_counts_html },
	{ "cluster-counts", "xml", cluster_counts_xml },
	{ "cluster-dc", "default", cluster_dc_text },
	{ "cluster-dc", "html", cluster_dc_html },
	{ "cluster-dc", "xml", cluster_dc_xml },
	{ "cluster-options", "default", cluster_options_text },
	{ "cluster-options", "html", cluster_options_html },
	{ "cluster-options", "log", cluster_options_log },
	{ "cluster-options", "xml", cluster_options_xml },
	{ "cluster-summary", "default", cluster_summary },
	{ "cluster-summary", "html", cluster_summary_html },
	{ "cluster-stack", "default", cluster_stack_text },
	{ "cluster-stack", "html", cluster_stack_html },
	{ "cluster-stack", "xml", cluster_stack_xml },
	{ "cluster-times", "default", cluster_times_text },
	{ "cluster-times", "html", cluster_times_html },
	{ "cluster-times", "xml", cluster_times_xml },
	{ "failed-action", "default", failed_action_default },
	{ "failed-action", "xml", failed_action_xml },
	{ "failed-action-list", "default", failed_action_list },
	{ "group", "default", pe__group_default},
	{ "group", "xml", pe__group_xml },
	{ "maint-mode", "text", cluster_maint_mode_text },
	{ "node", "default", node_text },
	{ "node", "html", node_html },
	{ "node", "xml", node_xml },
	{ "node-and-op", "default", node_and_op },
	{ "node-and-op", "xml", node_and_op_xml },
	{ "node-capacity", "default", node_capacity },
	{ "node-capacity", "xml", node_capacity_xml },
	{ "node-history-list", "default", node_history_list },
	{ "node-list", "default", node_list_text },
	{ "node-list", "html", node_list_html },
	{ "node-list", "xml", node_list_xml },
	{ "node-weight", "default", node_weight },
	{ "node-weight", "xml", node_weight_xml },
	{ "node-attribute", "default", node_attribute_text },
	{ "node-attribute", "html", node_attribute_html },
	{ "node-attribute", "xml", node_attribute_xml },
	{ "node-attribute-list", "default", node_attribute_list },
	{ "node-summary", "default", node_summary },
	{ "op-history", "default", op_history_text },
	{ "op-history", "xml", op_history_xml },
	{ "primitive", "default", pe__resource_text },
	{ "primitive", "xml", pe__resource_xml },
	{ "primitive", "html", pe__resource_html },
	{ "promotion-score", "default", promotion_score },
	{ "promotion-score", "xml", promotion_score_xml },
	{ "resource-config", "default", resource_config },
	{ "resource-config", "text", resource_config_text },
	{ "resource-history", "default", resource_history_text },
	{ "resource-history", "xml", resource_history_xml },
	{ "resource-list", "default", resource_list },
	{ "resource-operation-list", "default", resource_operation_list },
	{ "resource-util", "default", resource_util },
	{ "resource-util", "xml", resource_util_xml },
	{ "ticket", "default", ticket_text },
	{ "ticket", "html", ticket_html },
	{ "ticket", "xml", ticket_xml },
	{ "ticket-list", "default", ticket_list },

	{ NULL, NULL, NULL }
	};

	void
	pe__register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index 316a13fd16..8e96e7f22a 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,5132 +1,5136 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <glib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <pe_status_private.h>

	CRM_TRACE_INIT_DATA(pe_status);

	// A (parsed) resource action history entry
	struct action_history {
	pcmk_resource_t *rsc; // Resource that history is for
	pcmk_node_t *node; // Node that history is for
	xmlNode *xml; // History entry XML

	// Parsed from entry XML
	const char *id; // XML ID of history entry
	const char *key; // Operation key of action
	const char *task; // Action name
	const char *exit_reason; // Exit reason given for result
	guint interval_ms; // Action interval
	int call_id; // Call ID of action
	int expected_exit_status; // Expected exit status of action
	int exit_status; // Actual exit status of action
	int execution_status; // Execution status of action
	};

	/* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
	* use pcmk__set_scheduler_flags()/pcmk__clear_scheduler_flags() so that the
	* flag is stringified more readably in log messages.
	*/
	#define set_config_flag(scheduler, option, flag) do { \
	const char *scf_value = pe_pref((scheduler)->config_hash, (option)); \
	if (scf_value != NULL) { \
	if (crm_is_true(scf_value)) { \
	(scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} else { \
	(scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, "Scheduler", \
	crm_system_name, (scheduler)->flags, \
	(flag), #flag); \
	} \
	} \
	} while(0)

	static void unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node,
	xmlNode xml_op, xmlNode *last_failure,
	enum action_fail_response *failed);
	static void determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node);
	static void add_node_attrs(const xmlNode xml_obj, pcmk_node_t node,
	bool overwrite, pcmk_scheduler_t *scheduler);
	static void determine_online_status(const xmlNode *node_state,
	pcmk_node_t *this_node,
	pcmk_scheduler_t *scheduler);

	static void unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler);


	static gboolean
	is_dangling_guest_node(pcmk_node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (pe__is_guest_or_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_removed_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	/*!
	* \brief Schedule a fence action for a node
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in,out] node Node to fence
	* \param[in] reason Text description of why fencing is needed
	* \param[in] priority_delay Whether to consider
	* \c PCMK_OPT_PRIORITY_FENCING_DELAY
	*/
	void
	pe_fence_node(pcmk_scheduler_t scheduler, pcmk_node_t node,
	const char *reason, bool priority_delay)
	{
	CRM_CHECK(node, return);

	/* A guest node is fenced by marking its container as failed */
	if (pe__is_guest_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc->container;

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing guest node %s "
	"(otherwise would because %s): "
	"its guest resource %s is unmanaged",
	pcmk__node_name(node), reason, rsc->id);
	} else {
	pcmk__sched_warn("Guest node %s will be fenced "
	"(by recovering its guest resource %s): %s",
	pcmk__node_name(node), rsc->id, reason);

	/* We don't mark the node as unclean because that would prevent the
	* node from running resources. We want to allow it to run resources
	* in this transition if the recovery succeeds.
	*/
	node->details->remote_requires_reset = TRUE;
	pcmk__set_rsc_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	}

	} else if (is_dangling_guest_node(node)) {
	crm_info("Cleaning up dangling connection for guest node %s: "
	"fencing was already done because %s, "
	"and guest resource no longer exists",
	pcmk__node_name(node), reason);
	pcmk__set_rsc_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);

	} else if (pe__is_remote_node(node)) {
	pcmk_resource_t *rsc = node->details->remote_rsc;

	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	crm_notice("Not fencing remote node %s "
	"(otherwise would because %s): connection is unmanaged",
	pcmk__node_name(node), reason);
	} else if(node->details->remote_requires_reset == FALSE) {
	node->details->remote_requires_reset = TRUE;
	pcmk__sched_warn("Remote node %s %s: %s",
	pcmk__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	}
	node->details->unclean = TRUE;
	// No need to apply PCMK_OPT_PRIORITY_FENCING_DELAY for remote nodes
	pe_fence_op(node, NULL, TRUE, reason, FALSE, scheduler);

	} else if (node->details->unclean) {
	crm_trace("Cluster node %s %s because %s",
	pcmk__node_name(node),
	pe_can_fence(scheduler, node)? "would also be fenced" : "also is unclean",
	reason);

	} else {
	pcmk__sched_warn("Cluster node %s %s: %s",
	pcmk__node_name(node),
	pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
	reason);
	node->details->unclean = TRUE;
	pe_fence_op(node, NULL, TRUE, reason, priority_delay, scheduler);
	}
	}

	// @TODO xpaths can't handle templates, rules, or id-refs

	// nvpair with provides or requires set to unfencing
	#define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR \
	"[(@" PCMK_XA_NAME "='" PCMK_STONITH_PROVIDES "'" \
	"or @" PCMK_XA_NAME "='" PCMK_META_REQUIRES "') " \
	"and @" PCMK_XA_VALUE "='" PCMK__VALUE_UNFENCING "']"

	// unfencing in rsc_defaults or any resource
	#define XPATH_ENABLE_UNFENCING \
	"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \
	"//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR \
	"\|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RSC_DEFAULTS \
	"/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR

	static void
	set_if_xpath(uint64_t flag, const char xpath, pcmk_scheduler_t scheduler)
	{
	xmlXPathObjectPtr result = NULL;

	if (!pcmk_is_set(scheduler->flags, flag)) {
	result = xpath_search(scheduler->input, xpath);
	if (result && (numXpathResults(result) > 0)) {
	pcmk__set_scheduler_flags(scheduler, flag);
	}
	freeXpathObject(result);
	}
	}

	gboolean
	unpack_config(xmlNode config, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	guint interval_ms = 0U;
	GHashTable *config_hash = pcmk__strkey_table(free, free);

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	scheduler->config_hash = config_hash;

	pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
	CIB_OPTIONS_FIRST, FALSE, scheduler);

	verify_pe_options(scheduler->config_hash);

	set_config_flag(scheduler, PCMK_OPT_ENABLE_STARTUP_PROBES,
	pcmk_sched_probe_resources);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_probe_resources)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_info("Watchdog-based self-fencing will be performed via SBD if "
	"fencing is required and " PCMK_OPT_STONITH_WATCHDOG_TIMEOUT
	" is nonzero");
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_fencing);
	}

	/* Set certain flags via xpath here, so they can be used before the relevant
	* configuration sections are unpacked.
	*/
	set_if_xpath(pcmk_sched_enable_unfencing, XPATH_ENABLE_UNFENCING,
	scheduler);

	value = pe_pref(scheduler->config_hash, PCMK_OPT_STONITH_TIMEOUT);
	pcmk_parse_interval_spec(value, &interval_ms);

	if (interval_ms >= INT_MAX) {
	scheduler->stonith_timeout = INT_MAX;
	} else {
	scheduler->stonith_timeout = (int) interval_ms;
	}
	crm_debug("STONITH timeout: %d", scheduler->stonith_timeout);

	set_config_flag(scheduler, PCMK_OPT_STONITH_ENABLED,
	pcmk_sched_fencing_enabled);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	crm_debug("STONITH of failed nodes is enabled");
	} else {
	crm_debug("STONITH of failed nodes is disabled");
	}

	scheduler->stonith_action = pe_pref(scheduler->config_hash,
	PCMK_OPT_STONITH_ACTION);
	if (!strcmp(scheduler->stonith_action, "poweroff")) {
	pcmk__warn_once(pcmk__wo_poweroff,
	"Support for " PCMK_OPT_STONITH_ACTION " of "
	"'poweroff' is deprecated and will be removed in a "
	"future release (use 'off' instead)");
	scheduler->stonith_action = PCMK_ACTION_OFF;
	}
	crm_trace("STONITH will %s nodes", scheduler->stonith_action);

	set_config_flag(scheduler, PCMK_OPT_CONCURRENT_FENCING,
	pcmk_sched_concurrent_fencing);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_concurrent_fencing)) {
	crm_debug("Concurrent fencing is enabled");
	} else {
	crm_debug("Concurrent fencing is disabled");
	}

	value = pe_pref(scheduler->config_hash,
	PCMK_OPT_PRIORITY_FENCING_DELAY);
	if (value) {
	pcmk_parse_interval_spec(value, &interval_ms);
	scheduler->priority_fencing_delay = (int) (interval_ms / 1000);
	crm_trace("Priority fencing delay is %ds",
	scheduler->priority_fencing_delay);
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ALL_RESOURCES,
	pcmk_sched_stop_all);
	crm_debug("Stop all active resources: %s",
	pcmk__flag_text(scheduler->flags, pcmk_sched_stop_all));

	set_config_flag(scheduler, PCMK_OPT_SYMMETRIC_CLUSTER,
	pcmk_sched_symmetric_cluster);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_NO_QUORUM_POLICY);

	if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_ignore;

	} else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_freeze;

	} else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_demote;

	} else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	int do_panic = 0;

	crm_element_value_int(scheduler->input, PCMK_XA_NO_QUORUM_PANIC,
	&do_panic);
	if (do_panic \|\| pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
	scheduler->no_quorum_policy = pcmk_no_quorum_fence;
	} else {
	crm_notice("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop': cluster has never had quorum");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}
	} else {
	pcmk__config_err("Resetting " PCMK_OPT_NO_QUORUM_POLICY
	" to 'stop' because fencing is disabled");
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	} else {
	scheduler->no_quorum_policy = pcmk_no_quorum_stop;
	}

	switch (scheduler->no_quorum_policy) {
	case pcmk_no_quorum_freeze:
	crm_debug("On loss of quorum: Freeze resources");
	break;
	case pcmk_no_quorum_stop:
	crm_debug("On loss of quorum: Stop ALL resources");
	break;
	case pcmk_no_quorum_demote:
	crm_debug("On loss of quorum: "
	"Demote promotable resources and stop other resources");
	break;
	case pcmk_no_quorum_fence:
	crm_notice("On loss of quorum: Fence all remaining nodes");
	break;
	case pcmk_no_quorum_ignore:
	crm_notice("On loss of quorum: Ignore");
	break;
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_RESOURCES,
	pcmk_sched_stop_removed_resources);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	crm_trace("Orphan resources are stopped");
	} else {
	crm_trace("Orphan resources are ignored");
	}

	set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_ACTIONS,
	pcmk_sched_cancel_removed_actions);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_cancel_removed_actions)) {
	crm_trace("Orphan resource actions are stopped");
	} else {
	crm_trace("Orphan resource actions are ignored");
	}

	value = pe_pref(scheduler->config_hash, PCMK__OPT_REMOVE_AFTER_STOP);
	if (value != NULL) {
	if (crm_is_true(value)) {
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_remove_after_stop);
	#ifndef PCMK__COMPAT_2_0
	pcmk__warn_once(pcmk__wo_remove_after,
	"Support for the " PCMK__OPT_REMOVE_AFTER_STOP
	" cluster property is deprecated and will be "
	"removed in a future release");
	#endif
	} else {
	pcmk__clear_scheduler_flags(scheduler,
	pcmk_sched_remove_after_stop);
	}
	}

	set_config_flag(scheduler, PCMK_OPT_MAINTENANCE_MODE,
	pcmk_sched_in_maintenance);
	crm_trace("Maintenance mode: %s",
	pcmk__flag_text(scheduler->flags, pcmk_sched_in_maintenance));

	set_config_flag(scheduler, PCMK_OPT_START_FAILURE_IS_FATAL,
	pcmk_sched_start_failure_fatal);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
	crm_trace("Start failures are always fatal");
	} else {
	crm_trace("Start failures are handled by failcount");
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	set_config_flag(scheduler, PCMK_OPT_STARTUP_FENCING,
	pcmk_sched_startup_fencing);
	}
	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	crm_trace("Unseen nodes will be fenced");
	} else {
	pcmk__warn_once(pcmk__wo_blind,
	"Blind faith: not fencing unseen nodes");
	}

	pe__unpack_node_health_scores(scheduler);

	scheduler->placement_strategy = pe_pref(scheduler->config_hash,
	PCMK_OPT_PLACEMENT_STRATEGY);
	crm_trace("Placement strategy: %s", scheduler->placement_strategy);

	set_config_flag(scheduler, PCMK_OPT_SHUTDOWN_LOCK,
	pcmk_sched_shutdown_lock);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	value = pe_pref(scheduler->config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
	pcmk_parse_interval_spec(value, &(scheduler->shutdown_lock));
	scheduler->shutdown_lock /= 1000;
	crm_trace("Resources will be locked to nodes that were cleanly "
	"shut down (locks expire after %s)",
	pcmk__readable_interval(scheduler->shutdown_lock));
	} else {
	crm_trace("Resources will not be locked to nodes that were cleanly "
	"shut down");
	}

	value = pe_pref(scheduler->config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
	pcmk_parse_interval_spec(value, &(scheduler->node_pending_timeout));
	scheduler->node_pending_timeout /= 1000;
	if (scheduler->node_pending_timeout == 0) {
	crm_trace("Do not fence pending nodes");
	} else {
	crm_trace("Fence pending nodes after %s",
	pcmk__readable_interval(scheduler->node_pending_timeout
	* 1000));
	}

	return TRUE;
	}

	pcmk_node_t *
	pe_create_node(const char id, const char uname, const char *type,
	const char score, pcmk_scheduler_t scheduler)
	{
	pcmk_node_t *new_node = NULL;

	if (pe_find_node(scheduler->nodes, uname) != NULL) {
	pcmk__config_warn("More than one node entry has name '%s'", uname);
	}

	new_node = calloc(1, sizeof(pcmk_node_t));
	if (new_node == NULL) {
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->details = calloc(1, sizeof(struct pe_node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	pcmk__sched_err("Could not allocate memory for node %s", uname);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->data_set = scheduler;

	if (pcmk__str_eq(type, "member", pcmk__str_null_matches \| pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_cluster;

	} else if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
	new_node->details->type = pcmk_node_variant_remote;
	pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_remote_nodes);

	} else {
	/* @COMPAT 'ping' is the default for backward compatibility, but it
	* should be changed to 'member' at a compatibility break
	*/
	if (!pcmk__str_eq(type, "ping", pcmk__str_casei)) {
	pcmk__config_warn("Node %s has unrecognized type '%s', "
	"assuming 'ping'", pcmk__s(uname, "without name"),
	type);
	}
	pcmk__warn_once(pcmk__wo_ping_node,
	"Support for nodes of type 'ping' (such as %s) is "
	"deprecated and will be removed in a future release",
	pcmk__s(uname, "unnamed node"));
	new_node->details->type = node_ping;
	}

	new_node->details->attrs = pcmk__strkey_table(free, free);

	if (pe__is_guest_or_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("cluster"));
	}

	new_node->details->utilization = pcmk__strkey_table(free, free);
	new_node->details->digest_cache = pcmk__strkey_table(free,
	pe__free_digests);

	scheduler->nodes = g_list_insert_sorted(scheduler->nodes, new_node,
	pe__cmp_node_name);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, pcmk_scheduler_t *data)
	{
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	const char *is_managed = NULL;

	for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
	attr_set = pcmk__xe_next(attr_set)) {

	if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	continue;
	}

	for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
	attr = pcmk__xe_next(attr)) {
	const char *value = crm_element_value(attr, PCMK_XA_VALUE);
	const char *name = crm_element_value(attr, PCMK_XA_NAME);

	if (name == NULL) { // Sanity
	continue;
	}

	if (strcmp(name, PCMK_META_REMOTE_NODE) == 0) {
	remote_name = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ADDR) == 0) {
	remote_server = value;

	} else if (strcmp(name, PCMK_META_REMOTE_PORT) == 0) {
	remote_port = value;

	} else if (strcmp(name, PCMK_META_REMOTE_CONNECT_TIMEOUT) == 0) {
	connect_timeout = value;

	} else if (strcmp(name, PCMK_META_REMOTE_ALLOW_MIGRATE) == 0) {
	remote_allow_migrate = value;

	} else if (strcmp(name, PCMK_META_IS_MANAGED) == 0) {
	is_managed = value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (pe_find_resource(data->resources, remote_name) != NULL) {
	return NULL;
	}

	pe_create_remote_xml(parent, remote_name, container_id,
	remote_allow_migrate, is_managed,
	connect_timeout, remote_server, remote_port);
	return remote_name;
	}

	static void
	handle_startup_fencing(pcmk_scheduler_t scheduler, pcmk_node_t new_node)
	{
	if ((new_node->details->type == pcmk_node_variant_remote)
	&& (new_node->details->remote_rsc == NULL)) {
	/* Ignore fencing for remote nodes that don't have a connection resource
	* associated with them. This happens when remote node entries get left
	* in the nodes section after the connection resource is removed.
	*/
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_startup_fencing)) {
	// All nodes are unclean until we've seen their status entry
	new_node->details->unclean = TRUE;

	} else {
	// Blind faith ...
	new_node->details->unclean = FALSE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode xml_nodes, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	pcmk_node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, PCMK_XA_ID);
	uname = crm_element_value(xml_obj, PCMK_XA_UNAME);
	type = crm_element_value(xml_obj, PCMK_XA_TYPE);
	score = crm_element_value(xml_obj, PCMK_XA_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
	"> entry in configuration without id");
	continue;
	}
	new_node = pe_create_node(id, uname, type, score, scheduler);

	if (new_node == NULL) {
	return FALSE;
	}

	handle_startup_fencing(scheduler, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, scheduler);

	crm_trace("Done with node %s",
	crm_element_value(xml_obj, PCMK_XA_UNAME));
	}
	}

	if (scheduler->localhost
	&& (pe_find_node(scheduler->nodes, scheduler->localhost) == NULL)) {
	crm_info("Creating a fake local node");
	pe_create_node(scheduler->localhost, scheduler->localhost, NULL, 0,
	scheduler);
	}

	return TRUE;
	}

	static void
	setup_container(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) setup_container, scheduler);
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, PCMK__META_CONTAINER);
	if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
	pcmk_resource_t *container = pe_find_resource(scheduler->resources,
	container_id);

	if (container) {
	rsc->container = container;
	pcmk__set_rsc_flags(container, pcmk_rsc_has_filler);
	container->fillers = g_list_append(container->fillers, rsc);
	pcmk__rsc_trace(rsc, "Resource %s's container is %s",
	rsc->id, container_id);
	} else {
	pcmk__config_err("Resource %s: Unknown resource container (%s)",
	rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	/* Create remote nodes and guest nodes from the resource configuration
	* before unpacking resources.
	*/
	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	const char *new_node_id = NULL;

	/* Check for remote nodes, which are defined by ocf:pacemaker:remote
	* primitives.
	*/
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found remote node %s defined by resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes, which are defined by special meta-attributes
	* of a primitive of any type (for example, VirtualDomain or Xen).
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
	/* This will add an ocf:pacemaker:remote primitive to the
	* configuration for the guest node's connection, to be unpacked
	* later.
	*/
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources,
	scheduler);
	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s",
	new_node_id, ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	continue;
	}

	/* Check for guest nodes inside a group. Clones are currently not
	* supported as guest nodes.
	*/
	if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
	xmlNode *xml_obj2 = NULL;
	for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
	xml_obj2 = pcmk__xe_next(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources,
	scheduler);

	if (new_node_id
	&& (pe_find_node(scheduler->nodes, new_node_id) == NULL)) {
	crm_trace("Found guest node %s in resource %s inside group %s",
	new_node_id, ID(xml_obj2), ID(xml_obj));
	pe_create_node(new_node_id, new_node_id, "remote", NULL,
	scheduler);
	}
	}
	}
	}
	return TRUE;
	}

	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the scheduler calculations.
	*/
	static void
	link_rsc2remotenode(pcmk_scheduler_t scheduler, pcmk_resource_t new_rsc)
	{
	pcmk_node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	remote_node = pe_find_node(scheduler->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return);

	pcmk__rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
	new_rsc->id, pcmk__node_name(remote_node));
	remote_node->details->remote_rsc = new_rsc;

	if (new_rsc->container == NULL) {
	/* Handle start-up fencing for remote nodes (as opposed to guest nodes)
	* the same as is done for cluster nodes.
	*/
	handle_startup_fencing(scheduler, remote_node);

	} else {
	/* pe_create_node() marks the new node as "remote" or "cluster"; now
	* that we know the node is a guest node, update it correctly.
	*/
	g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
	strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	pcmk_tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	/*!
	* \internal
	* \brief Parse configuration XML for resource information
	*
	* \param[in] xml_resources Top of resource configuration XML
	* \param[in,out] scheduler Scheduler data
	*
	* \return TRUE
	*
	* \note unpack_remote_nodes() MUST be called before this, so that the nodes can
	* be used when pe__unpack_resource() calls resource_location()
	*/
	gboolean
	unpack_resources(const xmlNode xml_resources, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;
	GList *gIter = NULL;

	scheduler->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);

	for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	pcmk_resource_t *new_rsc = NULL;
	const char *id = ID(xml_obj);

	if (pcmk__str_empty(id)) {
	pcmk__config_err("Ignoring <%s> resource without ID",
	xml_obj->name);
	continue;
	}

	if (pcmk__str_eq((const char *) xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE,
	pcmk__str_none)) {
	if (g_hash_table_lookup_extended(scheduler->template_rsc_sets, id,
	NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(scheduler->template_rsc_sets, strdup(id),
	NULL);
	}
	continue;
	}

	crm_trace("Unpacking <%s " PCMK_XA_ID "='%s'>", xml_obj->name, id);
	if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
	scheduler) == pcmk_rc_ok) {
	scheduler->resources = g_list_append(scheduler->resources, new_rsc);
	pcmk__rsc_trace(new_rsc, "Added resource %s", new_rsc->id);

	} else {
	pcmk__config_err("Ignoring <%s> resource '%s' "
	"because configuration is invalid",
	xml_obj->name, id);
	}
	}

	for (gIter = scheduler->resources; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) gIter->data;

	setup_container(rsc, scheduler);
	link_rsc2remotenode(scheduler, rsc);
	}

	scheduler->resources = g_list_sort(scheduler->resources,
	pe__cmp_rsc_priority);
	if (pcmk_is_set(scheduler->flags, pcmk_sched_location_only)) {
	/* Ignore */

	} else if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_have_fencing)) {

	pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
	pcmk__config_err("Either configure some or disable STONITH with the "
	PCMK_OPT_STONITH_ENABLED " option");
	pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode xml_tags, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_tag = NULL;

	scheduler->tags = pcmk__strkey_table(free, destroy_tag);

	for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
	xml_tag = pcmk__xe_next(xml_tag)) {

	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
	continue;
	}

	if (tag_id == NULL) {
	pcmk__config_err("Ignoring <%s> without " PCMK_XA_ID,
	(const char *) xml_tag->name);
	continue;
	}

	for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
	xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {

	const char *obj_ref = ID(xml_obj_ref);

	if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
	continue;
	}

	if (obj_ref == NULL) {
	pcmk__config_err("Ignoring <%s> for tag '%s' without " PCMK_XA_ID,
	xml_obj_ref->name, tag_id);
	continue;
	}

	if (add_tag_ref(scheduler->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode xml_ticket, pcmk_scheduler_t scheduler)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	pcmk_ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (pcmk__str_empty(ticket_id)) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(scheduler->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, scheduler);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = pcmk__xml_attr_value(xIter);

	if (pcmk__str_eq(prop_name, PCMK_XA_ID, pcmk__str_none)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	long long last_granted_ll;

	pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
	ticket->last_granted = (time_t) last_granted_ll;
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode xml_tickets, pcmk_scheduler_t scheduler)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
	xml_obj = pcmk__xe_next(xml_obj)) {

	if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
	continue;
	}
	unpack_ticket_state(xml_obj, scheduler);
	}

	return TRUE;
	}

	static void
	unpack_handle_remote_attrs(pcmk_node_t this_node, const xmlNode state,
	pcmk_scheduler_t *scheduler)
	{
	const char *resource_discovery_enabled = NULL;
	const xmlNode *attrs = NULL;
	pcmk_resource_t *rsc = NULL;

	- if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	+ if (!pcmk__str_eq((const char *) state->name, PCMK__XE_NODE_STATE,
	+ pcmk__str_none)) {
	return;
	}

	if ((this_node == NULL) \|\| !pe__is_guest_or_remote_node(this_node)) {
	return;
	}
	crm_trace("Processing Pacemaker Remote node %s",
	pcmk__node_name(this_node));

	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
	&(this_node->details->remote_maintenance), 0);

	rsc = this_node->details->remote_rsc;
	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, scheduler);

	if (pe__shutdown_requested(this_node)) {
	crm_info("%s is shutting down", pcmk__node_name(this_node));
	this_node->details->shutdown = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
	crm_info("%s is in standby mode", pcmk__node_name(this_node));
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) \|\|
	((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_managed))) {
	crm_info("%s is in maintenance mode", pcmk__node_name(this_node));
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (pe__is_remote_node(this_node)
	&& !pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
	" attribute on Pacemaker Remote node %s"
	" because fencing is disabled",
	pcmk__node_name(this_node));
	} else {
	/* This is either a remote node with fencing enabled, or a guest
	* node. We don't care whether fencing is enabled when fencing guest
	* nodes, because they are "fenced" by recovering their containing
	* resource.
	*/
	crm_info("%s has resource discovery disabled",
	pcmk__node_name(this_node));
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack a cluster node's transient attributes
	*
	* \param[in] state CIB node state XML
	* \param[in,out] node Cluster node whose attributes are being unpacked
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_transient_attributes(const xmlNode state, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	const char *discovery = NULL;
	const xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS,
	FALSE);

	add_node_attrs(attrs, node, TRUE, scheduler);

	if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
	crm_info("%s is in standby mode", pcmk__node_name(node));
	node->details->standby = TRUE;
	}

	if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
	crm_info("%s is in maintenance mode", pcmk__node_name(node));
	node->details->maintenance = TRUE;
	}

	discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
	if ((discovery != NULL) && !crm_is_true(discovery)) {
	pcmk__config_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY " attribute "
	"for %s because disabling resource discovery is "
	"not allowed for cluster nodes",
	pcmk__node_name(node));
	}
	}

	/*!
	* \internal
	* \brief Unpack a node state entry (first pass)
	*
	* Unpack one node state entry from status. This unpacks information from the
	- * node_state element itself and node attributes inside it, but not the
	- * resource history inside it. Multiple passes through the status are needed to
	- * fully unpack everything.
	+ * \C PCMK__XE_NODE_STATE element itself and node attributes inside it, but not
	+ * the resource history inside it. Multiple passes through the status are needed
	+ * to fully unpack everything.
	*
	* \param[in] state CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_state(const xmlNode state, pcmk_scheduler_t scheduler)
	{
	const char *id = NULL;
	const char *uname = NULL;
	pcmk_node_t *this_node = NULL;

	id = crm_element_value(state, PCMK_XA_ID);
	if (id == NULL) {
	- pcmk__config_err("Ignoring invalid " XML_CIB_TAG_STATE " entry without "
	+ pcmk__config_err("Ignoring invalid " PCMK__XE_NODE_STATE " entry without "
	PCMK_XA_ID);
	crm_log_xml_info(state, "missing-id");
	return;
	}

	uname = crm_element_value(state, PCMK_XA_UNAME);
	if (uname == NULL) {
	/* If a joining peer makes the cluster acquire the quorum from corosync
	* meanwhile it has not joined CPG membership of pacemaker-controld yet,
	- * it's possible that the created node_state entry doesn't have an uname
	- * yet. We should recognize the node as `pending` and wait for it to
	- * join CPG.
	+ * it's possible that the created PCMK__XE_NODE_STATE entry doesn't have
	+ * a PCMK_XA_UNAME yet. We should recognize the node as `pending` and
	+ * wait for it to join CPG.
	*/
	- crm_trace("Handling " XML_CIB_TAG_STATE " entry with id=\"%s\" without "
	- PCMK_XA_UNAME, id);
	+ crm_trace("Handling " PCMK__XE_NODE_STATE " entry with id=\"%s\" "
	+ "without " PCMK_XA_UNAME,
	+ id);
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	pcmk__config_warn("Ignoring recorded node state for id=\"%s\" (%s) "
	"because it is no longer in the configuration",
	id, pcmk__s(uname, "uname unknown"));
	return;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	/* We can't determine the online status of Pacemaker Remote nodes until
	* after all resource history has been unpacked. In this first pass, we
	* do need to mark whether the node has been fenced, as this plays a
	* role during unpacking cluster node resource state.
	*/
	pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
	&(this_node->details->remote_was_fenced), 0);
	return;
	}

	unpack_transient_attributes(state, this_node, scheduler);

	/* Provisionally mark this cluster node as clean. We have at least seen it
	* in the current cluster's lifetime.
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;

	crm_trace("Determining online status of cluster node %s (id %s)",
	pcmk__node_name(this_node), id);
	determine_online_status(state, this_node, scheduler);

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
	&& this_node->details->online
	&& (scheduler->no_quorum_policy == pcmk_no_quorum_fence)) {
	/* Everything else should flow from this automatically
	* (at least until the scheduler becomes able to migrate off
	* healthy resources)
	*/
	pe_fence_node(scheduler, this_node, "cluster does not have quorum",
	FALSE);
	}
	}

	/*!
	* \internal
	* \brief Unpack nodes' resource history as much as possible
	*
	* Unpack as many nodes' resource history as possible in one pass through the
	* status. We need to process Pacemaker Remote nodes' connections/containers
	* before unpacking their history; the connection/container history will be
	* in another node's history, so it might take multiple passes to unpack
	* everything.
	*
	* \param[in] status CIB XML status section
	* \param[in] fence If true, treat any not-yet-unpacked nodes as unseen
	* \param[in,out] scheduler Scheduler data
	*
	* \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
	* or EAGAIN if more unpacking remains to be done)
	*/
	static int
	unpack_node_history(const xmlNode *status, bool fence,
	pcmk_scheduler_t *scheduler)
	{
	int rc = pcmk_rc_ok;

	- // Loop through all node_state entries in CIB status
	- for (const xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
	+ // Loop through all PCMK__XE_NODE_STATE entries in CIB status
	+ for (const xmlNode *state = first_named_child(status, PCMK__XE_NODE_STATE);
	state != NULL; state = crm_next_same_xml(state)) {

	const char *id = ID(state);
	const char *uname = crm_element_value(state, PCMK_XA_UNAME);
	pcmk_node_t *this_node = NULL;

	if ((id == NULL) \|\| (uname == NULL)) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history from malformed "
	- XML_CIB_TAG_STATE " without id and/or uname");
	+ PCMK__XE_NODE_STATE " without id and/or uname");
	continue;
	}

	this_node = pe_find_node_any(scheduler->nodes, id, uname);
	if (this_node == NULL) {
	// Warning already logged in first pass through status section
	crm_trace("Not unpacking resource history for node %s because "
	"no longer in configuration", id);
	continue;
	}

	if (this_node->details->unpacked) {
	crm_trace("Not unpacking resource history for node %s because "
	"already unpacked", id);
	continue;
	}

	if (fence) {
	// We're processing all remaining nodes

	} else if (pe__is_guest_node(this_node)) {
	/* We can unpack a guest node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection and containing resource are both up.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL) \|\| (rsc->role != pcmk_role_started)
	\|\| (rsc->container->role != pcmk_role_started)) {
	crm_trace("Not unpacking resource history for guest node %s "
	"because container and connection are not known to "
	"be up", id);
	continue;
	}

	} else if (pe__is_remote_node(this_node)) {
	/* We can unpack a remote node's history only after we've unpacked
	* other resource history to the point that we know that the node's
	* connection is up, with the exception of when shutdown locks are
	* in use.
	*/
	pcmk_resource_t *rsc = this_node->details->remote_rsc;

	if ((rsc == NULL)
	\|\| (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)
	&& (rsc->role != pcmk_role_started))) {
	crm_trace("Not unpacking resource history for remote node %s "
	"because connection is not known to be up", id);
	continue;
	}

	/* If fencing and shutdown locks are disabled and we're not processing
	* unseen nodes, then we don't want to unpack offline nodes until online
	* nodes have been unpacked. This allows us to number active clone
	* instances first.
	*/
	} else if (!pcmk_any_flags_set(scheduler->flags,
	pcmk_sched_fencing_enabled
	\|pcmk_sched_shutdown_lock)
	&& !this_node->details->online) {
	crm_trace("Not unpacking resource history for offline "
	"cluster node %s", id);
	continue;
	}

	if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);
	unpack_handle_remote_attrs(this_node, state, scheduler);
	}

	crm_trace("Unpacking resource history for %snode %s",
	(fence? "unseen " : ""), id);

	this_node->details->unpacked = TRUE;
	unpack_node_lrm(this_node, state, scheduler);

	rc = EAGAIN; // Other node histories might depend on this one
	}
	return rc;
	}

	/* remove nodes that are down, stopping */
	/* create positive rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode status, pcmk_scheduler_t scheduler)
	{
	xmlNode *state = NULL;

	crm_trace("Beginning unpack");

	if (scheduler->tickets == NULL) {
	scheduler->tickets = pcmk__strkey_table(free, destroy_ticket);
	}

	for (state = pcmk__xe_first_child(status); state != NULL;
	state = pcmk__xe_next(state)) {

	if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
	unpack_tickets_state((xmlNode *) state, scheduler);

	- } else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	+ } else if (pcmk__str_eq((const char *) state->name, PCMK__XE_NODE_STATE,
	+ pcmk__str_none)) {
	unpack_node_state(state, scheduler);
	}
	}

	while (unpack_node_history(status, FALSE, scheduler) == EAGAIN) {
	crm_trace("Another pass through node resource histories is needed");
	}

	// Now catch any nodes we didn't see
	unpack_node_history(status,
	pcmk_is_set(scheduler->flags,
	pcmk_sched_fencing_enabled),
	scheduler);

	/* Now that we know where resources are, we can schedule stops of containers
	* with failed bundle connections
	*/
	if (scheduler->stop_needed != NULL) {
	for (GList *item = scheduler->stop_needed; item; item = item->next) {
	pcmk_resource_t *container = item->data;
	pcmk_node_t *node = pcmk__current_node(container);

	if (node) {
	stop_action(container, node, FALSE);
	}
	}
	g_list_free(scheduler->stop_needed);
	scheduler->stop_needed = NULL;
	}

	/* Now that we know status of all Pacemaker Remote connections and nodes,
	* we can stop connections for node shutdowns, and check the online status
	* of remote/guest nodes that didn't have any node history to unpack.
	*/
	for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
	pcmk_node_t *this_node = gIter->data;

	if (!pe__is_guest_or_remote_node(this_node)) {
	continue;
	}
	if (this_node->details->shutdown
	&& (this_node->details->remote_rsc != NULL)) {
	pe__set_next_role(this_node->details->remote_rsc, pcmk_role_stopped,
	"remote shutdown");
	}
	if (!this_node->details->unpacked) {
	determine_remote_online_status(scheduler, this_node);
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became a member at the cluster layer
	*
	- * \param[in] node_state Node's node_state entry
	+ * \param[in] node_state Node's \c PCMK__XE_NODE_STATE entry
	* \param[in,out] scheduler Scheduler data
	*
	* \return Epoch time when node became a cluster member
	* (or scheduler effective time for legacy entries) if a member,
	* 0 if not a member, or -1 if no valid information available
	*/
	static long long
	unpack_node_member(const xmlNode node_state, pcmk_scheduler_t scheduler)
	{
	const char *member_time = crm_element_value(node_state, PCMK__XA_IN_CCM);
	int member = 0;

	if (member_time == NULL) {
	return -1LL;

	} else if (crm_str_to_boolean(member_time, &member) == 1) {
	/* If in_ccm=0, we'll return 0 here. If in_ccm=1, either the entry was
	* recorded as a boolean for a DC < 2.1.7, or the node is pending
	* shutdown and has left the CPG, in which case it was set to 1 to avoid
	* fencing for PCMK_OPT_NODE_PENDING_TIMEOUT.
	*
	* We return the effective time for in_ccm=1 because what's important to
	* avoid fencing is that effective time minus this value is less than
	* the pending node timeout.
	*/
	return member? (long long) get_effective_time(scheduler) : 0LL;

	} else {
	long long when_member = 0LL;

	if ((pcmk__scan_ll(member_time, &when_member,
	0LL) != pcmk_rc_ok) \|\| (when_member < 0LL)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_IN_CCM
	- " in " XML_CIB_TAG_STATE " entry", member_time);
	+ " in " PCMK__XE_NODE_STATE " entry", member_time);
	return -1LL;
	}
	return when_member;
	}
	}

	/*!
	* \internal
	* \brief Unpack node's time when it became online in process group
	*
	- * \param[in] node_state Node's node_state entry
	+ * \param[in] node_state Node's \c PCMK__XE_NODE_STATE entry
	*
	* \return Epoch time when node became online in process group (or 0 if not
	* online, or 1 for legacy online entries)
	*/
	static long long
	unpack_node_online(const xmlNode *node_state)
	{
	const char *peer_time = crm_element_value(node_state, PCMK__XA_CRMD);

	// @COMPAT Entries recorded for DCs < 2.1.7 have "online" or "offline"
	if (pcmk__str_eq(peer_time, OFFLINESTATUS,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	return 0LL;

	} else if (pcmk__str_eq(peer_time, ONLINESTATUS, pcmk__str_casei)) {
	return 1LL;

	} else {
	long long when_online = 0LL;

	if ((pcmk__scan_ll(peer_time, &when_online, 0LL) != pcmk_rc_ok)
	\|\| (when_online < 0)) {
	crm_warn("Unrecognized value '%s' for " PCMK__XA_CRMD " in "
	- XML_CIB_TAG_STATE " entry, assuming offline", peer_time);
	+ PCMK__XE_NODE_STATE " entry, assuming offline", peer_time);
	return 0LL;
	}
	return when_online;
	}
	}

	/*!
	* \internal
	* \brief Unpack node attribute for user-requested fencing
	*
	* \param[in] node Node to check
	- * \param[in] node_state Node's node_state entry in CIB status
	+ * \param[in] node_state Node's \c PCMK__XE_NODE_STATE entry in CIB status
	*
	* \return \c true if fencing has been requested for \p node, otherwise \c false
	*/
	static bool
	unpack_node_terminate(const pcmk_node_t node, const xmlNode node_state)
	{
	long long value = 0LL;
	int value_i = 0;
	const char *value_s = pe_node_attribute_raw(node, PCMK_NODE_ATTR_TERMINATE);

	// Value may be boolean or an epoch time
	if (crm_str_to_boolean(value_s, &value_i) == 1) {
	return (value_i != 0);
	}
	if (pcmk__scan_ll(value_s, &value, 0LL) == pcmk_rc_ok) {
	return (value > 0);
	}
	crm_warn("Ignoring unrecognized value '%s' for " PCMK_NODE_ATTR_TERMINATE
	"node attribute for %s", value_s, pcmk__node_name(node));
	return false;
	}

	static gboolean
	determine_online_status_no_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	if (when_member <= 0) {
	crm_trace("Node %s is %sdown", pcmk__node_name(this_node),
	((when_member < 0)? "presumed " : ""));

	} else if (when_online > 0) {
	if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	online = TRUE;
	} else {
	crm_debug("Node %s is not ready to run resources: %s",
	pcmk__node_name(this_node), join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("Node %s controller is down: "
	"member@%lld online@%lld join=%s expected=%s",
	pcmk__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));

	} else {
	/* mark it unclean */
	pe_fence_node(scheduler, this_node, "peer is unexpectedly down", FALSE);
	crm_info("Node %s member@%lld online@%lld join=%s expected=%s",
	pcmk__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
	}
	return online;
	}

	/*!
	* \internal
	* \brief Check whether a node has taken too long to join controller group
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node to check
	* \param[in] when_member Epoch time when node became a cluster member
	* \param[in] when_online Epoch time when node joined controller group
	*
	* \return true if node has been pending (on the way up) longer than
	* \c PCMK_OPT_NODE_PENDING_TIMEOUT, otherwise false
	* \note This will also update the cluster's recheck time if appropriate.
	*/
	static inline bool
	pending_too_long(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	long long when_member, long long when_online)
	{
	if ((scheduler->node_pending_timeout > 0)
	&& (when_member > 0) && (when_online <= 0)) {
	// There is a timeout on pending nodes, and node is pending

	time_t timeout = when_member + scheduler->node_pending_timeout;

	if (get_effective_time(node->details->data_set) >= timeout) {
	return true; // Node has timed out
	}

	// Node is pending, but still has time
	pe__update_recheck_time(timeout, scheduler, "pending node timeout");
	}
	return false;
	}

	static bool
	determine_online_status_fencing(pcmk_scheduler_t *scheduler,
	const xmlNode *node_state,
	pcmk_node_t *this_node)
	{
	bool termination_requested = unpack_node_terminate(this_node, node_state);
	const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);
	long long when_member = unpack_node_member(node_state, scheduler);
	long long when_online = unpack_node_online(node_state);

	/*
	- PCMK__XA_JOIN ::= member\|down\|pending\|banned
	- PCMK__XA_EXPECTED ::= member\|down

	@COMPAT with entries recorded for DCs < 2.1.7
	- PCMK__XA_IN_CCM ::= true\|false
	- PCMK__XA_CRMD ::= online\|offline

	Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
	- PCMK__XA_IN_CCM ::= <timestamp>\|0
	Since when node has been a cluster member. A value 0 of means the node is not
	a cluster member.

	- PCMK__XA_CRMD ::= <timestamp>\|0
	Since when peer has been online in CPG. A value 0 means the peer is offline
	in CPG.
	*/

	crm_trace("Node %s member@%lld online@%lld join=%s expected=%s%s",
	pcmk__node_name(this_node), when_member, when_online,
	pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"),
	(termination_requested? " (termination requested)" : ""));

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", pcmk__node_name(this_node));

	/* Slightly different criteria since we can't shut down a dead peer */
	return (when_online > 0);
	}

	if (when_member < 0) {
	pe_fence_node(scheduler, this_node,
	"peer has not been seen by the cluster", FALSE);
	return false;
	}

	if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_none)) {
	pe_fence_node(scheduler, this_node,
	"peer failed Pacemaker membership criteria", FALSE);

	} else if (termination_requested) {
	if ((when_member <= 0) && (when_online <= 0)
	&& pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_none)) {
	crm_info("%s was fenced as requested", pcmk__node_name(this_node));
	return false;
	}
	pe_fence_node(scheduler, this_node, "fencing was requested", false);

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN,
	pcmk__str_null_matches)) {

	if (pending_too_long(scheduler, this_node, when_member, when_online)) {
	pe_fence_node(scheduler, this_node,
	"peer pending timed out on joining the process group",
	FALSE);

	} else if ((when_member > 0) \|\| (when_online > 0)) {
	crm_info("- %s is not ready to run resources",
	pcmk__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up",
	pcmk__node_name(this_node));
	}

	} else if (when_member <= 0) {
	// Consider PCMK_OPT_PRIORITY_FENCING_DELAY for lost nodes
	pe_fence_node(scheduler, this_node,
	"peer is no longer part of the cluster", TRUE);

	} else if (when_online <= 0) {
	pe_fence_node(scheduler, this_node,
	"peer process is no longer available", FALSE);

	/* Everything is running at this point, now check join state */

	} else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_none)) {
	crm_info("%s is active", pcmk__node_name(this_node));

	} else if (pcmk__str_any_of(join, CRMD_JOINSTATE_PENDING,
	CRMD_JOINSTATE_DOWN, NULL)) {
	crm_info("%s is not ready to run resources",
	pcmk__node_name(this_node));
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(scheduler, this_node, "peer was in an unknown state",
	FALSE);
	}

	return (when_member > 0);
	}

	static void
	determine_remote_online_status(pcmk_scheduler_t *scheduler,
	pcmk_node_t *this_node)
	{
	pcmk_resource_t *rsc = this_node->details->remote_rsc;
	pcmk_resource_t *container = NULL;
	pcmk_node_t *host = NULL;

	/* If there is a node state entry for a (former) Pacemaker Remote node
	* but no resource creating that node, the node's connection resource will
	* be NULL. Consider it an offline remote node in that case.
	*/
	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	if (container && pcmk__list_of_1(rsc->running_on)) {
	host = rsc->running_on->data;
	}

	/* If the resource is currently started, mark it online. */
	if (rsc->role == pcmk_role_started) {
	crm_trace("%s node %s presumed ONLINE because connection resource is started",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if ((rsc->role == pcmk_role_started)
	&& (rsc->next_role == pcmk_role_stopped)) {

	crm_trace("%s node %s shutting down because connection resource is stopping",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && pcmk_is_set(container->flags, pcmk_rsc_failed)) {
	crm_trace("Guest node %s UNCLEAN because guest resource failed",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
	crm_trace("%s node %s OFFLINE because connection resource failed",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;

	} else if ((rsc->role == pcmk_role_stopped)
	\|\| ((container != NULL)
	&& (container->role == pcmk_role_stopped))) {

	crm_trace("%s node %s OFFLINE because its resource is stopped",
	(container? "Guest" : "Remote"), this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;

	} else if (host && (host->details->online == FALSE)
	&& host->details->unclean) {
	crm_trace("Guest node %s UNCLEAN because host is unclean",
	this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	}

	static void
	determine_online_status(const xmlNode node_state, pcmk_node_t this_node,
	pcmk_scheduler_t *scheduler)
	{
	gboolean online = FALSE;
	const char *exp_state = crm_element_value(node_state, PCMK__XA_EXPECTED);

	CRM_CHECK(this_node != NULL, return);

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;

	if (pe__shutdown_requested(this_node)) {
	this_node->details->shutdown = TRUE;

	} else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (!pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	online = determine_online_status_no_fencing(scheduler, node_state,
	this_node);

	} else {
	online = determine_online_status_fencing(scheduler, node_state,
	this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* don't run resources here */
	this_node->fixed = TRUE; // @COMPAT deprecated and unused
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("%s is not a Pacemaker node", pcmk__node_name(this_node));

	} else if (this_node->details->unclean) {
	pcmk__sched_warn("%s is unclean", pcmk__node_name(this_node));

	} else if (this_node->details->online) {
	crm_info("%s is %s", pcmk__node_name(this_node),
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("%s is offline", pcmk__node_name(this_node));
	}
	}

	/*!
	* \internal
	* \brief Find the end of a resource's name, excluding any clone suffix
	*
	* \param[in] id Resource ID to check
	*
	* \return Pointer to last character of resource's base name
	*/
	const char *
	pe_base_name_end(const char *id)
	{
	if (!pcmk__str_empty(id)) {
	const char *end = id + strlen(id) - 1;

	for (const char *s = end; s > id; --s) {
	switch (*s) {
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	return (s == end)? s : (s - 1);
	default:
	return end;
	}
	}
	return end;
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a resource name excluding any clone suffix
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_strip(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	char *basename = NULL;

	CRM_ASSERT(end);
	basename = strndup(last_rsc_id, end - last_rsc_id + 1);
	CRM_ASSERT(basename);
	return basename;
	}

	/*!
	* \internal
	* \brief Get the name of the first instance of a cloned resource
	*
	* \param[in] last_rsc_id Resource ID to check
	*
	* \return Pointer to newly allocated string with resource's base name plus :0
	* \note It is the caller's responsibility to free() the result.
	* This asserts on error, so callers can assume result is not NULL.
	*/
	char *
	clone_zero(const char *last_rsc_id)
	{
	const char *end = pe_base_name_end(last_rsc_id);
	size_t base_name_len = end - last_rsc_id + 1;
	char *zero = NULL;

	CRM_ASSERT(end);
	zero = calloc(base_name_len + 3, sizeof(char));
	CRM_ASSERT(zero);
	memcpy(zero, last_rsc_id, base_name_len);
	zero[base_name_len] = ':';
	zero[base_name_len + 1] = '0';
	return zero;
	}

	static pcmk_resource_t *
	create_fake_resource(const char rsc_id, const xmlNode rsc_entry,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (pe__unpack_resource(xml_rsc, &rsc, NULL, scheduler) != pcmk_rc_ok) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	pcmk_node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	node = pe_find_node(scheduler->nodes, rsc_id);
	if (node == NULL) {
	node = pe_create_node(rsc_id, rsc_id, "remote", NULL, scheduler);
	}
	link_rsc2remotenode(scheduler, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, PCMK__META_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_removed_filler);
	}
	pcmk__set_rsc_flags(rsc, pcmk_rsc_removed);
	scheduler->resources = g_list_append(scheduler->resources, rsc);
	return rsc;
	}

	/*!
	* \internal
	* \brief Create orphan instance for anonymous clone resource history
	*
	* \param[in,out] parent Clone resource that orphan will be added to
	* \param[in] rsc_id Orphan's resource ID
	* \param[in] node Where orphan is active (for logging only)
	* \param[in,out] scheduler Scheduler data
	*
	* \return Newly added orphaned instance of \p parent
	*/
	static pcmk_resource_t *
	create_anonymous_orphan(pcmk_resource_t parent, const char rsc_id,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	pcmk_resource_t *top = pe__create_clone_child(parent, scheduler);

	// find_rsc() because we might be a cloned group
	pcmk_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	pcmk__rsc_debug(parent, "Created orphan %s for %s: %s on %s",
	top->id, parent->id, rsc_id, pcmk__node_name(node));
	return orphan;
	}

	/*!
	* \internal
	* \brief Check a node for an instance of an anonymous clone
	*
	* Return a child instance of the specified anonymous clone, in order of
	* preference: (1) the instance running on the specified node, if any;
	* (2) an inactive instance (i.e. within the total of \c PCMK_META_CLONE_MAX
	* instances); (3) a newly created orphan (that is, \c PCMK_META_CLONE_MAX
	* instances are already active).
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] node Node on which to check for instance
	* \param[in,out] parent Clone to check
	* \param[in] rsc_id Name of cloned resource in history (no instance)
	*/
	static pcmk_resource_t *
	find_anonymous_clone(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	pcmk_resource_t parent, const char rsc_id)
	{
	GList *rIter = NULL;
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *inactive_instance = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(pe_rsc_is_clone(parent));
	CRM_ASSERT(!pcmk_is_set(parent->flags, pcmk_rsc_unique));

	// Check for active (or partially active, for cloned groups) instance
	pcmk__rsc_trace(parent, "Looking for %s on %s in %s",
	rsc_id, pcmk__node_name(node), parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GList *locations = NULL;
	pcmk_resource_t *child = rIter->data;

	/* Check whether this instance is already known to be active or pending
	* anywhere, at this stage of unpacking. Because this function is called
	* for a resource before the resource's individual operation history
	* entries are unpacked, locations will generally not contain the
	* desired node.
	*
	* However, there are three exceptions:
	* (1) when child is a cloned group and we have already unpacked the
	* history of another member of the group on the same node;
	* (2) when we've already unpacked the history of another numbered
	* instance on the same node (which can happen if
	* PCMK_META_GLOBALLY_UNIQUE was flipped from true to false); and
	* (3) when we re-run calculations on the same scheduler data as part of
	* a simulation.
	*/
	child->fns->location(child, &locations, 2);
	if (locations) {
	/* We should never associate the same numbered anonymous clone
	* instance with multiple nodes, and clone instances can't migrate,
	* so there must be only one location, regardless of history.
	*/
	CRM_LOG_ASSERT(locations->next == NULL);

	if (pcmk__same_node((pcmk_node_t *) locations->data, node)) {
	/* This child instance is active on the requested node, so check
	* for a corresponding configured resource. We use find_rsc()
	* instead of child because child may be a cloned group, and we
	* need the particular member corresponding to rsc_id.
	*
	* If the history entry is orphaned, rsc will be NULL.
	*/
	rsc = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);
	if (rsc) {
	/* If there are multiple instance history entries for an
	* anonymous clone in a single node's history (which can
	* happen if PCMK_META_GLOBALLY_UNIQUE is switched from true
	* to false), we want to consider the instances beyond the
	* first as orphans, even if there are inactive instance
	* numbers available.
	*/
	if (rsc->running_on) {
	crm_notice("Active (now-)anonymous clone %s has "
	"multiple (orphan) instance histories on %s",
	parent->id, pcmk__node_name(node));
	skip_inactive = TRUE;
	rsc = NULL;
	} else {
	pcmk__rsc_trace(parent, "Resource %s, active", rsc->id);
	}
	}
	}
	g_list_free(locations);

	} else {
	pcmk__rsc_trace(parent, "Resource %s, skip inactive", child->id);
	if (!skip_inactive && !inactive_instance
	&& !pcmk_is_set(child->flags, pcmk_rsc_blocked)) {
	// Remember one inactive instance in case we don't find active
	inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
	pcmk_rsc_match_clone_only);

	/* ... but don't use it if it was already associated with a
	* pending action on another node
	*/
	if ((inactive_instance != NULL) &&
	(inactive_instance->pending_node != NULL) &&
	!pcmk__same_node(inactive_instance->pending_node, node)) {
	inactive_instance = NULL;
	}
	}
	}
	}

	if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
	pcmk__rsc_trace(parent, "Resource %s, empty slot",
	inactive_instance->id);
	rsc = inactive_instance;
	}

	/* If the resource has "requires" set to "quorum" or "nothing", and we don't
	* have a clone instance for every node, we don't want to consume a valid
	* instance number for unclean nodes. Such instances may appear to be active
	* according to the history, but should be considered inactive, so we can
	* start an instance elsewhere. Treat such instances as orphans.
	*
	* An exception is instances running on guest nodes -- since guest node
	* "fencing" is actually just a resource stop, requires shouldn't apply.
	*
	* @TODO Ideally, we'd use an inactive instance number if it is not needed
	* for any clean instances. However, we don't know that at this point.
	*/
	if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)
	&& (!node->details->online \|\| node->details->unclean)
	&& !pe__is_guest_node(node)
	&& !pe__is_universal_clone(parent, scheduler)) {

	rsc = NULL;
	}

	if (rsc == NULL) {
	rsc = create_anonymous_orphan(parent, rsc_id, node, scheduler);
	pcmk__rsc_trace(parent, "Resource %s, orphan", rsc->id);
	}
	return rsc;
	}

	static pcmk_resource_t *
	unpack_find_resource(pcmk_scheduler_t scheduler, const pcmk_node_t node,
	const char *rsc_id)
	{
	pcmk_resource_t *rsc = NULL;
	pcmk_resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	/* If we didn't find the resource by its name in the operation history,
	* check it again as a clone instance. Even when PCMK_META_CLONE_MAX=0,
	* we create a single :0 orphan to match against here.
	*/
	char *clone0_id = clone_zero(rsc_id);
	pcmk_resource_t *clone0 = pe_find_resource(scheduler->resources,
	clone0_id);

	if (clone0 && !pcmk_is_set(clone0->flags, pcmk_rsc_unique)) {
	rsc = clone0;
	parent = uber_parent(clone0);
	crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
	} else {
	crm_trace("%s is not known as %s either (orphan)",
	rsc_id, clone0_id);
	}
	free(clone0_id);

	} else if (rsc->variant > pcmk_rsc_variant_primitive) {
	crm_trace("Resource history for %s is orphaned because it is no longer primitive",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (pe_rsc_is_anon_clone(parent)) {

	if (pe_rsc_is_bundled(parent)) {
	rsc = pe__find_bundle_replica(parent->parent, node);
	} else {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(scheduler, node, parent, base);
	free(base);
	CRM_ASSERT(rsc != NULL);
	}
	}

	if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
	&& !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {

	pcmk__str_update(&rsc->clone_name, rsc_id);
	pcmk__rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, pcmk__node_name(node), rsc->id,
	pcmk_is_set(rsc->flags, pcmk_rsc_removed)? " (ORPHAN)" : "");
	}
	return rsc;
	}

	static pcmk_resource_t *
	process_orphan_resource(const xmlNode rsc_entry, const pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);

	crm_debug("Detected orphan resource %s on %s",
	rsc_id, pcmk__node_name(node));
	rsc = create_fake_resource(rsc_id, rsc_entry, scheduler);
	if (rsc == NULL) {
	return NULL;
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_stop_removed_resources)) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);

	} else {
	CRM_CHECK(rsc != NULL, return NULL);
	pcmk__rsc_trace(rsc, "Added orphan %s", rsc->id);
	resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__",
	scheduler);
	}
	return rsc;
	}

	static void
	process_rsc_state(pcmk_resource_t rsc, pcmk_node_t node,
	enum action_fail_response on_fail)
	{
	pcmk_node_t *tmpnode = NULL;
	char *reason = NULL;
	enum action_fail_response save_on_fail = pcmk_on_fail_ignore;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), pcmk__node_name(node),
	fail2text(on_fail));

	/* process current state */
	if (rsc->role != pcmk_role_unknown) {
	pcmk_resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	pcmk_node_t *n = pe__copy_node(node);

	pcmk__rsc_trace(rsc, "%s%s%s known on %s",
	rsc->id,
	((rsc->clone_name == NULL)? "" : " also known as "),
	((rsc->clone_name == NULL)? "" : rsc->clone_name),
	pcmk__node_name(n));
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (pcmk_is_set(iter->flags, pcmk_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	/* If a managed resource is believed to be running, but node is down ... */
	if ((rsc->role > pcmk_role_stopped)
	&& node->details->online == FALSE
	&& node->details->maintenance == FALSE
	&& pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {

	gboolean should_fence = FALSE;

	/* If this is a guest node, fence it (regardless of whether fencing is
	* enabled, because guest node fencing is done by recovery of the
	* container resource rather than by the fencer). Mark the resource
	* we're processing as failed. When the guest comes back up, its
	* operation history in the CIB will be cleared, freeing the affected
	* resource to run again once we are sure we know its state.
	*/
	if (pe__is_guest_node(node)) {
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	should_fence = TRUE;

	} else if (pcmk_is_set(rsc->cluster->flags,
	pcmk_sched_fencing_enabled)) {
	if (pe__is_remote_node(node) && node->details->remote_rsc
	&& !pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_failed)) {

	/* Setting unseen means that fencing of the remote node will
	* occur only if the connection resource is not going to start
	* somewhere. This allows connection resources on a failed
	* cluster node to move to another node without requiring the
	* remote nodes to be fenced as well.
	*/
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("%s is active there (fencing will be"
	" revoked if remote connection can "
	"be re-established elsewhere)",
	rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
	}
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	}
	free(reason);
	}

	/* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
	save_on_fail = on_fail;

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = pcmk_on_fail_ignore;
	}

	switch (on_fail) {
	case pcmk_on_fail_ignore:
	/* nothing to do */
	break;

	case pcmk_on_fail_demote:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed);
	demote_action(rsc, node, FALSE);
	break;

	case pcmk_on_fail_fence_node:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	reason = crm_strdup_printf("%s failed there", rsc->id);
	pe_fence_node(rsc->cluster, node, reason, FALSE);
	free(reason);
	break;

	case pcmk_on_fail_standby_node:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case pcmk_on_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_blocked);
	break;

	case pcmk_on_fail_ban:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__",
	rsc->cluster);
	break;

	case pcmk_on_fail_stop:
	pe__set_next_role(rsc, pcmk_role_stopped, "on-fail=stop");
	break;

	case pcmk_on_fail_restart:
	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	pcmk__set_rsc_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_restart_container:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (rsc->container && pe_rsc_is_bundled(rsc)) {
	/* A bundle's remote connection can run on a different node than
	* the bundle's container. We don't necessarily know where the
	* container is running yet, so remember it and add a stop
	* action for it later.
	*/
	rsc->cluster->stop_needed =
	g_list_prepend(rsc->cluster->stop_needed, rsc->container);
	} else if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	stop_action(rsc, node, FALSE);
	}
	break;

	case pcmk_on_fail_reset_remote:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	}
	if (tmpnode &&
	pe__is_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* The remote connection resource failed in a way that
	* should result in fencing the remote node.
	*/
	pe_fence_node(rsc->cluster, tmpnode,
	"remote connection is unrecoverable", FALSE);
	}
	}

	/* require the stop action regardless if fencing is occurring or not. */
	if (rsc->role > pcmk_role_stopped) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_ms) {
	pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (pcmk_is_set(rsc->flags, pcmk_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if ((rsc->role != pcmk_role_stopped)
	&& (rsc->role != pcmk_role_unknown)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
	if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	pcmk__config_warn("Detected active orphan %s running on %s",
	rsc->id, pcmk__node_name(node));
	} else {
	pcmk__config_warn("Resource '%s' must be stopped manually on "
	"%s because cluster is configured not to "
	"stop active orphans",
	rsc->id, pcmk__node_name(node));
	}
	}

	native_add_running(rsc, node, rsc->cluster,
	(save_on_fail != pcmk_on_fail_ignore));
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	break;
	case pcmk_on_fail_demote:
	case pcmk_on_fail_block:
	pcmk__set_rsc_flags(rsc, pcmk_rsc_failed);
	break;
	default:
	pcmk__set_rsc_flags(rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	break;
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pcmk__rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)",
	rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	GList *possible_matches = pe__resource_actions(rsc, node,
	PCMK_ACTION_STOP, FALSE);
	GList *gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	pcmk_action_t stop = (pcmk_action_t ) gIter->data;

	pcmk__set_action_flags(stop, pcmk_action_optional);
	}

	g_list_free(possible_matches);
	}

	/* A successful stop after migrate_to on the migration source doesn't make
	* the partially migrated resource stopped on the migration target.
	*/
	if ((rsc->role == pcmk_role_stopped)
	&& rsc->partial_migration_source
	&& rsc->partial_migration_source->details == node->details
	&& rsc->partial_migration_target
	&& rsc->running_on) {

	rsc->role = pcmk_role_started;
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(pcmk_node_t node, pcmk_resource_t rsc,
	int start_index, int stop_index,
	GList sorted_op_list, pcmk_scheduler_t scheduler)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GList *gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pcmk__rsc_trace(rsc, "%s: Start index %d, stop index = %d",
	rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	guint interval_ms = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);

	counter++;

	if (node->details->online == FALSE) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: node is offline",
	rsc->id, pcmk__node_name(node));
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: resource is not active",
	id, pcmk__node_name(node));
	continue;

	} else if (counter < start_index) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: old %d",
	id, pcmk__node_name(node), counter);
	continue;
	}

	crm_element_value_ms(rsc_op, PCMK_META_INTERVAL, &interval_ms);
	if (interval_ms == 0) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: non-recurring",
	id, pcmk__node_name(node));
	continue;
	}

	status = crm_element_value(rsc_op, PCMK__XA_OP_STATUS);
	if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
	pcmk__rsc_trace(rsc, "Skipping %s on %s: status",
	id, pcmk__node_name(node));
	continue;
	}
	task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	/* create the action */
	key = pcmk__op_key(rsc->id, task, interval_ms);
	pcmk__rsc_trace(rsc, "Creating %s on %s", key, pcmk__node_name(node));
	custom_action(rsc, key, task, node, TRUE, scheduler);
	}
	}

	void
	calculate_active_ops(const GList sorted_op_list, int start_index,
	int *stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_clone_start = -1;
	const char *task = NULL;
	const char *status = NULL;

	*stop_index = -1;
	*start_index = -1;

	for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
	const xmlNode rsc_op = (const xmlNode ) iter->data;

	counter++;

	task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
	status = crm_element_value(rsc_op, PCMK__XA_OP_STATUS);

	if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
	&& pcmk__str_eq(status, "0", pcmk__str_casei)) {
	*stop_index = counter;

	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
	PCMK_ACTION_MIGRATE_FROM, NULL)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR,
	pcmk__str_casei)) {
	const char *rc = crm_element_value(rsc_op, PCMK__XA_RC_CODE);

	if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
	implied_monitor_start = counter;
	}
	} else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
	PCMK_ACTION_DEMOTE, NULL)) {
	implied_clone_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_clone_start != -1) {
	*start_index = implied_clone_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	// If resource history entry has shutdown lock, remember lock node and time
	static void
	unpack_shutdown_lock(const xmlNode rsc_entry, pcmk_resource_t rsc,
	const pcmk_node_t node, pcmk_scheduler_t scheduler)
	{
	time_t lock_time = 0; // When lock started (i.e. node shutdown time)

	if ((crm_element_value_epoch(rsc_entry, PCMK_OPT_SHUTDOWN_LOCK,
	&lock_time) == pcmk_ok) && (lock_time != 0)) {

	if ((scheduler->shutdown_lock > 0)
	&& (get_effective_time(scheduler)
	> (lock_time + scheduler->shutdown_lock))) {
	pcmk__rsc_info(rsc, "Shutdown lock for %s on %s expired",
	rsc->id, pcmk__node_name(node));
	pe__clear_resource_history(rsc, node);
	} else {
	/* @COMPAT I don't like breaking const signatures, but
	* rsc->lock_node should really be const -- we just can't change it
	* until the next API compatibility break.
	*/
	rsc->lock_node = (pcmk_node_t *) node;
	rsc->lock_time = lock_time;
	}
	}
	}

	/*!
	* \internal
	* \brief Unpack one lrm_resource entry from a node's CIB status
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] rsc_entry lrm_resource XML being unpacked
	* \param[in,out] scheduler Scheduler data
	*
	* \return Resource corresponding to the entry, or NULL if no operation history
	*/
	static pcmk_resource_t *
	unpack_lrm_resource(pcmk_node_t node, const xmlNode lrm_resource,
	pcmk_scheduler_t *scheduler)
	{
	GList *gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = pcmk_role_unknown;

	const char *rsc_id = ID(lrm_resource);

	pcmk_resource_t *rsc = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	xmlNode *rsc_op = NULL;
	xmlNode *last_failure = NULL;

	enum action_fail_response on_fail = pcmk_on_fail_ignore;
	enum rsc_role_e saved_role = pcmk_role_unknown;

	if (rsc_id == NULL) {
	pcmk__config_err("Ignoring invalid " XML_LRM_TAG_RESOURCE
	" entry: No " PCMK_XA_ID);
	crm_log_xml_info(lrm_resource, "missing-id");
	return NULL;
	}
	crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
	rsc_id, pcmk__node_name(node));

	// Build a list of individual lrm_rsc_op entries, so we can sort them
	for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {

	op_list = g_list_prepend(op_list, rsc_op);
	}

	if (!pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	}
	}

	/* find the resource */
	rsc = unpack_find_resource(scheduler, node, rsc_id);
	if (rsc == NULL) {
	if (op_list == NULL) {
	// If there are no operations, there is nothing to do
	return NULL;
	} else {
	rsc = process_orphan_resource(lrm_resource, node, scheduler);
	}
	}
	CRM_ASSERT(rsc != NULL);

	// Check whether the resource is "shutdown-locked" to this node
	if (pcmk_is_set(scheduler->flags, pcmk_sched_shutdown_lock)) {
	unpack_shutdown_lock(lrm_resource, rsc, node, scheduler);
	}

	/* process operations */
	saved_role = rsc->role;
	rsc->role = pcmk_role_unknown;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list,
	scheduler);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail);

	if (get_target_role(rsc, &req_role)) {
	if ((rsc->next_role == pcmk_role_unknown)
	\|\| (req_role < rsc->next_role)) {

	pe__set_next_role(rsc, req_role, PCMK_META_TARGET_ROLE);

	} else if (req_role > rsc->next_role) {
	pcmk__rsc_info(rsc,
	"%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role),
	role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(const xmlNode *lrm_rsc_list,
	pcmk_scheduler_t *scheduler)
	{
	for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list);
	rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {

	pcmk_resource_t *rsc;
	pcmk_resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, PCMK__META_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(scheduler->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(scheduler->resources, rsc_id);
	if ((rsc == NULL) \|\| (rsc->container != NULL)
	\|\| !pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	continue;
	}

	pcmk__rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
	rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	/*!
	* \internal
	* \brief Unpack one node's lrm status section
	*
	* \param[in,out] node Node whose status is being unpacked
	* \param[in] xml CIB node state XML
	* \param[in,out] scheduler Scheduler data
	*/
	static void
	unpack_node_lrm(pcmk_node_t node, const xmlNode xml,
	pcmk_scheduler_t *scheduler)
	{
	bool found_orphaned_container_filler = false;

	// Drill down to lrm_resources section
	xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
	if (xml == NULL) {
	return;
	}
	xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
	if (xml == NULL) {
	return;
	}

	// Unpack each lrm_resource entry
	for (const xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
	rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {

	pcmk_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, scheduler);

	if ((rsc != NULL)
	&& pcmk_is_set(rsc->flags, pcmk_rsc_removed_filler)) {
	found_orphaned_container_filler = true;
	}
	}

	/* Now that all resource state has been unpacked for this node, map any
	* orphaned container fillers to their container resource.
	*/
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(xml, scheduler);
	}
	}

	static void
	set_active(pcmk_resource_t *rsc)
	{
	const pcmk_resource_t *top = pe__const_top_resource(rsc, false);

	if (top && pcmk_is_set(top->flags, pcmk_rsc_promotable)) {
	rsc->role = pcmk_role_unpromoted;
	} else {
	rsc->role = pcmk_role_started;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk_node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define XPATH_NODE_STATE "/" PCMK_XE_CIB "/" PCMK_XE_STATUS \
	- "/" XML_CIB_TAG_STATE
	+ "/" PCMK__XE_NODE_STATE
	#define SUB_XPATH_LRM_RESOURCE "/" XML_CIB_TAG_LRM \
	"/" XML_LRM_TAG_RESOURCES \
	"/" XML_LRM_TAG_RESOURCE
	#define SUB_XPATH_LRM_RSC_OP "/" XML_LRM_TAG_RSC_OP

	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	int target_rc, pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
	return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", resource, "']"
	SUB_XPATH_LRM_RSC_OP "[@" PCMK_XA_OPERATION "='", op, "'",
	NULL);

	/* Need to check against transition_magic too? */
	if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" PCMK__META_MIGRATE_TARGET "='", source, "']",
	NULL);

	} else if ((source != NULL)
	&& (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
	pcmk__g_strcat(xpath,
	" and @" PCMK__META_MIGRATE_SOURCE "='", source, "']",
	NULL);
	} else {
	g_string_append_c(xpath, ']');
	}

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);
	g_string_free(xpath, TRUE);

	if (xml && target_rc >= 0) {
	int rc = PCMK_OCF_UNKNOWN_ERROR;
	int status = PCMK_EXEC_ERROR;

	crm_element_value_int(xml, PCMK__XA_RC_CODE, &rc);
	crm_element_value_int(xml, PCMK__XA_OP_STATUS, &status);
	if ((rc != target_rc) \|\| (status != PCMK_EXEC_DONE)) {
	return NULL;
	}
	}
	return xml;
	}

	static xmlNode *
	find_lrm_resource(const char rsc_id, const char node_name,
	pcmk_scheduler_t *scheduler)
	{
	GString *xpath = NULL;
	xmlNode *xml = NULL;

	CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);

	xpath = g_string_sized_new(256);
	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc_id, "']",
	NULL);

	xml = get_xpath_object((const char *) xpath->str, scheduler->input,
	LOG_DEBUG);

	g_string_free(xpath, TRUE);
	return xml;
	}

	/*!
	* \internal
	* \brief Check whether a resource has no completed action history on a node
	*
	* \param[in,out] rsc Resource to check
	* \param[in] node_name Node to check
	*
	* \return true if \p rsc_id is unknown on \p node_name, otherwise false
	*/
	static bool
	unknown_on_node(pcmk_resource_t rsc, const char node_name)
	{
	bool result = false;
	xmlXPathObjectPtr search;
	GString *xpath = g_string_sized_new(256);

	pcmk__g_strcat(xpath,
	XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node_name, "']"
	SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc->id, "']"
	SUB_XPATH_LRM_RSC_OP "[@" PCMK__XA_RC_CODE "!='193']",
	NULL);
	search = xpath_search(rsc->cluster->input, (const char *) xpath->str);
	result = (numXpathResults(search) == 0);
	freeXpathObject(search);
	g_string_free(xpath, TRUE);
	return result;
	}

	/*!
	* \brief Check whether a probe/monitor indicating the resource was not running
	* on a node happened after some event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a monitor happened after event, false otherwise
	*/
	static bool
	monitor_not_running_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	/* Any probe/monitor operation on the node indicating it was not running
	* there
	*/
	xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
	NULL, PCMK_OCF_NOT_RUNNING, scheduler);

	return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
	}

	/*!
	* \brief Check whether any non-monitor operation on a node happened after some
	* event
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] xml_op Event that non-monitor is being compared to
	* \param[in] same_node Whether the operations are on the same node
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	non_monitor_after(const char rsc_id, const char node_name,
	const xmlNode *xml_op, bool same_node,
	pcmk_scheduler_t *scheduler)
	{
	xmlNode *lrm_resource = NULL;

	lrm_resource = find_lrm_resource(rsc_id, node_name, scheduler);
	if (lrm_resource == NULL) {
	return false;
	}

	for (xmlNode *op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
	op != NULL; op = crm_next_same_xml(op)) {
	const char * task = NULL;

	if (op == xml_op) {
	continue;
	}

	task = crm_element_value(op, PCMK_XA_OPERATION);

	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	NULL)
	&& pe__is_newer_op(op, xml_op, same_node) > 0) {
	return true;
	}
	}

	return false;
	}

	/*!
	* \brief Check whether the resource has newer state on a node after a migration
	* attempt
	*
	* \param[in] rsc_id Resource being checked
	* \param[in] node_name Node being checked
	* \param[in] migrate_to Any migrate_to event that is being compared to
	* \param[in] migrate_from Any migrate_from event that is being compared to
	* \param[in,out] scheduler Scheduler data
	*
	* \return true if such a operation happened after event, false otherwise
	*/
	static bool
	newer_state_after_migrate(const char rsc_id, const char node_name,
	const xmlNode *migrate_to,
	const xmlNode *migrate_from,
	pcmk_scheduler_t *scheduler)
	{
	const xmlNode *xml_op = migrate_to;
	const char *source = NULL;
	const char *target = NULL;
	bool same_node = false;

	if (migrate_from) {
	xml_op = migrate_from;
	}

	source = crm_element_value(xml_op, PCMK__META_MIGRATE_SOURCE);
	target = crm_element_value(xml_op, PCMK__META_MIGRATE_TARGET);

	/* It's preferred to compare to the migrate event on the same node if
	* existing, since call ids are more reliable.
	*/
	if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
	if (migrate_from) {
	xml_op = migrate_from;
	same_node = true;

	} else {
	xml_op = migrate_to;
	}

	} else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
	if (migrate_to) {
	xml_op = migrate_to;
	same_node = true;

	} else {
	xml_op = migrate_from;
	}
	}

	/* If there's any newer non-monitor operation on the node, or any newer
	* probe/monitor operation on the node indicating it was not running there,
	* the migration events potentially no longer matter for the node.
	*/
	return non_monitor_after(rsc_id, node_name, xml_op, same_node, scheduler)
	\|\| monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
	scheduler);
	}

	/*!
	* \internal
	* \brief Parse migration source and target node names from history entry
	*
	* \param[in] entry Resource history entry for a migration action
	* \param[in] source_node If not NULL, source must match this node
	* \param[in] target_node If not NULL, target must match this node
	* \param[out] source_name Where to store migration source node name
	* \param[out] target_name Where to store migration target node name
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_migration_node_names(const xmlNode entry, const pcmk_node_t source_node,
	const pcmk_node_t *target_node,
	const char source_name, const char target_name)
	{
	*source_name = crm_element_value(entry, PCMK__META_MIGRATE_SOURCE);
	*target_name = crm_element_value(entry, PCMK__META_MIGRATE_TARGET);
	if ((source_name == NULL) \|\| (target_name == NULL)) {
	pcmk__config_err("Ignoring resource history entry %s without "
	PCMK__META_MIGRATE_SOURCE " and "
	PCMK__META_MIGRATE_TARGET, ID(entry));
	return pcmk_rc_unpack_error;
	}

	if ((source_node != NULL)
	&& !pcmk__str_eq(*source_name, source_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	PCMK__META_MIGRATE_SOURCE "='%s' does not match %s",
	ID(entry), *source_name, pcmk__node_name(source_node));
	return pcmk_rc_unpack_error;
	}

	if ((target_node != NULL)
	&& !pcmk__str_eq(*target_name, target_node->details->uname,
	pcmk__str_casei\|pcmk__str_null_matches)) {
	pcmk__config_err("Ignoring resource history entry %s because "
	PCMK__META_MIGRATE_TARGET "='%s' does not match %s",
	ID(entry), *target_name, pcmk__node_name(target_node));
	return pcmk_rc_unpack_error;
	}

	return pcmk_rc_ok;
	}

	/*
	* \internal
	* \brief Add a migration source to a resource's list of dangling migrations
	*
	* If the migrate_to and migrate_from actions in a live migration both
	* succeeded, but there is no stop on the source, the migration is considered
	* "dangling." Add the source to the resource's dangling migration list, which
	* will be used to schedule a stop on the source without affecting the target.
	*
	* \param[in,out] rsc Resource involved in migration
	* \param[in] node Migration source
	*/
	static void
	add_dangling_migration(pcmk_resource_t rsc, const pcmk_node_t node)
	{
	pcmk__rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
	rsc->id, pcmk__node_name(node));
	rsc->role = pcmk_role_stopped;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations,
	(gpointer) node);
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a successful migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_success(struct action_history *history)
	{
	/* A complete migration sequence is:
	* 1. migrate_to on source node (which succeeded if we get to this function)
	* 2. migrate_from on target node
	* 3. stop on source node
	*
	* If no migrate_from has happened, the migration is considered to be
	* "partial". If the migrate_from succeeded but no stop has happened, the
	* migration is considered to be "dangling".
	*
	* If a successful migrate_to and stop have happened on the source node, we
	* still need to check for a partial migration, due to scenarios (easier to
	* produce with batch-limit=1) like:
	*
	* - A resource is migrating from node1 to node2, and a migrate_to is
	* initiated for it on node1.
	*
	* - node2 goes into standby mode while the migrate_to is pending, which
	* aborts the transition.
	*
	* - Upon completion of the migrate_to, a new transition schedules a stop
	* on both nodes and a start on node1.
	*
	* - If the new transition is aborted for any reason while the resource is
	* stopping on node1, the transition after that stop completes will see
	* the migrate_to and stop on the source, but it's still a partial
	* migration, and the resource must be stopped on node2 because it is
	* potentially active there due to the migrate_to.
	*
	* We also need to take into account that either node's history may be
	* cleared at any point in the migration process.
	*/
	int from_rc = PCMK_OCF_OK;
	int from_status = PCMK_EXEC_PENDING;
	pcmk_node_t *target_node = NULL;
	xmlNode *migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;
	bool source_newer_op = false;
	bool target_newer_state = false;
	bool active_on_target = false;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	// Check for newer state on the source
	source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
	true, history->rsc->cluster);

	// Check for a migrate_from action from this source on the target
	migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
	target, source, -1, history->rsc->cluster);
	if (migrate_from != NULL) {
	if (source_newer_op) {
	/* There's a newer non-monitor operation on the source and a
	* migrate_from on the target, so this migrate_to is irrelevant to
	* the resource's state.
	*/
	return;
	}
	crm_element_value_int(migrate_from, PCMK__XA_RC_CODE, &from_rc);
	crm_element_value_int(migrate_from, PCMK__XA_OP_STATUS, &from_status);
	}

	/* If the resource has newer state on both the source and target after the
	* migration events, this migrate_to is irrelevant to the resource's state.
	*/
	target_newer_state = newer_state_after_migrate(history->rsc->id, target,
	history->xml, migrate_from,
	history->rsc->cluster);
	if (source_newer_op && target_newer_state) {
	return;
	}

	/* Check for dangling migration (migrate_from succeeded but stop not done).
	* We know there's no stop because we already returned if the target has a
	* migrate_from and the source has any newer non-monitor operation.
	*/
	if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
	add_dangling_migration(history->rsc, history->node);
	return;
	}

	/* Without newer state, this migrate_to implies the resource is active.
	* (Clones are not allowed to migrate, so role can't be promoted.)
	*/
	history->rsc->role = pcmk_role_started;

	target_node = pe_find_node(history->rsc->cluster->nodes, target);
	active_on_target = !target_newer_state && (target_node != NULL)
	&& target_node->details->online;

	if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
	if (active_on_target) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	TRUE);
	} else {
	// Mark resource as failed, require recovery, and prevent migration
	pcmk__set_rsc_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pcmk__clear_rsc_flags(history->rsc, pcmk_rsc_migratable);
	}
	return;
	}

	// The migrate_from is pending, complete but erased, or to be scheduled

	/* If there is no history at all for the resource on an online target, then
	* it was likely cleaned. Just return, and we'll schedule a probe. Once we
	* have the probe result, it will be reflected in target_newer_state.
	*/
	if ((target_node != NULL) && target_node->details->online
	&& unknown_on_node(history->rsc, target)) {
	return;
	}

	if (active_on_target) {
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	if ((source_node != NULL) && source_node->details->online) {
	/* This is a partial migration: the migrate_to completed
	* successfully on the source, but the migrate_from has not
	* completed. Remember the source and target; if the newly
	* chosen target remains the same when we schedule actions
	* later, we may continue with the migration.
	*/
	history->rsc->partial_migration_target = target_node;
	history->rsc->partial_migration_source = source_node;
	}

	} else if (!source_newer_op) {
	// Mark resource as failed, require recovery, and prevent migration
	pcmk__set_rsc_flags(history->rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	pcmk__clear_rsc_flags(history->rsc, pcmk_rsc_migratable);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_to action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_to_failure(struct action_history *history)
	{
	xmlNode *target_migrate_from = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, history->node, NULL, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for migrate_from on the target
	target_migrate_from = find_lrm_op(history->rsc->id,
	PCMK_ACTION_MIGRATE_FROM, target, source,
	PCMK_OCF_OK, history->rsc->cluster);

	if (/* If the resource state is unknown on the target, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, target)
	/* If the resource has newer state on the target after the migration
	* events, this migrate_to no longer matters for the target.
	*/
	&& !newer_state_after_migrate(history->rsc->id, target, history->xml,
	target_migrate_from,
	history->rsc->cluster)) {
	/* The resource has no newer state on the target, so assume it's still
	* active there.
	* (if it is up).
	*/
	pcmk_node_t *target_node = pe_find_node(history->rsc->cluster->nodes,
	target);

	if (target_node && target_node->details->online) {
	native_add_running(history->rsc, target_node, history->rsc->cluster,
	FALSE);
	}

	} else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
	history->rsc->cluster)) {
	/* We know the resource has newer state on the target, but this
	* migrate_to still matters for the source as long as there's no newer
	* non-monitor operation there.
	*/

	// Mark node as having dangling migration so we can force a stop later
	history->rsc->dangling_migrations =
	g_list_prepend(history->rsc->dangling_migrations,
	(gpointer) history->node);
	}
	}

	/*!
	* \internal
	* \brief Update resource role etc. after a failed migrate_from action
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	unpack_migrate_from_failure(struct action_history *history)
	{
	xmlNode *source_migrate_to = NULL;
	const char *source = NULL;
	const char *target = NULL;

	// Get source and target node names from XML
	if (get_migration_node_names(history->xml, NULL, history->node, &source,
	&target) != pcmk_rc_ok) {
	return;
	}

	/* If a migration failed, we have to assume the resource is active. Clones
	* are not allowed to migrate, so role can't be promoted.
	*/
	history->rsc->role = pcmk_role_started;

	// Check for a migrate_to on the source
	source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
	source, target, PCMK_OCF_OK,
	history->rsc->cluster);

	if (/* If the resource state is unknown on the source, it will likely be
	* probed there.
	* Don't just consider it running there. We will get back here anyway in
	* case the probe detects it's running there.
	*/
	!unknown_on_node(history->rsc, source)
	/* If the resource has newer state on the source after the migration
	* events, this migrate_from no longer matters for the source.
	*/
	&& !newer_state_after_migrate(history->rsc->id, source,
	source_migrate_to, history->xml,
	history->rsc->cluster)) {
	/* The resource has no newer state on the source, so assume it's still
	* active there (if it is up).
	*/
	pcmk_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
	source);

	if (source_node && source_node->details->online) {
	native_add_running(history->rsc, source_node, history->rsc->cluster,
	TRUE);
	}
	}
	}

	/*!
	* \internal
	* \brief Add an action to cluster's list of failed actions
	*
	* \param[in,out] history Parsed action result history
	*/
	static void
	record_failed_op(struct action_history *history)
	{
	if (!(history->node->details->online)) {
	return;
	}

	for (const xmlNode *xIter = history->rsc->cluster->failed->children;
	xIter != NULL; xIter = xIter->next) {

	const char *key = pcmk__xe_history_key(xIter);
	const char *uname = crm_element_value(xIter, PCMK_XA_UNAME);

	if (pcmk__str_eq(history->key, key, pcmk__str_none)
	&& pcmk__str_eq(uname, history->node->details->uname,
	pcmk__str_casei)) {
	crm_trace("Skipping duplicate entry %s on %s",
	history->key, pcmk__node_name(history->node));
	return;
	}
	}

	crm_trace("Adding entry for %s on %s to failed action list",
	history->key, pcmk__node_name(history->node));
	crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->details->uname);
	crm_xml_add(history->xml, PCMK__XA_RSC_ID, history->rsc->id);
	add_node_copy(history->rsc->cluster->failed, history->xml);
	}

	static char *
	last_change_str(const xmlNode *xml_op)
	{
	time_t when;
	char *result = NULL;

	if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
	&when) == pcmk_ok) {
	char *when_s = pcmk__epoch2str(&when, 0);
	const char *p = strchr(when_s, ' ');

	// Skip day of week to make message shorter
	if ((p != NULL) && (*(++p) != '\0')) {
	result = strdup(p);
	CRM_ASSERT(result != NULL);
	}
	free(when_s);
	}

	if (result == NULL) {
	result = strdup("unknown time");
	CRM_ASSERT(result != NULL);
	}

	return result;
	}

	/*!
	* \internal
	* \brief Compare two on-fail values
	*
	* \param[in] first One on-fail value to compare
	* \param[in] second The other on-fail value to compare
	*
	* \return A negative number if second is more severe than first, zero if they
	* are equal, or a positive number if first is more severe than second.
	* \note This is only needed until the action_fail_response values can be
	* renumbered at the next API compatibility break.
	*/
	static int
	cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
	{
	switch (first) {
	case pcmk_on_fail_demote:
	switch (second) {
	case pcmk_on_fail_ignore:
	return 1;
	case pcmk_on_fail_demote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_reset_remote:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return 1;
	case pcmk_on_fail_reset_remote:
	return 0;
	default:
	return -1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (second) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return 1;
	case pcmk_on_fail_restart_container:
	return 0;
	default:
	return -1;
	}
	break;

	default:
	break;
	}
	switch (second) {
	case pcmk_on_fail_demote:
	return (first == pcmk_on_fail_ignore)? -1 : 1;

	case pcmk_on_fail_reset_remote:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	return -1;
	default:
	return 1;
	}
	break;

	case pcmk_on_fail_restart_container:
	switch (first) {
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_reset_remote:
	return -1;
	default:
	return 1;
	}
	break;

	default:
	break;
	}
	return first - second;
	}

	/*!
	* \internal
	* \brief Ban a resource (or its clone if an anonymous instance) from all nodes
	*
	* \param[in,out] rsc Resource to ban
	*/
	static void
	ban_from_all_nodes(pcmk_resource_t *rsc)
	{
	int score = -INFINITY;
	pcmk_resource_t *fail_rsc = rsc;

	if (fail_rsc->parent != NULL) {
	pcmk_resource_t *parent = uber_parent(fail_rsc);

	if (pe_rsc_is_anon_clone(parent)) {
	/* For anonymous clones, if an operation with on-fail=stop fails for
	* any instance, the entire clone must stop.
	*/
	fail_rsc = parent;
	}
	}

	// Ban the resource from all nodes
	crm_notice("%s will not be started under current conditions", fail_rsc->id);
	if (fail_rsc->allowed_nodes != NULL) {
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	}
	fail_rsc->allowed_nodes = pe__node_list2table(rsc->cluster->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	/*!
	* \internal
	* \brief Get configured failure handling and role after failure for an action
	*
	* \param[in,out] history Unpacked action history entry
	* \param[out] on_fail Where to set configured failure handling
	* \param[out] fail_role Where to set to role after failure
	*/
	static void
	unpack_failure_handling(struct action_history *history,
	enum action_fail_response *on_fail,
	enum rsc_role_e *fail_role)
	{
	xmlNode *config = pcmk__find_action_config(history->rsc, history->task,
	history->interval_ms, true);

	GHashTable *meta = pcmk__unpack_action_meta(history->rsc, history->node,
	history->task,
	history->interval_ms, config);

	const char *on_fail_str = g_hash_table_lookup(meta, PCMK_META_ON_FAIL);

	*on_fail = pcmk__parse_on_fail(history->rsc, history->task,
	history->interval_ms, on_fail_str);
	fail_role = pcmk__role_after_failure(history->rsc, history->task, on_fail,
	meta);
	g_hash_table_destroy(meta);
	}

	/*!
	* \internal
	* \brief Update resource role, failure handling, etc., after a failed action
	*
	* \param[in,out] history Parsed action result history
	* \param[in] config_on_fail Action failure handling from configuration
	* \param[in] fail_role Resource's role after failure of this action
	* \param[out] last_failure This will be set to the history XML
	* \param[in,out] on_fail Actual handling of action result
	*/
	static void
	unpack_rsc_op_failure(struct action_history *history,
	enum action_fail_response config_on_fail,
	enum rsc_role_e fail_role, xmlNode **last_failure,
	enum action_fail_response *on_fail)
	{
	bool is_probe = false;
	char *last_change_s = NULL;

	*last_failure = history->xml;

	is_probe = pcmk_xe_is_probe(history->xml);
	last_change_s = last_change_str(history->xml);

	if (!pcmk_is_set(history->rsc->cluster->flags, pcmk_sched_symmetric_cluster)
	&& (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	crm_trace("Unexpected result (%s%s%s) was recorded for "
	"%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pcmk__node_name(history->node), last_change_s,
	history->exit_status, history->id);
	} else {
	pcmk__sched_warn("Unexpected result (%s%s%s) was recorded for %s of "
	"%s on %s at %s " CRM_XS " exit-status=%d id=%s",
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	(is_probe? "probe" : history->task), history->rsc->id,
	pcmk__node_name(history->node), last_change_s,
	history->exit_status, history->id);

	if (is_probe && (history->exit_status != PCMK_OCF_OK)
	&& (history->exit_status != PCMK_OCF_NOT_RUNNING)
	&& (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {

	/* A failed (not just unexpected) probe result could mean the user
	* didn't know resources will be probed even where they can't run.
	*/
	crm_notice("If it is not possible for %s to run on %s, see "
	"the " PCMK_XA_RESOURCE_DISCOVERY " option for location "
	"constraints",
	history->rsc->id, pcmk__node_name(history->node));
	}

	record_failed_op(history);
	}

	free(last_change_s);

	if (cmp_on_fail(*on_fail, config_on_fail) < 0) {
	pcmk__rsc_trace(history->rsc, "on-fail %s -> %s for %s",
	fail2text(*on_fail), fail2text(config_on_fail),
	history->key);
	*on_fail = config_on_fail;
	}

	if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
	resource_location(history->rsc, history->node, -INFINITY,
	"__stop_fail__", history->rsc->cluster);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
	unpack_migrate_to_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
	unpack_migrate_from_failure(history);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
	if (config_on_fail == pcmk_on_fail_block) {
	history->rsc->role = pcmk_role_promoted;
	pe__set_next_role(history->rsc, pcmk_role_stopped,
	"demote with on-fail=block");

	} else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
	history->rsc->role = pcmk_role_stopped;

	} else {
	/* Staying in the promoted role would put the scheduler and
	* controller into a loop. Setting the role to unpromoted is not
	* dangerous because the resource will be stopped as part of
	* recovery, and any promotion will be ordered after that stop.
	*/
	history->rsc->role = pcmk_role_unpromoted;
	}
	}

	if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
	/* leave stopped */
	pcmk__rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
	history->rsc->role = pcmk_role_stopped;

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
	set_active(history->rsc);
	}

	pcmk__rsc_trace(history->rsc,
	"Resource %s: role=%s unclean=%s on_fail=%s fail_role=%s",
	history->rsc->id, role2text(history->rsc->role),
	pcmk__btoa(history->node->details->unclean),
	fail2text(config_on_fail), role2text(fail_role));

	if ((fail_role != pcmk_role_started)
	&& (history->rsc->next_role < fail_role)) {
	pe__set_next_role(history->rsc, fail_role, "failure");
	}

	if (fail_role == pcmk_role_stopped) {
	ban_from_all_nodes(history->rsc);
	}
	}

	/*!
	* \internal
	* \brief Block a resource with a failed action if it cannot be recovered
	*
	* If resource action is a failed stop and fencing is not possible, mark the
	* resource as unmanaged and blocked, since recovery cannot be done.
	*
	* \param[in,out] history Parsed action history entry
	*/
	static void
	block_if_unrecoverable(struct action_history *history)
	{
	char *last_change_s = NULL;

	if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
	return; // All actions besides stop are always recoverable
	}
	if (pe_can_fence(history->node->details->data_set, history->node)) {
	return; // Failed stops are recoverable via fencing
	}

	last_change_s = last_change_str(history->xml);
	pcmk__sched_err("No further recovery can be attempted for %s "
	"because %s on %s failed (%s%s%s) at %s "
	CRM_XS " rc=%d id=%s",
	history->rsc->id, history->task,
	pcmk__node_name(history->node),
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""),
	last_change_s, history->exit_status, history->id);

	free(last_change_s);

	pcmk__clear_rsc_flags(history->rsc, pcmk_rsc_managed);
	pcmk__set_rsc_flags(history->rsc, pcmk_rsc_blocked);
	}

	/*!
	* \internal
	* \brief Update action history's execution status and why
	*
	* \param[in,out] history Parsed action history entry
	* \param[out] why Where to store reason for update
	* \param[in] value New value
	* \param[in] reason Description of why value was changed
	*/
	static inline void
	remap_because(struct action_history history, const char *why, int value,
	const char *reason)
	{
	if (history->execution_status != value) {
	history->execution_status = value;
	*why = reason;
	}
	}

	/*!
	* \internal
	* \brief Remap informational monitor results and operation status
	*
	* For the monitor results, certain OCF codes are for providing extended information
	* to the user about services that aren't yet failed but not entirely healthy either.
	* These must be treated as the "normal" result by Pacemaker.
	*
	* For operation status, the action result can be used to determine an appropriate
	* status for the purposes of responding to the action. The status provided by the
	* executor is not directly usable since the executor does not know what was expected.
	*
	* \param[in,out] history Parsed action history entry
	* \param[in,out] on_fail What should be done about the result
	* \param[in] expired Whether result is expired
	*
	* \note If the result is remapped and the node is not shutting down or failed,
	* the operation will be recorded in the scheduler data's list of failed
	* operations to highlight it for the user.
	*
	* \note This may update the resource's current and next role.
	*/
	static void
	remap_operation(struct action_history *history,
	enum action_fail_response *on_fail, bool expired)
	{
	bool is_probe = false;
	int orig_exit_status = history->exit_status;
	int orig_exec_status = history->execution_status;
	const char *why = NULL;
	const char *task = history->task;

	// Remap degraded results to their successful counterparts
	history->exit_status = pcmk__effective_rc(history->exit_status);
	if (history->exit_status != orig_exit_status) {
	why = "degraded result";
	if (!expired && (!history->node->details->shutdown
	\|\| history->node->details->online)) {
	record_failed_op(history);
	}
	}

	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& ((history->execution_status != PCMK_EXEC_DONE)
	\|\| (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
	history->execution_status = PCMK_EXEC_DONE;
	history->exit_status = PCMK_OCF_NOT_RUNNING;
	why = "equivalent probe result";
	}

	/* If the executor reported an execution status of anything but done or
	* error, consider that final. But for done or error, we know better whether
	* it should be treated as a failure or not, because we know the expected
	* result.
	*/
	switch (history->execution_status) {
	case PCMK_EXEC_DONE:
	case PCMK_EXEC_ERROR:
	break;

	// These should be treated as node-fatal
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"node-fatal error");
	goto remap_done;

	default:
	goto remap_done;
	}

	is_probe = pcmk_xe_is_probe(history->xml);
	if (is_probe) {
	task = "probe";
	}

	if (history->expected_exit_status < 0) {
	/* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
	* Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
	* expected exit status in the transition key, which (along with the
	* similar case of a corrupted transition key in the CIB) will be
	* reported to this function as -1. Pacemaker 2.0+ does not support
	* rolling upgrades from those versions or processing of saved CIB files
	* from those versions, so we do not need to care much about this case.
	*/
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"obsolete history format");
	pcmk__config_warn("Expected result not found for %s on %s "
	"(corrupt or obsolete CIB?)",
	history->key, pcmk__node_name(history->node));

	} else if (history->exit_status == history->expected_exit_status) {
	remap_because(history, &why, PCMK_EXEC_DONE, "expected result");

	} else {
	remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
	pcmk__rsc_debug(history->rsc,
	"%s on %s: expected %d (%s), got %d (%s%s%s)",
	history->key, pcmk__node_name(history->node),
	history->expected_exit_status,
	services_ocf_exitcode_str(history->expected_exit_status),
	history->exit_status,
	services_ocf_exitcode_str(history->exit_status),
	(pcmk__str_empty(history->exit_reason)? "" : ": "),
	pcmk__s(history->exit_reason, ""));
	}

	switch (history->exit_status) {
	case PCMK_OCF_OK:
	if (is_probe
	&& (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active on %s at %s",
	history->rsc->id, pcmk__node_name(history->node),
	last_change_s);
	free(last_change_s);
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe
	\|\| (history->expected_exit_status == history->exit_status)
	\|\| !pcmk_is_set(history->rsc->flags, pcmk_rsc_managed)) {

	/* For probes, recurring monitors for the Stopped role, and
	* unmanaged resources, "not running" is not considered a
	* failure.
	*/
	remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
	history->rsc->role = pcmk_role_stopped;
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"not running");
	}
	break;

	case PCMK_OCF_RUNNING_PROMOTED:
	if (is_probe
	&& (history->exit_status != history->expected_exit_status)) {
	char *last_change_s = last_change_str(history->xml);

	remap_because(history, &why, PCMK_EXEC_DONE, "probe");
	pcmk__rsc_info(history->rsc,
	"Probe found %s active and promoted on %s at %s",
	history->rsc->id,
	pcmk__node_name(history->node), last_change_s);
	free(last_change_s);
	}
	if (!expired
	\|\| (history->exit_status == history->expected_exit_status)) {
	history->rsc->role = pcmk_role_promoted;
	}
	break;

	case PCMK_OCF_FAILED_PROMOTED:
	if (!expired) {
	history->rsc->role = pcmk_role_promoted;
	}
	remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
	break;

	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	{
	guint interval_ms = 0;
	crm_element_value_ms(history->xml, PCMK_META_INTERVAL,
	&interval_ms);

	if (interval_ms == 0) {
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
	"exit status");
	} else {
	remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
	"exit status");
	}
	}
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	if (!expired) {
	block_if_unrecoverable(history);
	}
	remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
	break;

	default:
	if (history->execution_status == PCMK_EXEC_DONE) {
	char *last_change_s = last_change_str(history->xml);

	crm_info("Treating unknown exit status %d from %s of %s "
	"on %s at %s as failure",
	history->exit_status, task, history->rsc->id,
	pcmk__node_name(history->node), last_change_s);
	remap_because(history, &why, PCMK_EXEC_ERROR,
	"unknown exit status");
	free(last_change_s);
	}
	break;
	}

	remap_done:
	if (why != NULL) {
	pcmk__rsc_trace(history->rsc,
	"Remapped %s result from [%s: %s] to [%s: %s] "
	"because of %s",
	history->key, pcmk_exec_status_str(orig_exec_status),
	crm_exit_str(orig_exit_status),
	pcmk_exec_status_str(history->execution_status),
	crm_exit_str(history->exit_status), why);
	}
	}

	// return TRUE if start or monitor last failure but parameters changed
	static bool
	should_clear_for_param_change(const xmlNode xml_op, const char task,
	pcmk_resource_t rsc, pcmk_node_t node)
	{
	if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
	if (pe__bundle_needs_remote_name(rsc)) {
	/* We haven't allocated resources yet, so we can't reliably
	* substitute addr parameters for the REMOTE_CONTAINER_HACK.
	* When that's needed, defer the check until later.
	*/
	pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
	rsc->cluster);

	} else {
	pcmk__op_digest_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
	rsc->cluster);
	switch (digest_data->rc) {
	case pcmk__digest_unknown:
	crm_trace("Resource %s history entry %s on %s"
	" has no digest to compare",
	rsc->id, pcmk__xe_history_key(xml_op),
	node->details->id);
	break;
	case pcmk__digest_match:
	break;
	default:
	return TRUE;
	}
	}
	}
	return FALSE;
	}

	// Order action after fencing of remote node, given connection rsc
	static void
	order_after_remote_fencing(pcmk_action_t action, pcmk_resource_t remote_conn,
	pcmk_scheduler_t *scheduler)
	{
	pcmk_node_t *remote_node = pe_find_node(scheduler->nodes, remote_conn->id);

	if (remote_node) {
	pcmk_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
	FALSE, scheduler);

	order_actions(fence, action, pcmk__ar_first_implies_then);
	}
	}

	static bool
	should_ignore_failure_timeout(const pcmk_resource_t rsc, const char task,
	guint interval_ms, bool is_last_failure)
	{
	/* Clearing failures of recurring monitors has special concerns. The
	* executor reports only changes in the monitor result, so if the
	* monitor is still active and still getting the same failure result,
	* that will go undetected after the failure is cleared.
	*
	* Also, the operation history will have the time when the recurring
	* monitor result changed to the given code, not the time when the
	* result last happened.
	*
	* @TODO We probably should clear such failures only when the failure
	* timeout has passed since the last occurrence of the failed result.
	* However we don't record that information. We could maybe approximate
	* that by clearing only if there is a more recent successful monitor or
	* stop result, but we don't even have that information at this point
	* since we are still unpacking the resource's operation history.
	*
	* This is especially important for remote connection resources with a
	* reconnect interval, so in that case, we skip clearing failures
	* if the remote node hasn't been fenced.
	*/
	if (rsc->remote_reconnect_ms
	&& pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)
	&& (interval_ms != 0)
	&& pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {

	pcmk_node_t *remote_node = pe_find_node(rsc->cluster->nodes, rsc->id);

	if (remote_node && !remote_node->details->remote_was_fenced) {
	if (is_last_failure) {
	crm_info("Waiting to clear monitor failure for remote node %s"
	" until fencing has occurred", rsc->id);
	}
	return TRUE;
	}
	}
	return FALSE;
	}

	/*!
	* \internal
	* \brief Check operation age and schedule failure clearing when appropriate
	*
	* This function has two distinct purposes. The first is to check whether an
	* operation history entry is expired (i.e. the resource has a failure timeout,
	* the entry is older than the timeout, and the resource either has no fail
	* count or its fail count is entirely older than the timeout). The second is to
	* schedule fail count clearing when appropriate (i.e. the operation is expired
	* and either the resource has an expired fail count or the operation is a
	* last_failure for a remote connection resource with a reconnect interval,
	* or the operation is a last_failure for a start or monitor operation and the
	* resource's parameters have changed since the operation).
	*
	* \param[in,out] history Parsed action result history
	*
	* \return true if operation history entry is expired, otherwise false
	*/
	static bool
	check_operation_expiry(struct action_history *history)
	{
	bool expired = false;
	bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
	time_t last_run = 0;
	int unexpired_fail_count = 0;
	const char *clear_reason = NULL;

	if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not expired: "
	"Not Installed does not expire",
	history->id, pcmk__node_name(history->node));
	return false; // "Not installed" must always be cleared manually
	}

	if ((history->rsc->failure_timeout > 0)
	&& (crm_element_value_epoch(history->xml, PCMK_XA_LAST_RC_CHANGE,
	&last_run) == 0)) {

	/* Resource has a PCMK_META_FAILURE_TIMEOUT and history entry has a
	* timestamp
	*/

	time_t now = get_effective_time(history->rsc->cluster);
	time_t last_failure = 0;

	// Is this particular operation history older than the failure timeout?
	if ((now >= (last_run + history->rsc->failure_timeout))
	&& !should_ignore_failure_timeout(history->rsc, history->task,
	history->interval_ms,
	is_last_failure)) {
	expired = true;
	}

	// Does the resource as a whole have an unexpired fail count?
	unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
	&last_failure,
	pcmk__fc_effective,
	history->xml);

	// Update scheduler recheck time according to last failure
	crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
	" last-failure@%lld",
	history->id, (long long) last_run, (expired? "" : "not "),
	(long long) now, unexpired_fail_count,
	history->rsc->failure_timeout, (long long) last_failure);
	last_failure += history->rsc->failure_timeout + 1;
	if (unexpired_fail_count && (now < last_failure)) {
	pe__update_recheck_time(last_failure, history->rsc->cluster,
	"fail count expiration");
	}
	}

	if (expired) {
	if (pe_get_failcount(history->node, history->rsc, NULL,
	pcmk__fc_default, history->xml)) {
	// There is a fail count ignoring timeout

	if (unexpired_fail_count == 0) {
	// There is no fail count considering timeout
	clear_reason = "it expired";

	} else {
	/* This operation is old, but there is an unexpired fail count.
	* In a properly functioning cluster, this should only be
	* possible if this operation is not a failure (otherwise the
	* fail count should be expired too), so this is really just a
	* failsafe.
	*/
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Unexpired fail count",
	history->id, pcmk__node_name(history->node));
	expired = false;
	}

	} else if (is_last_failure
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* Clear any expired last failure when reconnect interval is set,
	* even if there is no fail count.
	*/
	clear_reason = "reconnect interval is set";
	}
	}

	if (!expired && is_last_failure
	&& should_clear_for_param_change(history->xml, history->task,
	history->rsc, history->node)) {
	clear_reason = "resource parameters have changed";
	}

	if (clear_reason != NULL) {
	pcmk_action_t *clear_op = NULL;

	// Schedule clearing of the fail count
	clear_op = pe__clear_failcount(history->rsc, history->node,
	clear_reason, history->rsc->cluster);

	if (pcmk_is_set(history->rsc->cluster->flags,
	pcmk_sched_fencing_enabled)
	&& (history->rsc->remote_reconnect_ms != 0)) {
	/* If we're clearing a remote connection due to a reconnect
	* interval, we want to wait until any scheduled fencing
	* completes.
	*
	* We could limit this to remote_node->details->unclean, but at
	* this point, that's always true (it won't be reliable until
	* after unpack_node_history() is done).
	*/
	crm_info("Clearing %s failure will wait until any scheduled "
	"fencing of %s completes",
	history->task, history->rsc->id);
	order_after_remote_fencing(clear_op, history->rsc,
	history->rsc->cluster);
	}
	}

	if (expired && (history->interval_ms == 0)
	&& pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	switch (history->exit_status) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_PROMOTED:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_PROMOTED:
	// Don't expire probes that return these values
	pcmk__rsc_trace(history->rsc,
	"Resource history entry %s on %s is not "
	"expired: Probe result",
	history->id, pcmk__node_name(history->node));
	expired = false;
	break;
	}
	}

	return expired;
	}

	int
	pe__target_rc_from_xml(const xmlNode *xml_op)
	{
	int target_rc = 0;
	const char *key = crm_element_value(xml_op, PCMK__XA_TRANSITION_KEY);

	if (key == NULL) {
	return -1;
	}
	decode_transition_key(key, NULL, NULL, NULL, &target_rc);
	return target_rc;
	}

	/*!
	* \internal
	* \brief Update a resource's state for an action result
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] exit_status Exit status to base new state on
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	update_resource_state(struct action_history *history, int exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	bool clear_past_failure = false;

	if ((exit_status == PCMK_OCF_NOT_INSTALLED)
	\|\| (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml))) {
	history->rsc->role = pcmk_role_stopped;

	} else if (exit_status == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
	pcmk__str_none)) {
	if ((last_failure != NULL)
	&& pcmk__str_eq(history->key, pcmk__xe_history_key(last_failure),
	pcmk__str_none)) {
	clear_past_failure = true;
	}
	if (history->rsc->role < pcmk_role_started) {
	set_active(history->rsc);
	}

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
	history->rsc->role = pcmk_role_stopped;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_promoted;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
	pcmk__str_none)) {
	if (*on_fail == pcmk_on_fail_demote) {
	// Demote clears an error only if on-fail=demote
	clear_past_failure = true;
	}
	history->rsc->role = pcmk_role_unpromoted;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
	pcmk__str_none)) {
	history->rsc->role = pcmk_role_started;
	clear_past_failure = true;

	} else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
	pcmk__str_none)) {
	unpack_migrate_to_success(history);

	} else if (history->rsc->role < pcmk_role_started) {
	pcmk__rsc_trace(history->rsc, "%s active on %s",
	history->rsc->id, pcmk__node_name(history->node));
	set_active(history->rsc);
	}

	if (!clear_past_failure) {
	return;
	}

	switch (*on_fail) {
	case pcmk_on_fail_stop:
	case pcmk_on_fail_ban:
	case pcmk_on_fail_standby_node:
	case pcmk_on_fail_fence_node:
	pcmk__rsc_trace(history->rsc,
	"%s (%s) is not cleared by a completed %s",
	history->rsc->id, fail2text(*on_fail),
	history->task);
	break;

	case pcmk_on_fail_block:
	case pcmk_on_fail_ignore:
	case pcmk_on_fail_demote:
	case pcmk_on_fail_restart:
	case pcmk_on_fail_restart_container:
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures");
	break;

	case pcmk_on_fail_reset_remote:
	if (history->rsc->remote_reconnect_ms == 0) {
	/* With no reconnect interval, the connection is allowed to
	* start again after the remote node is fenced and
	* completely stopped. (With a reconnect interval, we wait
	* for the failure to be cleared entirely before attempting
	* to reconnect.)
	*/
	*on_fail = pcmk_on_fail_ignore;
	pe__set_next_role(history->rsc, pcmk_role_unknown,
	"clear past failures and reset remote");
	}
	break;
	}
	}

	/*!
	* \internal
	* \brief Check whether a given history entry matters for resource state
	*
	* \param[in] history Parsed action history entry
	*
	* \return true if action can affect resource state, otherwise false
	*/
	static inline bool
	can_affect_state(struct action_history *history)
	{
	#if 0
	/* @COMPAT It might be better to parse only actions we know we're interested
	* in, rather than exclude a couple we don't. However that would be a
	* behavioral change that should be done at a major or minor series release.
	* Currently, unknown operations can affect whether a resource is considered
	* active and/or failed.
	*/
	return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
	PCMK_ACTION_START, PCMK_ACTION_STOP,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	"asyncmon", NULL);
	#else
	return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
	PCMK_ACTION_META_DATA, NULL);
	#endif
	}

	/*!
	* \internal
	* \brief Unpack execution/exit status and exit reason from a history entry
	*
	* \param[in,out] history Action history entry to unpack
	*
	* \return Standard Pacemaker return code
	*/
	static int
	unpack_action_result(struct action_history *history)
	{
	if ((crm_element_value_int(history->xml, PCMK__XA_OP_STATUS,
	&(history->execution_status)) < 0)
	\|\| (history->execution_status < PCMK_EXEC_PENDING)
	\|\| (history->execution_status > PCMK_EXEC_MAX)
	\|\| (history->execution_status == PCMK_EXEC_CANCELLED)) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " PCMK__XA_OP_STATUS " '%s'",
	history->id, history->rsc->id,
	pcmk__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	PCMK__XA_OP_STATUS),
	""));
	return pcmk_rc_unpack_error;
	}
	if ((crm_element_value_int(history->xml, PCMK__XA_RC_CODE,
	&(history->exit_status)) < 0)
	\|\| (history->exit_status < 0) \|\| (history->exit_status > CRM_EX_MAX)) {
	#if 0
	/* @COMPAT We should ignore malformed entries, but since that would
	* change behavior, it should be done at a major or minor series
	* release.
	*/
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"with invalid " PCMK__XA_RC_CODE " '%s'",
	history->id, history->rsc->id,
	pcmk__node_name(history->node),
	pcmk__s(crm_element_value(history->xml,
	PCMK__XA_RC_CODE),
	""));
	return pcmk_rc_unpack_error;
	#else
	history->exit_status = CRM_EX_ERROR;
	#endif
	}
	history->exit_reason = crm_element_value(history->xml, PCMK_XA_EXIT_REASON);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Process an action history entry whose result expired
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	*
	* \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
	* entry needs no further processing)
	*/
	static int
	process_expired_result(struct action_history *history, int orig_exit_status)
	{
	if (!pe_rsc_is_bundled(history->rsc)
	&& pcmk_xe_mask_probe_failure(history->xml)
	&& (orig_exit_status != history->expected_exit_status)) {

	if (history->rsc->role <= pcmk_role_stopped) {
	history->rsc->role = pcmk_role_unknown;
	}
	crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
	"Masked failure expired",
	history->id, history->rsc->id,
	pcmk__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->exit_status == history->expected_exit_status) {
	return pcmk_rc_undetermined; // Only failures expire
	}

	if (history->interval_ms == 0) {
	crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
	"Expired failure",
	history->id, history->task, history->rsc->id,
	pcmk__node_name(history->node));
	return pcmk_rc_ok;
	}

	if (history->node->details->online && !history->node->details->unclean) {
	/* Reschedule the recurring action. schedule_cancel() won't work at
	* this stage, so as a hacky workaround, forcibly change the restart
	* digest so pcmk__check_action_config() does what we want later.
	*
	* @TODO We should skip this if there is a newer successful monitor.
	* Also, this causes rescheduling only if the history entry
	* has a PCMK__XA_OP_DIGEST (which the expire-non-blocked-failure
	* scheduler regression test doesn't, but that may not be a
	* realistic scenario in production).
	*/
	crm_notice("Rescheduling %s-interval %s of %s on %s "
	"after failure expired",
	pcmk__readable_interval(history->interval_ms), history->task,
	history->rsc->id, pcmk__node_name(history->node));
	crm_xml_add(history->xml, PCMK__XA_OP_RESTART_DIGEST,
	"calculated-failure-timeout");
	return pcmk_rc_ok;
	}

	return pcmk_rc_undetermined;
	}

	/*!
	* \internal
	* \brief Process a masked probe failure
	*
	* \param[in,out] history Parsed action history entry
	* \param[in] orig_exit_status Action exit status before remapping
	* \param[in] last_failure Resource's last_failure entry, if known
	* \param[in,out] on_fail Resource's current failure handling
	*/
	static void
	mask_probe_failure(struct action_history *history, int orig_exit_status,
	const xmlNode *last_failure,
	enum action_fail_response *on_fail)
	{
	pcmk_resource_t *ban_rsc = history->rsc;

	if (!pcmk_is_set(history->rsc->flags, pcmk_rsc_unique)) {
	ban_rsc = uber_parent(history->rsc);
	}

	crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
	services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
	pcmk__node_name(history->node));
	update_resource_state(history, history->expected_exit_status, last_failure,
	on_fail);
	crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->details->uname);

	record_failed_op(history);
	resource_location(ban_rsc, history->node, -INFINITY, "masked-probe-failure",
	history->rsc->cluster);
	}

	/*!
	* \internal Check whether a given failure is for a given pending action
	*
	* \param[in] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*
	* \return true if \p last_failure is failure of pending action in \p history,
	* otherwise false
	* \note Both \p history and \p last_failure must come from the same
	* lrm_resource block, as node and resource are assumed to be the same.
	*/
	static bool
	failure_is_newer(const struct action_history *history,
	const xmlNode *last_failure)
	{
	guint failure_interval_ms = 0U;
	long long failure_change = 0LL;
	long long this_change = 0LL;

	if (last_failure == NULL) {
	return false; // Resource has no last_failure entry
	}

	if (!pcmk__str_eq(history->task,
	crm_element_value(last_failure, PCMK_XA_OPERATION),
	pcmk__str_none)) {
	return false; // last_failure is for different action
	}

	if ((crm_element_value_ms(last_failure, PCMK_META_INTERVAL,
	&failure_interval_ms) != pcmk_ok)
	\|\| (history->interval_ms != failure_interval_ms)) {
	return false; // last_failure is for action with different interval
	}

	if ((pcmk__scan_ll(crm_element_value(history->xml, PCMK_XA_LAST_RC_CHANGE),
	&this_change, 0LL) != pcmk_rc_ok)
	\|\| (pcmk__scan_ll(crm_element_value(last_failure,
	PCMK_XA_LAST_RC_CHANGE),
	&failure_change, 0LL) != pcmk_rc_ok)
	\|\| (failure_change < this_change)) {
	return false; // Failure is not known to be newer
	}

	return true;
	}

	/*!
	* \internal
	* \brief Update a resource's role etc. for a pending action
	*
	* \param[in,out] history Parsed history entry for pending action
	* \param[in] last_failure Resource's last_failure entry, if known
	*/
	static void
	process_pending_action(struct action_history *history,
	const xmlNode *last_failure)
	{
	/* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
	* and there might be a RSC_monitor_INTERVAL entry with the last successful
	* or pending result.
	*
	* If last_failure contains the failure of the pending recurring monitor
	* we're processing here, and is newer, the action is no longer pending.
	* (Pending results have call ID -1, which sorts last, so the last failure
	* if any should be known.)
	*/
	if (failure_is_newer(history, last_failure)) {
	return;
	}

	if (strcmp(history->task, PCMK_ACTION_START) == 0) {
	pcmk__set_rsc_flags(history->rsc, pcmk_rsc_start_pending);
	set_active(history->rsc);

	} else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
	history->rsc->role = pcmk_role_promoted;

	} else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
	&& history->node->details->unclean) {
	/* A migrate_to action is pending on a unclean source, so force a stop
	* on the target.
	*/
	const char *migrate_target = NULL;
	pcmk_node_t *target = NULL;

	migrate_target = crm_element_value(history->xml,
	PCMK__META_MIGRATE_TARGET);
	target = pe_find_node(history->rsc->cluster->nodes, migrate_target);
	if (target != NULL) {
	stop_action(history->rsc, target, FALSE);
	}
	}

	if (history->rsc->pending_task != NULL) {
	/* There should never be multiple pending actions, but as a failsafe,
	* just remember the first one processed for display purposes.
	*/
	return;
	}

	if (pcmk_is_probe(history->task, history->interval_ms)) {
	/* Pending probes are currently never displayed, even if pending
	* operations are requested. If we ever want to change that,
	* enable the below and the corresponding part of
	* native.c:native_pending_task().
	*/
	#if 0
	history->rsc->pending_task = strdup("probe");
	history->rsc->pending_node = history->node;
	#endif
	} else {
	history->rsc->pending_task = strdup(history->task);
	history->rsc->pending_node = history->node;
	}
	}

	static void
	unpack_rsc_op(pcmk_resource_t rsc, pcmk_node_t node, xmlNode *xml_op,
	xmlNode *last_failure, enum action_fail_response on_fail)
	{
	int old_rc = 0;
	bool expired = false;
	pcmk_resource_t *parent = rsc;
	enum rsc_role_e fail_role = pcmk_role_unknown;
	enum action_fail_response failure_strategy = pcmk_on_fail_restart;

	struct action_history history = {
	.rsc = rsc,
	.node = node,
	.xml = xml_op,
	.execution_status = PCMK_EXEC_UNKNOWN,
	};

	CRM_CHECK(rsc && node && xml_op, return);

	history.id = ID(xml_op);
	if (history.id == NULL) {
	pcmk__config_err("Ignoring resource history entry for %s on %s "
	"without ID", rsc->id, pcmk__node_name(node));
	return;
	}

	// Task and interval
	history.task = crm_element_value(xml_op, PCMK_XA_OPERATION);
	if (history.task == NULL) {
	pcmk__config_err("Ignoring resource history entry %s for %s on %s "
	"without " PCMK_XA_OPERATION,
	history.id, rsc->id, pcmk__node_name(node));
	return;
	}
	crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &(history.interval_ms));
	if (!can_affect_state(&history)) {
	pcmk__rsc_trace(rsc,
	"Ignoring resource history entry %s for %s on %s "
	"with irrelevant action '%s'",
	history.id, rsc->id, pcmk__node_name(node),
	history.task);
	return;
	}

	if (unpack_action_result(&history) != pcmk_rc_ok) {
	return; // Error already logged
	}

	history.expected_exit_status = pe__target_rc_from_xml(xml_op);
	history.key = pcmk__xe_history_key(xml_op);
	crm_element_value_int(xml_op, PCMK__XA_CALL_ID, &(history.call_id));

	pcmk__rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
	history.id, history.task, history.call_id,
	pcmk__node_name(node),
	pcmk_exec_status_str(history.execution_status),
	crm_exit_str(history.exit_status));

	if (node->details->unclean) {
	pcmk__rsc_trace(rsc,
	"%s is running on %s, which is unclean (further action "
	"depends on value of stop's on-fail attribute)",
	rsc->id, pcmk__node_name(node));
	}

	expired = check_operation_expiry(&history);
	old_rc = history.exit_status;

	remap_operation(&history, on_fail, expired);

	if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
	goto done;
	}

	if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
	mask_probe_failure(&history, old_rc, *last_failure, on_fail);
	goto done;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	switch (history.execution_status) {
	case PCMK_EXEC_PENDING:
	process_pending_action(&history, *last_failure);
	goto done;

	case PCMK_EXEC_DONE:
	update_resource_state(&history, history.exit_status, *last_failure,
	on_fail);
	goto done;

	case PCMK_EXEC_NOT_INSTALLED:
	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if (failure_strategy == pcmk_on_fail_ignore) {
	crm_warn("Cannot ignore failed %s of %s on %s: "
	"Resource agent doesn't exist "
	CRM_XS " status=%d rc=%d id=%s",
	history.task, rsc->id, pcmk__node_name(node),
	history.execution_status, history.exit_status,
	history.id);
	/* Also for printing it as "FAILED" by marking it as
	* pcmk_rsc_failed later
	*/
	*on_fail = pcmk_on_fail_ban;
	}
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);
	goto done;

	case PCMK_EXEC_NOT_CONNECTED:
	if (pe__is_guest_or_remote_node(node)
	&& pcmk_is_set(node->details->remote_rsc->flags,
	pcmk_rsc_managed)) {
	/* We should never get into a situation where a managed remote
	* connection resource is considered OK but a resource action
	* behind the connection gets a "not connected" status. But as a
	* fail-safe in case a bug or unusual circumstances do lead to
	* that, ensure the remote connection is considered failed.
	*/
	pcmk__set_rsc_flags(node->details->remote_rsc,
	pcmk_rsc_failed\|pcmk_rsc_stop_if_failed);
	}
	break; // Not done, do error handling

	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_ERROR_HARD:
	case PCMK_EXEC_ERROR_FATAL:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_INVALID:
	break; // Not done, do error handling

	default: // No other value should be possible at this point
	break;
	}

	unpack_failure_handling(&history, &failure_strategy, &fail_role);
	if ((failure_strategy == pcmk_on_fail_ignore)
	\|\| ((failure_strategy == pcmk_on_fail_restart_container)
	&& (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {

	char *last_change_s = last_change_str(xml_op);

	crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
	CRM_XS " %s",
	history.task, services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), rsc->id,
	pcmk__node_name(node), last_change_s, history.id);
	free(last_change_s);

	update_resource_state(&history, history.expected_exit_status,
	*last_failure, on_fail);
	crm_xml_add(xml_op, PCMK_XA_UNAME, node->details->uname);
	pcmk__set_rsc_flags(rsc, pcmk_rsc_ignore_failure);

	record_failed_op(&history);

	if ((failure_strategy == pcmk_on_fail_restart_container)
	&& cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(&history, failure_strategy, fail_role,
	last_failure, on_fail);

	if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
	uint8_t log_level = LOG_ERR;

	if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
	log_level = LOG_NOTICE;
	}
	do_crm_log(log_level,
	"Preventing %s from restarting on %s because "
	"of hard failure (%s%s%s) " CRM_XS " %s",
	parent->id, pcmk__node_name(node),
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, node, -INFINITY, "hard-error",
	rsc->cluster);

	} else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
	pcmk__sched_err("Preventing %s from restarting anywhere because "
	"of fatal failure (%s%s%s) " CRM_XS " %s",
	parent->id,
	services_ocf_exitcode_str(history.exit_status),
	(pcmk__str_empty(history.exit_reason)? "" : ": "),
	pcmk__s(history.exit_reason, ""), history.id);
	resource_location(parent, NULL, -INFINITY, "fatal-error",
	rsc->cluster);
	}
	}

	done:
	pcmk__rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
	rsc->id, pcmk__node_name(node), history.id,
	role2text(rsc->role), role2text(rsc->next_role));
	}

	static void
	add_node_attrs(const xmlNode xml_obj, pcmk_node_t node, bool overwrite,
	pcmk_scheduler_t *scheduler)
	{
	const char *cluster_name = NULL;

	pe_rule_eval_data_t rule_data = {
	.node_hash = NULL,
	.role = pcmk_role_unknown,
	.now = scheduler->now,
	.match_data = NULL,
	.rsc_data = NULL,
	.op_data = NULL
	};

	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_UNAME), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
	strdup(node->details->id));
	if (pcmk__str_eq(node->details->id, scheduler->dc_uuid, pcmk__str_casei)) {
	scheduler->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(PCMK_VALUE_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_IS_DC), strdup(PCMK_VALUE_FALSE));
	}

	cluster_name = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_CLUSTER_NAME);
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
	strdup(cluster_name));
	}

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
	node->details->attrs, NULL, overwrite,
	scheduler);

	pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
	node->details->utilization, NULL,
	FALSE, scheduler);

	if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
	const char *site_name = pe_node_attribute_raw(node, "site-name");

	if (site_name) {
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs,
	strdup(CRM_ATTR_SITE_NAME),
	strdup(cluster_name));
	}
	}
	}

	static GList *
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GList *gIter = NULL;
	GList *op_list = NULL;
	GList *sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = pcmk__xe_first_child(rsc_entry);
	rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {

	if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
	pcmk__str_none)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, PCMK_XA_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GList *
	find_operations(const char rsc, const char node, gboolean active_filter,
	pcmk_scheduler_t *scheduler)
	{
	GList *output = NULL;
	GList *intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(scheduler->input, PCMK_XE_STATUS, TRUE);

	pcmk_node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = pcmk__xe_first_child(status); node_state != NULL;
	node_state = pcmk__xe_next(node_state)) {

	- if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
	+ if (pcmk__str_eq((const char *) node_state->name, PCMK__XE_NODE_STATE,
	+ pcmk__str_none)) {
	const char *uname = crm_element_value(node_state, PCMK_XA_UNAME);

	if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
	continue;
	}

	this_node = pe_find_node(scheduler->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (pe__is_guest_or_remote_node(this_node)) {
	determine_remote_online_status(scheduler, this_node);

	} else {
	determine_online_status(node_state, this_node, scheduler);
	}

	if (this_node->details->online
	\|\| pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = pcmk__xe_next(lrm_rsc)) {

	if (pcmk__str_eq((const char *)lrm_rsc->name,
	XML_LRM_TAG_RESOURCE, pcmk__str_none)) {

	const char *rsc_id = crm_element_value(lrm_rsc,
	PCMK_XA_ID);

	if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/tools/crm_mon.c b/tools/crm_mon.c
	index a988c0e637..2fbd29d2d5 100644
	--- a/tools/crm_mon.c
	+++ b/tools/crm_mon.c
	@@ -1,2288 +1,2288 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdint.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>
	#include <signal.h>
	#include <sys/utsname.h>

	#include <crm/msg_xml.h>
	#include <crm/services.h>
	#include <crm/lrmd.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/internal.h> // pcmk__ends_with_ext()
	#include <crm/common/ipc.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/output.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/results.h>
	#include <crm/common/util.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/cib/internal.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <pacemaker-internal.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>

	#include "crm_mon.h"

	#define SUMMARY "Provides a summary of cluster's current state.\n\n" \
	"Outputs varying levels of detail in a number of different formats."

	/*
	* Definitions indicating which items to print
	*/

	static uint32_t show;
	static uint32_t show_opts = pcmk_show_pending;

	/*
	* Definitions indicating how to output
	*/

	static mon_output_format_t output_format = mon_output_unset;

	/* other globals */
	static GIOChannel *io_channel = NULL;
	static GMainLoop *mainloop = NULL;
	static guint reconnect_timer = 0;
	static mainloop_timer_t *refresh_timer = NULL;

	static enum pcmk_pacemakerd_state pcmkd_state = pcmk_pacemakerd_state_invalid;
	static cib_t *cib = NULL;
	static stonith_t *st = NULL;
	static xmlNode *current_cib = NULL;

	static GError *error = NULL;
	static pcmk__common_args_t *args = NULL;
	static pcmk__output_t *out = NULL;
	static GOptionContext *context = NULL;
	static gchar **processed_args = NULL;

	static time_t last_refresh = 0;
	volatile crm_trigger_t *refresh_trigger = NULL;

	static enum pcmk__fence_history fence_history = pcmk__fence_history_none;

	int interactive_fence_level = 0;

	static pcmk__supported_format_t formats[] = {
	#if CURSES_ENABLED
	CRM_MON_SUPPORTED_FORMAT_CURSES,
	#endif
	PCMK__SUPPORTED_FORMAT_HTML,
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
	"enum pcmk_pacemakerd_state")
	static int
	crm_mon_disconnected_default(pcmk__output_t *out, va_list args)
	{
	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
	"enum pcmk_pacemakerd_state")
	static int
	crm_mon_disconnected_html(pcmk__output_t *out, va_list args)
	{
	const char desc = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);

	if (out->dest != stdout) {
	out->reset(out);
	}

	pcmk__output_create_xml_text_node(out, "span", "Not connected to CIB");

	if (desc != NULL) {
	pcmk__output_create_xml_text_node(out, "span", ": ");
	pcmk__output_create_xml_text_node(out, "span", desc);
	}

	if (state != pcmk_pacemakerd_state_invalid) {
	const char *state_s = pcmk__pcmkd_state_enum2friendly(state);

	pcmk__output_create_xml_text_node(out, "span", " (");
	pcmk__output_create_xml_text_node(out, "span", state_s);
	pcmk__output_create_xml_text_node(out, "span", ")");
	}

	out->finish(out, CRM_EX_DISCONNECT, true, NULL);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
	"enum pcmk_pacemakerd_state")
	static int
	crm_mon_disconnected_text(pcmk__output_t *out, va_list args)
	{
	const char desc = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	int rc = pcmk_rc_ok;

	if (out->dest != stdout) {
	out->reset(out);
	}

	if (state != pcmk_pacemakerd_state_invalid) {
	rc = out->info(out, "Not connected to CIB%s%s (%s)",
	(desc != NULL)? ": " : "", pcmk__s(desc, ""),
	pcmk__pcmkd_state_enum2friendly(state));
	} else {
	rc = out->info(out, "Not connected to CIB%s%s",
	(desc != NULL)? ": " : "", pcmk__s(desc, ""));
	}

	out->finish(out, CRM_EX_DISCONNECT, true, NULL);
	return rc;
	}

	PCMK__OUTPUT_ARGS("crm-mon-disconnected", "const char *",
	"enum pcmk_pacemakerd_state")
	static int
	crm_mon_disconnected_xml(pcmk__output_t *out, va_list args)
	{
	const char desc = va_arg(args, const char );
	enum pcmk_pacemakerd_state state =
	(enum pcmk_pacemakerd_state) va_arg(args, int);
	const char *state_s = NULL;

	if (out->dest != stdout) {
	out->reset(out);
	}

	if (state != pcmk_pacemakerd_state_invalid) {
	state_s = pcmk_pacemakerd_api_daemon_state_enum2text(state);
	}

	pcmk__output_create_xml_node(out, "crm-mon-disconnected",
	PCMK_XA_DESCRIPTION, desc,
	"pacemakerd-state", state_s,
	NULL);

	out->finish(out, CRM_EX_DISCONNECT, true, NULL);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "crm-mon-disconnected", "default", crm_mon_disconnected_default },
	{ "crm-mon-disconnected", "html", crm_mon_disconnected_html },
	{ "crm-mon-disconnected", "text", crm_mon_disconnected_text },
	{ "crm-mon-disconnected", "xml", crm_mon_disconnected_xml },
	{ NULL, NULL, NULL },
	};

	/* Define exit codes for monitoring-compatible output
	* For nagios plugins, the possibilities are
	* OK=0, WARN=1, CRIT=2, and UNKNOWN=3
	*/
	#define MON_STATUS_WARN CRM_EX_ERROR
	#define MON_STATUS_CRIT CRM_EX_INVALID_PARAM
	#define MON_STATUS_UNKNOWN CRM_EX_UNIMPLEMENT_FEATURE

	#define RECONNECT_MSECS 5000

	struct {
	guint reconnect_ms;
	enum mon_exec_mode exec_mode;
	gboolean fence_connect;
	gboolean print_pending;
	gboolean show_bans;
	gboolean watch_fencing;
	char *pid_file;
	char *external_agent;
	char *external_recipient;
	char *neg_location_prefix;
	char *only_node;
	char *only_rsc;
	GSList *user_includes_excludes;
	GSList *includes_excludes;
	} options = {
	.reconnect_ms = RECONNECT_MSECS,
	.exec_mode = mon_exec_unset,
	.fence_connect = TRUE,
	};

	static crm_exit_t clean_up(crm_exit_t exit_code);
	static void crm_diff_update(const char event, xmlNode msg);
	static void clean_up_on_connection_failure(int rc);
	static int mon_refresh_display(gpointer user_data);
	static int setup_cib_connection(void);
	static int setup_fencer_connection(void);
	static int setup_api_connections(void);
	static void mon_st_callback_event(stonith_t * st, stonith_event_t * e);
	static void mon_st_callback_display(stonith_t * st, stonith_event_t * e);
	static void refresh_after_event(gboolean data_updated, gboolean enforce);

	static uint32_t
	all_includes(mon_output_format_t fmt) {
	if (fmt == mon_output_monitor \|\| fmt == mon_output_plain \|\| fmt == mon_output_console) {
	return ~pcmk_section_options;
	} else {
	return pcmk_section_all;
	}
	}

	static uint32_t
	default_includes(mon_output_format_t fmt) {
	switch (fmt) {
	case mon_output_monitor:
	case mon_output_plain:
	case mon_output_console:
	case mon_output_html:
	case mon_output_cgi:
	return pcmk_section_summary
	\|pcmk_section_nodes
	\|pcmk_section_resources
	\|pcmk_section_failures;

	case mon_output_xml:
	return all_includes(fmt);

	default:
	return 0;
	}
	}

	struct {
	const char *name;
	uint32_t bit;
	} sections[] = {
	{ "attributes", pcmk_section_attributes },
	{ "bans", pcmk_section_bans },
	{ "counts", pcmk_section_counts },
	{ "dc", pcmk_section_dc },
	{ "failcounts", pcmk_section_failcounts },
	{ "failures", pcmk_section_failures },
	{ PCMK__VALUE_FENCING, pcmk_section_fencing_all },
	{ "fencing-failed", pcmk_section_fence_failed },
	{ "fencing-pending", pcmk_section_fence_pending },
	{ "fencing-succeeded", pcmk_section_fence_worked },
	{ "maint-mode", pcmk_section_maint_mode },
	{ "nodes", pcmk_section_nodes },
	{ "operations", pcmk_section_operations },
	{ "options", pcmk_section_options },
	{ "resources", pcmk_section_resources },
	{ "stack", pcmk_section_stack },
	{ "summary", pcmk_section_summary },
	{ "tickets", pcmk_section_tickets },
	{ "times", pcmk_section_times },
	{ NULL }
	};

	static uint32_t
	find_section_bit(const char *name) {
	for (int i = 0; sections[i].name != NULL; i++) {
	if (pcmk__str_eq(sections[i].name, name, pcmk__str_casei)) {
	return sections[i].bit;
	}
	}

	return 0;
	}

	static gboolean
	apply_exclude(const gchar excludes, GError *error) {
	char **parts = NULL;
	gboolean result = TRUE;

	parts = g_strsplit(excludes, ",", 0);
	for (char *s = parts; s != NULL; s++) {
	uint32_t bit = find_section_bit(*s);

	if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
	show = 0;
	} else if (pcmk__str_eq(*s, PCMK__VALUE_NONE, pcmk__str_none)) {
	show = all_includes(output_format);
	} else if (bit != 0) {
	show &= ~bit;
	} else {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--exclude options: all, attributes, bans, counts, dc, "
	"failcounts, failures, fencing, fencing-failed, "
	"fencing-pending, fencing-succeeded, maint-mode, nodes, "
	PCMK__VALUE_NONE ", operations, options, resources, "
	"stack, summary, tickets, times");
	result = FALSE;
	break;
	}
	}
	g_strfreev(parts);
	return result;
	}

	static gboolean
	apply_include(const gchar includes, GError *error) {
	char **parts = NULL;
	gboolean result = TRUE;

	parts = g_strsplit(includes, ",", 0);
	for (char *s = parts; s != NULL; s++) {
	uint32_t bit = find_section_bit(*s);

	if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
	show = all_includes(output_format);
	} else if (pcmk__starts_with(*s, "bans")) {
	show \|= pcmk_section_bans;
	if (options.neg_location_prefix != NULL) {
	free(options.neg_location_prefix);
	options.neg_location_prefix = NULL;
	}

	if (strlen(s) > 4 && (s)[4] == ':') {
	options.neg_location_prefix = strdup(*s+5);
	}
	} else if (pcmk__str_any_of(*s, "default", "defaults", NULL)) {
	show \|= default_includes(output_format);
	} else if (pcmk__str_eq(*s, PCMK__VALUE_NONE, pcmk__str_none)) {
	show = 0;
	} else if (bit != 0) {
	show \|= bit;
	} else {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--include options: all, attributes, bans[:PREFIX], counts, dc, "
	"default, failcounts, failures, fencing, fencing-failed, "
	"fencing-pending, fencing-succeeded, maint-mode, nodes, "
	PCMK__VALUE_NONE ", operations, options, resources, "
	"stack, summary, tickets, times");
	result = FALSE;
	break;
	}
	}
	g_strfreev(parts);
	return result;
	}

	static gboolean
	apply_include_exclude(GSList lst, GError *error) {
	gboolean rc = TRUE;
	GSList *node = lst;

	while (node != NULL) {
	char *s = node->data;

	if (pcmk__starts_with(s, "--include=")) {
	rc = apply_include(s+10, error);
	} else if (pcmk__starts_with(s, "-I=")) {
	rc = apply_include(s+3, error);
	} else if (pcmk__starts_with(s, "--exclude=")) {
	rc = apply_exclude(s+10, error);
	} else if (pcmk__starts_with(s, "-U=")) {
	rc = apply_exclude(s+3, error);
	}

	if (rc != TRUE) {
	break;
	}

	node = node->next;
	}

	return rc;
	}

	static gboolean
	user_include_exclude_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	char *s = crm_strdup_printf("%s=%s", option_name, optarg);

	options.user_includes_excludes = g_slist_append(options.user_includes_excludes, s);
	return TRUE;
	}

	static gboolean
	include_exclude_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	char *s = crm_strdup_printf("%s=%s", option_name, optarg);

	options.includes_excludes = g_slist_append(options.includes_excludes, s);
	return TRUE;
	}

	static gboolean
	as_cgi_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&args->output_ty, "html");
	output_format = mon_output_cgi;
	options.exec_mode = mon_exec_one_shot;
	return TRUE;
	}

	static gboolean
	as_html_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&args->output_dest, optarg);
	pcmk__str_update(&args->output_ty, "html");
	output_format = mon_output_html;
	umask(S_IWGRP \| S_IWOTH); // World-readable HTML
	return TRUE;
	}

	static gboolean
	as_simple_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&args->output_ty, "text");
	output_format = mon_output_monitor;
	options.exec_mode = mon_exec_one_shot;
	return TRUE;
	}

	static gboolean
	as_xml_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&args->output_ty, "xml");
	output_format = mon_output_legacy_xml;
	return TRUE;
	}

	static gboolean
	fence_history_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	if (optarg == NULL) {
	interactive_fence_level = 2;
	} else {
	pcmk__scan_min_int(optarg, &interactive_fence_level, 0);
	}

	switch (interactive_fence_level) {
	case 3:
	options.fence_connect = TRUE;
	fence_history = pcmk__fence_history_full;
	return include_exclude_cb("--include", PCMK__VALUE_FENCING, data,
	err);

	case 2:
	options.fence_connect = TRUE;
	fence_history = pcmk__fence_history_full;
	return include_exclude_cb("--include", PCMK__VALUE_FENCING, data,
	err);

	case 1:
	options.fence_connect = TRUE;
	fence_history = pcmk__fence_history_full;
	return include_exclude_cb("--include", "fencing-failed,fencing-pending", data, err);

	case 0:
	options.fence_connect = FALSE;
	fence_history = pcmk__fence_history_none;
	return include_exclude_cb("--exclude", PCMK__VALUE_FENCING, data,
	err);

	default:
	g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Fence history must be 0-3");
	return FALSE;
	}
	}

	static gboolean
	group_by_node_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	show_opts \|= pcmk_show_rscs_by_node;
	return TRUE;
	}

	static gboolean
	hide_headers_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	return user_include_exclude_cb("--exclude", "summary", data, err);
	}

	static gboolean
	inactive_resources_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	show_opts \|= pcmk_show_inactive_rscs;
	return TRUE;
	}

	static gboolean
	no_curses_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	pcmk__str_update(&args->output_ty, "text");
	output_format = mon_output_plain;
	return TRUE;
	}

	static gboolean
	print_brief_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	show_opts \|= pcmk_show_brief;
	return TRUE;
	}

	static gboolean
	print_detail_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	show_opts \|= pcmk_show_details;
	return TRUE;
	}

	static gboolean
	print_description_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	show_opts \|= pcmk_show_description;
	return TRUE;
	}

	static gboolean
	print_timing_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	show_opts \|= pcmk_show_timing;
	return user_include_exclude_cb("--include", "operations", data, err);
	}

	static gboolean
	reconnect_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	int rc = crm_get_msec(optarg);

	if (rc == -1) {
	g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid value for -i: %s", optarg);
	return FALSE;
	} else {
	pcmk_parse_interval_spec(optarg, &options.reconnect_ms);

	if (options.exec_mode != mon_exec_daemonized) {
	// Reconnect interval applies to daemonized too, so don't override
	options.exec_mode = mon_exec_update;
	}
	}

	return TRUE;
	}

	/*!
	* \internal
	* \brief Enable one-shot mode
	*
	* \param[in] option_name Name of option being parsed (ignored)
	* \param[in] optarg Value to be parsed (ignored)
	* \param[in] data User data (ignored)
	* \param[out] err Where to store error (ignored)
	*/
	static gboolean
	one_shot_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **err)
	{
	options.exec_mode = mon_exec_one_shot;
	return TRUE;
	}

	/*!
	* \internal
	* \brief Enable daemonized mode
	*
	* \param[in] option_name Name of option being parsed (ignored)
	* \param[in] optarg Value to be parsed (ignored)
	* \param[in] data User data (ignored)
	* \param[out] err Where to store error (ignored)
	*/
	static gboolean
	daemonize_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **err)
	{
	options.exec_mode = mon_exec_daemonized;
	return TRUE;
	}

	static gboolean
	show_attributes_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	return user_include_exclude_cb("--include", "attributes", data, err);
	}

	static gboolean
	show_bans_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	if (optarg != NULL) {
	char *s = crm_strdup_printf("bans:%s", optarg);
	gboolean rc = user_include_exclude_cb("--include", s, data, err);
	free(s);
	return rc;
	} else {
	return user_include_exclude_cb("--include", "bans", data, err);
	}
	}

	static gboolean
	show_failcounts_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	return user_include_exclude_cb("--include", "failcounts", data, err);
	}

	static gboolean
	show_operations_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	return user_include_exclude_cb("--include", "failcounts,operations", data, err);
	}

	static gboolean
	show_tickets_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	return user_include_exclude_cb("--include", "tickets", data, err);
	}

	static gboolean
	use_cib_file_cb(const gchar option_name, const gchar optarg, gpointer data, GError **err) {
	setenv("CIB_file", optarg, 1);
	options.exec_mode = mon_exec_one_shot;
	return TRUE;
	}

	#define INDENT " "

	/* INDENT-OFF */
	static GOptionEntry addl_entries[] = {
	{ "interval", 'i', 0, G_OPTION_ARG_CALLBACK, reconnect_cb,
	"Update frequency (default is 5 seconds)",
	"TIMESPEC" },

	{ "one-shot", '1', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	one_shot_cb,
	"Display the cluster status once and exit",
	NULL },

	{ "daemonize", 'd', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	daemonize_cb,
	"Run in the background as a daemon.\n"
	INDENT "Requires at least one of --output-to and --external-agent.",
	NULL },

	{ "pid-file", 'p', 0, G_OPTION_ARG_FILENAME, &options.pid_file,
	"(Advanced) Daemon pid file location",
	"FILE" },

	{ "external-agent", 'E', 0, G_OPTION_ARG_FILENAME, &options.external_agent,
	"A program to run when resource operations take place",
	"FILE" },

	{ "external-recipient", 'e', 0, G_OPTION_ARG_STRING, &options.external_recipient,
	"A recipient for your program (assuming you want the program to send something to someone).",
	"RCPT" },

	{ "watch-fencing", 'W', 0, G_OPTION_ARG_NONE, &options.watch_fencing,
	"Listen for fencing events. For use with --external-agent.",
	NULL },

	{ "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, use_cib_file_cb,
	NULL,
	NULL },

	{ NULL }
	};

	static GOptionEntry display_entries[] = {
	{ "include", 'I', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
	"A list of sections to include in the output.\n"
	INDENT "See `Output Control` help for more information.",
	"SECTION(s)" },

	{ "exclude", 'U', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
	"A list of sections to exclude from the output.\n"
	INDENT "See `Output Control` help for more information.",
	"SECTION(s)" },

	{ "node", 0, 0, G_OPTION_ARG_STRING, &options.only_node,
	"When displaying information about nodes, show only what's related to the given\n"
	INDENT "node, or to all nodes tagged with the given tag",
	"NODE" },

	{ "resource", 0, 0, G_OPTION_ARG_STRING, &options.only_rsc,
	"When displaying information about resources, show only what's related to the given\n"
	INDENT "resource, or to all resources tagged with the given tag",
	"RSC" },

	{ "group-by-node", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, group_by_node_cb,
	"Group resources by node",
	NULL },

	{ "inactive", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, inactive_resources_cb,
	"Display inactive resources",
	NULL },

	{ "failcounts", 'f', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_failcounts_cb,
	"Display resource fail counts",
	NULL },

	{ "operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_operations_cb,
	"Display resource operation history",
	NULL },

	{ "timing-details", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_timing_cb,
	"Display resource operation history with timing details",
	NULL },

	{ "tickets", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_tickets_cb,
	"Display cluster tickets",
	NULL },

	{ "fence-history", 'm', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, fence_history_cb,
	"Show fence history:\n"
	INDENT "0=off, 1=failures and pending (default without option),\n"
	INDENT "2=add successes (default without value for option),\n"
	INDENT "3=show full history without reduction to most recent of each flavor",
	"LEVEL" },

	{ "neg-locations", 'L', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, show_bans_cb,
	"Display negative location constraints [optionally filtered by id prefix]",
	NULL },

	{ "show-node-attributes", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_attributes_cb,
	"Display node attributes",
	NULL },

	{ "hide-headers", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, hide_headers_cb,
	"Hide all headers",
	NULL },

	{ "show-detail", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_detail_cb,
	"Show more details (node IDs, individual clone instances)",
	NULL },

	{ "show-description", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_description_cb,
	"Show resource descriptions",
	NULL },

	{ "brief", 'b', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_brief_cb,
	"Brief output",
	NULL },

	{ "pending", 'j', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.print_pending,
	"Display pending state if '" PCMK_META_RECORD_PENDING "' is enabled",
	NULL },

	{ NULL }
	};

	static GOptionEntry deprecated_entries[] = {
	{ "as-html", 'h', G_OPTION_FLAG_FILENAME, G_OPTION_ARG_CALLBACK, as_html_cb,
	"Write cluster status to the named HTML file.\n"
	INDENT "Use --output-as=html --output-to=FILE instead.",
	"FILE" },

	{ "as-xml", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_xml_cb,
	"Write cluster status as XML to stdout. This will enable one-shot mode.\n"
	INDENT "Use --output-as=xml instead.",
	NULL },

	{ "simple-status", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	as_simple_cb,
	"Display the cluster status once as a simple one line output\n"
	INDENT "(suitable for nagios)",
	NULL },

	{ "disable-ncurses", 'N', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, no_curses_cb,
	"Disable the use of ncurses.\n"
	INDENT "Use --output-as=text instead.",
	NULL },

	{ "web-cgi", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_cgi_cb,
	"Web mode with output suitable for CGI (preselected when run as *.cgi).\n"
	INDENT "Use --output-as=html --html-cgi instead.",
	NULL },

	{ NULL }
	};
	/* INDENT-ON */

	/* Reconnect to the CIB and fencing agent after reconnect_ms has passed. This sounds
	* like it would be more broadly useful, but only ever happens after a disconnect via
	* mon_cib_connection_destroy.
	*/
	static gboolean
	reconnect_after_timeout(gpointer data)
	{
	#if CURSES_ENABLED
	if (output_format == mon_output_console) {
	clear();
	refresh();
	}
	#endif

	out->transient(out, "Reconnecting...");
	if (setup_api_connections() == pcmk_rc_ok) {
	// Trigger redrawing the screen (needs reconnect_timer == 0)
	reconnect_timer = 0;
	refresh_after_event(FALSE, TRUE);
	return G_SOURCE_REMOVE;
	}

	out->message(out, "crm-mon-disconnected",
	"Latest connection attempt failed", pcmkd_state);

	reconnect_timer = g_timeout_add(options.reconnect_ms,
	reconnect_after_timeout, NULL);
	return G_SOURCE_REMOVE;
	}

	/* Called from various places when we are disconnected from the CIB or from the
	* fencing agent. If the CIB connection is still valid, this function will also
	* attempt to sign off and reconnect.
	*/
	static void
	mon_cib_connection_destroy(gpointer user_data)
	{
	const char *msg = "Connection to the cluster lost";

	pcmkd_state = pcmk_pacemakerd_state_invalid;

	/* No crm-mon-disconnected message for console; a working implementation
	* is not currently worth the effort
	*/
	out->transient(out, "%s", msg);

	out->message(out, "crm-mon-disconnected", msg, pcmkd_state);

	if (refresh_timer != NULL) {
	/* we'll trigger a refresh after reconnect */
	mainloop_timer_stop(refresh_timer);
	}
	if (reconnect_timer) {
	/* we'll trigger a new reconnect-timeout at the end */
	g_source_remove(reconnect_timer);
	reconnect_timer = 0;
	}

	/* the client API won't properly reconnect notifications if they are still
	* in the table - so remove them
	*/
	stonith_api_delete(st);
	st = NULL;

	if (cib) {
	cib->cmds->signoff(cib);
	reconnect_timer = g_timeout_add(options.reconnect_ms,
	reconnect_after_timeout, NULL);
	}
	}

	/* Signal handler installed into the mainloop for normal program shutdown */
	static void
	mon_shutdown(int nsig)
	{
	clean_up(CRM_EX_OK);
	}

	#if CURSES_ENABLED
	static volatile sighandler_t ncurses_winch_handler;

	/* Signal handler installed the regular way (not into the main loop) for when
	* the screen is resized. Commonly, this happens when running in an xterm and
	* the user changes its size.
	*/
	static void
	mon_winresize(int nsig)
	{
	static int not_done;
	int lines = 0, cols = 0;

	if (!not_done++) {
	if (ncurses_winch_handler)
	/* the original ncurses WINCH signal handler does the
	* magic of retrieving the new window size;
	* otherwise, we'd have to use ioctl or tgetent */
	(*ncurses_winch_handler) (SIGWINCH);
	getmaxyx(stdscr, lines, cols);
	resizeterm(lines, cols);
	/* Alert the mainloop code we'd like the refresh_trigger to run next
	* time the mainloop gets around to checking.
	*/
	mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
	}
	not_done--;
	}
	#endif

	static int
	setup_fencer_connection(void)
	{
	int rc = pcmk_ok;

	if (options.fence_connect && st == NULL) {
	st = stonith_api_new();
	}

	if (!options.fence_connect \|\| st == NULL \|\| st->state != stonith_disconnected) {
	return rc;
	}

	rc = st->cmds->connect(st, crm_system_name, NULL);
	if (rc == pcmk_ok) {
	crm_trace("Setting up stonith callbacks");
	if (options.watch_fencing) {
	st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
	mon_st_callback_event);
	st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback_event);
	} else {
	st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
	mon_st_callback_display);
	st->cmds->register_notification(st, T_STONITH_NOTIFY_HISTORY, mon_st_callback_display);
	}
	} else {
	stonith_api_delete(st);
	st = NULL;
	}

	return rc;
	}

	static int
	setup_cib_connection(void)
	{
	int rc = pcmk_rc_ok;

	CRM_CHECK(cib != NULL, return EINVAL);

	if (cib->state != cib_disconnected) {
	// Already connected with notifications registered for CIB updates
	return rc;
	}

	rc = cib__signon_query(out, &cib, &current_cib);

	if (rc == pcmk_rc_ok) {
	rc = pcmk_legacy2rc(cib->cmds->set_connection_dnotify(cib,
	mon_cib_connection_destroy));
	if (rc == EPROTONOSUPPORT) {
	out->err(out,
	"CIB client does not support connection loss "
	"notifications; crm_mon will be unable to reconnect after "
	"connection loss");
	rc = pcmk_rc_ok;
	}

	if (rc == pcmk_rc_ok) {
	cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY,
	crm_diff_update);
	rc = pcmk_legacy2rc(cib->cmds->add_notify_callback(cib,
	T_CIB_DIFF_NOTIFY, crm_diff_update));
	}

	if (rc != pcmk_rc_ok) {
	if (rc == EPROTONOSUPPORT) {
	out->err(out,
	"CIB client does not support CIB diff "
	"notifications");
	} else {
	out->err(out, "CIB diff notification setup failed");
	}

	out->err(out, "Cannot monitor CIB changes; exiting");
	cib__clean_up_connection(&cib);
	stonith_api_delete(st);
	st = NULL;
	}
	}
	return rc;
	}

	/* This is used to set up the fencing options after the interactive UI has been stared.
	* fence_history_cb can't be used because it builds up a list of includes/excludes that
	* then have to be processed with apply_include_exclude and that could affect other
	* things.
	*/
	static void
	set_fencing_options(int level)
	{
	switch (level) {
	case 3:
	options.fence_connect = TRUE;
	fence_history = pcmk__fence_history_full;
	show \|= pcmk_section_fencing_all;
	break;

	case 2:
	options.fence_connect = TRUE;
	fence_history = pcmk__fence_history_full;
	show \|= pcmk_section_fencing_all;
	break;

	case 1:
	options.fence_connect = TRUE;
	fence_history = pcmk__fence_history_full;
	show \|= pcmk_section_fence_failed \| pcmk_section_fence_pending;
	break;

	default:
	interactive_fence_level = 0;
	options.fence_connect = FALSE;
	fence_history = pcmk__fence_history_none;
	show &= ~pcmk_section_fencing_all;
	break;
	}
	}

	static int
	setup_api_connections(void)
	{
	int rc = pcmk_rc_ok;

	CRM_CHECK(cib != NULL, return EINVAL);

	if (cib->state != cib_disconnected) {
	return rc;
	}

	if (cib->variant == cib_native) {
	rc = pcmk__pacemakerd_status(out, crm_system_name,
	options.reconnect_ms / 2, false,
	&pcmkd_state);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	switch (pcmkd_state) {
	case pcmk_pacemakerd_state_running:
	case pcmk_pacemakerd_state_remote:
	case pcmk_pacemakerd_state_shutting_down:
	/* Fencer and CIB may still be available while shutting down or
	* running on a Pacemaker Remote node
	*/
	break;
	default:
	// Fencer and CIB are definitely unavailable
	return ENOTCONN;
	}

	setup_fencer_connection();
	}

	rc = setup_cib_connection();
	return rc;
	}

	#if CURSES_ENABLED
	static const char *
	get_option_desc(char c)
	{
	const char *desc = "No help available";

	for (GOptionEntry *entry = display_entries; entry != NULL; entry++) {
	if (entry->short_name == c) {
	desc = entry->description;
	break;
	}
	}
	return desc;
	}

	#define print_option_help(out, option, condition) \
	curses_formatted_printf(out, "%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));

	/* This function is called from the main loop when there is something to be read
	* on stdin, like an interactive user's keystroke. All it does is read the keystroke,
	* set flags (or show the page showing which keystrokes are valid), and redraw the
	* screen. It does not do anything with connections to the CIB or fencing agent
	* agent what would happen in mon_refresh_display.
	*/
	static gboolean
	detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer user_data)
	{
	int c;
	gboolean config_mode = FALSE;

	while (1) {

	/* Get user input */
	c = getchar();

	switch (c) {
	case 'm':
	interactive_fence_level++;
	if (interactive_fence_level > 3) {
	interactive_fence_level = 0;
	}

	set_fencing_options(interactive_fence_level);
	break;
	case 'c':
	show ^= pcmk_section_tickets;
	break;
	case 'f':
	show ^= pcmk_section_failcounts;
	break;
	case 'n':
	show_opts ^= pcmk_show_rscs_by_node;
	break;
	case 'o':
	show ^= pcmk_section_operations;
	if (!pcmk_is_set(show, pcmk_section_operations)) {
	show_opts &= ~pcmk_show_timing;
	}
	break;
	case 'r':
	show_opts ^= pcmk_show_inactive_rscs;
	break;
	case 'R':
	show_opts ^= pcmk_show_details;
	#ifdef PCMK__COMPAT_2_0
	// Keep failed action output the same as 2.0.x
	show_opts \|= pcmk_show_failed_detail;
	#endif
	break;
	case 't':
	show_opts ^= pcmk_show_timing;
	if (pcmk_is_set(show_opts, pcmk_show_timing)) {
	show \|= pcmk_section_operations;
	}
	break;
	case 'A':
	show ^= pcmk_section_attributes;
	break;
	case 'L':
	show ^= pcmk_section_bans;
	break;
	case 'D':
	/* If any header is shown, clear them all, otherwise set them all */
	if (pcmk_any_flags_set(show, pcmk_section_summary)) {
	show &= ~pcmk_section_summary;
	} else {
	show \|= pcmk_section_summary;
	}
	/* Regardless, we don't show options in console mode. */
	show &= ~pcmk_section_options;
	break;
	case 'b':
	show_opts ^= pcmk_show_brief;
	break;
	case 'j':
	show_opts ^= pcmk_show_pending;
	break;
	case '?':
	config_mode = TRUE;
	break;
	default:
	/* All other keys just redraw the screen. */
	goto refresh;
	}

	if (!config_mode)
	goto refresh;

	clear();
	refresh();

	curses_formatted_printf(out, "%s", "Display option change mode\n");
	print_option_help(out, 'c', pcmk_is_set(show, pcmk_section_tickets));
	print_option_help(out, 'f', pcmk_is_set(show, pcmk_section_failcounts));
	print_option_help(out, 'n', pcmk_is_set(show_opts, pcmk_show_rscs_by_node));
	print_option_help(out, 'o', pcmk_is_set(show, pcmk_section_operations));
	print_option_help(out, 'r', pcmk_is_set(show_opts, pcmk_show_inactive_rscs));
	print_option_help(out, 't', pcmk_is_set(show_opts, pcmk_show_timing));
	print_option_help(out, 'A', pcmk_is_set(show, pcmk_section_attributes));
	print_option_help(out, 'L', pcmk_is_set(show, pcmk_section_bans));
	print_option_help(out, 'D', !pcmk_is_set(show, pcmk_section_summary));
	#ifdef PCMK__COMPAT_2_0
	print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details & ~pcmk_show_failed_detail));
	#else
	print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details));
	#endif
	print_option_help(out, 'b', pcmk_is_set(show_opts, pcmk_show_brief));
	print_option_help(out, 'j', pcmk_is_set(show_opts, pcmk_show_pending));
	curses_formatted_printf(out, "%d m: \t%s\n", interactive_fence_level, get_option_desc('m'));
	curses_formatted_printf(out, "%s", "\nToggle fields via field letter, type any other key to return\n");
	}

	refresh:
	refresh_after_event(FALSE, TRUE);

	return TRUE;
	}
	#endif // CURSES_ENABLED

	// Basically crm_signal_handler(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
	static void
	avoid_zombies(void)
	{
	struct sigaction sa;

	memset(&sa, 0, sizeof(struct sigaction));
	if (sigemptyset(&sa.sa_mask) < 0) {
	crm_warn("Cannot avoid zombies: %s", pcmk_rc_str(errno));
	return;
	}
	sa.sa_handler = SIG_IGN;
	sa.sa_flags = SA_RESTART\|SA_NOCLDWAIT;
	if (sigaction(SIGCHLD, &sa, NULL) < 0) {
	crm_warn("Cannot avoid zombies: %s", pcmk_rc_str(errno));
	}
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
	"Be less descriptive in output.",
	NULL },

	{ NULL }
	};

	#if CURSES_ENABLED
	const char *fmts = "console (default), html, text, xml, none";
	#else
	const char *fmts = "text (default), html, xml, none";
	#endif // CURSES_ENABLED
	const char *desc = NULL;

	desc = "Notes:\n\n"
	"If this program is called as crm_mon.cgi, --output-as=html and\n"
	"--html-cgi are automatically added to the command line\n"
	"arguments.\n\n"

	"Time Specification:\n\n"
	"The TIMESPEC in any command line option can be specified in many\n"
	"different formats. It can be an integer number of seconds, a\n"
	"number plus units (us/usec/ms/msec/s/sec/m/min/h/hr), or an ISO\n"
	"8601 period specification.\n\n"

	"Output Control:\n\n"
	"By default, a particular set of sections are written to the\n"
	"output destination. The default varies based on the output\n"
	"format: XML includes all sections by default, while other output\n"
	"formats include less. This set can be modified with the --include\n"
	"and --exclude command line options. Each option may be passed\n"
	"multiple times, and each can specify a comma-separated list of\n"
	"sections. The options are applied to the default set, in order\n"
	"from left to right as they are passed on the command line. For a\n"
	"list of valid sections, pass --include=list or --exclude=list.\n\n"

	"Interactive Use:\n\n"
	#if CURSES_ENABLED
	"When run interactively, crm_mon can be told to hide and show\n"
	"various sections of output. To see a help screen explaining the\n"
	"options, press '?'. Any key stroke aside from those listed will\n"
	"cause the screen to refresh.\n\n"
	#else
	"The local installation of Pacemaker was built without support for\n"
	"interactive (console) mode. A curses library must be available at\n"
	"build time to support interactive mode.\n\n"
	#endif // CURSES_ENABLED

	"Examples:\n\n"
	#if CURSES_ENABLED
	"Display the cluster status on the console with updates as they\n"
	"occur:\n\n"
	"\tcrm_mon\n\n"
	#endif // CURSES_ENABLED

	"Display the cluster status once and exit:\n\n"
	"\tcrm_mon -1\n\n"

	"Display the cluster status, group resources by node, and include\n"
	"inactive resources in the list:\n\n"
	"\tcrm_mon --group-by-node --inactive\n\n"

	"Start crm_mon as a background daemon and have it write the\n"
	"cluster status to an HTML file:\n\n"
	"\tcrm_mon --daemonize --output-as html "
	"--output-to /path/to/docroot/filename.html\n\n"

	"Display the cluster status as XML:\n\n"
	"\tcrm_mon --output-as xml\n\n";

	context = pcmk__build_arg_context(args, fmts, group, NULL);
	pcmk__add_main_args(context, extra_prog_entries);
	g_option_context_set_description(context, desc);

	pcmk__add_arg_group(context, "display", "Display Options:",
	"Show display options", display_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
	"Show deprecated options", deprecated_entries);

	return context;
	}

	/* If certain format options were specified, we want to set some extra
	* options. We can just process these like they were given on the
	* command line.
	*/
	static void
	add_output_args(void) {
	GError *err = NULL;

	if (output_format == mon_output_plain) {
	if (!pcmk__force_args(context, &err, "%s --text-fancy", g_get_prgname())) {
	g_propagate_error(&error, err);
	clean_up(CRM_EX_USAGE);
	}
	} else if (output_format == mon_output_cgi) {
	if (!pcmk__force_args(context, &err, "%s --html-cgi", g_get_prgname())) {
	g_propagate_error(&error, err);
	clean_up(CRM_EX_USAGE);
	}
	} else if (output_format == mon_output_xml) {
	if (!pcmk__force_args(context, &err, "%s --xml-simple-list --xml-substitute", g_get_prgname())) {
	g_propagate_error(&error, err);
	clean_up(CRM_EX_USAGE);
	}
	} else if (output_format == mon_output_legacy_xml) {
	output_format = mon_output_xml;
	if (!pcmk__force_args(context, &err, "%s --xml-legacy --xml-substitute", g_get_prgname())) {
	g_propagate_error(&error, err);
	clean_up(CRM_EX_USAGE);
	}
	}
	}

	/*!
	* \internal
	* \brief Set output format based on \p --output-as arguments and mode arguments
	*
	* When the deprecated output format arguments (\p --as-cgi, \p --as-html,
	* \p --simple-status, \p --as-xml) are parsed, callback functions set
	* \p output_format (and the umask if appropriate). If none of the deprecated
	* arguments were specified, this function does the same based on the current
	* \p --output-as arguments and the \p --one-shot and \p --daemonize arguments.
	*
	* \param[in,out] args Command line arguments
	*/
	static void
	reconcile_output_format(pcmk__common_args_t *args)
	{
	if (output_format != mon_output_unset) {
	/* One of the deprecated arguments was used, and we're finished. Note
	* that this means the deprecated arguments take precedence.
	*/
	return;
	}

	if (pcmk__str_eq(args->output_ty, "none", pcmk__str_none)) {
	output_format = mon_output_none;

	} else if (pcmk__str_eq(args->output_ty, "html", pcmk__str_none)) {
	output_format = mon_output_html;
	umask(S_IWGRP \| S_IWOTH); // World-readable HTML

	} else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
	output_format = mon_output_xml;

	#if CURSES_ENABLED
	} else if (pcmk__str_eq(args->output_ty, "console",
	pcmk__str_null_matches)) {
	/* Console is the default format if no conflicting options are given.
	*
	* Use text output instead if one of the following conditions is met:
	* * We've requested daemonized or one-shot mode (console output is
	* incompatible with modes other than mon_exec_update)
	* * We requested the version, which is effectively one-shot
	* * We specified a non-stdout output destination (console mode is
	* compatible only with stdout)
	*/
	if ((options.exec_mode == mon_exec_daemonized)
	\|\| (options.exec_mode == mon_exec_one_shot)
	\|\| args->version
	\|\| !pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches)) {

	pcmk__str_update(&args->output_ty, "text");
	output_format = mon_output_plain;
	} else {
	pcmk__str_update(&args->output_ty, "console");
	output_format = mon_output_console;
	crm_enable_stderr(FALSE);
	}
	#endif // CURSES_ENABLED

	} else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches)) {
	/* Text output was explicitly requested, or it's the default because
	* curses is not enabled
	*/
	pcmk__str_update(&args->output_ty, "text");
	output_format = mon_output_plain;
	}

	// Otherwise, invalid format. Let pcmk__output_new() throw an error.
	}

	/*!
	* \internal
	* \brief Set execution mode to the output format's default if appropriate
	*
	* \param[in,out] args Command line arguments
	*/
	static void
	set_default_exec_mode(const pcmk__common_args_t *args)
	{
	if (output_format == mon_output_console) {
	/* Update is the only valid mode for console, but set here instead of
	* reconcile_output_format() for isolation and consistency
	*/
	options.exec_mode = mon_exec_update;

	} else if (options.exec_mode == mon_exec_unset) {
	// Default to one-shot mode for all other formats
	options.exec_mode = mon_exec_one_shot;

	} else if ((options.exec_mode == mon_exec_update)
	&& pcmk__str_eq(args->output_dest, "-",
	pcmk__str_null_matches)) {
	// If not using console format, update mode cannot be used with stdout
	options.exec_mode = mon_exec_one_shot;
	}
	}

	static void
	clean_up_on_connection_failure(int rc)
	{
	if (output_format == mon_output_monitor) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "CLUSTER CRIT: Connection to cluster failed: %s",
	pcmk_rc_str(rc));
	clean_up(MON_STATUS_CRIT);
	} else if (rc == ENOTCONN) {
	if (pcmkd_state == pcmk_pacemakerd_state_remote) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: remote-node not connected to cluster");
	} else {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");
	}
	} else {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Connection to cluster failed: %s", pcmk_rc_str(rc));
	}

	clean_up(pcmk_rc2exitc(rc));
	}

	static void
	one_shot(void)
	{
	int rc = pcmk__status(out, cib, fence_history, show, show_opts,
	options.only_node, options.only_rsc,
	options.neg_location_prefix,
	output_format == mon_output_monitor, 0);

	if (rc == pcmk_rc_ok) {
	clean_up(pcmk_rc2exitc(rc));
	} else {
	clean_up_on_connection_failure(rc);
	}
	}

	static void
	exit_on_invalid_cib(void)
	{
	if (cib != NULL) {
	return;
	}

	// Shouldn't really be possible
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Invalid CIB source");
	clean_up(CRM_EX_ERROR);
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	GOptionGroup *output_group = NULL;

	args = pcmk__new_common_args(SUMMARY);
	context = build_arg_context(args, &output_group);
	pcmk__register_formats(output_group, formats);

	options.pid_file = strdup("/tmp/ClusterMon.pid");
	pcmk__cli_init_logging("crm_mon", 0);

	// Avoid needing to wait for subprocesses forked for -E/--external-agent
	avoid_zombies();

	if (pcmk__ends_with_ext(argv[0], ".cgi")) {
	output_format = mon_output_cgi;
	options.exec_mode = mon_exec_one_shot;
	}

	processed_args = pcmk__cmdline_preproc(argv, "ehimpxEILU");

	fence_history_cb("--fence-history", "1", NULL, NULL);

	/* Set an HTML title regardless of what format we will eventually use. This can't
	* be done in add_output_args. That function is called after command line
	* arguments are processed in the next block, which means it'll override whatever
	* title the user provides. Doing this here means the user can give their own
	* title on the command line.
	*/
	if (!pcmk__force_args(context, &error, "%s --html-title \"Cluster Status\"",
	g_get_prgname())) {
	return clean_up(CRM_EX_USAGE);
	}

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	return clean_up(CRM_EX_USAGE);
	}

	for (int i = 0; i < args->verbosity; i++) {
	crm_bump_log_level(argc, argv);
	}

	if (!args->version) {
	if (args->quiet) {
	include_exclude_cb("--exclude", "times", NULL, NULL);
	}

	if (options.watch_fencing) {
	fence_history_cb("--fence-history", "0", NULL, NULL);
	options.fence_connect = TRUE;
	}

	/* create the cib-object early to be able to do further
	* decisions based on the cib-source
	*/
	cib = cib_new();

	exit_on_invalid_cib();

	switch (cib->variant) {
	case cib_native:
	// Everything (fencer, CIB, pcmkd status) should be available
	break;

	case cib_file:
	// Live fence history is not meaningful
	fence_history_cb("--fence-history", "0", NULL, NULL);

	/* Notifications are unsupported; nothing to monitor
	* @COMPAT: Let setup_cib_connection() handle this by exiting?
	*/
	options.exec_mode = mon_exec_one_shot;
	break;

	case cib_remote:
	// We won't receive any fencing updates
	fence_history_cb("--fence-history", "0", NULL, NULL);
	break;

	default:
	/* something is odd */
	exit_on_invalid_cib();
	break;
	}

	if ((options.exec_mode == mon_exec_daemonized)
	&& !options.external_agent
	&& pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches)) {

	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--daemonize requires at least one of --output-to "
	"(with value not set to '-') and --external-agent");
	return clean_up(CRM_EX_USAGE);
	}
	}

	reconcile_output_format(args);
	set_default_exec_mode(args);
	add_output_args();

	/* output_format MUST NOT BE CHANGED AFTER THIS POINT. */

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error creating output format %s: %s",
	args->output_ty, pcmk_rc_str(rc));
	return clean_up(CRM_EX_ERROR);
	}

	/* If we had a valid format for pcmk__output_new(), output_format should be
	* set by now.
	*/
	CRM_ASSERT(output_format != mon_output_unset);

	if (options.exec_mode == mon_exec_daemonized) {
	if (!options.external_agent && (output_format == mon_output_none)) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"--daemonize requires --external-agent if used with "
	"--output-as=none");
	return clean_up(CRM_EX_USAGE);
	}
	crm_enable_stderr(FALSE);
	cib_delete(cib);
	cib = NULL;
	pcmk__daemonize(crm_system_name, options.pid_file);
	cib = cib_new();
	exit_on_invalid_cib();
	}

	show = default_includes(output_format);

	/* Apply --include/--exclude flags we used internally. There's no error reporting
	* here because this would be a programming error.
	*/
	apply_include_exclude(options.includes_excludes, &error);

	/* And now apply any --include/--exclude flags the user gave on the command line.
	* These are done in a separate pass from the internal ones because we want to
	* make sure whatever the user specifies overrides whatever we do.
	*/
	if (!apply_include_exclude(options.user_includes_excludes, &error)) {
	return clean_up(CRM_EX_USAGE);
	}

	/* Sync up the initial value of interactive_fence_level with whatever was set with
	* --include/--exclude= options.
	*/
	if (pcmk_all_flags_set(show, pcmk_section_fencing_all)) {
	interactive_fence_level = 3;
	} else if (pcmk_is_set(show, pcmk_section_fence_worked)) {
	interactive_fence_level = 2;
	} else if (pcmk_any_flags_set(show, pcmk_section_fence_failed \| pcmk_section_fence_pending)) {
	interactive_fence_level = 1;
	} else {
	interactive_fence_level = 0;
	}

	pcmk__register_lib_messages(out);
	crm_mon_register_messages(out);
	pe__register_messages(out);
	stonith__register_messages(out);

	// Messages internal to this file, nothing curses-specific
	pcmk__register_messages(out, fmt_functions);

	if (args->version) {
	out->version(out, false);
	return clean_up(CRM_EX_OK);
	}

	/* Extra sanity checks when in CGI mode */
	if (output_format == mon_output_cgi) {
	if (cib->variant == cib_file) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode used with CIB file");
	return clean_up(CRM_EX_USAGE);
	} else if (options.external_agent != NULL) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with --external-agent");
	return clean_up(CRM_EX_USAGE);
	} else if (options.exec_mode == mon_exec_daemonized) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with -d");
	return clean_up(CRM_EX_USAGE);
	}
	}

	if (output_format == mon_output_xml) {
	show_opts \|= pcmk_show_inactive_rscs \| pcmk_show_timing;
	}

	if ((output_format == mon_output_html \|\| output_format == mon_output_cgi) &&
	out->dest != stdout) {
	pcmk__html_add_header("meta", "http-equiv", "refresh", "content",
	pcmk__itoa(options.reconnect_ms / 1000), NULL);
	}

	#ifdef PCMK__COMPAT_2_0
	// Keep failed action output the same as 2.0.x
	show_opts \|= pcmk_show_failed_detail;
	#endif

	crm_info("Starting %s", crm_system_name);

	cib__set_output(cib, out);

	if (options.exec_mode == mon_exec_one_shot) {
	one_shot();
	}

	out->message(out, "crm-mon-disconnected",
	"Waiting for initial connection", pcmkd_state);
	do {
	out->transient(out, "Connecting to cluster...");
	rc = setup_api_connections();

	if (rc != pcmk_rc_ok) {
	if ((rc == ENOTCONN) \|\| (rc == ECONNREFUSED)) {
	out->transient(out, "Connection failed. Retrying in %ums...",
	options.reconnect_ms);
	}

	// Give some time to view all output even if we won't retry
	pcmk__sleep_ms(options.reconnect_ms);
	#if CURSES_ENABLED
	if (output_format == mon_output_console) {
	clear();
	refresh();
	}
	#endif
	}
	} while ((rc == ENOTCONN) \|\| (rc == ECONNREFUSED));

	if (rc != pcmk_rc_ok) {
	clean_up_on_connection_failure(rc);
	}

	set_fencing_options(interactive_fence_level);
	mon_refresh_display(NULL);

	mainloop = g_main_loop_new(NULL, FALSE);

	mainloop_add_signal(SIGTERM, mon_shutdown);
	mainloop_add_signal(SIGINT, mon_shutdown);
	#if CURSES_ENABLED
	if (output_format == mon_output_console) {
	ncurses_winch_handler = crm_signal_handler(SIGWINCH, mon_winresize);
	if (ncurses_winch_handler == SIG_DFL \|\|
	ncurses_winch_handler == SIG_IGN \|\| ncurses_winch_handler == SIG_ERR)
	ncurses_winch_handler = NULL;

	io_channel = g_io_channel_unix_new(STDIN_FILENO);
	g_io_add_watch(io_channel, G_IO_IN, detect_user_input, NULL);
	}
	#endif

	/* When refresh_trigger->trigger is set to TRUE, call mon_refresh_display. In
	* this file, that is anywhere mainloop_set_trigger is called.
	*/
	refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);

	g_main_loop_run(mainloop);
	g_main_loop_unref(mainloop);

	if (io_channel != NULL) {
	g_io_channel_shutdown(io_channel, TRUE, NULL);
	}

	crm_info("Exiting %s", crm_system_name);

	return clean_up(CRM_EX_OK);
	}

	static int
	send_custom_trap(const char node, const char rsc, const char *task, int target_rc, int rc,
	int status, const char *desc)
	{
	pid_t pid;

	/setenv needs chars, these are ints /
	char *rc_s = pcmk__itoa(rc);
	char *status_s = pcmk__itoa(status);
	char *target_rc_s = pcmk__itoa(target_rc);

	crm_debug("Sending external notification to '%s' via '%s'", options.external_recipient, options.external_agent);

	if(rsc) {
	setenv("CRM_notify_rsc", rsc, 1);
	}
	if (options.external_recipient) {
	setenv("CRM_notify_recipient", options.external_recipient, 1);
	}
	setenv("CRM_notify_node", node, 1);
	setenv("CRM_notify_task", task, 1);
	setenv("CRM_notify_desc", desc, 1);
	setenv("CRM_notify_rc", rc_s, 1);
	setenv("CRM_notify_target_rc", target_rc_s, 1);
	setenv("CRM_notify_status", status_s, 1);

	pid = fork();
	if (pid == -1) {
	out->err(out, "notification fork() failed: %s", strerror(errno));
	}
	if (pid == 0) {
	/* crm_debug("notification: I am the child. Executing the nofitication program."); */
	execl(options.external_agent, options.external_agent, NULL);
	exit(CRM_EX_ERROR);
	}

	crm_trace("Finished running custom notification program '%s'.", options.external_agent);
	free(target_rc_s);
	free(status_s);
	free(rc_s);
	return 0;
	}

	static int
	handle_rsc_op(xmlNode xml, void userdata)
	{
	const char node_id = (const char ) userdata;
	int rc = -1;
	int status = -1;
	int target_rc = -1;
	gboolean notify = TRUE;

	char *rsc = NULL;
	char *task = NULL;
	const char *desc = NULL;
	const char *magic = NULL;
	const char *id = NULL;
	const char *node = NULL;

	xmlNode *n = xml;
	xmlNode * rsc_op = xml;

	if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
	pcmk__xe_foreach_child(xml, NULL, handle_rsc_op, (void *) node_id);
	return pcmk_rc_ok;
	}

	id = pcmk__xe_history_key(rsc_op);

	magic = crm_element_value(rsc_op, PCMK__XA_TRANSITION_MAGIC);
	if (magic == NULL) {
	/* non-change */
	return pcmk_rc_ok;
	}

	if (!decode_transition_magic(magic, NULL, NULL, NULL, &status, &rc,
	&target_rc)) {
	crm_err("Invalid event %s detected for %s", magic, id);
	return pcmk_rc_ok;
	}

	if (parse_op_key(id, &rsc, &task, NULL) == FALSE) {
	crm_err("Invalid event detected for %s", id);
	goto bail;
	}

	node = crm_element_value(rsc_op, PCMK__META_ON_NODE);

	- while ((n != NULL) && !pcmk__xe_is(n, XML_CIB_TAG_STATE)) {
	+ while ((n != NULL) && !pcmk__xe_is(n, PCMK__XE_NODE_STATE)) {
	n = n->parent;
	}

	if(node == NULL && n) {
	node = crm_element_value(n, PCMK_XA_UNAME);
	}

	if (node == NULL && n) {
	node = ID(n);
	}

	if (node == NULL) {
	node = node_id;
	}

	if (node == NULL) {
	crm_err("No node detected for event %s (%s)", magic, id);
	goto bail;
	}

	/* look up where we expected it to be? */
	desc = pcmk_rc_str(pcmk_rc_ok);
	if ((status == PCMK_EXEC_DONE) && (target_rc == rc)) {
	crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
	if (rc == PCMK_OCF_NOT_RUNNING) {
	notify = FALSE;
	}

	} else if (status == PCMK_EXEC_DONE) {
	desc = services_ocf_exitcode_str(rc);
	crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);

	} else {
	desc = pcmk_exec_status_str(status);
	crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
	}

	if (notify && options.external_agent) {
	send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
	}

	bail:
	free(rsc);
	free(task);
	return pcmk_rc_ok;
	}

	/* This function is just a wrapper around mainloop_set_trigger so that it can be
	* called from a mainloop directly. It's simply another way of ensuring the screen
	* gets redrawn.
	*/
	static gboolean
	mon_trigger_refresh(gpointer user_data)
	{
	mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
	return FALSE;
	}

	static int
	handle_op_for_node(xmlNode xml, void userdata)
	{
	const char *node = crm_element_value(xml, PCMK_XA_UNAME);

	if (node == NULL) {
	node = ID(xml);
	}

	handle_rsc_op(xml, (void *) node);
	return pcmk_rc_ok;
	}

	static void
	crm_diff_update_v2(const char event, xmlNode msg)
	{
	xmlNode *change = NULL;
	xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	for (change = pcmk__xml_first_child(diff); change != NULL;
	change = pcmk__xml_next(change)) {
	const char *name = NULL;
	const char *op = crm_element_value(change, PCMK_XA_OPERATION);
	const char *xpath = crm_element_value(change, PCMK_XA_PATH);
	xmlNode *match = NULL;
	const char *node = NULL;

	if(op == NULL) {
	continue;

	} else if(strcmp(op, "create") == 0) {
	match = change->children;

	} else if(strcmp(op, "move") == 0) {
	continue;

	} else if(strcmp(op, "delete") == 0) {
	continue;

	} else if(strcmp(op, "modify") == 0) {
	match = first_named_child(change, XML_DIFF_RESULT);
	if(match) {
	match = match->children;
	}
	}

	if(match) {
	name = (const char *)match->name;
	}

	crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
	if(xpath == NULL) {
	/* Version field, ignore */

	} else if(name == NULL) {
	crm_debug("No result for %s operation to %s", op, xpath);
	CRM_ASSERT(strcmp(op, "delete") == 0 \|\| strcmp(op, "move") == 0);

	} else if (strcmp(name, PCMK_XE_CIB) == 0) {
	pcmk__xe_foreach_child(first_named_child(match, PCMK_XE_STATUS),
	NULL, handle_op_for_node, NULL);

	} else if (strcmp(name, PCMK_XE_STATUS) == 0) {
	pcmk__xe_foreach_child(match, NULL, handle_op_for_node, NULL);

	- } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
	+ } else if (strcmp(name, PCMK__XE_NODE_STATE) == 0) {
	node = crm_element_value(match, PCMK_XA_UNAME);
	if (node == NULL) {
	node = ID(match);
	}
	handle_rsc_op(match, (void *) node);

	} else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
	node = ID(match);
	handle_rsc_op(match, (void *) node);

	} else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	handle_rsc_op(match, local_node);
	free(local_node);

	} else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	handle_rsc_op(match, local_node);
	free(local_node);

	} else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
	char *local_node = pcmk__xpath_node_id(xpath, "lrm");

	handle_rsc_op(match, local_node);
	free(local_node);

	} else {
	crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
	}
	}
	}

	static void
	crm_diff_update_v1(const char event, xmlNode msg)
	{
	/* Process operation updates */
	xmlXPathObject *xpathObj = xpath_search(msg,
	"//" F_CIB_UPDATE_RESULT
	"//" PCMK__XE_DIFF_ADDED
	"//" XML_LRM_TAG_RSC_OP);
	int lpc = 0, max = numXpathResults(xpathObj);

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *rsc_op = getXpathResult(xpathObj, lpc);

	handle_rsc_op(rsc_op, NULL);
	}
	freeXpathObject(xpathObj);
	}

	static void
	crm_diff_update(const char event, xmlNode msg)
	{
	int rc = -1;
	static bool stale = FALSE;
	gboolean cib_updated = FALSE;
	xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	out->progress(out, false);

	if (current_cib != NULL) {
	rc = xml_apply_patchset(current_cib, diff, TRUE);

	switch (rc) {
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(current_cib); current_cib = NULL;
	break;
	case pcmk_ok:
	cib_updated = TRUE;
	break;
	default:
	crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(current_cib); current_cib = NULL;
	}
	}

	if (current_cib == NULL) {
	crm_trace("Re-requesting the full cib");
	cib->cmds->query(cib, NULL, &current_cib, cib_scope_local \| cib_sync_call);
	}

	if (options.external_agent) {
	int format = 0;
	crm_element_value_int(diff, PCMK_XA_FORMAT, &format);
	switch(format) {
	case 1:
	crm_diff_update_v1(event, msg);
	break;
	case 2:
	crm_diff_update_v2(event, msg);
	break;
	default:
	crm_err("Unknown patch format: %d", format);
	}
	}

	if (current_cib == NULL) {
	if(!stale) {
	out->info(out, "--- Stale data ---");
	}
	stale = TRUE;
	return;
	}

	stale = FALSE;
	refresh_after_event(cib_updated, FALSE);
	}

	static int
	mon_refresh_display(gpointer user_data)
	{
	int rc = pcmk_rc_ok;

	last_refresh = time(NULL);

	if (output_format == mon_output_none) {
	return G_SOURCE_REMOVE;
	}

	if (fence_history == pcmk__fence_history_full &&
	!pcmk_all_flags_set(show, pcmk_section_fencing_all) &&
	output_format != mon_output_xml) {
	fence_history = pcmk__fence_history_reduced;
	}

	// Get an up-to-date pacemakerd status for the cluster summary
	if (cib->variant == cib_native) {
	pcmk__pacemakerd_status(out, crm_system_name, options.reconnect_ms / 2,
	false, &pcmkd_state);
	}

	if (out->dest != stdout) {
	out->reset(out);
	}

	rc = pcmk__output_cluster_status(out, st, cib, current_cib, pcmkd_state,
	fence_history, show, show_opts,
	options.only_node,options.only_rsc,
	options.neg_location_prefix,
	output_format == mon_output_monitor);

	if (output_format == mon_output_monitor && rc != pcmk_rc_ok) {
	clean_up(MON_STATUS_WARN);
	return G_SOURCE_REMOVE;
	} else if (rc == pcmk_rc_schema_validation) {
	clean_up(CRM_EX_CONFIG);
	return G_SOURCE_REMOVE;
	}

	if (out->dest != stdout) {
	out->finish(out, CRM_EX_OK, true, NULL);
	}

	return G_SOURCE_CONTINUE;
	}

	/* This function is called for fencing events (see setup_fencer_connection() for
	* which ones) when --watch-fencing is used on the command line
	*/
	static void
	mon_st_callback_event(stonith_t * st, stonith_event_t * e)
	{
	if (st->state == stonith_disconnected) {
	/* disconnect cib as well and have everything reconnect */
	mon_cib_connection_destroy(NULL);
	} else if (options.external_agent) {
	char *desc = stonith__event_description(e);

	send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
	free(desc);
	}
	}

	/* Cause the screen to be redrawn (via mainloop_set_trigger) when various conditions are met:
	*
	* - If the last update occurred more than reconnect_ms ago (defaults to 5s, but
	* can be changed via the -i command line option), or
	* - After every 10 CIB updates, or
	* - If it's been 2s since the last update
	*
	* This function sounds like it would be more broadly useful, but it is only called when a
	* fencing event is received or a CIB diff occurrs.
	*/
	static void
	refresh_after_event(gboolean data_updated, gboolean enforce)
	{
	static int updates = 0;
	time_t now = time(NULL);

	if (data_updated) {
	updates++;
	}

	if(refresh_timer == NULL) {
	refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
	}

	if (reconnect_timer > 0) {
	/* we will receive a refresh request after successful reconnect */
	mainloop_timer_stop(refresh_timer);
	return;
	}

	/* as we're not handling initial failure of fencer-connection as
	* fatal give it a retry here
	* not getting here if cib-reconnection is already on the way
	*/
	setup_fencer_connection();

	if (enforce \|\|
	((now - last_refresh) > (options.reconnect_ms / 1000)) \|\|
	updates >= 10) {
	mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
	mainloop_timer_stop(refresh_timer);
	updates = 0;

	} else {
	mainloop_timer_start(refresh_timer);
	}
	}

	/* This function is called for fencing events (see setup_fencer_connection() for
	* which ones) when --watch-fencing is NOT used on the command line
	*/
	static void
	mon_st_callback_display(stonith_t * st, stonith_event_t * e)
	{
	if (st->state == stonith_disconnected) {
	/* disconnect cib as well and have everything reconnect */
	mon_cib_connection_destroy(NULL);
	} else {
	out->progress(out, false);
	refresh_after_event(TRUE, FALSE);
	}
	}

	/*
	* De-init ncurses, disconnect from the CIB manager, disconnect fencing,
	* deallocate memory and show usage-message if requested.
	*
	* We don't actually return, but nominally returning crm_exit_t allows a usage
	* like "return clean_up(exit_code);" which helps static analysis understand the
	* code flow.
	*/
	static crm_exit_t
	clean_up(crm_exit_t exit_code)
	{
	/* Quitting crm_mon is much more complicated than it ought to be. */

	/* (1) Close connections, free things, etc. */
	cib__clean_up_connection(&cib);
	stonith_api_delete(st);
	free(options.neg_location_prefix);
	free(options.only_node);
	free(options.only_rsc);
	free(options.pid_file);
	g_slist_free_full(options.includes_excludes, free);

	g_strfreev(processed_args);

	/* (2) If this is abnormal termination and we're in curses mode, shut down
	* curses first. Any messages displayed to the screen before curses is shut
	* down will be lost because doing the shut down will also restore the
	* screen to whatever it looked like before crm_mon was started.
	*/
	if (((error != NULL) \|\| (exit_code == CRM_EX_USAGE))
	&& (output_format == mon_output_console)
	&& (out != NULL)) {

	out->finish(out, exit_code, false, NULL);
	pcmk__output_free(out);
	out = NULL;
	}

	/* (3) If this is a command line usage related failure, print the usage
	* message.
	*/
	if (exit_code == CRM_EX_USAGE && (output_format == mon_output_console \|\| output_format == mon_output_plain)) {
	char *help = g_option_context_get_help(context, TRUE, NULL);

	fprintf(stderr, "%s", help);
	g_free(help);
	}

	pcmk__free_arg_context(context);

	/* (4) If this is any kind of error, print the error out and exit. Make
	* sure to handle situations both before and after formatted output is
	* set up. We want errors to appear formatted if at all possible.
	*/
	if (error != NULL) {
	if (out != NULL) {
	out->err(out, "%s: %s", g_get_prgname(), error->message);
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	} else {
	fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
	}

	g_clear_error(&error);
	crm_exit(exit_code);
	}

	/* (5) Print formatted output to the screen if we made it far enough in
	* crm_mon to be able to do so.
	*/
	if (out != NULL) {
	if (options.exec_mode != mon_exec_daemonized) {
	out->finish(out, exit_code, true, NULL);
	}

	pcmk__output_free(out);
	pcmk__unregister_formats();
	}

	crm_exit(exit_code);
	}
	diff --git a/tools/crm_node.c b/tools/crm_node.c
	index 3c51e9b7f8..746f919149 100644
	--- a/tools/crm_node.c
	+++ b/tools/crm_node.c
	@@ -1,883 +1,883 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/msg_xml.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/attrd_internal.h>

	#include <pacemaker-internal.h>

	#define SUMMARY "crm_node - Tool for displaying low-level node information"

	struct {
	gboolean corosync;
	gboolean dangerous_cmd;
	gboolean force_flag;
	char command;
	int nodeid;
	char *target_uname;
	} options = {
	.command = '\0',
	.force_flag = FALSE
	};

	gboolean command_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean name_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean remove_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	static GError *error = NULL;
	static GMainLoop *mainloop = NULL;
	static crm_exit_t exit_code = CRM_EX_OK;
	static pcmk__output_t *out = NULL;

	#define INDENT " "

	static GOptionEntry command_entries[] = {
	{ "cluster-id", 'i', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display this node's cluster id",
	NULL },
	{ "list", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display all known members (past and present) of this cluster",
	NULL },
	{ "name", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the name used by the cluster for this node",
	NULL },
	{ "partition", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the members of this partition",
	NULL },
	{ "quorum", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display a 1 if our partition has quorum, 0 if not",
	NULL },
	{ "name-for-id", 'N', 0, G_OPTION_ARG_CALLBACK, name_cb,
	"Display the name used by the cluster for the node with the specified ID",
	"ID" },
	{ "remove", 'R', 0, G_OPTION_ARG_CALLBACK, remove_cb,
	"(Advanced) Remove the (stopped) node with the specified name from Pacemaker's\n"
	INDENT "configuration and caches (the node must already have been removed from\n"
	INDENT "the underlying cluster stack configuration",
	"NAME" },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "force", 'f', 0, G_OPTION_ARG_NONE, &options.force_flag,
	NULL,
	NULL },
	#if SUPPORT_COROSYNC
	/* Unused and deprecated */
	{ "corosync", 'C', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.corosync,
	NULL,
	NULL },
	#endif

	// @TODO add timeout option for when IPC replies are needed

	{ NULL }
	};

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	gboolean
	command_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_eq("-i", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--cluster-id", option_name, pcmk__str_casei)) {
	options.command = 'i';
	} else if (pcmk__str_eq("-l", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--list", option_name, pcmk__str_casei)) {
	options.command = 'l';
	} else if (pcmk__str_eq("-n", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--name", option_name, pcmk__str_casei)) {
	options.command = 'n';
	} else if (pcmk__str_eq("-p", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--partition", option_name, pcmk__str_casei)) {
	options.command = 'p';
	} else if (pcmk__str_eq("-q", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--quorum", option_name, pcmk__str_casei)) {
	options.command = 'q';
	} else {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Unknown param passed to command_cb: %s", option_name);
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	name_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.command = 'N';
	pcmk__scan_min_int(optarg, &(options.nodeid), 0);
	return TRUE;
	}

	gboolean
	remove_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (optarg == NULL) {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "-R option requires an argument");
	return FALSE;
	}

	options.command = 'R';
	options.dangerous_cmd = TRUE;
	pcmk__str_update(&options.target_uname, optarg);
	return TRUE;
	}

	PCMK__OUTPUT_ARGS("node-id", "uint32_t")
	static int
	node_id_default(pcmk__output_t *out, va_list args) {
	uint32_t node_id = va_arg(args, uint32_t);

	out->info(out, "%" PRIu32, node_id);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-id", "uint32_t")
	static int
	node_id_xml(pcmk__output_t *out, va_list args) {
	uint32_t node_id = va_arg(args, uint32_t);

	char *id_s = crm_strdup_printf("%" PRIu32, node_id);

	pcmk__output_create_xml_node(out, "node-info",
	"nodeid", id_s,
	NULL);

	free(id_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList *")
	static int
	node_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;
	out->info(out, "%" PRIu32 " %s %s", node->id, pcmk__s(node->uname, ""),
	pcmk__s(node->state, ""));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-list", "GList *")
	static int
	node_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	out->begin_list(out, NULL, NULL, PCMK_XE_NODES);

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;
	char *id_s = crm_strdup_printf("%" PRIu32, node->id);

	pcmk__output_create_xml_node(out, "node",
	PCMK_XA_ID, id_s,
	PCMK_XA_NAME, node->uname,
	"state", node->state,
	NULL);

	free(id_s);
	}

	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-name", "uint32_t", "const char *")
	static int
	node_name_default(pcmk__output_t *out, va_list args) {
	uint32_t node_id G_GNUC_UNUSED = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );

	out->info(out, "%s", node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-name", "uint32_t", "const char *")
	static int
	node_name_xml(pcmk__output_t *out, va_list args) {
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );

	char *id_s = crm_strdup_printf("%" PRIu32, node_id);

	pcmk__output_create_xml_node(out, "node-info",
	"nodeid", id_s,
	PCMK_XA_UNAME, node_name,
	NULL);

	free(id_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("partition-list", "GList *")
	static int
	partition_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	GString *buffer = NULL;

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;
	if (pcmk__str_eq(node->state, "member", pcmk__str_none)) {
	pcmk__add_separated_word(&buffer, 128, pcmk__s(node->uname, ""), " ");
	}
	}

	if (buffer != NULL) {
	out->info(out, "%s", buffer->str);
	g_string_free(buffer, TRUE);
	return pcmk_rc_ok;
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("partition-list", "GList *")
	static int
	partition_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	out->begin_list(out, NULL, NULL, PCMK_XE_NODES);

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;

	if (pcmk__str_eq(node->state, "member", pcmk__str_none)) {
	char *id_s = crm_strdup_printf("%" PRIu32, node->id);

	pcmk__output_create_xml_node(out, "node",
	PCMK_XA_ID, id_s,
	PCMK_XA_NAME, node->uname,
	"state", node->state,
	NULL);
	free(id_s);
	}
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("quorum", "bool")
	static int
	quorum_default(pcmk__output_t *out, va_list args) {
	bool have_quorum = va_arg(args, int);

	out->info(out, "%d", have_quorum);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("quorum", "bool")
	static int
	quorum_xml(pcmk__output_t *out, va_list args) {
	bool have_quorum = va_arg(args, int);

	pcmk__output_create_xml_node(out, "cluster-info",
	"quorum", pcmk__btoa(have_quorum),
	NULL);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "node-id", "default", node_id_default },
	{ "node-id", "xml", node_id_xml },
	{ "node-list", "default", node_list_default },
	{ "node-list", "xml", node_list_xml },
	{ "node-name", "default", node_name_default },
	{ "node-name", "xml", node_name_xml },
	{ "quorum", "default", quorum_default },
	{ "quorum", "xml", quorum_xml },
	{ "partition-list", "default", partition_list_default },
	{ "partition-list", "xml", partition_list_xml },

	{ NULL, NULL, NULL }
	};

	static gint
	sort_node(gconstpointer a, gconstpointer b)
	{
	const pcmk_controld_api_node_t *node_a = a;
	const pcmk_controld_api_node_t *node_b = b;

	return pcmk__numeric_strcasecmp((node_a->uname? node_a->uname : ""),
	(node_b->uname? node_b->uname : ""));
	}

	static void
	controller_event_cb(pcmk_ipc_api_t *controld_api,
	enum pcmk_ipc_event event_type, crm_exit_t status,
	void event_data, void user_data)
	{
	pcmk_controld_api_reply_t *reply = event_data;

	switch (event_type) {
	case pcmk_ipc_event_disconnect:
	if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Lost connection to controller");
	}
	goto done;
	break;

	case pcmk_ipc_event_reply:
	break;

	default:
	return;
	}

	if (status != CRM_EX_OK) {
	exit_code = status;
	g_set_error(&error, PCMK__EXITC_ERROR, status,
	"Bad reply from controller: %s",
	crm_exit_str(status));
	goto done;
	}

	if (reply->reply_type != pcmk_controld_reply_nodes) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_INDETERMINATE,
	"Unknown reply type %d from controller",
	reply->reply_type);
	goto done;
	}

	reply->data.nodes = g_list_sort(reply->data.nodes, sort_node);

	if (options.command == 'p') {
	out->message(out, "partition-list", reply->data.nodes);
	} else if (options.command == 'l') {
	out->message(out, "node-list", reply->data.nodes);
	}

	// Success
	exit_code = CRM_EX_OK;
	done:
	pcmk_disconnect_ipc(controld_api);
	pcmk_quit_main_loop(mainloop, 10);
	}

	static void
	run_controller_mainloop(void)
	{
	pcmk_ipc_api_t *controld_api = NULL;
	int rc;

	// Set disconnect exit code to handle unexpected disconnects
	exit_code = CRM_EX_DISCONNECT;

	// Create controller IPC object
	rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not connect to controller: %s",
	pcmk_rc_str(rc));
	return;
	}
	pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);

	// Connect to controller
	rc = pcmk__connect_ipc(controld_api, pcmk_ipc_dispatch_main, 5);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not connect to %s: %s",
	pcmk_ipc_name(controld_api, true), pcmk_rc_str(rc));
	return;
	}

	rc = pcmk_controld_api_list_nodes(controld_api);

	if (rc != pcmk_rc_ok) {
	pcmk_disconnect_ipc(controld_api);
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not ping controller: %s", pcmk_rc_str(rc));
	return;
	}

	// Run main loop to get controller reply via controller_event_cb()
	mainloop = g_main_loop_new(NULL, FALSE);
	g_main_loop_run(mainloop);
	g_main_loop_unref(mainloop);
	mainloop = NULL;
	pcmk_free_ipc_api(controld_api);
	}

	static void
	print_node_id(void)
	{
	uint32_t nodeid;
	int rc = pcmk__query_node_info(out, &nodeid, NULL, NULL, NULL, NULL, NULL,
	false, 0);

	if (rc != pcmk_rc_ok) {
	/* pcmk__query_node_info already sets an error message on the output object,
	* so there's no need to call g_set_error here. That would just create a
	* duplicate error message in the output.
	*/
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	rc = out->message(out, "node-id", nodeid);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print node ID: %s",
	pcmk_rc_str(rc));
	}

	exit_code = pcmk_rc2exitc(rc);
	}

	static void
	print_node_name(uint32_t nodeid)
	{
	int rc = pcmk_rc_ok;
	char *node_name = NULL;

	if (nodeid == 0) {
	// Check environment first (i.e. when called by resource agent)
	const char *name = getenv("OCF_RESKEY_" CRM_META "_"
	PCMK__META_ON_NODE);

	if (name != NULL) {
	rc = out->message(out, "node-name", 0, name);
	goto done;
	}
	}

	// Otherwise ask the controller

	/* pcmk__query_node_name already sets an error message on the output object,
	* so there's no need to call g_set_error here. That would just create a
	* duplicate error message in the output.
	*/
	rc = pcmk__query_node_name(out, nodeid, &node_name, 0);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	rc = out->message(out, "node-name", 0, node_name);

	done:
	if (node_name != NULL) {
	free(node_name);
	}

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print node name: %s",
	pcmk_rc_str(rc));
	}

	exit_code = pcmk_rc2exitc(rc);
	}

	static void
	print_quorum(void)
	{
	bool quorum;
	int rc = pcmk__query_node_info(out, NULL, NULL, NULL, NULL, &quorum, NULL,
	false, 0);

	if (rc != pcmk_rc_ok) {
	/* pcmk__query_node_info already sets an error message on the output object,
	* so there's no need to call g_set_error here. That would just create a
	* duplicate error message in the output.
	*/
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	rc = out->message(out, "quorum", quorum);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print quorum status: %s",
	pcmk_rc_str(rc));
	}

	exit_code = pcmk_rc2exitc(rc);
	}

	/*!
	* \internal
	* \brief Extend a transaction by removing a node from a CIB section
	*
	* \param[in,out] cib Active CIB connection
	* \param[in] element CIB element containing node name and/or ID
	* \param[in] section CIB section that \p element is in
	* \param[in] node_name Name of node to purge (NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	remove_from_section(cib_t cib, const char element, const char *section,
	const char *node_name, long node_id)
	{
	xmlNode *xml = NULL;
	int rc = pcmk_rc_ok;

	xml = create_xml_node(NULL, element);
	if (xml == NULL) {
	return pcmk_rc_error;
	}
	crm_xml_add(xml, PCMK_XA_UNAME, node_name);
	if (node_id > 0) {
	crm_xml_set_id(xml, "%ld", node_id);
	}
	rc = cib->cmds->remove(cib, section, xml, cib_transaction);
	free_xml(xml);
	return (rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(rc);
	}

	/*!
	* \internal
	* \brief Purge a node from CIB
	*
	* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	purge_node_from_cib(const char *node_name, long node_id)
	{
	int rc = pcmk_rc_ok;
	int commit_rc = pcmk_rc_ok;
	cib_t *cib = NULL;

	// Connect to CIB and start a transaction
	cib = cib_new();
	if (cib == NULL) {
	return ENOTCONN;
	}
	rc = cib->cmds->signon(cib, crm_system_name, cib_command);
	if (rc == pcmk_ok) {
	rc = cib->cmds->init_transaction(cib);
	}
	if (rc != pcmk_ok) {
	rc = pcmk_legacy2rc(rc);
	cib__clean_up_connection(&cib);
	return rc;
	}

	// Remove from configuration and status
	rc = remove_from_section(cib, XML_CIB_TAG_NODE, PCMK_XE_NODES, node_name,
	node_id);
	if (rc == pcmk_rc_ok) {
	- rc = remove_from_section(cib, XML_CIB_TAG_STATE, PCMK_XE_STATUS,
	+ rc = remove_from_section(cib, PCMK__XE_NODE_STATE, PCMK_XE_STATUS,
	node_name, node_id);
	}

	// Commit the transaction
	commit_rc = cib->cmds->end_transaction(cib, (rc == pcmk_rc_ok),
	cib_sync_call);
	cib__clean_up_connection(&cib);

	if ((rc == pcmk_rc_ok) && (commit_rc == pcmk_ok)) {
	crm_debug("Purged node %s (%ld) from CIB",
	pcmk__s(node_name, "by ID"), node_id);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Purge a node from a single server's peer cache
	*
	* \param[in] server IPC server to send request to
	* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	purge_node_from(enum pcmk_ipc_server server, const char *node_name,
	long node_id)
	{
	pcmk_ipc_api_t *api = NULL;
	int rc;

	rc = pcmk_new_ipc_api(&api, server);
	if (rc != pcmk_rc_ok) {
	goto done;
	}

	rc = pcmk__connect_ipc(api, pcmk_ipc_dispatch_sync, 5);
	if (rc != pcmk_rc_ok) {
	goto done;
	}

	rc = pcmk_ipc_purge_node(api, node_name, node_id);
	done:
	if (rc != pcmk_rc_ok) { // Debug message already logged on success
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not purge node %s from %s: %s",
	pcmk__s(node_name, "by ID"), pcmk_ipc_name(api, true),
	pcmk_rc_str(rc));
	}
	pcmk_free_ipc_api(api);
	return rc;
	}

	/*!
	* \internal
	* \brief Purge a node from the fencer's peer cache
	*
	* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	purge_node_from_fencer(const char *node_name, long node_id)
	{
	int rc = pcmk_rc_ok;
	crm_ipc_t *conn = NULL;
	xmlNode *cmd = NULL;

	conn = crm_ipc_new("stonith-ng", 0);
	if (conn == NULL) {
	rc = ENOTCONN;
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not connect to fencer to purge node %s",
	pcmk__s(node_name, "by ID"));
	return rc;
	}

	rc = pcmk__connect_generic_ipc(conn);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not connect to fencer to purge node %s: %s",
	pcmk__s(node_name, "by ID"), pcmk_rc_str(rc));
	crm_ipc_destroy(conn);
	return rc;
	}

	cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, "stonith-ng",
	crm_system_name, NULL);
	if (node_id > 0) {
	crm_xml_set_id(cmd, "%ld", node_id);
	}
	crm_xml_add(cmd, PCMK_XA_UNAME, node_name);

	rc = crm_ipc_send(conn, cmd, 0, 0, NULL);
	if (rc >= 0) {
	rc = pcmk_rc_ok;
	crm_debug("Purged node %s (%ld) from fencer",
	pcmk__s(node_name, "by ID"), node_id);
	} else {
	rc = pcmk_legacy2rc(rc);
	fprintf(stderr, "Could not purge node %s from fencer: %s\n",
	pcmk__s(node_name, "by ID"), pcmk_rc_str(rc));
	}
	free_xml(cmd);
	crm_ipc_close(conn);
	crm_ipc_destroy(conn);
	return rc;
	}

	static void
	remove_node(const char *target_uname)
	{
	int rc = pcmk_rc_ok;
	long nodeid = 0;
	const char *node_name = NULL;
	char *endptr = NULL;
	const enum pcmk_ipc_server servers[] = {
	pcmk_ipc_controld,
	pcmk_ipc_attrd,
	};

	// Check whether node was specified by name or numeric ID
	errno = 0;
	nodeid = strtol(target_uname, &endptr, 10);
	if ((errno != 0) \|\| (endptr == target_uname) \|\| (*endptr != '\0')
	\|\| (nodeid <= 0)) {
	// It's not a positive integer, so assume it's a node name
	nodeid = 0;
	node_name = target_uname;
	}

	for (int i = 0; i < PCMK__NELEM(servers); ++i) {
	rc = purge_node_from(servers[i], node_name, nodeid);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	return;
	}
	}

	// The fencer hasn't been converted to pcmk_ipc_api_t yet
	rc = purge_node_from_fencer(node_name, nodeid);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	// Lastly, purge the node from the CIB itself
	rc = purge_node_from_cib(node_name, nodeid);
	exit_code = pcmk_rc2exitc(rc);
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
	"Be less descriptive in output.",
	NULL },

	{ NULL }
	};

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);

	/* Add the -q option, which cannot be part of the globally supported options
	* because some tools use that flag for something else.
	*/
	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "commands", "Commands:",
	"Show command help", command_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "NR");
	GOptionContext *context = build_arg_context(args, &output_group);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_node", args->verbosity);

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error creating output format %s: %s", args->output_ty,
	pcmk_rc_str(rc));
	goto done;
	}

	if (!pcmk__force_args(context, &error, "%s --xml-simple-list", g_get_prgname())) {
	exit_code = CRM_EX_SOFTWARE;
	goto done;
	}

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (options.command == 0) {
	char *help = g_option_context_get_help(context, TRUE, NULL);

	out->err(out, "%s", help);
	g_free(help);
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	if (options.dangerous_cmd && options.force_flag == FALSE) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command is considered dangerous."
	" To prevent accidental destruction of the cluster,"
	" the --force flag is required in order to proceed.");
	goto done;
	}

	pcmk__register_lib_messages(out);
	pcmk__register_messages(out, fmt_functions);

	switch (options.command) {
	case 'i':
	print_node_id();
	break;

	case 'n':
	print_node_name(0);
	break;

	case 'q':
	print_quorum();
	break;

	case 'N':
	print_node_name(options.nodeid);
	break;

	case 'R':
	remove_node(options.target_uname);
	break;

	case 'l':
	case 'p':
	run_controller_mainloop();
	break;

	default:
	break;
	}

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}
	pcmk__unregister_formats();
	return crm_exit(exit_code);
	}
	diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
	index f5945f946e..1d5163d7e6 100644
	--- a/tools/crm_resource_runtime.c
	+++ b/tools/crm_resource_runtime.c
	@@ -1,2252 +1,2254 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>
	#include <crm/common/ipc_attrd_internal.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/lists_internal.h>
	#include <crm/services_internal.h>

	static GList *
	build_node_info_list(const pcmk_resource_t *rsc)
	{
	GList *retval = NULL;

	for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;

	for (const GList *iter2 = child->running_on;
	iter2 != NULL; iter2 = iter2->next) {

	const pcmk_node_t node = (const pcmk_node_t ) iter2->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));

	ni->node_name = node->details->uname;
	ni->promoted = pcmk_is_set(rsc->flags, pcmk_rsc_promotable) &&
	child->fns->state(child, TRUE) == pcmk_role_promoted;

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	GList *
	cli_resource_search(pcmk_resource_t rsc, const char requested_name,
	pcmk_scheduler_t *scheduler)
	{
	GList *retval = NULL;
	const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);

	if (pe_rsc_is_clone(rsc)) {
	retval = build_node_info_list(rsc);

	/* The anonymous clone children's common ID is supplied */
	} else if (pe_rsc_is_clone(parent)
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_unique)
	&& rsc->clone_name
	&& pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
	&& !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {

	retval = build_node_info_list(parent);

	} else if (rsc->running_on != NULL) {
	for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;
	node_info_t *ni = calloc(1, sizeof(node_info_t));
	ni->node_name = node->details->uname;
	ni->promoted = (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted);

	retval = g_list_prepend(retval, ni);
	}
	}

	return retval;
	}

	// \return Standard Pacemaker return code
	static int
	find_resource_attr(pcmk__output_t out, cib_t the_cib, const char *attr,
	const char rsc, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, char **value)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_search = NULL;
	GString *xpath = NULL;
	const char *xpath_base = NULL;

	if(value) {
	*value = NULL;
	}

	if(the_cib == NULL) {
	return ENOTCONN;
	}

	xpath_base = pcmk_cib_xpath_for(PCMK_XE_RESOURCES);
	if (xpath_base == NULL) {
	crm_err(PCMK_XE_RESOURCES " CIB element not known (bug?)");
	return ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	pcmk__g_strcat(xpath,
	xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);

	if (attr_set_type != NULL) {
	pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
	if (set_name != NULL) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
	NULL);
	}
	}

	g_string_append(xpath, "//" XML_CIB_TAG_NVPAIR "[");
	if (attr_id != NULL) {
	pcmk__g_strcat(xpath, "@" PCMK_XA_ID "=\"", attr_id, "\"", NULL);
	}

	if (attr_name != NULL) {
	if (attr_id != NULL) {
	g_string_append(xpath, " and ");
	}
	pcmk__g_strcat(xpath, "@" PCMK_XA_NAME "=\"", attr_name, "\"", NULL);
	}
	g_string_append_c(xpath, ']');

	rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
	cib_sync_call \| cib_scope_local \| cib_xpath);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto done;
	}

	crm_log_xml_debug(xml_search, "Match");
	if (xml_search->children != NULL) {
	xmlNode *child = NULL;

	rc = ENOTUNIQ;
	out->info(out, "Multiple attributes match name=%s", attr_name);

	for (child = pcmk__xml_first_child(xml_search); child != NULL;
	child = pcmk__xml_next(child)) {
	out->info(out, " Value: %s \t(id=%s)",
	crm_element_value(child, PCMK_XA_VALUE), ID(child));
	}

	out->spacer(out);

	} else if(value) {
	pcmk__str_update(value, crm_element_value(xml_search, attr));
	}

	done:
	g_string_free(xpath, TRUE);
	free_xml(xml_search);
	return rc;
	}

	/* PRIVATE. Use the find_matching_attr_resources instead. */
	static void
	find_matching_attr_resources_recursive(pcmk__output_t *out,
	GList /* <pcmk_resource_t> / **result,
	pcmk_resource_t rsc, const char rsc_id,
	const char * attr_set, const char * attr_set_type,
	const char * attr_id, const char * attr_name,
	cib_t * cib, const char * cmd, int depth)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = clone_strip(rsc->id);
	char *local_attr_id = NULL;

	/* visit the children */
	for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
	find_matching_attr_resources_recursive(out, result,
	(pcmk_resource_t *) gIter->data,
	rsc_id, attr_set, attr_set_type,
	attr_id, attr_name, cib, cmd, depth+1);
	/* do it only once for clones */
	if (rsc->variant == pcmk_rsc_variant_clone) {
	break;
	}
	}

	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	/* Post-order traversal.
	* The root is always on the list and it is the last item. */
	if((0 == depth) \|\| (pcmk_rc_ok == rc)) {
	/* push the head */
	result = g_list_append(result, rsc);
	}

	free(local_attr_id);
	free(lookup_id);
	}


	/* The result is a linearized pre-ordered tree of resources. */
	static GList/<pcmk_resource_t>/
	find_matching_attr_resources(pcmk__output_t out, pcmk_resource_t rsc,
	const char * rsc_id, const char * attr_set,
	const char * attr_set_type, const char * attr_id,
	const char * attr_name, cib_t * cib, const char * cmd,
	gboolean force)
	{
	int rc = pcmk_rc_ok;
	char *lookup_id = NULL;
	char *local_attr_id = NULL;
	GList * result = NULL;
	/* If --force is used, update only the requested resource (clone or primitive).
	* Otherwise, if the primitive has the attribute, use that.
	* Otherwise use the clone. */
	if(force == TRUE) {
	return g_list_append(result, rsc);
	}
	if ((rsc->parent != NULL)
	&& (rsc->parent->variant == pcmk_rsc_variant_clone)) {
	int rc = pcmk_rc_ok;
	char *local_attr_id = NULL;
	rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
	attr_set, attr_id, attr_name, &local_attr_id);
	free(local_attr_id);

	if(rc != pcmk_rc_ok) {
	rsc = rsc->parent;
	out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
	cmd, attr_name, rsc->id, rsc_id);
	}
	return g_list_append(result, rsc);

	} else if ((rsc->parent == NULL) && (rsc->children != NULL)
	&& (rsc->variant == pcmk_rsc_variant_clone)) {
	pcmk_resource_t *child = rsc->children->data;

	if (child->variant == pcmk_rsc_variant_primitive) {
	lookup_id = clone_strip(child->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
	attr_set_type, attr_set, attr_id, attr_name,
	&local_attr_id);

	if(rc == pcmk_rc_ok) {
	rsc = child;
	out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
	attr_name, lookup_id, cmd, rsc_id);
	}

	free(local_attr_id);
	free(lookup_id);
	}
	return g_list_append(result, rsc);
	}
	/* If the resource is a group ==> children inherit the attribute if defined. */
	find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
	attr_set_type, attr_id, attr_name,
	cib, cmd, 0);
	return result;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_update_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	const char *attr_value, gboolean recursive,
	cib_t *cib, int cib_options, gboolean force)
	{
	pcmk__output_t *out = rsc->cluster->priv;
	int rc = pcmk_rc_ok;

	char *found_attr_id = NULL;

	GList/<pcmk_resource_t>/ resources = NULL;
	const char *top_id = pe__const_top_resource(rsc, false)->id;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
	attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
	if (!force) {
	rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
	XML_TAG_META_SETS, attr_set, attr_id,
	attr_name, &found_attr_id);
	if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
	out->err(out,
	"WARNING: There is already a meta attribute "
	"for '%s' called '%s' (id=%s)",
	top_id, attr_name, found_attr_id);
	out->err(out,
	" Delete '%s' first or use the force option "
	"to override", found_attr_id);
	}
	free(found_attr_id);
	if (rc == pcmk_rc_ok) {
	return ENOTUNIQ;
	}
	}
	resources = g_list_append(resources, rsc);

	} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	crm_xml_add(rsc->xml, attr_name, attr_value);
	CRM_ASSERT(cib != NULL);
	rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->xml, cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Set attribute: name=%s value=%s",
	attr_name, attr_value);
	}
	return rc;

	} else {
	resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"update", force);
	}

	/* If the user specified attr_set or attr_id, the intent is to modify a
	* single resource, which will be the last item in the list.
	*/
	if ((attr_set != NULL) \|\| (attr_id != NULL)) {
	GList *last = g_list_last(resources);

	resources = g_list_remove_link(resources, last);
	g_list_free(resources);
	resources = last;
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	char *local_attr_set = NULL;
	const char *rsc_attr_id = attr_id;
	const char *rsc_attr_set = attr_set;

	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	found_attr_id = NULL;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &found_attr_id);

	switch (rc) {
	case pcmk_rc_ok:
	crm_debug("Found a match for name=%s: id=%s",
	attr_name, found_attr_id);
	rsc_attr_id = found_attr_id;
	break;

	case ENXIO:
	if (rsc_attr_set == NULL) {
	local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
	attr_set_type);
	rsc_attr_set = local_attr_set;
	}
	if (rsc_attr_id == NULL) {
	found_attr_id = crm_strdup_printf("%s-%s",
	rsc_attr_set, attr_name);
	rsc_attr_id = found_attr_id;
	}

	xml_top = create_xml_node(NULL, (const char *) rsc->xml->name);
	crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);

	xml_obj = create_xml_node(xml_top, attr_set_type);
	crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
	break;

	default:
	free(lookup_id);
	free(found_attr_id);
	g_list_free(resources);
	return rc;
	}

	xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
	attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_debug(xml_top, "Update");

	rc = cib->cmds->modify(cib, PCMK_XE_RESOURCES, xml_top, cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s",
	lookup_id, found_attr_id,
	((rsc_attr_set == NULL)? "" : " set="),
	pcmk__s(rsc_attr_set, ""),
	((attr_name == NULL)? "" : " name="),
	pcmk__s(attr_name, ""), attr_value);
	}

	free_xml(xml_top);

	free(lookup_id);
	free(found_attr_id);
	free(local_attr_set);

	if (recursive
	&& pcmk__str_eq(attr_set_type, XML_TAG_META_SETS,
	pcmk__str_casei)) {
	GList *lpc = NULL;
	static bool need_init = true;

	if (need_init) {
	need_init = false;
	pcmk__unpack_constraints(rsc->cluster);
	pe__clear_resource_flags_on_all(rsc->cluster,
	pcmk_rsc_detect_loop);
	}

	/* We want to set the attribute only on resources explicitly
	* colocated with this one, so we use rsc->rsc_cons_lhs directly
	* rather than the with_this_colocations() method.
	*/
	pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
	for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
	pcmk__colocation_t cons = (pcmk__colocation_t ) lpc->data;

	crm_debug("Checking %s %d", cons->id, cons->score);
	if (!pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)
	&& (cons->score > 0)) {
	crm_debug("Setting %s=%s for dependent resource %s",
	attr_name, attr_value, cons->dependent->id);
	cli_resource_update_attribute(cons->dependent,
	cons->dependent->id, NULL,
	attr_set_type, NULL,
	attr_name, attr_value,
	recursive, cib, cib_options,
	force);
	}
	}
	}
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete_attribute(pcmk_resource_t rsc, const char requested_name,
	const char attr_set, const char attr_set_type,
	const char attr_id, const char attr_name,
	cib_t *cib, int cib_options, gboolean force)
	{
	pcmk__output_t *out = rsc->cluster->priv;
	int rc = pcmk_rc_ok;
	GList/<pcmk_resource_t>/ resources = NULL;

	if ((attr_id == NULL) && !force) {
	find_resource_attr(out, cib, PCMK_XA_ID,
	pe__const_top_resource(rsc, false)->id, NULL,
	NULL, NULL, attr_name, NULL);
	}

	if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
	resources = find_matching_attr_resources(out, rsc, requested_name,
	attr_set, attr_set_type,
	attr_id, attr_name, cib,
	"delete", force);

	} else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
	xml_remove_prop(rsc->xml, attr_name);
	CRM_ASSERT(cib != NULL);
	rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->xml, cib_options);
	rc = pcmk_legacy2rc(rc);
	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted attribute: %s", attr_name);
	}
	return rc;

	} else {
	resources = g_list_append(resources, rsc);
	}

	for (GList *iter = resources; iter != NULL; iter = iter->next) {
	char *lookup_id = NULL;
	xmlNode *xml_obj = NULL;
	char *found_attr_id = NULL;
	const char *rsc_attr_id = attr_id;

	rsc = (pcmk_resource_t *) iter->data;

	lookup_id = clone_strip(rsc->id);
	rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
	attr_set, attr_id, attr_name, &found_attr_id);
	switch (rc) {
	case pcmk_rc_ok:
	break;

	case ENXIO:
	free(lookup_id);
	rc = pcmk_rc_ok;
	continue;

	default:
	free(lookup_id);
	g_list_free(resources);
	return rc;
	}

	if (rsc_attr_id == NULL) {
	rsc_attr_id = found_attr_id;
	}

	xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
	crm_log_xml_debug(xml_obj, "Delete");

	CRM_ASSERT(cib);
	rc = cib->cmds->remove(cib, PCMK_XE_RESOURCES, xml_obj, cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted '%s' option: id=%s%s%s%s%s",
	lookup_id, found_attr_id,
	((attr_set == NULL)? "" : " set="),
	pcmk__s(attr_set, ""),
	((attr_name == NULL)? "" : " name="),
	pcmk__s(attr_name, ""));
	}

	free(lookup_id);
	free_xml(xml_obj);
	free(found_attr_id);
	}
	g_list_free(resources);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
	const char host_uname, const char rsc_id,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	const char *router_node = host_uname;
	const char *rsc_api_id = NULL;
	const char *rsc_long_id = NULL;
	const char *rsc_class = NULL;
	const char *rsc_provider = NULL;
	const char *rsc_type = NULL;
	bool cib_only = false;
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);

	if (rsc == NULL) {
	out->err(out, "Resource %s not found", rsc_id);
	return ENXIO;

	} else if (rsc->variant != pcmk_rsc_variant_primitive) {
	out->err(out, "We can only process primitive resources, not %s", rsc_id);
	return EINVAL;
	}

	rsc_class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	rsc_provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
	rsc_type = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	if ((rsc_class == NULL) \|\| (rsc_type == NULL)) {
	out->err(out, "Resource %s does not have a class and type", rsc_id);
	return EINVAL;
	}

	{
	pcmk_node_t *node = pe_find_node(scheduler->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Node %s not found", host_uname);
	return pcmk_rc_node_unknown;
	}

	if (!(node->details->online)) {
	if (do_fail_resource) {
	out->err(out, "Node %s is not online", host_uname);
	return ENOTCONN;
	} else {
	cib_only = true;
	}
	}
	if (!cib_only && pe__is_guest_or_remote_node(node)) {
	node = pcmk__current_node(node->details->remote_rsc);
	if (node == NULL) {
	out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
	host_uname);
	return ENOTCONN;
	}
	router_node = node->details->uname;
	}
	}

	if (rsc->clone_name) {
	rsc_api_id = rsc->clone_name;
	rsc_long_id = rsc->id;
	} else {
	rsc_api_id = rsc->id;
	}
	if (do_fail_resource) {
	return pcmk_controld_api_fail(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id,
	rsc_class, rsc_provider, rsc_type);
	} else {
	return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
	rsc_api_id, rsc_long_id, rsc_class,
	rsc_provider, rsc_type, cib_only);
	}
	}

	/*!
	* \internal
	* \brief Get resource name as used in failure-related node attributes
	*
	* \param[in] rsc Resource to check
	*
	* \return Newly allocated string containing resource's fail name
	* \note The caller is responsible for freeing the result.
	*/
	static inline char *
	rsc_fail_name(const pcmk_resource_t *rsc)
	{
	const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
	return strdup(name);
	}
	return clone_strip(name);
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_history(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;

	/* Erase the resource's entire LRM history in the CIB, even if we're only
	* clearing a single operation's fail count. If we erased only entries for a
	* single operation, we might wind up with a wrong idea of the current
	* resource state, and we might not re-probe the resource.
	*/
	rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	crm_trace("Processing %d mainloop inputs",
	pcmk_controld_api_replies_expected(controld_api));
	while (g_main_context_iteration(NULL, FALSE)) {
	crm_trace("Processed mainloop input, %d still remaining",
	pcmk_controld_api_replies_expected(controld_api));
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_failures(pcmk__output_t out, pcmk_ipc_api_t controld_api,
	const char node_name, const char rsc_id, const char *operation,
	const char interval_spec, pcmk_scheduler_t scheduler)
	{
	int rc = pcmk_rc_ok;
	const char *failed_value = NULL;
	const char *failed_id = NULL;
	char *interval_ms_s = NULL;
	GHashTable *rscs = NULL;
	GHashTableIter iter;

	/* Create a hash table to use as a set of resources to clean. This lets us
	* clean each resource only once (per node) regardless of how many failed
	* operations it has.
	*/
	rscs = pcmk__strkey_table(NULL, NULL);

	// Normalize interval to milliseconds for comparison to history entry
	if (operation) {
	guint interval_ms = 0U;

	pcmk_parse_interval_spec(interval_spec, &interval_ms);
	interval_ms_s = crm_strdup_printf("%u", interval_ms);
	}

	for (xmlNode *xml_op = pcmk__xml_first_child(scheduler->failed);
	xml_op != NULL;
	xml_op = pcmk__xml_next(xml_op)) {

	failed_id = crm_element_value(xml_op, PCMK__XA_RSC_ID);
	if (failed_id == NULL) {
	// Malformed history entry, should never happen
	continue;
	}

	// No resource specified means all resources match
	if (rsc_id) {
	pcmk_resource_t *fail_rsc = NULL;

	fail_rsc = pe_find_resource_with_flags(scheduler->resources,
	failed_id,
	pcmk_rsc_match_history
	\|pcmk_rsc_match_anon_basename);
	if (!fail_rsc \|\| !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
	continue;
	}
	}

	// Host name should always have been provided by this point
	failed_value = crm_element_value(xml_op, PCMK_XA_UNAME);
	if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
	continue;
	}

	// No operation specified means all operations match
	if (operation) {
	failed_value = crm_element_value(xml_op, PCMK_XA_OPERATION);
	if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
	continue;
	}

	// Interval (if operation was specified) defaults to 0 (not all)
	failed_value = crm_element_value(xml_op, PCMK_META_INTERVAL);
	if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
	continue;
	}
	}

	g_hash_table_add(rscs, (gpointer) failed_id);
	}

	free(interval_ms_s);

	g_hash_table_iter_init(&iter, rscs);
	while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
	crm_debug("Erasing failures of %s on %s", failed_id, node_name);
	rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	g_hash_table_destroy(rscs);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	clear_rsc_fail_attrs(const pcmk_resource_t rsc, const char operation,
	const char interval_spec, const pcmk_node_t node)
	{
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	char *rsc_name = rsc_fail_name(rsc);

	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
	operation, interval_spec, NULL,
	attr_options);
	free(rsc_name);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_delete(pcmk_ipc_api_t controld_api, const char host_uname,
	const pcmk_resource_t rsc, const char operation,
	const char *interval_spec, bool just_failures,
	pcmk_scheduler_t *scheduler, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	pcmk_node_t *node = NULL;

	if (rsc == NULL) {
	return ENXIO;

	} else if (rsc->children) {

	for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) lpc->data;

	rc = cli_resource_delete(controld_api, host_uname, child, operation,
	interval_spec, just_failures, scheduler,
	force);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	return pcmk_rc_ok;

	} else if (host_uname == NULL) {
	GList *lpc = NULL;
	GList *nodes = g_hash_table_get_values(rsc->known_on);

	if(nodes == NULL && force) {
	nodes = pcmk__copy_node_list(scheduler->nodes, false);

	} else if(nodes == NULL && rsc->exclusive_discover) {
	GHashTableIter iter;
	pcmk_node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
	if(node->weight >= 0) {
	nodes = g_list_prepend(nodes, node);
	}
	}

	} else if(nodes == NULL) {
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	}

	for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node = (pcmk_node_t *) lpc->data;

	if (node->details->online) {
	rc = cli_resource_delete(controld_api, node->details->uname, rsc,
	operation, interval_spec, just_failures,
	scheduler, force);
	}
	if (rc != pcmk_rc_ok) {
	g_list_free(nodes);
	return rc;
	}
	}

	g_list_free(nodes);
	return pcmk_rc_ok;
	}

	node = pe_find_node(scheduler->nodes, host_uname);

	if (node == NULL) {
	out->err(out, "Unable to clean up %s because node %s not found",
	rsc->id, host_uname);
	return ENODEV;
	}

	if (!node->details->rsc_discovery_enabled) {
	out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
	rsc->id, host_uname);
	return EOPNOTSUPP;
	}

	if (controld_api == NULL) {
	out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
	rsc->id, host_uname);
	return pcmk_rc_ok;
	}

	rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up %s failures on %s: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	return rc;
	}

	if (just_failures) {
	rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
	interval_spec, scheduler);
	} else {
	rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
	}
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
	rsc->id, host_uname, pcmk_rc_str(rc));
	} else {
	out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_cleanup_all(pcmk_ipc_api_t controld_api, const char node_name,
	const char operation, const char interval_spec,
	pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	int attr_options = pcmk__node_attr_none;
	const char *display_name = node_name? node_name : "all nodes";

	if (controld_api == NULL) {
	out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
	display_name);
	return rc;
	}

	if (node_name) {
	pcmk_node_t *node = pe_find_node(scheduler->nodes, node_name);

	if (node == NULL) {
	out->err(out, "Unknown node: %s", node_name);
	return ENXIO;
	}
	if (pe__is_guest_or_remote_node(node)) {
	attr_options \|= pcmk__node_attr_remote;
	}
	}

	rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
	interval_spec, NULL, attr_options);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Unable to clean up all failures on %s: %s",
	display_name, pcmk_rc_str(rc));
	return rc;
	}

	if (node_name) {
	rc = clear_rsc_failures(out, controld_api, node_name, NULL,
	operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
	node_name, pcmk_rc_str(rc));
	return rc;
	}
	} else {
	for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
	pcmk_node_t node = (pcmk_node_t ) iter->data;

	rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
	operation, interval_spec, scheduler);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
	pcmk_rc_str(rc));
	return rc;
	}
	}
	}

	out->info(out, "Cleaned up all resources on %s", display_name);
	return rc;
	}

	static void
	check_role(resource_checks_t *checks)
	{
	const char *role_s = g_hash_table_lookup(checks->rsc->meta,
	PCMK_META_TARGET_ROLE);

	if (role_s == NULL) {
	return;
	}
	switch (text2role(role_s)) {
	case pcmk_role_stopped:
	checks->flags \|= rsc_remain_stopped;
	break;

	case pcmk_role_unpromoted:
	if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
	pcmk_rsc_promotable)) {
	checks->flags \|= rsc_unpromotable;
	}
	break;

	default:
	break;
	}
	}

	static void
	check_managed(resource_checks_t *checks)
	{
	const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
	PCMK_META_IS_MANAGED);

	if ((managed_s != NULL) && !crm_is_true(managed_s)) {
	checks->flags \|= rsc_unmanaged;
	}
	}

	static void
	check_locked(resource_checks_t *checks)
	{
	if (checks->rsc->lock_node != NULL) {
	checks->flags \|= rsc_locked;
	checks->lock_node = checks->rsc->lock_node->details->uname;
	}
	}

	static bool
	node_is_unhealthy(pcmk_node_t *node)
	{
	switch (pe__health_strategy(node->details->data_set)) {
	case pcmk__health_strategy_none:
	break;

	case pcmk__health_strategy_no_red:
	if (pe__node_health(node) < 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_only_green:
	if (pe__node_health(node) <= 0) {
	return true;
	}
	break;

	case pcmk__health_strategy_progressive:
	case pcmk__health_strategy_custom:
	/* @TODO These are finite scores, possibly with rules, and possibly
	* combining with other scores, so attributing these as a cause is
	* nontrivial.
	*/
	break;
	}
	return false;
	}

	static void
	check_node_health(resource_checks_t checks, pcmk_node_t node)
	{
	if (node == NULL) {
	GHashTableIter iter;
	bool allowed = false;
	bool all_nodes_unhealthy = true;

	g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
	allowed = true;
	if (!node_is_unhealthy(node)) {
	all_nodes_unhealthy = false;
	break;
	}
	}
	if (allowed && all_nodes_unhealthy) {
	checks->flags \|= rsc_node_health;
	}

	} else if (node_is_unhealthy(node)) {
	checks->flags \|= rsc_node_health;
	}
	}

	int
	cli_resource_check(pcmk__output_t out, pcmk_resource_t rsc, pcmk_node_t *node)
	{
	resource_checks_t checks = { .rsc = rsc };

	check_role(&checks);
	check_managed(&checks);
	check_locked(&checks);
	check_node_health(&checks, node);

	return out->message(out, "resource-check-list", &checks);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_fail(pcmk_ipc_api_t controld_api, const char host_uname,
	const char rsc_id, pcmk_scheduler_t scheduler)
	{
	crm_notice("Failing %s on %s", rsc_id, host_uname);
	return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
	}

	static GHashTable *
	generate_resource_params(pcmk_resource_t rsc, pcmk_node_t node,
	pcmk_scheduler_t *scheduler)
	{
	GHashTable *params = NULL;
	GHashTable *meta = NULL;
	GHashTable *combined = NULL;
	GHashTableIter iter;
	char *key = NULL;
	char *value = NULL;

	combined = pcmk__strkey_table(free, free);

	params = pe_rsc_params(rsc, node, scheduler);
	if (params != NULL) {
	g_hash_table_iter_init(&iter, params);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	g_hash_table_insert(combined, strdup(key), strdup(value));
	}
	}

	meta = pcmk__strkey_table(free, free);
	get_meta_attributes(meta, rsc, node, scheduler);
	if (meta != NULL) {
	g_hash_table_iter_init(&iter, meta);
	while (g_hash_table_iter_next(&iter, (gpointer ) & key, (gpointer ) & value)) {
	char *crm_name = crm_meta_name(key);

	g_hash_table_insert(combined, crm_name, strdup(value));
	}
	g_hash_table_destroy(meta);
	}

	return combined;
	}

	bool resource_is_running_on(pcmk_resource_t rsc, const char host)
	{
	bool found = true;
	GList *hIter = NULL;
	GList *hosts = NULL;

	if (rsc == NULL) {
	return false;
	}

	rsc->fns->location(rsc, &hosts, TRUE);
	for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
	pcmk_node_t node = (pcmk_node_t ) hIter->data;

	if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
	crm_trace("Resource %s is running on %s\n", rsc->id, host);
	goto done;
	}
	}

	if (host != NULL) {
	crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
	found = false;

	} else if(host == NULL && hosts == NULL) {
	crm_trace("Resource %s is not running\n", rsc->id);
	found = false;
	}

	done:
	g_list_free(hosts);
	return found;
	}

	/*!
	* \internal
	* \brief Create a list of all resources active on host from a given list
	*
	* \param[in] host Name of host to check whether resources are active
	* \param[in] rsc_list List of resources to check
	*
	* \return New list of resources from list that are active on host
	*/
	static GList *
	get_active_resources(const char host, GList rsc_list)
	{
	GList *rIter = NULL;
	GList *active = NULL;

	for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) rIter->data;

	/* Expand groups to their members, because if we're restarting a member
	* other than the first, we can't otherwise tell which resources are
	* stopping and starting.
	*/
	if (rsc->variant == pcmk_rsc_variant_group) {
	active = g_list_concat(active,
	get_active_resources(host, rsc->children));
	} else if (resource_is_running_on(rsc, host)) {
	active = g_list_append(active, strdup(rsc->id));
	}
	}
	return active;
	}

	static void dump_list(GList items, const char tag)
	{
	int lpc = 0;
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
	lpc++;
	}
	}

	static void display_list(pcmk__output_t out, GList items, const char *tag)
	{
	GList *item = NULL;

	for (item = items; item != NULL; item = item->next) {
	out->info(out, "%s%s", tag, (const char *)item->data);
	}
	}

	/*!
	* \internal
	* \brief Upgrade XML to latest schema version and use it as scheduler input
	*
	* This also updates the scheduler timestamp to the current time.
	*
	* \param[in,out] scheduler Scheduler data to update
	* \param[in,out] xml XML to use as input
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->now.
	* \todo This follows the example of other callers of cli_config_update()
	* and returns ENOKEY ("Required key not available") if that fails,
	* but perhaps pcmk_rc_schema_validation would be better in that case.
	*/
	int
	update_scheduler_input(pcmk_scheduler_t scheduler, xmlNode *xml)
	{
	if (cli_config_update(xml, NULL, FALSE) == FALSE) {
	return ENOKEY;
	}
	scheduler->input = *xml;
	scheduler->now = crm_time_new(NULL);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Update scheduler XML input based on a CIB query
	*
	* \param[in] scheduler Scheduler data to initialize
	* \param[in] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	* \note On success, caller is responsible for freeing memory allocated for
	* scheduler->input and scheduler->now.
	*/
	static int
	update_scheduler_input_to_cib(pcmk__output_t out, pcmk_scheduler_t scheduler,
	cib_t *cib)
	{
	xmlNode *cib_xml_copy = NULL;
	int rc = pcmk_rc_ok;

	rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local \| cib_sync_call);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
	return rc;
	}
	rc = update_scheduler_input(scheduler, &cib_xml_copy);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not upgrade the current CIB XML");
	free_xml(cib_xml_copy);
	return rc;
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	update_dataset(cib_t cib, pcmk_scheduler_t scheduler, bool simulate)
	{
	char *pid = NULL;
	char *shadow_file = NULL;
	cib_t *shadow_cib = NULL;
	int rc = pcmk_rc_ok;

	pcmk__output_t *out = scheduler->priv;

	pe_reset_working_set(scheduler);
	pcmk__set_scheduler_flags(scheduler,
	pcmk_sched_no_counts\|pcmk_sched_no_compat);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if(simulate) {
	bool prev_quiet = false;

	pid = pcmk__getpid_s();
	shadow_cib = cib_shadow_new(pid);
	shadow_file = get_shadow_file(pid);

	if (shadow_cib == NULL) {
	out->err(out, "Could not create shadow cib: '%s'", pid);
	rc = ENXIO;
	goto done;
	}

	rc = write_xml_file(scheduler->input, shadow_file, FALSE);

	if (rc < 0) {
	out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
	goto done;
	}

	rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_rc_str(rc), rc);
	goto done;
	}

	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	prev_quiet = out->is_quiet(out);
	out->quiet = true;
	pcmk__simulate_transition(scheduler, shadow_cib, NULL);
	out->quiet = prev_quiet;

	rc = update_dataset(shadow_cib, scheduler, false);

	} else {
	cluster_status(scheduler);
	}

	done:
	// Do not free scheduler->input here, we need rsc->xml to be valid later on
	cib_delete(shadow_cib);
	free(pid);

	if(shadow_file) {
	unlink(shadow_file);
	free(shadow_file);
	}

	return rc;
	}

	/*!
	* \internal
	* \brief Find the maximum stop timeout of a resource and its children (if any)
	*
	* \param[in,out] rsc Resource to get timeout for
	*
	* \return Maximum stop timeout for \p rsc (in milliseconds)
	*/
	static int
	max_rsc_stop_timeout(pcmk_resource_t *rsc)
	{
	long long result_ll;
	int max_delay = 0;
	xmlNode *config = NULL;
	GHashTable *meta = NULL;

	if (rsc == NULL) {
	return 0;
	}

	// If resource is collective, use maximum of its children's stop timeouts
	if (rsc->children != NULL) {
	for (GList *iter = rsc->children; iter; iter = iter->next) {
	pcmk_resource_t *child = iter->data;
	int delay = max_rsc_stop_timeout(child);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Maximum stop timeout for %s is now %s "
	"due to %s", rsc->id,
	pcmk__readable_interval(delay), child->id);
	max_delay = delay;
	}
	}
	return max_delay;
	}

	// Get resource's stop action configuration from CIB
	config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);

	/* Get configured timeout for stop action (fully evaluated for rules,
	* defaults, etc.).
	*
	* @TODO This currently ignores node (which might matter for rules)
	*/
	meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
	if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
	&result_ll, -1LL) == pcmk_rc_ok)
	&& (result_ll >= 0) && (result_ll <= INT_MAX)) {
	max_delay = (int) result_ll;
	}
	g_hash_table_destroy(meta);

	return max_delay;
	}

	/*!
	* \internal
	* \brief Find a reasonable waiting time for stopping any one resource in a list
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] resources List of names of resources that will be stopped
	*
	* \return Rough estimate of a reasonable time to wait (in seconds) to stop any
	* one resource in \p resources
	* \note This estimate is very rough, simply the maximum stop timeout of all
	* given resources and their children, plus a small fudge factor. It does
	* not account for children that must be stopped in sequence, action
	* throttling, or any demotions needed. It checks the stop timeout, even
	* if the resources in question are actually being started.
	*/
	static int
	wait_time_estimate(pcmk_scheduler_t scheduler, const GList resources)
	{
	int max_delay = 0;

	// Find maximum stop timeout in milliseconds
	for (const GList *item = resources; item != NULL; item = item->next) {
	pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
	(const char *) item->data);
	int delay = max_rsc_stop_timeout(rsc);

	if (delay > max_delay) {
	pcmk__rsc_trace(rsc,
	"Wait time is now %s due to %s",
	pcmk__readable_interval(delay), rsc->id);
	max_delay = delay;
	}
	}

	return (max_delay / 1000) + 5;
	}

	#define waiting_for_starts(d, r, h) ((d != NULL) \|\| \
	(!resource_is_running_on((r), (h))))

	/*!
	* \internal
	* \brief Restart a resource (on a particular host if requested).
	*
	* \param[in,out] out Output object
	* \param[in,out] rsc The resource to restart
	* \param[in] node Node to restart resource on (NULL for all)
	* \param[in] move_lifetime If not NULL, how long constraint should
	* remain in effect (as ISO 8601 string)
	* \param[in] timeout_ms Consider failed if actions do not complete
	* in this time (specified in milliseconds,
	* but a two-second granularity is actually
	* used; if 0, it will be calculated based on
	* the resource timeout)
	* \param[in,out] cib Connection to the CIB manager
	* \param[in] cib_options Group of enum cib_call_options flags to
	* use with CIB calls
	* \param[in] promoted_role_only If true, limit to promoted instances
	* \param[in] force If true, apply only to requested instance
	* if part of a collective resource
	*
	* \return Standard Pacemaker return code (exits on certain failures)
	*/
	int
	cli_resource_restart(pcmk__output_t out, pcmk_resource_t rsc,
	const pcmk_node_t node, const char move_lifetime,
	int timeout_ms, cib_t *cib, int cib_options,
	gboolean promoted_role_only, gboolean force)
	{
	int rc = pcmk_rc_ok;
	int lpc = 0;
	int before = 0;
	int step_timeout_s = 0;
	int sleep_interval = 2;
	int timeout = timeout_ms / 1000;

	bool stop_via_ban = false;
	char *rsc_id = NULL;
	char *lookup_id = NULL;
	char *orig_target_role = NULL;

	GList *list_delta = NULL;
	GList *target_active = NULL;
	GList *current_active = NULL;
	GList *restart_target_active = NULL;

	pcmk_scheduler_t *scheduler = NULL;
	pcmk_resource_t *parent = uber_parent(rsc);

	bool running = false;
	const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
	const char *host = node ? node->details->uname : NULL;

	/* If the implicit resource or primitive resource of a bundle is given, operate on the
	* bundle itself instead.
	*/
	if (pe_rsc_is_bundled(rsc)) {
	rsc = parent->parent;
	}

	running = resource_is_running_on(rsc, host);

	if (pe_rsc_is_clone(parent) && !running) {
	if (pe_rsc_is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	rsc = parent->fns->find_rsc(parent, lookup_id, node,
	pcmk_rsc_match_basename
	\|pcmk_rsc_match_current_node);
	free(lookup_id);
	running = resource_is_running_on(rsc, host);
	}

	if (!running) {
	if (host) {
	out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
	} else {
	out->err(out, "%s is not running anywhere and so cannot be restarted", id);
	}
	return ENXIO;
	}

	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	out->err(out, "Unmanaged resources cannot be restarted.");
	return EAGAIN;
	}

	rsc_id = strdup(rsc->id);

	if (pe_rsc_is_unique_clone(parent)) {
	lookup_id = strdup(rsc->id);
	} else {
	lookup_id = clone_strip(rsc->id);
	}

	if (host) {
	if (pe_rsc_is_clone(rsc) \|\| pe_bundle_replicas(rsc)) {
	stop_via_ban = true;
	} else if (pe_rsc_is_clone(parent)) {
	stop_via_ban = true;
	free(lookup_id);
	lookup_id = strdup(parent->id);
	}
	}

	/*
	grab full cib
	determine originally active resources
	disable or ban
	poll cib and watch for affected resources to get stopped
	without --timeout, calculate the stop timeout for each step and wait for that
	if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
	if everything stopped, re-enable or un-ban
	poll cib and watch for affected resources to get started
	without --timeout, calculate the start timeout for each step and wait for that
	if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
	report success

	Optimizations:
	- use constraints to determine ordered list of affected resources
	- Allow a --no-deps option (aka. --force-restart)
	*/

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = errno;
	out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}

	scheduler->priv = out;
	rc = update_dataset(cib, scheduler, false);

	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
	goto done;
	}

	restart_target_active = get_active_resources(host, scheduler->resources);
	current_active = get_active_resources(host, scheduler->resources);

	dump_list(current_active, "Origin");

	if (stop_via_ban) {
	/* Stop the clone or bundle instance by banning it from the host */
	out->quiet = true;
	rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
	cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);
	} else {
	/* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
	* Remember any existing PCMK_META_TARGET_ROLE so we can restore it
	* later (though it only makes any difference if it's Unpromoted).
	*/

	find_resource_attr(out, cib, PCMK_XA_VALUE, lookup_id, NULL, NULL, NULL,
	PCMK_META_TARGET_ROLE, &orig_target_role);
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE,
	PCMK_ACTION_STOPPED, FALSE, cib,
	cib_options, force);
	}
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	current_active = NULL;
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	restart_target_active = NULL;
	}
	goto done;
	}

	rc = update_dataset(cib, scheduler, true);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources would be stopped");
	goto failure;
	}

	target_active = get_active_resources(host, scheduler->resources);
	dump_list(target_active, "Target");

	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (list_delta != NULL) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}
	rc = update_dataset(cib, scheduler, FALSE);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were stopped");
	goto failure;
	}

	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(host, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);

	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
	if(before == g_list_length(list_delta)) {
	/* aborted during stop phase, print the contents of list_delta */
	out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	if (stop_via_ban) {
	rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);

	} else if (orig_target_role) {
	rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE,
	orig_target_role, FALSE, cib,
	cib_options, force);
	free(orig_target_role);
	orig_target_role = NULL;
	} else {
	rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE, cib,
	cib_options, force);
	}

	if(rc != pcmk_rc_ok) {
	out->err(out,
	"Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
	rsc_id, pcmk_rc_str(rc), rc);
	goto done;
	}

	if (target_active != NULL) {
	g_list_free_full(target_active, free);
	}
	target_active = restart_target_active;

	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
	display_list(out, list_delta, " * ");

	step_timeout_s = timeout / sleep_interval;
	while (waiting_for_starts(list_delta, rsc, host)) {
	before = g_list_length(list_delta);
	if(timeout_ms == 0) {
	step_timeout_s = wait_time_estimate(scheduler, list_delta)
	/ sleep_interval;
	}

	/* We probably don't need the entire step timeout */
	for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {

	sleep(sleep_interval);
	if(timeout) {
	timeout -= sleep_interval;
	crm_trace("%ds remaining", timeout);
	}

	rc = update_dataset(cib, scheduler, false);
	if(rc != pcmk_rc_ok) {
	out->err(out, "Could not determine which resources were started");
	goto failure;
	}

	/* It's OK if dependent resources moved to a different node,
	* so we check active resources on all nodes.
	*/
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	current_active = get_active_resources(NULL, scheduler->resources);

	g_list_free(list_delta);
	list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
	dump_list(current_active, "Current");
	dump_list(list_delta, "Delta");
	}

	if(before == g_list_length(list_delta)) {
	/* aborted during start phase, print the contents of list_delta */
	out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
	display_list(out, list_delta, " * ");
	rc = ETIME;
	goto failure;
	}

	}

	rc = pcmk_rc_ok;
	goto done;

	failure:
	if (stop_via_ban) {
	cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
	} else if (orig_target_role) {
	cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
	PCMK_META_TARGET_ROLE, orig_target_role,
	FALSE, cib, cib_options, force);
	free(orig_target_role);
	} else {
	cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
	NULL, PCMK_META_TARGET_ROLE, cib,
	cib_options, force);
	}

	done:
	if (list_delta != NULL) {
	g_list_free(list_delta);
	}
	if (current_active != NULL) {
	g_list_free_full(current_active, free);
	}
	if (target_active != NULL && (target_active != restart_target_active)) {
	g_list_free_full(target_active, free);
	}
	if (restart_target_active != NULL) {
	g_list_free_full(restart_target_active, free);
	}
	free(rsc_id);
	free(lookup_id);
	pe_free_working_set(scheduler);
	return rc;
	}

	static inline bool
	action_is_pending(const pcmk_action_t *action)
	{
	if (pcmk_any_flags_set(action->flags,
	pcmk_action_optional\|pcmk_action_pseudo)
	\|\| !pcmk_is_set(action->flags, pcmk_action_runnable)
	\|\| pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether any actions in a list are pending
	*
	* \param[in] actions List of actions to check
	*
	* \return true if any actions in the list are pending, otherwise false
	*/
	static bool
	actions_are_pending(const GList *actions)
	{
	for (const GList *action = actions; action != NULL; action = action->next) {
	const pcmk_action_t a = (const pcmk_action_t ) action->data;

	if (action_is_pending(a)) {
	crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
	return true;
	}
	}
	return false;
	}

	static void
	print_pending_actions(pcmk__output_t out, GList actions)
	{
	GList *action;

	out->info(out, "Pending actions:");
	for (action = actions; action != NULL; action = action->next) {
	pcmk_action_t a = (pcmk_action_t ) action->data;

	if (!action_is_pending(a)) {
	continue;
	}

	if (a->node) {
	out->info(out, "\tAction %d: %s\ton %s",
	a->id, a->uuid, pcmk__node_name(a->node));
	} else {
	out->info(out, "\tAction %d: %s", a->id, a->uuid);
	}
	}
	}

	/* For --wait, timeout (in seconds) to use if caller doesn't specify one */
	#define WAIT_DEFAULT_TIMEOUT_S (60 * 60)

	/* For --wait, how long to sleep between cluster state checks */
	#define WAIT_SLEEP_S (2)

	/*!
	* \internal
	* \brief Wait until all pending cluster actions are complete
	*
	* This waits until either the CIB's transition graph is idle or a timeout is
	* reached.
	*
	* \param[in,out] out Output object
	* \param[in] timeout_ms Consider failed if actions do not complete in
	* this time (specified in milliseconds, but
	* one-second granularity is actually used; if 0, a
	* default will be used)
	* \param[in,out] cib Connection to the CIB manager
	*
	* \return Standard Pacemaker return code
	*/
	int
	wait_till_stable(pcmk__output_t out, int timeout_ms, cib_t cib)
	{
	pcmk_scheduler_t *scheduler = NULL;
	xmlXPathObjectPtr search;
	int rc = pcmk_rc_ok;
	bool pending_unknown_state_resources;
	int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
	time_t expire_time = time(NULL) + timeout_s;
	time_t time_diff;
	bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	return ENOMEM;
	}

	do {
	/* Abort if timeout is reached */
	time_diff = expire_time - time(NULL);
	if (time_diff > 0) {
	crm_info("Waiting up to %lld seconds for cluster actions to complete", (long long) time_diff);
	} else {
	print_pending_actions(out, scheduler->actions);
	pe_free_working_set(scheduler);
	return ETIME;
	}
	if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
	sleep(WAIT_SLEEP_S);
	}

	/* Get latest transition graph */
	pe_reset_working_set(scheduler);
	rc = update_scheduler_input_to_cib(out, scheduler, cib);
	if (rc != pcmk_rc_ok) {
	pe_free_working_set(scheduler);
	return rc;
	}
	pcmk__schedule_actions(scheduler->input,
	pcmk_sched_no_counts\|pcmk_sched_no_compat,
	scheduler);

	if (!printed_version_warning) {
	/* If the DC has a different version than the local node, the two
	* could come to different conclusions about what actions need to be
	* done. Warn the user in this case.
	*
	* @TODO A possible long-term solution would be to reimplement the
	* wait as a new controller operation that would be forwarded to the
	* DC. However, that would have potential problems of its own.
	*/
	const char *dc_version = g_hash_table_lookup(scheduler->config_hash,
	PCMK_OPT_DC_VERSION);

	if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
	out->info(out, "warning: wait option may not work properly in "
	"mixed-version cluster");
	printed_version_warning = true;
	}
	}

	search = xpath_search(scheduler->input,
	- "/cib/status/node_state/lrm/lrm_resources"
	- "/lrm_resource/" XML_LRM_TAG_RSC_OP
	+ "/" PCMK_XE_CIB "/" PCMK_XE_STATUS
	+ "/" PCMK__XE_NODE_STATE "/" XML_CIB_TAG_LRM
	+ "/" XML_LRM_TAG_RESOURCES "/" XML_LRM_TAG_RESOURCE
	+ "/" XML_LRM_TAG_RSC_OP
	"[@" PCMK__XA_RC_CODE "='193']");
	pending_unknown_state_resources = (numXpathResults(search) > 0);
	freeXpathObject(search);
	} while (actions_are_pending(scheduler->actions) \|\| pending_unknown_state_resources);

	pe_free_working_set(scheduler);
	return rc;
	}

	static const char *
	get_action(const char *rsc_action) {
	const char *action = NULL;

	if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
	action = PCMK_ACTION_VALIDATE_ALL;

	} else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
	action = PCMK_ACTION_MONITOR;

	} else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
	"force-demote", "force-promote", NULL)) {
	action = rsc_action+6;
	} else {
	action = rsc_action;
	}

	return action;
	}

	/*!
	* \brief Set up environment variables as expected by resource agents
	*
	* When the cluster executes resource agents, it adds certain environment
	* variables (directly or via resource meta-attributes) expected by some
	* resource agents. Add the essential ones that many resource agents expect, so
	* the behavior is the same for command-line execution.
	*
	* \param[in,out] params Resource parameters that will be passed to agent
	* \param[in] timeout_ms Action timeout (in milliseconds)
	* \param[in] check_level OCF check level
	* \param[in] verbosity Verbosity level
	*/
	static void
	set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
	int verbosity)
	{
	g_hash_table_insert(params, crm_meta_name(PCMK_META_TIMEOUT),
	crm_strdup_printf("%d", timeout_ms));

	g_hash_table_insert(params, strdup(PCMK_XA_CRM_FEATURE_SET),
	strdup(CRM_FEATURE_SET));

	if (check_level >= 0) {
	char *level = crm_strdup_printf("%d", check_level);

	setenv("OCF_CHECK_LEVEL", level, 1);
	free(level);
	}

	pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
	if (verbosity > 1) {
	setenv("OCF_TRACE_RA", "1", 1);
	}

	/* A resource agent using the standard ocf-shellfuncs library will not print
	* messages to stderr if it doesn't have a controlling terminal (e.g. if
	* crm_resource is called via script or ssh). This forces it to do so.
	*/
	setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
	}

	/*!
	* \internal
	* \brief Apply command-line overrides to resource parameters
	*
	* \param[in,out] params Parameters to be passed to agent
	* \param[in] overrides Parameters to override (or NULL if none)
	*/
	static void
	apply_overrides(GHashTable params, GHashTable overrides)
	{
	if (overrides != NULL) {
	GHashTableIter iter;
	char *name = NULL;
	char *value = NULL;

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer *) &name,
	(gpointer *) &value)) {
	g_hash_table_replace(params, strdup(name), strdup(value));
	}
	}
	}

	crm_exit_t
	cli_resource_execute_from_params(pcmk__output_t out, const char rsc_name,
	const char rsc_class, const char rsc_prov,
	const char rsc_type, const char rsc_action,
	GHashTable params, GHashTable override_hash,
	int timeout_ms, int resource_verbose, gboolean force,
	int check_level)
	{
	const char *class = rsc_class;
	const char *action = get_action(rsc_action);
	crm_exit_t exit_code = CRM_EX_OK;
	svc_action_t *op = NULL;

	// If no timeout was provided, use the same default as the cluster
	if (timeout_ms == 0) {
	timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
	}

	set_agent_environment(params, timeout_ms, check_level, resource_verbose);
	apply_overrides(params, override_hash);

	op = services__create_resource_action(rsc_name? rsc_name : "test",
	rsc_class, rsc_prov, rsc_type, action,
	0, timeout_ms, params, 0);
	if (op == NULL) {
	out->err(out, "Could not execute %s using %s%s%s:%s: %s",
	action, rsc_class, (rsc_prov? ":" : ""),
	(rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
	g_hash_table_destroy(params);
	return CRM_EX_OSERR;
	}

	if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
	class = resources_find_service_class(rsc_type);
	}
	if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
	PCMK_RESOURCE_CLASS_LSB, NULL)) {
	services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
	"Manual execution of the %s standard is "
	"unsupported", pcmk__s(class, "unspecified"));
	}

	if (op->rc != PCMK_OCF_UNKNOWN) {
	exit_code = op->rc;
	goto done;
	}

	services_action_sync(op);

	// Map results to OCF codes for consistent reporting to user
	{
	enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);

	// Cast variable instead of function return to keep compilers happy
	exit_code = (crm_exit_t) ocf_code;
	}

	done:
	out->message(out, "resource-agent-action", resource_verbose, rsc_class,
	rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
	exit_code, op->status, services__exit_reason(op),
	op->stdout_data, op->stderr_data);
	services_action_free(op);
	return exit_code;
	}

	crm_exit_t
	cli_resource_execute(pcmk_resource_t rsc, const char requested_name,
	const char rsc_action, GHashTable override_hash,
	int timeout_ms, cib_t cib, pcmk_scheduler_t scheduler,
	int resource_verbose, gboolean force, int check_level)
	{
	pcmk__output_t *out = scheduler->priv;
	crm_exit_t exit_code = CRM_EX_OK;
	const char *rid = NULL;
	const char *rtype = NULL;
	const char *rprov = NULL;
	const char *rclass = NULL;
	GHashTable *params = NULL;

	if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
	"force-promote", NULL)) {
	if(pe_rsc_is_clone(rsc)) {
	GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
	if(nodes != NULL && force == FALSE) {
	out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
	rsc_action, rsc->id);
	out->err(out,
	"Try setting "
	PCMK_META_TARGET_ROLE "=" PCMK__ROLE_STOPPED
	" first or specifying the force option");
	return CRM_EX_UNSAFE;
	}

	g_list_free_full(nodes, free);
	}
	}

	if(pe_rsc_is_clone(rsc)) {
	/* Grab the first child resource in the hope it's not a group */
	rsc = rsc->children->data;
	}

	if (rsc->variant == pcmk_rsc_variant_group) {
	out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	} else if (pe_rsc_is_bundled(rsc)) {
	out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
	return CRM_EX_UNIMPLEMENT_FEATURE;
	}

	rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);

	params = generate_resource_params(rsc, NULL /* @TODO use local node */,
	scheduler);

	if (timeout_ms == 0) {
	timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action),
	scheduler);
	}

	rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;

	exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
	params, override_hash, timeout_ms,
	resource_verbose, force, check_level);
	return exit_code;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_move(const pcmk_resource_t rsc, const char rsc_id,
	const char host_name, const char move_lifetime, cib_t *cib,
	int cib_options, pcmk_scheduler_t *scheduler,
	gboolean promoted_role_only, gboolean force)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_ok;
	unsigned int count = 0;
	pcmk_node_t *current = NULL;
	pcmk_node_t *dest = pe_find_node(scheduler->nodes, host_name);
	bool cur_is_dest = false;

	if (dest == NULL) {
	return pcmk_rc_node_unknown;
	}

	if (promoted_role_only
	&& !pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {

	const pcmk_resource_t *p = pe__const_top_resource(rsc, false);

	if (pcmk_is_set(p->flags, pcmk_rsc_promotable)) {
	out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
	rsc_id = p->id;
	rsc = p;

	} else {
	out->info(out, "Ignoring --promoted option: %s is not promotable",
	rsc_id);
	promoted_role_only = FALSE;
	}
	}

	current = pe__find_active_requires(rsc, &count);

	if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
	unsigned int promoted_count = 0;
	pcmk_node_t *promoted_node = NULL;

	for (const GList *iter = rsc->children; iter; iter = iter->next) {
	const pcmk_resource_t child = (const pcmk_resource_t ) iter->data;
	enum rsc_role_e child_role = child->fns->state(child, TRUE);

	if (child_role == pcmk_role_promoted) {
	rsc = child;
	promoted_node = pcmk__current_node(child);
	promoted_count++;
	}
	}
	if (promoted_role_only \|\| (promoted_count != 0)) {
	count = promoted_count;
	current = promoted_node;
	}

	}

	if (count > 1) {
	if (pe_rsc_is_clone(rsc)) {
	current = NULL;
	} else {
	return pcmk_rc_multiple;
	}
	}

	if (pcmk__same_node(current, dest)) {
	cur_is_dest = true;
	if (force) {
	crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
	rsc_id, promoted_role_only?"promoted":"active",
	pcmk__node_name(dest));
	} else {
	return pcmk_rc_already;
	}
	}

	/* Clear any previous prefer constraints across all nodes. */
	cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, cib_options, false,
	force);

	/* Clear any previous ban constraints on 'dest'. */
	cli_resource_clear(rsc_id, dest->details->uname, scheduler->nodes, cib,
	cib_options, TRUE, force);

	/* Record an explicit preference for 'dest' */
	rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);

	crm_trace("%s%s now prefers %s%s",
	rsc->id, (promoted_role_only? " (promoted)" : ""),
	pcmk__node_name(dest), force?"(forced)":"");

	/* only ban the previous location if current location != destination location.
	* it is possible to use -M to enforce a location without regard of where the
	* resource is currently located */
	if (force && !cur_is_dest) {
	/* Ban the original location if possible */
	if(current) {
	(void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
	cib, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED);
	} else if(count > 1) {
	out->info(out, "Resource '%s' is currently %s in %d locations. "
	"One may now move to %s",
	rsc_id, (promoted_role_only? "promoted" : "active"),
	count, pcmk__node_name(dest));
	out->info(out, "To prevent '%s' from being %s at a specific location, "
	"specify a node.",
	rsc_id, (promoted_role_only? "promoted" : "active"));

	} else {
	crm_trace("Not banning %s from its current location: not active", rsc_id);
	}
	}

	return rc;
	}

File Metadata

Mime Type: text/x-diff
Expires: Sat, Nov 23, 6:28 AM (11 h, 16 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1018291
Default Alt Text: (983 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions