No OneTemporary
Actions

Size

400 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/attrd/attrd_cib.c b/daemons/attrd/attrd_cib.c
	index 01e27a4f2a..80afec4427 100644
	--- a/daemons/attrd/attrd_cib.c
	+++ b/daemons/attrd/attrd_cib.c
	@@ -1,684 +1,685 @@
	/*
	* Copyright 2013-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <errno.h>
	#include <inttypes.h> // PRIu32
	#include <stdbool.h>
	#include <stdlib.h>
	#include <glib.h>

	#include <crm/cib/internal.h> // cib__*
	#include <crm/common/logging.h>
	#include <crm/common/results.h>
	#include <crm/common/strings_internal.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h> // pcmk__get_node()

	#include "pacemaker-attrd.h"

	static int last_cib_op_done = 0;

	static void write_attribute(attribute_t *a, bool ignore_delay);

	static void
	attrd_cib_destroy_cb(gpointer user_data)
	{
	cib_t *cib = user_data;

	cib->cmds->signoff(cib);

	if (attrd_shutting_down(false)) {
	crm_info("Disconnected from the CIB manager");

	} else {
	// @TODO This should trigger a reconnect, not a shutdown
	crm_crit("Lost connection to the CIB manager, shutting down");
	attrd_exit_status = CRM_EX_DISCONNECT;
	attrd_shutdown(0);
	}
	}

	static void
	attrd_cib_updated_cb(const char event, xmlNode msg)
	{
	const xmlNode *patchset = NULL;
	const char *client_name = NULL;
	bool status_changed = false;

	if (attrd_shutting_down(true)) {
	crm_debug("Ignoring CIB change during shutdown");
	return;
	}

	if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
	return;
	}

	if (cib__element_in_patchset(patchset, PCMK_XE_ALERTS)) {
	mainloop_set_trigger(attrd_config_read);
	}

	status_changed = cib__element_in_patchset(patchset, PCMK_XE_STATUS);

	client_name = crm_element_value(msg, PCMK__XA_CIB_CLIENTNAME);
	if (!cib__client_triggers_refresh(client_name)) {
	/* This change came from a source that ensured the CIB is consistent
	* with our attributes table, so we don't need to write anything out.
	*/
	return;
	}

	if (!attrd_election_won()) {
	// Don't write attributes if we're not the writer
	return;
	}

	if (status_changed \|\| cib__element_in_patchset(patchset, PCMK_XE_NODES)) {
	/* An unsafe client modified the PCMK_XE_NODES or PCMK_XE_STATUS
	* section. Write transient attributes to ensure they're up-to-date in
	* the CIB.
	*/
	if (client_name == NULL) {
	client_name = crm_element_value(msg, PCMK__XA_CIB_CLIENTID);
	}
	crm_notice("Updating all attributes after %s event triggered by %s",
	event, pcmk__s(client_name, "(unidentified client)"));

	attrd_write_attributes(attrd_write_all);
	}
	}

	int
	attrd_cib_connect(int max_retry)
	{
	static int attempts = 0;

	int rc = -ENOTCONN;

	the_cib = cib_new();
	if (the_cib == NULL) {
	return -ENOTCONN;
	}

	do {
	if (attempts > 0) {
	sleep(attempts);
	}
	attempts++;
	crm_debug("Connection attempt %d to the CIB manager", attempts);
	rc = the_cib->cmds->signon(the_cib, PCMK__VALUE_ATTRD, cib_command);

	} while ((rc != pcmk_ok) && (attempts < max_retry));

	if (rc != pcmk_ok) {
	crm_err("Connection to the CIB manager failed: %s " QB_XS " rc=%d",
	pcmk_strerror(rc), rc);
	goto cleanup;
	}

	crm_debug("Connected to the CIB manager after %d attempts", attempts);

	rc = the_cib->cmds->set_connection_dnotify(the_cib, attrd_cib_destroy_cb);
	if (rc != pcmk_ok) {
	crm_err("Could not set disconnection callback");
	goto cleanup;
	}

	rc = the_cib->cmds->add_notify_callback(the_cib,
	PCMK__VALUE_CIB_DIFF_NOTIFY,
	attrd_cib_updated_cb);
	if (rc != pcmk_ok) {
	crm_err("Could not set CIB notification callback");
	goto cleanup;
	}

	return pcmk_ok;

	cleanup:
	cib__clean_up_connection(&the_cib);
	return -ENOTCONN;
	}

	void
	attrd_cib_disconnect(void)
	{
	CRM_CHECK(the_cib != NULL, return);
	the_cib->cmds->del_notify_callback(the_cib, PCMK__VALUE_CIB_DIFF_NOTIFY,
	attrd_cib_updated_cb);
	cib__clean_up_connection(&the_cib);
	mainloop_destroy_trigger(attrd_config_read);
	}

	static void
	attrd_erase_cb(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	const char node = pcmk__s((const char ) user_data, "a node");

	if (rc == pcmk_ok) {
	crm_info("Cleared transient node attributes for %s from CIB", node);
	} else {
	crm_err("Unable to clear transient node attributes for %s from CIB: %s",
	node, pcmk_strerror(rc));
	}
	}

	#define XPATH_TRANSIENT "//" PCMK__XE_NODE_STATE \
	"[@" PCMK_XA_UNAME "='%s']" \
	"/" PCMK__XE_TRANSIENT_ATTRIBUTES

	/*!
	* \internal
	* \brief Wipe all transient node attributes for a node from the CIB
	*
	* \param[in] node Node to clear attributes for
	*/
	void
	attrd_cib_erase_transient_attrs(const char *node)
	{
	int call_id = 0;
	char *xpath = NULL;

	CRM_CHECK(node != NULL, return);

	xpath = crm_strdup_printf(XPATH_TRANSIENT, node);

	crm_debug("Clearing transient node attributes for %s from CIB using %s",
	node, xpath);

	call_id = the_cib->cmds->remove(the_cib, xpath, NULL, cib_xpath);
	free(xpath);

	the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE,
	pcmk__str_copy(node),
	"attrd_erase_cb", attrd_erase_cb,
	free);
	}

	/*!
	* \internal
	* \brief Prepare the CIB after cluster is connected
	*/
	void
	attrd_cib_init(void)
	{
	/* We have no attribute values in memory, so wipe the CIB to match. This is
	* normally done by the DC's controller when this node leaves the cluster, but
	* this handles the case where the node restarted so quickly that the
	* cluster layer didn't notice.
	*
	* \todo If pacemaker-attrd respawns after crashing (see PCMK_ENV_RESPAWNED),
	* ideally we'd skip this and sync our attributes from the writer.
	* However, currently we reject any values for us that the writer has, in
	* attrd_peer_update().
	*/
	attrd_cib_erase_transient_attrs(attrd_cluster->uname);

	// Set a trigger for reading the CIB (for the alerts section)
	attrd_config_read = mainloop_add_trigger(G_PRIORITY_HIGH, attrd_read_options, NULL);

	// Always read the CIB at start-up
	mainloop_set_trigger(attrd_config_read);
	}

	static gboolean
	attribute_timer_cb(gpointer data)
	{
	attribute_t *a = data;
	crm_trace("Dampen interval expired for %s", a->id);
	attrd_write_or_elect_attribute(a);
	return FALSE;
	}

	static void
	attrd_cib_callback(xmlNode msg, int call_id, int rc, xmlNode output, void *user_data)
	{
	int level = LOG_ERR;
	GHashTableIter iter;
	const char *peer = NULL;
	attribute_value_t *v = NULL;

	char *name = user_data;
	attribute_t *a = g_hash_table_lookup(attributes, name);

	if(a == NULL) {
	crm_info("Attribute %s no longer exists", name);
	return;
	}

	a->update = 0;
	if (rc == pcmk_ok && call_id < 0) {
	rc = call_id;
	}

	switch (rc) {
	case pcmk_ok:
	level = LOG_INFO;
	last_cib_op_done = call_id;
	if (a->timer && !a->timeout_ms) {
	// Remove temporary dampening for failed writes
	mainloop_timer_del(a->timer);
	a->timer = NULL;
	}
	break;

	case -pcmk_err_diff_failed: /* When an attr changes while the CIB is syncing */
	case -ETIME: /* When an attr changes while there is a DC election */
	case -ENXIO: /* When an attr changes while the CIB is syncing a
	* newer config from a node that just came up
	*/
	level = LOG_WARNING;
	break;
	}

	do_crm_log(level, "CIB update %d result for %s: %s " QB_XS " rc=%d",
	call_id, a->id, pcmk_strerror(rc), rc);

	g_hash_table_iter_init(&iter, a->values);
	while (g_hash_table_iter_next(&iter, (gpointer ) & peer, (gpointer ) & v)) {
	if (rc == pcmk_ok) {
	crm_info("* Wrote %s[%s]=%s",
	a->id, peer, pcmk__s(v->requested, "(unset)"));
	pcmk__str_update(&(v->requested), NULL);
	} else {
	do_crm_log(level, "* Could not write %s[%s]=%s",
	a->id, peer, pcmk__s(v->requested, "(unset)"));
	/* Reattempt write below if we are still the writer */
	attrd_set_attr_flags(a, attrd_attr_changed);
	}
	}

	if (pcmk_is_set(a->flags, attrd_attr_changed) && attrd_election_won()) {
	if (rc == pcmk_ok) {
	/* We deferred a write of a new update because this update was in
	* progress. Write out the new value without additional delay.
	*/
	crm_debug("Pending update for %s can be written now", a->id);
	write_attribute(a, false);

	/* We're re-attempting a write because the original failed; delay
	* the next attempt so we don't potentially flood the CIB manager
	* and logs with a zillion attempts per second.
	*
	* @TODO We could elect a new writer instead. However, we'd have to
	* somehow downgrade our vote, and we'd still need something like this
	* if all peers similarly fail to write this attribute (which may
	* indicate a corrupted attribute entry rather than a CIB issue).
	*/
	} else if (a->timer) {
	// Attribute has a dampening value, so use that as delay
	if (!mainloop_timer_running(a->timer)) {
	crm_trace("Delayed re-attempted write for %s by %s",
	name, pcmk__readable_interval(a->timeout_ms));
	mainloop_timer_start(a->timer);
	}
	} else {
	/* Set a temporary dampening of 2 seconds (timer will continue
	* to exist until the attribute's dampening gets set or the
	* write succeeds).
	*/
	a->timer = attrd_add_timer(a->id, 2000, a);
	mainloop_timer_start(a->timer);
	}
	}
	}

	/*!
	* \internal
	* \brief Add a set-attribute update request to the current CIB transaction
	*
	* \param[in] attr Attribute to update
	* \param[in] attr_id ID of attribute to update
	* \param[in] node_id ID of node for which to update attribute value
	* \param[in] set_id ID of attribute set
	* \param[in] value New value for attribute
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_set_attr_update(const attribute_t attr, const char attr_id,
	const char node_id, const char set_id, const char *value)
	{
	xmlNode *update = pcmk__xe_create(NULL, PCMK__XE_NODE_STATE);
	xmlNode *child = update;
	int rc = ENOMEM;

	crm_xml_add(child, PCMK_XA_ID, node_id);

	child = pcmk__xe_create(child, PCMK__XE_TRANSIENT_ATTRIBUTES);
	crm_xml_add(child, PCMK_XA_ID, node_id);

	child = pcmk__xe_create(child, attr->set_type);
	crm_xml_add(child, PCMK_XA_ID, set_id);

	child = pcmk__xe_create(child, PCMK_XE_NVPAIR);
	crm_xml_add(child, PCMK_XA_ID, attr_id);
	crm_xml_add(child, PCMK_XA_NAME, attr->id);
	crm_xml_add(child, PCMK_XA_VALUE, value);

	rc = the_cib->cmds->modify(the_cib, PCMK_XE_STATUS, update,
	cib_can_create\|cib_transaction);
	rc = pcmk_legacy2rc(rc);

	pcmk__xml_free(update);
	return rc;
	}

	/*!
	* \internal
	* \brief Add an unset-attribute update request to the current CIB transaction
	*
	* \param[in] attr Attribute to update
	* \param[in] attr_id ID of attribute to update
	* \param[in] node_id ID of node for which to update attribute value
	* \param[in] set_id ID of attribute set
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_unset_attr_update(const attribute_t attr, const char attr_id,
	const char node_id, const char set_id)
	{
	char *xpath = crm_strdup_printf("/" PCMK_XE_CIB
	"/" PCMK_XE_STATUS
	"/" PCMK__XE_NODE_STATE
	"[@" PCMK_XA_ID "='%s']"
	"/" PCMK__XE_TRANSIENT_ATTRIBUTES
	"[@" PCMK_XA_ID "='%s']"
	"/%s[@" PCMK_XA_ID "='%s']"
	"/" PCMK_XE_NVPAIR
	"[@" PCMK_XA_ID "='%s' "
	"and @" PCMK_XA_NAME "='%s']",
	node_id, node_id, attr->set_type, set_id,
	attr_id, attr->id);

	int rc = the_cib->cmds->remove(the_cib, xpath, NULL,
	cib_xpath\|cib_transaction);

	free(xpath);
	return pcmk_legacy2rc(rc);
	}

	/*!
	* \internal
	* \brief Add an attribute update request to the current CIB transaction
	*
	* \param[in] attr Attribute to update
	* \param[in] value New value for attribute
	* \param[in] node_id ID of node for which to update attribute value
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_attr_update(const attribute_t attr, const char value, const char *node_id)
	{
	char *set_id = attrd_set_id(attr, node_id);
	char *nvpair_id = attrd_nvpair_id(attr, node_id);
	int rc = pcmk_rc_ok;

	if (value == NULL) {
	rc = add_unset_attr_update(attr, nvpair_id, node_id, set_id);
	} else {
	rc = add_set_attr_update(attr, nvpair_id, node_id, set_id, value);
	}
	free(set_id);
	free(nvpair_id);
	return rc;
	}

	static void
	send_alert_attributes_value(attribute_t a, GHashTable t)
	{
	int rc = 0;
	attribute_value_t *at = NULL;
	GHashTableIter vIter;

	g_hash_table_iter_init(&vIter, t);

	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & at)) {
	rc = attrd_send_attribute_alert(at->nodename, at->nodeid,
	a->id, at->current);
	crm_trace("Sent alerts for %s[%s]=%s: nodeid=%d rc=%d",
	a->id, at->nodename, at->current, at->nodeid, rc);
	}
	}

	static void
	set_alert_attribute_value(GHashTable t, attribute_value_t v)
	{
	attribute_value_t *a_v = pcmk__assert_alloc(1, sizeof(attribute_value_t));

	a_v->nodeid = v->nodeid;
	a_v->nodename = pcmk__str_copy(v->nodename);
	a_v->current = pcmk__str_copy(v->current);

	g_hash_table_replace(t, a_v->nodename, a_v);
	}

	mainloop_timer_t *
	attrd_add_timer(const char id, int timeout_ms, attribute_t attr)
	{
	return mainloop_timer_add(id, timeout_ms, FALSE, attribute_timer_cb, attr);
	}

	/*!
	* \internal
	* \brief Write an attribute's values to the CIB if appropriate
	*
	* \param[in,out] a Attribute to write
	* \param[in] ignore_delay If true, write attribute now regardless of any
	* configured delay
	*/
	static void
	write_attribute(attribute_t *a, bool ignore_delay)
	{
	int private_updates = 0, cib_updates = 0;
	attribute_value_t *v = NULL;
	GHashTableIter iter;
	GHashTable *alert_attribute_value = NULL;
	int rc = pcmk_ok;

	if (a == NULL) {
	return;
	}

	/* If this attribute will be written to the CIB ... */
	if (!stand_alone && !pcmk_is_set(a->flags, attrd_attr_is_private)) {
	/* Defer the write if now's not a good time */
	if (a->update && (a->update < last_cib_op_done)) {
	crm_info("Write out of '%s' continuing: update %d considered lost",
	a->id, a->update);
	a->update = 0; // Don't log this message again

	} else if (a->update) {
	crm_info("Write out of '%s' delayed: update %d in progress",
	a->id, a->update);
	goto done;

	} else if (mainloop_timer_running(a->timer)) {
	if (ignore_delay) {
	mainloop_timer_stop(a->timer);
	crm_debug("Overriding '%s' write delay", a->id);
	} else {
	crm_info("Delaying write of '%s'", a->id);
	goto done;
	}
	}

	// Initiate a transaction for all the peer value updates
	CRM_CHECK(the_cib != NULL, goto done);
	the_cib->cmds->set_user(the_cib, a->user);
	rc = the_cib->cmds->init_transaction(the_cib);
	if (rc != pcmk_ok) {
	crm_err("Failed to write %s (set %s): Could not initiate "
	"CIB transaction",
	a->id, pcmk__s(a->set_id, "unspecified"));
	goto done;
	}
	}

	/* Attribute will be written shortly, so clear changed flag and force
	* write flag, and initialize UUID missing flag to false.
	*/
	attrd_clear_attr_flags(a, attrd_attr_changed\|attrd_attr_uuid_missing\|attrd_attr_force_write);

	/* Make the table for the attribute trap */
	alert_attribute_value = pcmk__strikey_table(NULL,
	attrd_free_attribute_value);

	/* Iterate over each peer value of this attribute */
	g_hash_table_iter_init(&iter, a->values);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &v)) {
	const char *uuid = NULL;

	if (pcmk_is_set(v->flags, attrd_value_remote)) {
	/* If this is a Pacemaker Remote node, the node's UUID is the same
	* as its name, which we already have.
	*/
	uuid = v->nodename;

	} else {
	// This will create a cluster node cache entry if none exists
	pcmk__node_status_t *peer = pcmk__get_node(v->nodeid, v->nodename,
	NULL,
	pcmk__node_search_any);

	uuid = peer->uuid;

	// Remember peer's node ID if we're just now learning it
	- if ((peer->id != 0) && (v->nodeid == 0)) {
	- crm_trace("Learned ID %u for node %s", peer->id, v->nodename);
	- v->nodeid = peer->id;
	+ if ((peer->cluster_layer_id != 0) && (v->nodeid == 0)) {
	+ crm_trace("Learned ID %" PRIu32 " for node %s",
	+ peer->cluster_layer_id, v->nodename);
	+ v->nodeid = peer->cluster_layer_id;
	}
	}

	/* If this is a private attribute, no update needs to be sent */
	if (stand_alone \|\| pcmk_is_set(a->flags, attrd_attr_is_private)) {
	private_updates++;
	continue;
	}

	// Defer write if this is a cluster node that's never been seen
	if (uuid == NULL) {
	attrd_set_attr_flags(a, attrd_attr_uuid_missing);
	crm_notice("Cannot update %s[%s]='%s' now because node's UUID is "
	"unknown (will retry if learned)",
	a->id, v->nodename, v->current);
	continue;
	}

	// Update this value as part of the CIB transaction we're building
	rc = add_attr_update(a, v->current, uuid);
	if (rc != pcmk_rc_ok) {
	crm_err("Failed to update %s[%s]='%s': %s "
	QB_XS " node uuid=%s id=%" PRIu32,
	a->id, v->nodename, v->current, pcmk_rc_str(rc),
	uuid, v->nodeid);
	continue;
	}

	crm_debug("Writing %s[%s]=%s (node-state-id=%s node-id=%" PRIu32 ")",
	a->id, v->nodename, pcmk__s(v->current, "(unset)"),
	uuid, v->nodeid);
	cib_updates++;

	/* Preservation of the attribute to transmit alert */
	set_alert_attribute_value(alert_attribute_value, v);

	// Save this value so we can log it when write completes
	pcmk__str_update(&(v->requested), v->current);
	}

	if (private_updates) {
	crm_info("Processed %d private change%s for %s (set %s)",
	private_updates, pcmk__plural_s(private_updates),
	a->id, pcmk__s(a->set_id, "unspecified"));
	}
	if (cib_updates > 0) {
	char *id = pcmk__str_copy(a->id);

	// Commit transaction
	a->update = the_cib->cmds->end_transaction(the_cib, true, cib_none);

	crm_info("Sent CIB request %d with %d change%s for %s (set %s)",
	a->update, cib_updates, pcmk__plural_s(cib_updates),
	a->id, pcmk__s(a->set_id, "unspecified"));

	if (the_cib->cmds->register_callback_full(the_cib, a->update,
	CIB_OP_TIMEOUT_S, FALSE, id,
	"attrd_cib_callback",
	attrd_cib_callback, free)) {
	// Transmit alert of the attribute
	send_alert_attributes_value(a, alert_attribute_value);
	}
	}

	done:
	// Discard transaction (if any)
	if (the_cib != NULL) {
	the_cib->cmds->end_transaction(the_cib, false, cib_none);
	the_cib->cmds->set_user(the_cib, NULL);
	}

	if (alert_attribute_value != NULL) {
	g_hash_table_destroy(alert_attribute_value);
	}
	}

	/*!
	* \internal
	* \brief Write out attributes
	*
	* \param[in] options Group of enum attrd_write_options
	*/
	void
	attrd_write_attributes(uint32_t options)
	{
	GHashTableIter iter;
	attribute_t *a = NULL;

	crm_debug("Writing out %s attributes",
	pcmk_is_set(options, attrd_write_all)? "all" : "changed");
	g_hash_table_iter_init(&iter, attributes);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & a)) {
	if (!pcmk_is_set(options, attrd_write_all) &&
	pcmk_is_set(a->flags, attrd_attr_uuid_missing)) {
	// Try writing this attribute again, in case peer ID was learned
	attrd_set_attr_flags(a, attrd_attr_changed);
	} else if (pcmk_is_set(a->flags, attrd_attr_force_write)) {
	/* If the force_write flag is set, write the attribute. */
	attrd_set_attr_flags(a, attrd_attr_changed);
	}

	if (pcmk_is_set(options, attrd_write_all) \|\|
	pcmk_is_set(a->flags, attrd_attr_changed)) {
	bool ignore_delay = pcmk_is_set(options, attrd_write_no_delay);

	if (pcmk_is_set(a->flags, attrd_attr_force_write)) {
	// Always ignore delay when forced write flag is set
	ignore_delay = true;
	}
	write_attribute(a, ignore_delay);
	} else {
	crm_trace("Skipping unchanged attribute %s", a->id);
	}
	}
	}

	void
	attrd_write_or_elect_attribute(attribute_t *a)
	{
	if (attrd_election_won()) {
	write_attribute(a, false);
	} else {
	attrd_start_election_if_needed();
	}
	}
	diff --git a/daemons/controld/controld_alerts.c b/daemons/controld/controld_alerts.c
	index 858f1011f8..919c059451 100644
	--- a/daemons/controld/controld_alerts.c
	+++ b/daemons/controld/controld_alerts.c
	@@ -1,88 +1,88 @@
	/*
	* Copyright 2012-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <libxml/tree.h>

	#include <crm/fencing/internal.h>
	#include <crm/lrmd.h>
	#include <crm/lrmd_internal.h>
	#include <crm/pengine/rules_internal.h>
	#include <crm/pengine/status.h>
	#include <crm/stonith-ng.h>

	#include <pacemaker-controld.h>

	static GList *crmd_alert_list = NULL;

	void
	crmd_unpack_alerts(xmlNode *alerts)
	{
	pe_free_alert_list(crmd_alert_list);
	crmd_alert_list = pe_unpack_alerts(alerts);
	}

	void
	crmd_alert_node_event(pcmk__node_status_t *node)
	{
	lrm_state_t *lrm_state;

	if (crmd_alert_list == NULL) {
	return;
	}

	lrm_state = lrm_state_find(controld_globals.our_nodename);
	if (lrm_state == NULL) {
	return;
	}

	lrmd_send_node_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
	- node->uname, node->id, node->state);
	+ node->uname, node->cluster_layer_id, node->state);
	}

	void
	crmd_alert_fencing_op(stonith_event_t * e)
	{
	char *desc;
	lrm_state_t *lrm_state;

	if (crmd_alert_list == NULL) {
	return;
	}

	lrm_state = lrm_state_find(controld_globals.our_nodename);
	if (lrm_state == NULL) {
	return;
	}

	desc = stonith__event_description(e);
	lrmd_send_fencing_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
	e->target, e->operation, desc, e->result);
	free(desc);
	}

	void
	crmd_alert_resource_op(const char node, lrmd_event_data_t op)
	{
	lrm_state_t *lrm_state;

	if (crmd_alert_list == NULL) {
	return;
	}

	lrm_state = lrm_state_find(controld_globals.our_nodename);
	if (lrm_state == NULL) {
	return;
	}

	lrmd_send_resource_alert((lrmd_t *) lrm_state->conn, crmd_alert_list, node,
	op);
	}
	diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c
	index fd4ef18cfd..c7a59cd431 100644
	--- a/daemons/controld/controld_callbacks.c
	+++ b/daemons/controld/controld_callbacks.c
	@@ -1,378 +1,384 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	+#include <inttypes.h> // PRIu32
	+#include <stdbool.h> // bool
	+#include <stdio.h> // NULL
	+
	#include <sys/param.h>
	#include <string.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/cluster.h>
	#include <crm/cib.h>

	#include <pacemaker-controld.h>

	/* From join_dc... */
	extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	void
	crmd_ha_msg_filter(xmlNode * msg)
	{
	if (AM_I_DC) {
	const char *sys_from = crm_element_value(msg, PCMK__XA_CRM_SYS_FROM);

	if (pcmk__str_eq(sys_from, CRM_SYSTEM_DC, pcmk__str_casei)) {
	const char *from = crm_element_value(msg, PCMK__XA_SRC);

	if (!pcmk__str_eq(from, controld_globals.our_nodename,
	pcmk__str_casei)) {
	int level = LOG_INFO;
	const char *op = crm_element_value(msg, PCMK__XA_CRM_TASK);

	/* make sure the election happens NOW */
	if (controld_globals.fsa_state != S_ELECTION) {
	ha_msg_input_t new_input;

	level = LOG_WARNING;
	new_input.msg = msg;
	register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
	__func__);
	}

	do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
	goto done;
	}
	}

	} else {
	const char *sys_to = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);

	if (pcmk__str_eq(sys_to, CRM_SYSTEM_DC, pcmk__str_casei)) {
	return;
	}
	}

	/* crm_log_xml_trace(msg, "HA[inbound]"); */
	route_message(C_HA_MESSAGE, msg);

	done:
	controld_trigger_fsa();
	}

	/*!
	* \internal
	* \brief Check whether a node is online
	*
	* \param[in] node Node to check
	*
	* \retval -1 if completely dead
	* \retval 0 if partially alive
	* \retval 1 if completely alive
	*/
	static int
	node_alive(const pcmk__node_status_t *node)
	{
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	// Pacemaker Remote nodes can't be partially alive
	return pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei) ? 1: -1;

	} else if (pcmk__cluster_is_node_active(node)) {
	// Completely up cluster node: both cluster member and peer
	return 1;

	} else if (!pcmk_is_set(node->processes, crm_get_cluster_proc())
	&& !pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
	// Completely down cluster node: neither cluster member nor peer
	return -1;
	}

	// Partially up cluster node: only cluster member or only peer
	return 0;
	}

	#define state_text(state) ((state)? (const char *)(state) : "in unknown state")

	void
	peer_update_callback(enum crm_status_type type, pcmk__node_status_t *node,
	const void *data)
	{
	uint32_t old = 0;
	bool appeared = FALSE;
	bool is_remote = pcmk_is_set(node->flags, crm_remote_node);

	controld_node_pending_timer(node);

	/* The controller waits to receive some information from the membership
	* layer before declaring itself operational. If this is being called for a
	* cluster node, indicate that we have it.
	*/
	if (!is_remote) {
	controld_set_fsa_input_flags(R_PEER_DATA);
	}

	if (type == crm_status_processes
	&& pcmk_is_set(node->processes, crm_get_cluster_proc())
	&& !AM_I_DC
	&& !is_remote) {
	/*
	* This is a hack until we can send to a nodeid and/or we fix node name lookups
	* These messages are ignored in crmd_ha_msg_filter()
	*/
	xmlNode *query = create_request(CRM_OP_HELLO, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	- crm_debug("Sending hello to node %u so that it learns our node name", node->id);
	+ crm_debug("Sending hello to node %" PRIu32 " so that it learns our "
	+ "node name",
	+ node->cluster_layer_id);
	pcmk__cluster_send_message(node, crm_msg_crmd, query);

	pcmk__xml_free(query);
	}

	if (node->uname == NULL) {
	return;
	}

	switch (type) {
	case crm_status_uname:
	/* If we've never seen the node, then it also won't be in the status section */
	crm_info("%s node %s is now %s",
	(is_remote? "Remote" : "Cluster"),
	node->uname, state_text(node->state));
	return;

	case crm_status_nstate:
	/* This callback should not be called unless the state actually
	* changed, but here's a failsafe just in case.
	*/
	CRM_CHECK(!pcmk__str_eq(data, node->state, pcmk__str_casei),
	return);

	crm_info("%s node %s is now %s (was %s)",
	(is_remote? "Remote" : "Cluster"),
	node->uname, state_text(node->state), state_text(data));

	if (pcmk__str_eq(CRM_NODE_MEMBER, node->state, pcmk__str_casei)) {
	appeared = TRUE;
	if (!is_remote) {
	remove_stonith_cleanup(node->uname);
	}
	} else {
	controld_remove_failed_sync_node(node->uname);
	controld_remove_voter(node->uname);
	}

	crmd_alert_node_event(node);
	break;

	case crm_status_processes:
	CRM_CHECK(data != NULL, return);
	old = (const uint32_t )data;
	appeared = pcmk_is_set(node->processes, crm_get_cluster_proc());

	{
	const char *dc_s = controld_globals.dc_name;

	if ((dc_s == NULL) && AM_I_DC) {
	dc_s = PCMK_VALUE_TRUE;
	}

	crm_info("Node %s is %s a peer " QB_XS
	" DC=%s old=%#07x new=%#07x",
	node->uname, (appeared? "now" : "no longer"),
	pcmk__s(dc_s, "<none>"), old, node->processes);
	}

	if (!pcmk_is_set((node->processes ^ old), crm_get_cluster_proc())) {
	/* Peer status did not change. This should not be possible,
	* since we don't track process flags other than peer status.
	*/
	crm_trace("Process flag %#7x did not change from %#7x to %#7x",
	crm_get_cluster_proc(), old, node->processes);
	return;

	}

	if (!appeared) {
	node->peer_lost = time(NULL);
	controld_remove_failed_sync_node(node->uname);
	controld_remove_voter(node->uname);
	}

	if (!pcmk_is_set(controld_globals.fsa_input_register,
	R_CIB_CONNECTED)) {
	crm_trace("Ignoring peer status change because not connected to CIB");
	return;

	} else if (controld_globals.fsa_state == S_STOPPING) {
	crm_trace("Ignoring peer status change because stopping");
	return;
	}

	if (!appeared
	&& pcmk__str_eq(node->uname, controld_globals.our_nodename,
	pcmk__str_casei)) {
	/* Did we get evicted? */
	crm_notice("Our peer connection failed");
	register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);

	} else if (pcmk__str_eq(node->uname, controld_globals.dc_name,
	pcmk__str_casei)
	&& !pcmk__cluster_is_node_active(node)) {

	/* The DC has left, so delete its transient attributes and
	* trigger a new election.
	*
	* A DC sends its shutdown request to all peers, who update the
	* DC's expected state to down. This avoids fencing upon
	* deletion of its transient attributes.
	*/
	crm_notice("Our peer on the DC (%s) is dead",
	controld_globals.dc_name);

	register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
	controld_delete_node_state(node->uname, controld_section_attrs,
	cib_none);

	} else if (AM_I_DC
	\|\| pcmk_is_set(controld_globals.flags, controld_dc_left)
	\|\| (controld_globals.dc_name == NULL)) {
	/* This only needs to be done once, so normally the DC should do
	* it. However if there is no DC, every node must do it, since
	* there is no other way to ensure some one node does it.
	*/
	if (appeared) {
	te_trigger_stonith_history_sync(FALSE);
	} else {
	controld_delete_node_state(node->uname,
	controld_section_attrs,
	cib_none);
	}
	}
	break;
	}

	if (AM_I_DC) {
	xmlNode *update = NULL;
	int flags = node_update_peer;
	int alive = node_alive(node);
	pcmk__graph_action_t *down = match_down_event(node->uuid);

	crm_trace("Alive=%d, appeared=%d, down=%d",
	alive, appeared, (down? down->id : -1));

	if (appeared && (alive > 0) && !is_remote) {
	register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
	}

	if (down) {
	const char *task = crm_element_value(down->xml, PCMK_XA_OPERATION);

	if (pcmk__str_eq(task, PCMK_ACTION_STONITH, pcmk__str_casei)) {

	/* tengine_stonith_callback() confirms fence actions */
	crm_trace("Updating CIB %s fencer reported fencing of %s complete",
	(pcmk_is_set(down->flags, pcmk__graph_action_confirmed)? "after" : "before"), node->uname);

	} else if (!appeared && pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_casei)) {

	// Shutdown actions are immediately confirmed (i.e. no_wait)
	if (!is_remote) {
	flags \|= node_update_join \| node_update_expected;
	crmd_peer_down(node, FALSE);
	check_join_state(controld_globals.fsa_state, __func__);
	}
	if (alive >= 0) {
	crm_info("%s of peer %s is in progress " QB_XS " action=%d",
	task, node->uname, down->id);
	} else {
	crm_notice("%s of peer %s is complete " QB_XS " action=%d",
	task, node->uname, down->id);
	pcmk__update_graph(controld_globals.transition_graph, down);
	trigger_graph();
	}

	} else {
	crm_trace("Node %s is %s, was expected to %s (op %d)",
	node->uname,
	((alive > 0)? "alive" :
	((alive < 0)? "dead" : "partially alive")),
	task, down->id);
	}

	} else if (appeared == FALSE) {
	if ((controld_globals.transition_graph == NULL)
	\|\| (controld_globals.transition_graph->id == -1)) {
	crm_info("Stonith/shutdown of node %s is unknown to the "
	"current DC", node->uname);
	} else {
	crm_warn("Stonith/shutdown of node %s was not expected",
	node->uname);
	}
	if (!is_remote) {
	crm_update_peer_join(__func__, node, crm_join_none);
	check_join_state(controld_globals.fsa_state, __func__);
	}
	abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
	"Node failure", NULL);
	fail_incompletable_actions(controld_globals.transition_graph,
	node->uuid);

	} else {
	crm_trace("Node %s came up, was not expected to be down",
	node->uname);
	}

	if (is_remote) {
	/* A pacemaker_remote node won't have its cluster status updated
	* in the CIB by membership-layer callbacks, so do it here.
	*/
	flags \|= node_update_cluster;

	/* Trigger resource placement on newly integrated nodes */
	if (appeared) {
	abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
	"Pacemaker Remote node integrated", NULL);
	}
	}

	if (!appeared && (type == crm_status_processes)
	&& (node->when_member > 1)) {
	/* The node left CPG but is still a cluster member. Set its
	* membership time to 1 to record it in the cluster state as a
	* boolean, so we don't fence it due to
	* PCMK_OPT_NODE_PENDING_TIMEOUT.
	*/
	node->when_member = 1;
	flags \|= node_update_cluster;
	controld_node_pending_timer(node);
	}

	/* Update the CIB node state */
	update = create_node_state_update(node, flags, NULL, __func__);
	if (update == NULL) {
	crm_debug("Node state update not yet possible for %s", node->uname);
	} else {
	fsa_cib_anon_update(PCMK_XE_STATUS, update);
	}
	pcmk__xml_free(update);
	}

	controld_trigger_fsa();
	}

	gboolean
	crm_fsa_trigger(gpointer user_data)
	{
	crm_trace("Invoked (queue len: %d)",
	g_list_length(controld_globals.fsa_message_queue));
	s_crmd_fsa(C_FSA_INTERNAL);
	crm_trace("Exited (queue len: %d)",
	g_list_length(controld_globals.fsa_message_queue));
	return TRUE;
	}
	diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c
	index 1d686bfb9f..1a3f6eb6b9 100644
	--- a/daemons/controld/controld_corosync.c
	+++ b/daemons/controld/controld_corosync.c
	@@ -1,162 +1,167 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	+#include <inttypes.h> // PRIu32
	+#include <stdio.h> // NULL
	+#include <stdlib.h> // free(), etc.
	+
	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	#if SUPPORT_COROSYNC

	extern void post_cache_update(int seq);

	/* A_HA_CONNECT */

	static void
	crmd_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	const char *from = NULL;
	char *data = pcmk__cpg_message_data(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	pcmk__node_status_t *peer = NULL;
	xmlNode *xml = pcmk__xml_parse(data);

	if (xml == NULL) {
	crm_err("Could not parse message content (%d): %.100s", kind, data);
	free(data);
	return;
	}

	crm_xml_add(xml, PCMK__XA_SRC, from);

	peer = pcmk__get_node(0, from, NULL, pcmk__node_search_cluster_member);
	if (!pcmk_is_set(peer->processes, crm_proc_cpg)) {
	/* If we can still talk to our peer process on that node,
	* then it must be part of the corosync membership
	*/
	- crm_warn("Receiving messages from a node we think is dead: %s[%d]",
	- peer->uname, peer->id);
	+ crm_warn("Receiving messages from a node we think is dead: "
	+ "%s[%" PRIu32 "]",
	+ peer->uname, peer->cluster_layer_id);
	crm_update_peer_proc(__func__, peer, crm_proc_cpg,
	PCMK_VALUE_ONLINE);
	}
	crmd_ha_msg_filter(xml);
	pcmk__xml_free(xml);
	} else {
	crm_err("Invalid message class (%d): %.100s", kind, data);
	}
	free(data);
	}

	static gboolean
	crmd_quorum_callback(unsigned long long seq, gboolean quorate)
	{
	crm_update_quorum(quorate, FALSE);
	post_cache_update(seq);
	return TRUE;
	}

	static void
	crmd_cs_destroy(gpointer user_data)
	{
	if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
	crm_crit("Lost connection to cluster layer, shutting down");
	crmd_exit(CRM_EX_DISCONNECT);
	}
	}

	/*!
	* \brief Handle a Corosync notification of a CPG configuration change
	*
	* \param[in] handle CPG connection
	* \param[in] cpg_name CPG group name
	* \param[in] member_list List of current CPG members
	* \param[in] member_list_entries Number of entries in \p member_list
	* \param[in] left_list List of CPG members that left
	* \param[in] left_list_entries Number of entries in \p left_list
	* \param[in] joined_list List of CPG members that joined
	* \param[in] joined_list_entries Number of entries in \p joined_list
	*/
	static void
	cpg_membership_callback(cpg_handle_t handle, const struct cpg_name *cpg_name,
	const struct cpg_address *member_list,
	size_t member_list_entries,
	const struct cpg_address *left_list,
	size_t left_list_entries,
	const struct cpg_address *joined_list,
	size_t joined_list_entries)
	{
	/* When nodes leave CPG, the DC clears their transient node attributes.
	*
	* However if there is no DC, or the DC is among the nodes that left, each
	* remaining node needs to do the clearing, to ensure it gets done.
	* Otherwise, the attributes would persist when the nodes rejoin, which
	* could have serious consequences for unfencing, agents that use attributes
	* for internal logic, etc.
	*
	* Here, we set a global boolean if the DC is among the nodes that left, for
	* use by the peer callback.
	*/
	if (controld_globals.dc_name != NULL) {
	pcmk__node_status_t *peer = NULL;

	peer = pcmk__search_node_caches(0, controld_globals.dc_name,
	pcmk__node_search_cluster_member);
	if (peer != NULL) {
	for (int i = 0; i < left_list_entries; ++i) {
	- if (left_list[i].nodeid == peer->id) {
	+ if (left_list[i].nodeid == peer->cluster_layer_id) {
	controld_set_global_flags(controld_dc_left);
	break;
	}
	}
	}
	}

	// Process the change normally, which will call the peer callback as needed
	pcmk__cpg_confchg_cb(handle, cpg_name, member_list, member_list_entries,
	left_list, left_list_entries,
	joined_list, joined_list_entries);

	controld_clear_global_flags(controld_dc_left);
	}

	extern gboolean crm_connect_corosync(pcmk_cluster_t *cluster);

	gboolean
	crm_connect_corosync(pcmk_cluster_t *cluster)
	{
	if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) {
	pcmk__cluster_set_status_callback(&peer_update_callback);

	pcmk_cluster_set_destroy_fn(cluster, crmd_cs_destroy);
	pcmk_cpg_set_deliver_fn(cluster, crmd_cs_dispatch);
	pcmk_cpg_set_confchg_fn(cluster, cpg_membership_callback);

	if (pcmk_cluster_connect(cluster) == pcmk_rc_ok) {
	pcmk__corosync_quorum_connect(crmd_quorum_callback,
	crmd_cs_destroy);
	return TRUE;
	}
	}
	return FALSE;
	}

	#endif
	diff --git a/daemons/controld/controld_join_dc.c b/daemons/controld/controld_join_dc.c
	index 2ba19681be..58ad6310a2 100644
	--- a/daemons/controld/controld_join_dc.c
	+++ b/daemons/controld/controld_join_dc.c
	@@ -1,1047 +1,1058 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	+#include <inttypes.h> // PRIu32
	+#include <stdbool.h> // bool, true, false
	+#include <stdio.h> // NULL
	+#include <stdlib.h> // free(), etc.
	+
	+#include <glib.h> // gboolean, etc.
	+#include <libxml/tree.h> // xmlNode
	+
	#include <crm/crm.h>

	#include <crm/common/xml.h>
	#include <crm/cluster.h>

	#include <pacemaker-controld.h>

	static char *max_generation_from = NULL;
	static xmlNodePtr max_generation_xml = NULL;

	/*!
	* \internal
	* \brief Nodes from which a CIB sync has failed since the peer joined
	*
	* This table is of the form (<tt>node_name -> join_id</tt>). \p node_name is
	* the name of a client node from which a CIB \p sync_from() call has failed in
	* \p do_dc_join_finalize() since the client joined the cluster as a peer.
	* \p join_id is the ID of the join round in which the \p sync_from() failed,
	* and is intended for use in nack log messages.
	*/
	static GHashTable *failed_sync_nodes = NULL;

	void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
	void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
	gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);

	/* Numeric counter used to identify join rounds (an unsigned int would be
	* appropriate, except we get and set it in XML as int)
	*/
	static int current_join_id = 0;

	/*!
	* \internal
	* \brief Destroy the hash table containing failed sync nodes
	*/
	void
	controld_destroy_failed_sync_table(void)
	{
	if (failed_sync_nodes != NULL) {
	g_hash_table_destroy(failed_sync_nodes);
	failed_sync_nodes = NULL;
	}
	}

	/*!
	* \internal
	* \brief Remove a node from the failed sync nodes table if present
	*
	* \param[in] node_name Node name to remove
	*/
	void
	controld_remove_failed_sync_node(const char *node_name)
	{
	if (failed_sync_nodes != NULL) {
	g_hash_table_remove(failed_sync_nodes, (gchar *) node_name);
	}
	}

	/*!
	* \internal
	* \brief Add to a hash table a node whose CIB failed to sync
	*
	* \param[in] node_name Name of node whose CIB failed to sync
	* \param[in] join_id Join round when the failure occurred
	*/
	static void
	record_failed_sync_node(const char *node_name, gint join_id)
	{
	if (failed_sync_nodes == NULL) {
	failed_sync_nodes = pcmk__strikey_table(g_free, NULL);
	}

	/* If the node is already in the table then we failed to nack it during the
	* filter offer step
	*/
	CRM_LOG_ASSERT(g_hash_table_insert(failed_sync_nodes, g_strdup(node_name),
	GINT_TO_POINTER(join_id)));
	}

	/*!
	* \internal
	* \brief Look up a node name in the failed sync table
	*
	* \param[in] node_name Name of node to look up
	* \param[out] join_id Where to store the join ID of when the sync failed
	*
	* \return Standard Pacemaker return code. Specifically, \p pcmk_rc_ok if the
	* node name was found, or \p pcmk_rc_node_unknown otherwise.
	* \note \p *join_id is set to -1 if the node is not found.
	*/
	static int
	lookup_failed_sync_node(const char node_name, gint join_id)
	{
	*join_id = -1;

	if (failed_sync_nodes != NULL) {
	gpointer result = g_hash_table_lookup(failed_sync_nodes,
	(gchar *) node_name);
	if (result != NULL) {
	*join_id = GPOINTER_TO_INT(result);
	return pcmk_rc_ok;
	}
	}
	return pcmk_rc_node_unknown;
	}

	void
	crm_update_peer_join(const char source, pcmk__node_status_t node,
	enum crm_join_phase phase)
	{
	enum crm_join_phase last = 0;

	CRM_CHECK(node != NULL, return);

	/* Remote nodes do not participate in joins */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return;
	}

	last = node->join;

	if(phase == last) {
	crm_trace("Node %s join-%d phase is still %s "
	- QB_XS " nodeid=%u source=%s",
	+ QB_XS " nodeid=%" PRIu32 " source=%s",
	node->uname, current_join_id, controld_join_phase_text(last),
	- node->id, source);
	+ node->cluster_layer_id, source);

	} else if ((phase <= crm_join_none) \|\| (phase == (last + 1))) {
	node->join = phase;
	crm_trace("Node %s join-%d phase is now %s (was %s) "
	- QB_XS " nodeid=%u source=%s",
	+ QB_XS " nodeid=%" PRIu32 " source=%s",
	node->uname, current_join_id, controld_join_phase_text(phase),
	- controld_join_phase_text(last), node->id, source);
	+ controld_join_phase_text(last), node->cluster_layer_id,
	+ source);

	} else {
	crm_warn("Rejecting join-%d phase update for node %s because "
	- "can't go from %s to %s " QB_XS " nodeid=%u source=%s",
	+ "can't go from %s to %s " QB_XS " nodeid=%" PRIu32
	+ " source=%s",
	current_join_id, node->uname, controld_join_phase_text(last),
	- controld_join_phase_text(phase), node->id, source);
	+ controld_join_phase_text(phase), node->cluster_layer_id,
	+ source);
	}
	}

	static void
	start_join_round(void)
	{
	GHashTableIter iter;
	pcmk__node_status_t *peer = NULL;

	crm_debug("Starting new join round join-%d", current_join_id);

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	crm_update_peer_join(__func__, peer, crm_join_none);
	}
	if (max_generation_from != NULL) {
	free(max_generation_from);
	max_generation_from = NULL;
	}
	if (max_generation_xml != NULL) {
	pcmk__xml_free(max_generation_xml);
	max_generation_xml = NULL;
	}
	controld_clear_fsa_input_flags(R_HAVE_CIB);
	}

	/*!
	* \internal
	* \brief Create a join message from the DC
	*
	* \param[in] join_op Join operation name
	* \param[in] host_to Recipient of message
	*/
	static xmlNode *
	create_dc_message(const char join_op, const char host_to)
	{
	xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
	CRM_SYSTEM_DC, NULL);

	/* Identify which election this is a part of */
	crm_xml_add_int(msg, PCMK__XA_JOIN_ID, current_join_id);

	/* Add a field specifying whether the DC is shutting down. This keeps the
	* joining node from fencing the old DC if it becomes the new DC.
	*/
	pcmk__xe_set_bool_attr(msg, PCMK__XA_DC_LEAVING,
	pcmk_is_set(controld_globals.fsa_input_register,
	R_SHUTDOWN));
	return msg;
	}

	static void
	join_make_offer(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *offer = NULL;
	pcmk__node_status_t member = (pcmk__node_status_t ) value;

	CRM_ASSERT(member != NULL);
	if (!pcmk__cluster_is_node_active(member)) {
	crm_info("Not making join-%d offer to inactive node %s",
	current_join_id,
	(member->uname? member->uname : "with unknown name"));
	if(member->expected == NULL && pcmk__str_eq(member->state, CRM_NODE_LOST, pcmk__str_casei)) {
	/* You would think this unsafe, but in fact this plus an
	* active resource is what causes it to be fenced.
	*
	* Yes, this does mean that any node that dies at the same
	* time as the old DC and is not running resource (still)
	* won't be fenced.
	*
	* I'm not happy about this either.
	*/
	pcmk__update_peer_expected(__func__, member, CRMD_JOINSTATE_DOWN);
	}
	return;
	}

	if (member->uname == NULL) {
	crm_info("Not making join-%d offer to node uuid %s with unknown name",
	current_join_id, member->uuid);
	return;
	}

	if (controld_globals.membership_id != crm_peer_seq) {
	controld_globals.membership_id = crm_peer_seq;
	crm_info("Making join-%d offers based on membership event %llu",
	current_join_id, crm_peer_seq);
	}

	if(user_data && member->join > crm_join_none) {
	crm_info("Not making join-%d offer to already known node %s (%s)",
	current_join_id, member->uname,
	controld_join_phase_text(member->join));
	return;
	}

	crm_update_peer_join(__func__, (pcmk__node_status_t*) member,
	crm_join_none);

	offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname);

	// Advertise our feature set so the joining node can bail if not compatible
	crm_xml_add(offer, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);

	crm_info("Sending join-%d offer to %s", current_join_id, member->uname);
	pcmk__cluster_send_message(member, crm_msg_crmd, offer);
	pcmk__xml_free(offer);

	crm_update_peer_join(__func__, member, crm_join_welcomed);
	}

	/* A_DC_JOIN_OFFER_ALL */
	void
	do_dc_join_offer_all(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int count;

	/* Reset everyone's status back to down or in_ccm in the CIB.
	* Any nodes that are active in the CIB but not in the cluster membership
	* will be seen as offline by the scheduler anyway.
	*/
	current_join_id++;
	start_join_round();

	update_dc(NULL);
	if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
	crm_info("A new node joined the cluster");
	}
	g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);

	count = crmd_join_phase_count(crm_join_welcomed);
	crm_info("Waiting on join-%d requests from %d outstanding node%s",
	current_join_id, count, pcmk__plural_s(count));

	// Don't waste time by invoking the scheduler yet
	}

	/* A_DC_JOIN_OFFER_ONE */
	void
	do_dc_join_offer_one(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	pcmk__node_status_t *member = NULL;
	ha_msg_input_t *welcome = NULL;
	int count;
	const char *join_to = NULL;

	if (msg_data->data == NULL) {
	crm_info("Making join-%d offers to any unconfirmed nodes "
	"because an unknown node joined", current_join_id);
	g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
	check_join_state(cur_state, __func__);
	return;
	}

	welcome = fsa_typed_data(fsa_dt_ha_msg);
	if (welcome == NULL) {
	// fsa_typed_data() already logged an error
	return;
	}

	join_to = crm_element_value(welcome->msg, PCMK__XA_SRC);
	if (join_to == NULL) {
	crm_err("Can't make join-%d offer to unknown node", current_join_id);
	return;
	}
	member = pcmk__get_node(0, join_to, NULL, pcmk__node_search_cluster_member);

	/* It is possible that a node will have been sick or starting up when the
	* original offer was made. However, it will either re-announce itself in
	* due course, or we can re-store the original offer on the client.
	*/

	crm_update_peer_join(__func__, member, crm_join_none);
	join_make_offer(NULL, member, NULL);

	/* If the offer isn't to the local node, make an offer to the local node as
	* well, to ensure the correct value for max_generation_from.
	*/
	if (strcasecmp(join_to, controld_globals.our_nodename) != 0) {
	member = pcmk__get_node(0, controld_globals.our_nodename, NULL,
	pcmk__node_search_cluster_member);
	join_make_offer(NULL, member, NULL);
	}

	/* This was a genuine join request; cancel any existing transition and
	* invoke the scheduler.
	*/
	abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, "Node join",
	NULL);

	count = crmd_join_phase_count(crm_join_welcomed);
	crm_info("Waiting on join-%d requests from %d outstanding node%s",
	current_join_id, count, pcmk__plural_s(count));

	// Don't waste time by invoking the scheduler yet
	}

	static int
	compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
	{
	const char *elem_l = crm_element_value(left, field);
	const char *elem_r = crm_element_value(right, field);

	long long int_elem_l;
	long long int_elem_r;

	pcmk__scan_ll(elem_l, &int_elem_l, -1LL);
	pcmk__scan_ll(elem_r, &int_elem_r, -1LL);

	if (int_elem_l < int_elem_r) {
	return -1;

	} else if (int_elem_l > int_elem_r) {
	return 1;
	}

	return 0;
	}

	/* A_DC_JOIN_PROCESS_REQ */
	void
	do_dc_join_filter_offer(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *generation = NULL;

	int cmp = 0;
	int join_id = -1;
	int count = 0;
	gint value = 0;
	gboolean ack_nack_bool = TRUE;
	ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);

	const char *join_from = crm_element_value(join_ack->msg, PCMK__XA_SRC);
	const char *ref = crm_element_value(join_ack->msg, PCMK_XA_REFERENCE);
	const char *join_version = crm_element_value(join_ack->msg,
	PCMK_XA_CRM_FEATURE_SET);
	pcmk__node_status_t *join_node = NULL;

	if (join_from == NULL) {
	crm_err("Ignoring invalid join request without node name");
	return;
	}
	join_node = pcmk__get_node(0, join_from, NULL,
	pcmk__node_search_cluster_member);

	crm_element_value_int(join_ack->msg, PCMK__XA_JOIN_ID, &join_id);
	if (join_id != current_join_id) {
	crm_debug("Ignoring join-%d request from %s because we are on join-%d",
	join_id, join_from, current_join_id);
	check_join_state(cur_state, __func__);
	return;
	}

	generation = join_ack->xml;
	if (max_generation_xml != NULL && generation != NULL) {
	int lpc = 0;

	const char *attributes[] = {
	PCMK_XA_ADMIN_EPOCH,
	PCMK_XA_EPOCH,
	PCMK_XA_NUM_UPDATES,
	};

	/* It's not obvious that join_ack->xml is the PCMK__XE_GENERATION_TUPLE
	* element from the join client. The "if" guard is for clarity.
	*/
	if (pcmk__xe_is(generation, PCMK__XE_GENERATION_TUPLE)) {
	for (lpc = 0; cmp == 0 && lpc < PCMK__NELEM(attributes); lpc++) {
	cmp = compare_int_fields(max_generation_xml, generation,
	attributes[lpc]);
	}

	} else { // Should always be PCMK__XE_GENERATION_TUPLE
	CRM_LOG_ASSERT(false);
	}
	}

	if (ref == NULL) {
	ref = "none"; // for logging only
	}

	if (lookup_failed_sync_node(join_from, &value) == pcmk_rc_ok) {
	crm_err("Rejecting join-%d request from node %s because we failed to "
	"sync its CIB in join-%d " QB_XS " ref=%s",
	join_id, join_from, value, ref);
	ack_nack_bool = FALSE;

	} else if (!pcmk__cluster_is_node_active(join_node)) {
	if (match_down_event(join_from) != NULL) {
	/* The join request was received after the node was fenced or
	* otherwise shutdown in a way that we're aware of. No need to log
	* an error in this rare occurrence; we know the client was recently
	* shut down, and receiving a lingering in-flight request is not
	* cause for alarm.
	*/
	crm_debug("Rejecting join-%d request from inactive node %s "
	QB_XS " ref=%s", join_id, join_from, ref);
	} else {
	crm_err("Rejecting join-%d request from inactive node %s "
	QB_XS " ref=%s", join_id, join_from, ref);
	}
	ack_nack_bool = FALSE;

	} else if (generation == NULL) {
	crm_err("Rejecting invalid join-%d request from node %s "
	"missing CIB generation " QB_XS " ref=%s",
	join_id, join_from, ref);
	ack_nack_bool = FALSE;

	} else if ((join_version == NULL)
	\|\| !feature_set_compatible(CRM_FEATURE_SET, join_version)) {
	crm_err("Rejecting join-%d request from node %s because feature set %s"
	" is incompatible with ours (%s) " QB_XS " ref=%s",
	join_id, join_from, (join_version? join_version : "pre-3.1.0"),
	CRM_FEATURE_SET, ref);
	ack_nack_bool = FALSE;

	} else if (max_generation_xml == NULL) {
	const char *validation = crm_element_value(generation,
	PCMK_XA_VALIDATE_WITH);

	if (pcmk__get_schema(validation) == NULL) {
	crm_err("Rejecting join-%d request from %s (with first CIB "
	"generation) due to unknown schema version %s "
	QB_XS " ref=%s",
	join_id, join_from, pcmk__s(validation, "(missing)"), ref);
	ack_nack_bool = FALSE;

	} else {
	crm_debug("Accepting join-%d request from %s (with first CIB "
	"generation) " QB_XS " ref=%s",
	join_id, join_from, ref);
	max_generation_xml = pcmk__xml_copy(NULL, generation);
	pcmk__str_update(&max_generation_from, join_from);
	}

	} else if ((cmp < 0)
	\|\| ((cmp == 0)
	&& pcmk__str_eq(join_from, controld_globals.our_nodename,
	pcmk__str_casei))) {
	const char *validation = crm_element_value(generation,
	PCMK_XA_VALIDATE_WITH);

	if (pcmk__get_schema(validation) == NULL) {
	crm_err("Rejecting join-%d request from %s (with better CIB "
	"generation than current best from %s) due to unknown "
	"schema version %s " QB_XS " ref=%s",
	join_id, join_from, max_generation_from,
	pcmk__s(validation, "(missing)"), ref);
	ack_nack_bool = FALSE;

	} else {
	crm_debug("Accepting join-%d request from %s (with better CIB "
	"generation than current best from %s) " QB_XS " ref=%s",
	join_id, join_from, max_generation_from, ref);
	crm_log_xml_debug(max_generation_xml, "Old max generation");
	crm_log_xml_debug(generation, "New max generation");

	pcmk__xml_free(max_generation_xml);
	max_generation_xml = pcmk__xml_copy(NULL, join_ack->xml);
	pcmk__str_update(&max_generation_from, join_from);
	}

	} else {
	crm_debug("Accepting join-%d request from %s " QB_XS " ref=%s",
	join_id, join_from, ref);
	}

	if (!ack_nack_bool) {
	if (compare_version(join_version, "3.17.0") < 0) {
	/* Clients with CRM_FEATURE_SET < 3.17.0 may respawn infinitely
	* after a nack message, don't send one
	*/
	crm_update_peer_join(__func__, join_node, crm_join_nack_quiet);
	} else {
	crm_update_peer_join(__func__, join_node, crm_join_nack);
	}
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_NACK);

	} else {
	crm_update_peer_join(__func__, join_node, crm_join_integrated);
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);
	}

	count = crmd_join_phase_count(crm_join_integrated);
	crm_debug("%d node%s currently integrated in join-%d",
	count, pcmk__plural_s(count), join_id);

	if (check_join_state(cur_state, __func__) == FALSE) {
	// Don't waste time by invoking the scheduler yet
	count = crmd_join_phase_count(crm_join_welcomed);
	crm_debug("Waiting on join-%d requests from %d outstanding node%s",
	join_id, count, pcmk__plural_s(count));
	}
	}

	/* A_DC_JOIN_FINALIZE */
	void
	do_dc_join_finalize(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	char *sync_from = NULL;
	int rc = pcmk_ok;
	int count_welcomed = crmd_join_phase_count(crm_join_welcomed);
	int count_finalizable = crmd_join_phase_count(crm_join_integrated)
	+ crmd_join_phase_count(crm_join_nack)
	+ crmd_join_phase_count(crm_join_nack_quiet);

	/* This we can do straight away and avoid clients timing us out
	* while we compute the latest CIB
	*/
	if (count_welcomed != 0) {
	crm_debug("Waiting on join-%d requests from %d outstanding node%s "
	"before finalizing join", current_join_id, count_welcomed,
	pcmk__plural_s(count_welcomed));
	crmd_join_phase_log(LOG_DEBUG);
	/* crmd_fsa_stall(FALSE); Needed? */
	return;

	} else if (count_finalizable == 0) {
	crm_debug("Finalization not needed for join-%d at the current time",
	current_join_id);
	crmd_join_phase_log(LOG_DEBUG);
	check_join_state(controld_globals.fsa_state, __func__);
	return;
	}

	controld_clear_fsa_input_flags(R_HAVE_CIB);
	if (pcmk__str_eq(max_generation_from, controld_globals.our_nodename,
	pcmk__str_null_matches\|pcmk__str_casei)) {
	controld_set_fsa_input_flags(R_HAVE_CIB);
	}

	if (!controld_globals.transition_graph->complete) {
	crm_warn("Delaying join-%d finalization while transition in progress",
	current_join_id);
	crmd_join_phase_log(LOG_DEBUG);
	crmd_fsa_stall(FALSE);
	return;
	}

	if (pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
	// Send our CIB out to everyone
	sync_from = pcmk__str_copy(controld_globals.our_nodename);
	crm_debug("Finalizing join-%d for %d node%s (sync'ing from local CIB)",
	current_join_id, count_finalizable,
	pcmk__plural_s(count_finalizable));
	crm_log_xml_debug(max_generation_xml, "Requested CIB version");

	} else {
	// Ask for the agreed best CIB
	sync_from = pcmk__str_copy(max_generation_from);
	crm_notice("Finalizing join-%d for %d node%s (sync'ing CIB from %s)",
	current_join_id, count_finalizable,
	pcmk__plural_s(count_finalizable), sync_from);
	crm_log_xml_notice(max_generation_xml, "Requested CIB version");
	}
	crmd_join_phase_log(LOG_DEBUG);

	rc = controld_globals.cib_conn->cmds->sync_from(controld_globals.cib_conn,
	sync_from, NULL, cib_none);
	fsa_register_cib_callback(rc, sync_from, finalize_sync_callback);
	}

	void
	free_max_generation(void)
	{
	free(max_generation_from);
	max_generation_from = NULL;

	pcmk__xml_free(max_generation_xml);
	max_generation_xml = NULL;
	}

	void
	finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	CRM_LOG_ASSERT(-EPERM != rc);

	if (rc != pcmk_ok) {
	const char sync_from = (const char ) user_data;

	do_crm_log(((rc == -pcmk_err_old_data)? LOG_WARNING : LOG_ERR),
	"Could not sync CIB from %s in join-%d: %s",
	sync_from, current_join_id, pcmk_strerror(rc));

	if (rc != -pcmk_err_old_data) {
	record_failed_sync_node(sync_from, current_join_id);
	}

	/* restart the whole join process */
	register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL,
	__func__);

	} else if (!AM_I_DC) {
	crm_debug("Sync'ed CIB for join-%d but no longer DC", current_join_id);

	} else if (controld_globals.fsa_state != S_FINALIZE_JOIN) {
	crm_debug("Sync'ed CIB for join-%d but no longer in S_FINALIZE_JOIN "
	"(%s)", current_join_id,
	fsa_state2string(controld_globals.fsa_state));

	} else {
	controld_set_fsa_input_flags(R_HAVE_CIB);

	/* make sure dc_uuid is re-set to us */
	if (!check_join_state(controld_globals.fsa_state, __func__)) {
	int count_finalizable = 0;

	count_finalizable = crmd_join_phase_count(crm_join_integrated)
	+ crmd_join_phase_count(crm_join_nack)
	+ crmd_join_phase_count(crm_join_nack_quiet);

	crm_debug("Notifying %d node%s of join-%d results",
	count_finalizable, pcmk__plural_s(count_finalizable),
	current_join_id);
	g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
	}
	}
	}

	static void
	join_node_state_commit_callback(xmlNode *msg, int call_id, int rc,
	xmlNode output, void user_data)
	{
	const char *node = user_data;

	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL; // for register_fsa_error() macro

	crm_crit("join-%d node history update (via CIB call %d) for node %s "
	"failed: %s",
	current_join_id, call_id, node, pcmk_strerror(rc));
	crm_log_xml_debug(msg, "failed");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}

	crm_debug("join-%d node history update (via CIB call %d) for node %s "
	"complete",
	current_join_id, call_id, node);
	check_join_state(controld_globals.fsa_state, __func__);
	}

	/* A_DC_JOIN_PROCESS_ACK */
	void
	do_dc_join_ack(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int join_id = -1;
	ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);

	const char *op = crm_element_value(join_ack->msg, PCMK__XA_CRM_TASK);
	char *join_from = crm_element_value_copy(join_ack->msg, PCMK__XA_SRC);
	pcmk__node_status_t *peer = NULL;

	enum controld_section_e section = controld_section_lrm;
	char *xpath = NULL;
	xmlNode *state = join_ack->xml;
	xmlNode *execd_state = NULL;

	cib_t *cib = controld_globals.cib_conn;
	int rc = pcmk_ok;

	// Sanity checks
	if (join_from == NULL) {
	crm_warn("Ignoring message received without node identification");
	goto done;
	}
	if (op == NULL) {
	crm_warn("Ignoring message received from %s without task", join_from);
	goto done;
	}

	if (strcmp(op, CRM_OP_JOIN_CONFIRM)) {
	crm_debug("Ignoring '%s' message from %s while waiting for '%s'",
	op, join_from, CRM_OP_JOIN_CONFIRM);
	goto done;
	}

	if (crm_element_value_int(join_ack->msg, PCMK__XA_JOIN_ID, &join_id) != 0) {
	crm_warn("Ignoring join confirmation from %s without valid join ID",
	join_from);
	goto done;
	}

	peer = pcmk__get_node(0, join_from, NULL, pcmk__node_search_cluster_member);
	if (peer->join != crm_join_finalized) {
	crm_info("Ignoring out-of-sequence join-%d confirmation from %s "
	"(currently %s not %s)",
	join_id, join_from, controld_join_phase_text(peer->join),
	controld_join_phase_text(crm_join_finalized));
	goto done;
	}

	if (join_id != current_join_id) {
	crm_err("Rejecting join-%d confirmation from %s "
	"because currently on join-%d",
	join_id, join_from, current_join_id);
	crm_update_peer_join(__func__, peer, crm_join_nack);
	goto done;
	}

	crm_update_peer_join(__func__, peer, crm_join_confirmed);

	/* Update CIB with node's current executor state. A new transition will be
	* triggered later, when the CIB manager notifies us of the change.
	*
	* The delete and modify requests are part of an atomic transaction.
	*/
	rc = cib->cmds->init_transaction(cib);
	if (rc != pcmk_ok) {
	goto done;
	}

	// Delete relevant parts of node's current executor state from CIB
	if (pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
	section = controld_section_lrm_unlocked;
	}
	controld_node_state_deletion_strings(join_from, section, &xpath, NULL);

	rc = cib->cmds->remove(cib, xpath, NULL,
	cib_xpath\|cib_multiple\|cib_transaction);
	if (rc != pcmk_ok) {
	goto done;
	}

	// Update CIB with node's latest known executor state
	if (pcmk__str_eq(join_from, controld_globals.our_nodename,
	pcmk__str_casei)) {

	// Use the latest possible state if processing our own join ack
	execd_state = controld_query_executor_state();

	if (execd_state != NULL) {
	crm_debug("Updating local node history for join-%d from query "
	"result",
	current_join_id);
	state = execd_state;

	} else {
	crm_warn("Updating local node history from join-%d confirmation "
	"because query failed",
	current_join_id);
	}

	} else {
	crm_debug("Updating node history for %s from join-%d confirmation",
	join_from, current_join_id);
	}

	rc = cib->cmds->modify(cib, PCMK_XE_STATUS, state,
	cib_can_create\|cib_transaction);
	pcmk__xml_free(execd_state);
	if (rc != pcmk_ok) {
	goto done;
	}

	// Commit the transaction
	rc = cib->cmds->end_transaction(cib, true, cib_none);
	fsa_register_cib_callback(rc, join_from, join_node_state_commit_callback);

	if (rc > 0) {
	// join_from will be freed after callback
	join_from = NULL;
	rc = pcmk_ok;
	}

	done:
	if (rc != pcmk_ok) {
	crm_crit("join-%d node history update for node %s failed: %s",
	current_join_id, join_from, pcmk_strerror(rc));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	free(join_from);
	free(xpath);
	}

	void
	finalize_join_for(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNode *acknak = NULL;
	xmlNode *tmp1 = NULL;
	pcmk__node_status_t *join_node = value;
	const char *join_to = join_node->uname;
	bool integrated = false;

	switch (join_node->join) {
	case crm_join_integrated:
	integrated = true;
	break;
	case crm_join_nack:
	case crm_join_nack_quiet:
	break;
	default:
	crm_trace("Not updating non-integrated and non-nacked node %s (%s) "
	"for join-%d", join_to,
	controld_join_phase_text(join_node->join),
	current_join_id);
	return;
	}

	/* Update the <node> element with the node's name and UUID, in case they
	* weren't known before
	*/
	crm_trace("Updating node name and UUID in CIB for %s", join_to);
	tmp1 = pcmk__xe_create(NULL, PCMK_XE_NODE);
	crm_xml_add(tmp1, PCMK_XA_ID, pcmk__cluster_node_uuid(join_node));
	crm_xml_add(tmp1, PCMK_XA_UNAME, join_to);
	fsa_cib_anon_update(PCMK_XE_NODES, tmp1);
	pcmk__xml_free(tmp1);

	if (join_node->join == crm_join_nack_quiet) {
	crm_trace("Not sending nack message to node %s with feature set older "
	"than 3.17.0", join_to);
	return;
	}

	join_node = pcmk__get_node(0, join_to, NULL,
	pcmk__node_search_cluster_member);
	if (!pcmk__cluster_is_node_active(join_node)) {
	/*
	* NACK'ing nodes that the membership layer doesn't know about yet
	* simply creates more churn
	*
	* Better to leave them waiting and let the join restart when
	* the new membership event comes in
	*
	* All other NACKs (due to versions etc) should still be processed
	*/
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_PENDING);
	return;
	}

	// Acknowledge or nack node's join request
	crm_debug("%sing join-%d request from %s",
	integrated? "Acknowledg" : "Nack", current_join_id, join_to);
	acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
	pcmk__xe_set_bool_attr(acknak, CRM_OP_JOIN_ACKNAK, integrated);

	if (integrated) {
	// No change needed for a nacked node
	crm_update_peer_join(__func__, join_node, crm_join_finalized);
	pcmk__update_peer_expected(__func__, join_node, CRMD_JOINSTATE_MEMBER);

	/* Iterate through the remote peer cache and add information on which
	* node hosts each to the ACK message. This keeps new controllers in
	* sync with what has already happened.
	*/
	if (pcmk__cluster_num_remote_nodes() > 0) {
	GHashTableIter iter;
	pcmk__node_status_t *node = NULL;
	xmlNode *remotes = pcmk__xe_create(acknak, PCMK_XE_NODES);

	g_hash_table_iter_init(&iter, crm_remote_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	xmlNode *remote = NULL;

	if (!node->conn_host) {
	continue;
	}

	remote = pcmk__xe_create(remotes, PCMK_XE_NODE);
	pcmk__xe_set_props(remote,
	PCMK_XA_ID, node->uname,
	PCMK__XA_NODE_STATE, node->state,
	PCMK__XA_CONNECTION_HOST, node->conn_host,
	NULL);
	}
	}
	}
	pcmk__cluster_send_message(join_node, crm_msg_crmd, acknak);
	pcmk__xml_free(acknak);
	return;
	}

	gboolean
	check_join_state(enum crmd_fsa_state cur_state, const char *source)
	{
	static unsigned long long highest_seq = 0;

	if (controld_globals.membership_id != crm_peer_seq) {
	crm_debug("join-%d: Membership changed from %llu to %llu "
	QB_XS " highest=%llu state=%s for=%s",
	current_join_id, controld_globals.membership_id, crm_peer_seq,
	highest_seq, fsa_state2string(cur_state), source);
	if(highest_seq < crm_peer_seq) {
	/* Don't spam the FSA with duplicates */
	highest_seq = crm_peer_seq;
	register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
	}

	} else if (cur_state == S_INTEGRATION) {
	if (crmd_join_phase_count(crm_join_welcomed) == 0) {
	int count = crmd_join_phase_count(crm_join_integrated);

	crm_debug("join-%d: Integration of %d peer%s complete "
	QB_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
	return TRUE;
	}

	} else if (cur_state == S_FINALIZE_JOIN) {
	if (!pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
	crm_debug("join-%d: Delaying finalization until we have CIB "
	QB_XS " state=%s for=%s",
	current_join_id, fsa_state2string(cur_state), source);
	return TRUE;

	} else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
	int count = crmd_join_phase_count(crm_join_welcomed);

	crm_debug("join-%d: Still waiting on %d welcomed node%s "
	QB_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else if (crmd_join_phase_count(crm_join_integrated) != 0) {
	int count = crmd_join_phase_count(crm_join_integrated);

	crm_debug("join-%d: Still waiting on %d integrated node%s "
	QB_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else if (crmd_join_phase_count(crm_join_finalized) != 0) {
	int count = crmd_join_phase_count(crm_join_finalized);

	crm_debug("join-%d: Still waiting on %d finalized node%s "
	QB_XS " state=%s for=%s",
	current_join_id, count, pcmk__plural_s(count),
	fsa_state2string(cur_state), source);
	crmd_join_phase_log(LOG_DEBUG);

	} else {
	crm_debug("join-%d: Complete " QB_XS " state=%s for=%s",
	current_join_id, fsa_state2string(cur_state), source);
	register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
	return TRUE;
	}
	}

	return FALSE;
	}

	void
	do_dc_join_final(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
	crm_update_quorum(pcmk__cluster_has_quorum(), TRUE);
	}

	int crmd_join_phase_count(enum crm_join_phase phase)
	{
	int count = 0;
	pcmk__node_status_t *peer;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	if(peer->join == phase) {
	count++;
	}
	}
	return count;
	}

	void crmd_join_phase_log(int level)
	{
	pcmk__node_status_t *peer;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
	do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->uname,
	controld_join_phase_text(peer->join));
	}
	}
	diff --git a/daemons/controld/controld_messages.c b/daemons/controld/controld_messages.c
	index e4293c3fc1..951508b15b 100644
	--- a/daemons/controld/controld_messages.c
	+++ b/daemons/controld/controld_messages.c
	@@ -1,1358 +1,1359 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <string.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/cluster/internal.h>
	#include <crm/cib.h>
	#include <crm/common/ipc_internal.h>

	#include <pacemaker-controld.h>

	static enum crmd_fsa_input handle_message(xmlNode *msg,
	enum crmd_fsa_cause cause);
	static void handle_response(xmlNode *stored_msg);
	static enum crmd_fsa_input handle_request(xmlNode *stored_msg,
	enum crmd_fsa_cause cause);
	static enum crmd_fsa_input handle_shutdown_request(xmlNode *stored_msg);
	static void send_msg_via_ipc(xmlNode * msg, const char *sys);

	/* debug only, can wrap all it likes */
	static int last_data_id = 0;

	void
	register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	fsa_data_t * cur_data, void new_data, const char raised_from)
	{
	/* save the current actions if any */
	if (controld_globals.fsa_actions != A_NOTHING) {
	register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
	I_NULL, cur_data ? cur_data->data : NULL,
	controld_globals.fsa_actions, TRUE, __func__);
	}

	/* reset the action list */
	crm_info("Resetting the current action list");
	fsa_dump_actions(controld_globals.fsa_actions, "Drop");
	controld_globals.fsa_actions = A_NOTHING;

	/* register the error */
	register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
	}

	void
	register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	void *data, uint64_t with_actions,
	gboolean prepend, const char *raised_from)
	{
	unsigned old_len = g_list_length(controld_globals.fsa_message_queue);
	fsa_data_t *fsa_data = NULL;

	if (raised_from == NULL) {
	raised_from = "<unknown>";
	}

	if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
	/* no point doing anything */
	crm_err("Cannot add entry to queue: no input and no action");
	return;
	}

	if (input == I_WAIT_FOR_EVENT) {
	controld_set_global_flags(controld_fsa_is_stalled);
	crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
	raised_from, fsa_cause2string(cause), data, old_len);

	if (old_len > 0) {
	fsa_dump_queue(LOG_TRACE);
	prepend = FALSE;
	}

	if (data == NULL) {
	controld_set_fsa_action_flags(with_actions);
	fsa_dump_actions(with_actions, "Restored");
	return;
	}

	/* Store everything in the new event and reset
	* controld_globals.fsa_actions
	*/
	with_actions \|= controld_globals.fsa_actions;
	controld_globals.fsa_actions = A_NOTHING;
	}

	last_data_id++;
	crm_trace("%s %s FSA input %d (%s) due to %s, %s data",
	raised_from, (prepend? "prepended" : "appended"), last_data_id,
	fsa_input2string(input), fsa_cause2string(cause),
	(data? "with" : "without"));

	fsa_data = pcmk__assert_alloc(1, sizeof(fsa_data_t));
	fsa_data->id = last_data_id;
	fsa_data->fsa_input = input;
	fsa_data->fsa_cause = cause;
	fsa_data->origin = raised_from;
	fsa_data->data = NULL;
	fsa_data->data_type = fsa_dt_none;
	fsa_data->actions = with_actions;

	if (with_actions != A_NOTHING) {
	crm_trace("Adding actions %.16llx to input",
	(unsigned long long) with_actions);
	}

	if (data != NULL) {
	switch (cause) {
	case C_FSA_INTERNAL:
	case C_CRMD_STATUS_CALLBACK:
	case C_IPC_MESSAGE:
	case C_HA_MESSAGE:
	CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
	crm_err("Bogus data from %s", raised_from));
	crm_trace("Copying %s data from %s as cluster message data",
	fsa_cause2string(cause), raised_from);
	fsa_data->data = copy_ha_msg_input(data);
	fsa_data->data_type = fsa_dt_ha_msg;
	break;

	case C_LRM_OP_CALLBACK:
	crm_trace("Copying %s data from %s as lrmd_event_data_t",
	fsa_cause2string(cause), raised_from);
	fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
	fsa_data->data_type = fsa_dt_lrm;
	break;

	case C_TIMER_POPPED:
	case C_SHUTDOWN:
	case C_UNKNOWN:
	case C_STARTUP:
	crm_crit("Copying %s data (from %s) is not yet implemented",
	fsa_cause2string(cause), raised_from);
	crmd_exit(CRM_EX_SOFTWARE);
	break;
	}
	}

	/* make sure to free it properly later */
	if (prepend) {
	controld_globals.fsa_message_queue
	= g_list_prepend(controld_globals.fsa_message_queue, fsa_data);
	} else {
	controld_globals.fsa_message_queue
	= g_list_append(controld_globals.fsa_message_queue, fsa_data);
	}

	crm_trace("FSA message queue length is %d",
	g_list_length(controld_globals.fsa_message_queue));

	/* fsa_dump_queue(LOG_TRACE); */

	if (old_len == g_list_length(controld_globals.fsa_message_queue)) {
	crm_err("Couldn't add message to the queue");
	}

	if (input != I_WAIT_FOR_EVENT) {
	controld_trigger_fsa();
	}
	}

	void
	fsa_dump_queue(int log_level)
	{
	int offset = 0;

	for (GList *iter = controld_globals.fsa_message_queue; iter != NULL;
	iter = iter->next) {
	fsa_data_t data = (fsa_data_t ) iter->data;

	do_crm_log_unlikely(log_level,
	"queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
	offset++, data->id, fsa_input2string(data->fsa_input),
	data->origin, data->data, data->data_type,
	fsa_cause2string(data->fsa_cause));
	}
	}

	ha_msg_input_t *
	copy_ha_msg_input(ha_msg_input_t * orig)
	{
	xmlNode *wrapper = NULL;

	ha_msg_input_t *copy = pcmk__assert_alloc(1, sizeof(ha_msg_input_t));

	copy->msg = (orig != NULL)? pcmk__xml_copy(NULL, orig->msg) : NULL;

	wrapper = pcmk__xe_first_child(copy->msg, PCMK__XE_CRM_XML, NULL, NULL);
	copy->xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
	return copy;
	}

	void
	delete_fsa_input(fsa_data_t * fsa_data)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *foo = NULL;

	if (fsa_data == NULL) {
	return;
	}
	crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));

	if (fsa_data->data != NULL) {
	switch (fsa_data->data_type) {
	case fsa_dt_ha_msg:
	delete_ha_msg_input(fsa_data->data);
	break;

	case fsa_dt_xml:
	foo = fsa_data->data;
	pcmk__xml_free(foo);
	break;

	case fsa_dt_lrm:
	op = (lrmd_event_data_t *) fsa_data->data;
	lrmd_free_event(op);
	break;

	case fsa_dt_none:
	if (fsa_data->data != NULL) {
	crm_err("Don't know how to free %s data from %s",
	fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
	crmd_exit(CRM_EX_SOFTWARE);
	}
	break;
	}
	crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
	}

	free(fsa_data);
	}

	/* returns the next message */
	fsa_data_t *
	get_message(void)
	{
	fsa_data_t *message
	= (fsa_data_t *) controld_globals.fsa_message_queue->data;

	controld_globals.fsa_message_queue
	= g_list_remove(controld_globals.fsa_message_queue, message);
	crm_trace("Processing input %d", message->id);
	return message;
	}

	void *
	fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
	{
	void *ret_val = NULL;

	if (fsa_data == NULL) {
	crm_err("%s: No FSA data available", caller);

	} else if (fsa_data->data == NULL) {
	crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);

	} else if (fsa_data->data_type != a_type) {
	crm_crit("%s: Message data was the wrong type! %d vs. requested=%d. Origin: %s",
	caller, fsa_data->data_type, a_type, fsa_data->origin);
	CRM_ASSERT(fsa_data->data_type == a_type);
	} else {
	ret_val = fsa_data->data;
	}

	return ret_val;
	}

	/* A_MSG_ROUTE */
	void
	do_msg_route(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);

	route_message(msg_data->fsa_cause, input->msg);
	}

	void
	route_message(enum crmd_fsa_cause cause, xmlNode * input)
	{
	ha_msg_input_t fsa_input;
	enum crmd_fsa_input result = I_NULL;

	fsa_input.msg = input;
	CRM_CHECK(cause == C_IPC_MESSAGE \|\| cause == C_HA_MESSAGE, return);

	/* try passing the buck first */
	if (relay_message(input, cause == C_IPC_MESSAGE)) {
	return;
	}

	/* handle locally */
	result = handle_message(input, cause);

	/* done or process later? */
	switch (result) {
	case I_NULL:
	case I_CIB_OP:
	case I_ROUTER:
	case I_NODE_JOIN:
	case I_JOIN_REQUEST:
	case I_JOIN_RESULT:
	break;
	default:
	/* Defering local processing of message */
	register_fsa_input_later(cause, result, &fsa_input);
	return;
	}

	if (result != I_NULL) {
	/* add to the front of the queue */
	register_fsa_input(cause, result, &fsa_input);
	}
	}

	gboolean
	relay_message(xmlNode * msg, gboolean originated_locally)
	{
	enum crm_ais_msg_types dest = crm_msg_none;
	bool is_for_dc = false;
	bool is_for_dcib = false;
	bool is_for_te = false;
	bool is_for_crm = false;
	bool is_for_cib = false;
	bool is_local = false;
	bool broadcast = false;
	const char *host_to = NULL;
	const char *sys_to = NULL;
	const char *sys_from = NULL;
	const char *type = NULL;
	const char *task = NULL;
	const char *ref = NULL;
	pcmk__node_status_t *node_to = NULL;

	CRM_CHECK(msg != NULL, return TRUE);

	host_to = crm_element_value(msg, PCMK__XA_CRM_HOST_TO);
	sys_to = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
	sys_from = crm_element_value(msg, PCMK__XA_CRM_SYS_FROM);
	type = crm_element_value(msg, PCMK__XA_T);
	task = crm_element_value(msg, PCMK__XA_CRM_TASK);
	ref = crm_element_value(msg, PCMK_XA_REFERENCE);

	broadcast = pcmk__str_empty(host_to);

	if (ref == NULL) {
	ref = "without reference ID";
	}

	if (pcmk__str_eq(task, CRM_OP_HELLO, pcmk__str_casei)) {
	crm_trace("Received hello %s from %s (no processing needed)",
	ref, pcmk__s(sys_from, "unidentified source"));
	crm_log_xml_trace(msg, "hello");
	return TRUE;
	}

	// Require message type (set by create_request())
	if (!pcmk__str_eq(type, PCMK__VALUE_CRMD, pcmk__str_none)) {
	crm_warn("Ignoring invalid message %s with type '%s' "
	"(not '" PCMK__VALUE_CRMD "')",
	ref, pcmk__s(type, ""));
	crm_log_xml_trace(msg, "ignored");
	return TRUE;
	}

	// Require a destination subsystem (also set by create_request())
	if (sys_to == NULL) {
	crm_warn("Ignoring invalid message %s with no " PCMK__XA_CRM_SYS_TO,
	ref);
	crm_log_xml_trace(msg, "ignored");
	return TRUE;
	}

	// Get the message type appropriate to the destination subsystem
	if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) {
	dest = pcmk__cluster_parse_msg_type(sys_to);
	if (dest == crm_msg_none) {
	/* Unrecognized value, use a sane default
	*
	* @TODO Maybe we should bail instead
	*/
	dest = crm_msg_crmd;
	}
	}

	is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
	is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
	is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
	is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
	is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);

	// Check whether message should be processed locally
	is_local = false;
	if (broadcast) {
	if (is_for_dc \|\| is_for_te) {
	is_local = false;

	} else if (is_for_crm) {
	if (pcmk__strcase_any_of(task, CRM_OP_NODE_INFO,
	PCMK__CONTROLD_CMD_NODES, NULL)) {
	/* Node info requests do not specify a host, which is normally
	* treated as "all hosts", because the whole point is that the
	* client may not know the local node name. Always handle these
	* requests locally.
	*/
	is_local = true;
	} else {
	is_local = !originated_locally;
	}

	} else {
	is_local = true;
	}

	} else if (pcmk__str_eq(controld_globals.our_nodename, host_to,
	pcmk__str_casei)) {
	is_local = true;

	} else if (is_for_crm && pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
	xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL,
	NULL);
	xmlNode *msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
	const char *mode = crm_element_value(msg_data, PCMK__XA_MODE);

	if (pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) {
	// Local delete of an offline node's resource history
	is_local = true;
	}
	}

	// Check whether message should be relayed

	if (is_for_dc \|\| is_for_dcib \|\| is_for_te) {
	if (AM_I_DC) {
	if (is_for_te) {
	crm_trace("Route message %s locally as transition request",
	ref);
	crm_log_xml_trace(msg, sys_to);
	send_msg_via_ipc(msg, sys_to);
	return TRUE; // No further processing of message is needed
	}
	crm_trace("Route message %s locally as DC request", ref);
	return FALSE; // More to be done by caller
	}

	if (originated_locally
	&& !pcmk__strcase_any_of(sys_from, CRM_SYSTEM_PENGINE,
	CRM_SYSTEM_TENGINE, NULL)) {
	crm_trace("Relay message %s to DC (via %s)",
	ref, pcmk__s(host_to, "broadcast"));
	crm_log_xml_trace(msg, "relayed");
	if (!broadcast) {
	node_to = pcmk__get_node(0, host_to, NULL,
	pcmk__node_search_cluster_member);
	}
	pcmk__cluster_send_message(node_to, dest, msg);
	return TRUE;
	}

	/* Transition engine and scheduler messages are sent only to the DC on
	* the same node. If we are no longer the DC, discard this message.
	*/
	crm_trace("Ignoring message %s because we are no longer DC", ref);
	crm_log_xml_trace(msg, "ignored");
	return TRUE; // No further processing of message is needed
	}

	if (is_local) {
	if (is_for_crm \|\| is_for_cib) {
	crm_trace("Route message %s locally as controller request", ref);
	return FALSE; // More to be done by caller
	}
	crm_trace("Relay message %s locally to %s", ref, sys_to);
	crm_log_xml_trace(msg, "IPC-relay");
	send_msg_via_ipc(msg, sys_to);
	return TRUE;
	}

	if (!broadcast) {
	node_to = pcmk__search_node_caches(0, host_to,
	pcmk__node_search_cluster_member);
	if (node_to == NULL) {
	crm_warn("Ignoring message %s because node %s is unknown",
	ref, host_to);
	crm_log_xml_trace(msg, "ignored");
	return TRUE;
	}
	}

	crm_trace("Relay message %s to %s",
	ref, pcmk__s(host_to, "all peers"));
	crm_log_xml_trace(msg, "relayed");
	pcmk__cluster_send_message(node_to, dest, msg);
	return TRUE;
	}

	// Return true if field contains a positive integer
	static bool
	authorize_version(xmlNode message_data, const char field,
	const char client_name, const char ref, const char *uuid)
	{
	const char *version = crm_element_value(message_data, field);
	long long version_num;

	if ((pcmk__scan_ll(version, &version_num, -1LL) != pcmk_rc_ok)
	\|\| (version_num < 0LL)) {

	crm_warn("Rejected IPC hello from %s: '%s' is not a valid protocol %s "
	QB_XS " ref=%s uuid=%s",
	client_name, ((version == NULL)? "" : version),
	field, (ref? ref : "none"), uuid);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Check whether a client IPC message is acceptable
	*
	* If a given client IPC message is a hello, "authorize" it by ensuring it has
	* valid information such as a protocol version, and return false indicating
	* that nothing further needs to be done with the message. If the message is not
	* a hello, just return true to indicate it needs further processing.
	*
	* \param[in] client_msg XML of IPC message
	* \param[in,out] curr_client If IPC is not proxied, client that sent message
	* \param[in] proxy_session If IPC is proxied, the session ID
	*
	* \return true if message needs further processing, false if it doesn't
	*/
	bool
	controld_authorize_ipc_message(const xmlNode client_msg, pcmk__client_t curr_client,
	const char *proxy_session)
	{
	xmlNode *wrapper = NULL;
	xmlNode *message_data = NULL;
	const char *client_name = NULL;
	const char *op = crm_element_value(client_msg, PCMK__XA_CRM_TASK);
	const char *ref = crm_element_value(client_msg, PCMK_XA_REFERENCE);
	const char *uuid = (curr_client? curr_client->id : proxy_session);

	if (uuid == NULL) {
	crm_warn("IPC message from client rejected: No client identifier "
	QB_XS " ref=%s", (ref? ref : "none"));
	goto rejected;
	}

	if (!pcmk__str_eq(CRM_OP_HELLO, op, pcmk__str_casei)) {
	// Only hello messages need to be authorized
	return true;
	}

	wrapper = pcmk__xe_first_child(client_msg, PCMK__XE_CRM_XML, NULL, NULL);
	message_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	client_name = crm_element_value(message_data, PCMK__XA_CLIENT_NAME);
	if (pcmk__str_empty(client_name)) {
	crm_warn("IPC hello from client rejected: No client name",
	QB_XS " ref=%s uuid=%s", (ref? ref : "none"), uuid);
	goto rejected;
	}
	if (!authorize_version(message_data, PCMK__XA_MAJOR_VERSION, client_name,
	ref, uuid)) {
	goto rejected;
	}
	if (!authorize_version(message_data, PCMK__XA_MINOR_VERSION, client_name,
	ref, uuid)) {
	goto rejected;
	}

	crm_trace("Validated IPC hello from client %s", client_name);
	crm_log_xml_trace(client_msg, "hello");
	if (curr_client) {
	curr_client->userdata = pcmk__str_copy(client_name);
	}
	controld_trigger_fsa();
	return false;

	rejected:
	crm_log_xml_trace(client_msg, "rejected");
	if (curr_client) {
	qb_ipcs_disconnect(curr_client->ipcs);
	}
	return false;
	}

	static enum crmd_fsa_input
	handle_message(xmlNode *msg, enum crmd_fsa_cause cause)
	{
	const char *type = NULL;

	CRM_CHECK(msg != NULL, return I_NULL);

	type = crm_element_value(msg, PCMK__XA_SUBT);
	if (pcmk__str_eq(type, PCMK__VALUE_REQUEST, pcmk__str_none)) {
	return handle_request(msg, cause);
	}

	if (pcmk__str_eq(type, PCMK__VALUE_RESPONSE, pcmk__str_none)) {
	handle_response(msg);
	return I_NULL;
	}

	crm_warn("Ignoring message with unknown " PCMK__XA_SUBT" '%s'",
	pcmk__s(type, ""));
	crm_log_xml_trace(msg, "bad");
	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_failcount_op(xmlNode * stored_msg)
	{
	const char *rsc = NULL;
	const char *uname = NULL;
	const char *op = NULL;
	char *interval_spec = NULL;
	guint interval_ms = 0;
	gboolean is_remote_node = FALSE;

	xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML, NULL,
	NULL);
	xmlNode *xml_op = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	if (xml_op) {
	xmlNode *xml_rsc = pcmk__xe_first_child(xml_op, PCMK_XE_PRIMITIVE, NULL,
	NULL);
	xmlNode *xml_attrs = pcmk__xe_first_child(xml_op, PCMK__XE_ATTRIBUTES,
	NULL, NULL);

	if (xml_rsc) {
	rsc = pcmk__xe_id(xml_rsc);
	}
	if (xml_attrs) {
	op = crm_element_value(xml_attrs,
	CRM_META "_" PCMK__META_CLEAR_FAILURE_OP);
	crm_element_value_ms(xml_attrs,
	CRM_META "_" PCMK__META_CLEAR_FAILURE_INTERVAL,
	&interval_ms);
	}
	}
	uname = crm_element_value(xml_op, PCMK__META_ON_NODE);

	if ((rsc == NULL) \|\| (uname == NULL)) {
	crm_log_xml_warn(stored_msg, "invalid failcount op");
	return I_NULL;
	}

	if (crm_element_value(xml_op, PCMK__XA_ROUTER_NODE)) {
	is_remote_node = TRUE;
	}

	crm_debug("Clearing failures for %s-interval %s on %s "
	"from attribute manager, CIB, and executor state",
	pcmk__readable_interval(interval_ms), rsc, uname);

	if (interval_ms) {
	interval_spec = crm_strdup_printf("%ums", interval_ms);
	}
	update_attrd_clear_failures(uname, rsc, op, interval_spec, is_remote_node);
	free(interval_spec);

	controld_cib_delete_last_failure(rsc, uname, op, interval_ms);

	lrm_clear_last_failure(rsc, uname, op, interval_ms);

	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_lrm_delete(xmlNode *stored_msg)
	{
	const char *mode = NULL;
	xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML, NULL,
	NULL);
	xmlNode *msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	CRM_CHECK(msg_data != NULL, return I_NULL);

	/* CRM_OP_LRM_DELETE has two distinct modes. The default behavior is to
	* relay the operation to the affected node, which will unregister the
	* resource from the local executor, clear the resource's history from the
	* CIB, and do some bookkeeping in the controller.
	*
	* However, if the affected node is offline, the client will specify
	* mode=PCMK__VALUE_CIB which means the controller receiving the operation
	* should clear the resource's history from the CIB and nothing else. This
	* is used to clear shutdown locks.
	*/
	mode = crm_element_value(msg_data, PCMK__XA_MODE);
	if (!pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) {
	// Relay to affected node
	crm_xml_add(stored_msg, PCMK__XA_CRM_SYS_TO, CRM_SYSTEM_LRMD);
	return I_ROUTER;

	} else {
	// Delete CIB history locally (compare with do_lrm_delete())
	const char *from_sys = NULL;
	const char *user_name = NULL;
	const char *rsc_id = NULL;
	const char *node = NULL;
	xmlNode *rsc_xml = NULL;
	int rc = pcmk_rc_ok;

	rsc_xml = pcmk__xe_first_child(msg_data, PCMK_XE_PRIMITIVE, NULL, NULL);
	CRM_CHECK(rsc_xml != NULL, return I_NULL);

	rsc_id = pcmk__xe_id(rsc_xml);
	from_sys = crm_element_value(stored_msg, PCMK__XA_CRM_SYS_FROM);
	node = crm_element_value(msg_data, PCMK__META_ON_NODE);
	user_name = pcmk__update_acl_user(stored_msg, PCMK__XA_CRM_USER, NULL);
	crm_debug("Handling " CRM_OP_LRM_DELETE " for %s on %s locally%s%s "
	"(clearing CIB resource history only)", rsc_id, node,
	(user_name? " for user " : ""), (user_name? user_name : ""));
	rc = controld_delete_resource_history(rsc_id, node, user_name,
	cib_dryrun\|cib_sync_call);
	if (rc == pcmk_rc_ok) {
	rc = controld_delete_resource_history(rsc_id, node, user_name,
	crmd_cib_smart_opt());
	}

	/* Notify client. Also notify tengine if mode=PCMK__VALUE_CIB and
	* op=CRM_OP_LRM_DELETE.
	*/
	if (from_sys) {
	lrmd_event_data_t *op = NULL;
	const char *from_host = crm_element_value(stored_msg, PCMK__XA_SRC);
	const char *transition;

	if (strcmp(from_sys, CRM_SYSTEM_TENGINE)) {
	transition = crm_element_value(msg_data,
	PCMK__XA_TRANSITION_KEY);
	} else {
	transition = crm_element_value(stored_msg,
	PCMK__XA_TRANSITION_KEY);
	}

	crm_info("Notifying %s on %s that %s was%s deleted",
	from_sys, (from_host? from_host : "local node"), rsc_id,
	((rc == pcmk_rc_ok)? "" : " not"));
	op = lrmd_new_event(rsc_id, PCMK_ACTION_DELETE, 0);
	op->type = lrmd_event_exec_complete;
	op->user_data = pcmk__str_copy(pcmk__s(transition, FAKE_TE_ID));
	op->params = pcmk__strkey_table(free, free);
	pcmk__insert_dup(op->params, PCMK_XA_CRM_FEATURE_SET,
	CRM_FEATURE_SET);
	controld_rc2event(op, rc);
	controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
	lrmd_free_event(op);
	controld_trigger_delete_refresh(from_sys, rsc_id);
	}
	return I_NULL;
	}
	}

	/*!
	* \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_remote_state(const xmlNode *msg)
	{
	const char *conn_host = NULL;
	const char *remote_uname = pcmk__xe_id(msg);
	pcmk__node_status_t *remote_peer;
	bool remote_is_up = false;
	int rc = pcmk_rc_ok;

	rc = pcmk__xe_get_bool_attr(msg, PCMK__XA_IN_CCM, &remote_is_up);

	CRM_CHECK(remote_uname && rc == pcmk_rc_ok, return I_NULL);

	remote_peer = pcmk__cluster_lookup_remote_node(remote_uname);
	CRM_CHECK(remote_peer, return I_NULL);

	pcmk__update_peer_state(__func__, remote_peer,
	remote_is_up ? CRM_NODE_MEMBER : CRM_NODE_LOST,
	0);

	conn_host = crm_element_value(msg, PCMK__XA_CONNECTION_HOST);
	if (conn_host) {
	pcmk__str_update(&remote_peer->conn_host, conn_host);
	} else if (remote_peer->conn_host) {
	free(remote_peer->conn_host);
	remote_peer->conn_host = NULL;
	}

	return I_NULL;
	}

	/*!
	* \brief Handle a CRM_OP_PING message
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_ping(const xmlNode *msg)
	{
	const char *value = NULL;
	xmlNode *ping = NULL;
	xmlNode *reply = NULL;

	// Build reply

	ping = pcmk__xe_create(NULL, PCMK__XE_PING_RESPONSE);
	value = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
	crm_xml_add(ping, PCMK__XA_CRM_SUBSYSTEM, value);

	// Add controller state
	value = fsa_state2string(controld_globals.fsa_state);
	crm_xml_add(ping, PCMK__XA_CRMD_STATE, value);
	crm_notice("Current ping state: %s", value); // CTS needs this

	// Add controller health
	// @TODO maybe do some checks to determine meaningful status
	crm_xml_add(ping, PCMK_XA_RESULT, "ok");

	// Send reply
	reply = create_reply(msg, ping);
	pcmk__xml_free(ping);
	if (reply != NULL) {
	(void) relay_message(reply, TRUE);
	pcmk__xml_free(reply);
	}

	// Nothing further to do
	return I_NULL;
	}

	/*!
	* \brief Handle a PCMK__CONTROLD_CMD_NODES message
	*
	* \param[in] request Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_node_list(const xmlNode *request)
	{
	GHashTableIter iter;
	pcmk__node_status_t *node = NULL;
	xmlNode *reply = NULL;
	xmlNode *reply_data = NULL;

	// Create message data for reply
	reply_data = pcmk__xe_create(NULL, PCMK_XE_NODES);
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
	xmlNode *xml = pcmk__xe_create(reply_data, PCMK_XE_NODE);

	- crm_xml_add_ll(xml, PCMK_XA_ID, (long long) node->id); // uint32_t
	+ crm_xml_add_ll(xml, PCMK_XA_ID,
	+ (long long) node->cluster_layer_id); // uint32_t
	crm_xml_add(xml, PCMK_XA_UNAME, node->uname);
	crm_xml_add(xml, PCMK__XA_IN_CCM, node->state);
	}

	// Create and send reply
	reply = create_reply(request, reply_data);
	pcmk__xml_free(reply_data);
	if (reply) {
	(void) relay_message(reply, TRUE);
	pcmk__xml_free(reply);
	}

	// Nothing further to do
	return I_NULL;
	}

	/*!
	* \brief Handle a CRM_OP_NODE_INFO request
	*
	* \param[in] msg Message XML
	*
	* \return Next FSA input
	*/
	static enum crmd_fsa_input
	handle_node_info_request(const xmlNode *msg)
	{
	const char *value = NULL;
	pcmk__node_status_t *node = NULL;
	int node_id = 0;
	xmlNode *reply = NULL;
	xmlNode *reply_data = NULL;

	// Build reply

	reply_data = pcmk__xe_create(NULL, PCMK_XE_NODE);
	crm_xml_add(reply_data, PCMK__XA_CRM_SUBSYSTEM, CRM_SYSTEM_CRMD);

	// Add whether current partition has quorum
	pcmk__xe_set_bool_attr(reply_data, PCMK_XA_HAVE_QUORUM,
	pcmk_is_set(controld_globals.flags,
	controld_has_quorum));

	// Check whether client requested node info by ID and/or name
	crm_element_value_int(msg, PCMK_XA_ID, &node_id);
	if (node_id < 0) {
	node_id = 0;
	}
	value = crm_element_value(msg, PCMK_XA_UNAME);

	// Default to local node if none given
	if ((node_id == 0) && (value == NULL)) {
	value = controld_globals.our_nodename;
	}

	node = pcmk__search_node_caches(node_id, value, pcmk__node_search_any);
	if (node) {
	crm_xml_add(reply_data, PCMK_XA_ID, node->uuid);
	crm_xml_add(reply_data, PCMK_XA_UNAME, node->uname);
	crm_xml_add(reply_data, PCMK_XA_CRMD, node->state);
	pcmk__xe_set_bool_attr(reply_data, PCMK_XA_REMOTE_NODE,
	pcmk_is_set(node->flags, crm_remote_node));
	}

	// Send reply
	reply = create_reply(msg, reply_data);
	pcmk__xml_free(reply_data);
	if (reply != NULL) {
	(void) relay_message(reply, TRUE);
	pcmk__xml_free(reply);
	}

	// Nothing further to do
	return I_NULL;
	}

	static void
	verify_feature_set(xmlNode *msg)
	{
	const char *dc_version = crm_element_value(msg, PCMK_XA_CRM_FEATURE_SET);

	if (dc_version == NULL) {
	/* All we really know is that the DC feature set is older than 3.1.0,
	* but that's also all that really matters.
	*/
	dc_version = "3.0.14";
	}

	if (feature_set_compatible(dc_version, CRM_FEATURE_SET)) {
	crm_trace("Local feature set (%s) is compatible with DC's (%s)",
	CRM_FEATURE_SET, dc_version);
	} else {
	crm_err("Local feature set (%s) is incompatible with DC's (%s)",
	CRM_FEATURE_SET, dc_version);

	// Nothing is likely to improve without administrator involvement
	controld_set_fsa_input_flags(R_STAYDOWN);
	crmd_exit(CRM_EX_FATAL);
	}
	}

	// DC gets own shutdown all-clear
	static enum crmd_fsa_input
	handle_shutdown_self_ack(xmlNode *stored_msg)
	{
	const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	// The expected case -- we initiated own shutdown sequence
	crm_info("Shutting down controller");
	return I_STOP;
	}

	if (pcmk__str_eq(host_from, controld_globals.dc_name, pcmk__str_casei)) {
	// Must be logic error -- DC confirming its own unrequested shutdown
	crm_err("Shutting down controller immediately due to "
	"unexpected shutdown confirmation");
	return I_TERMINATE;
	}

	if (controld_globals.fsa_state != S_STOPPING) {
	// Shouldn't happen -- non-DC confirming unrequested shutdown
	crm_err("Starting new DC election because %s is "
	"confirming shutdown we did not request",
	(host_from? host_from : "another node"));
	return I_ELECTION;
	}

	// Shouldn't happen, but we are already stopping anyway
	crm_debug("Ignoring unexpected shutdown confirmation from %s",
	(host_from? host_from : "another node"));
	return I_NULL;
	}

	// Non-DC gets shutdown all-clear from DC
	static enum crmd_fsa_input
	handle_shutdown_ack(xmlNode *stored_msg)
	{
	const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);

	if (host_from == NULL) {
	crm_warn("Ignoring shutdown request without origin specified");
	return I_NULL;
	}

	if (pcmk__str_eq(host_from, controld_globals.dc_name,
	pcmk__str_null_matches\|pcmk__str_casei)) {

	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	crm_info("Shutting down controller after confirmation from %s",
	host_from);
	} else {
	crm_err("Shutting down controller after unexpected "
	"shutdown request from %s", host_from);
	controld_set_fsa_input_flags(R_STAYDOWN);
	}
	return I_STOP;
	}

	crm_warn("Ignoring shutdown request from %s because DC is %s",
	host_from, controld_globals.dc_name);
	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_request(xmlNode *stored_msg, enum crmd_fsa_cause cause)
	{
	xmlNode *msg = NULL;
	const char *op = crm_element_value(stored_msg, PCMK__XA_CRM_TASK);

	/* Optimize this for the DC - it has the most to do */

	crm_log_xml_trace(stored_msg, "request");
	if (op == NULL) {
	crm_warn("Ignoring request without " PCMK__XA_CRM_TASK);
	return I_NULL;
	}

	if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	const char *from = crm_element_value(stored_msg, PCMK__XA_SRC);
	pcmk__node_status_t *node =
	pcmk__search_node_caches(0, from, pcmk__node_search_cluster_member);

	pcmk__update_peer_expected(__func__, node, CRMD_JOINSTATE_DOWN);
	if(AM_I_DC == FALSE) {
	return I_NULL; /* Done */
	}
	}

	/========== DC-Only Actions ==========/
	if (AM_I_DC) {
	if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
	return I_NODE_JOIN;

	} else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
	return I_JOIN_REQUEST;

	} else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	return handle_shutdown_self_ack(stored_msg);

	} else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	// Another controller wants to shut down its node
	return handle_shutdown_request(stored_msg);
	}
	}

	/========== common actions ==========/
	if (strcmp(op, CRM_OP_NOVOTE) == 0) {
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE,
	__func__);

	} else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) {
	/* a remote connection host is letting us know the node state */
	return handle_remote_state(stored_msg);

	} else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
	throttle_update(stored_msg);
	if (AM_I_DC && (controld_globals.transition_graph != NULL)
	&& !controld_globals.transition_graph->complete) {

	crm_debug("The throttle changed. Trigger a graph.");
	trigger_graph();
	}
	return I_NULL;

	} else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
	return handle_failcount_op(stored_msg);

	} else if (strcmp(op, CRM_OP_VOTE) == 0) {
	/* count the vote and decide what to do after that */
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE,
	__func__);

	/* Sometimes we _must_ go into S_ELECTION */
	if (controld_globals.fsa_state == S_HALT) {
	crm_debug("Forcing an election from S_HALT");
	return I_ELECTION;
	}

	} else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
	verify_feature_set(stored_msg);
	crm_debug("Raising I_JOIN_OFFER: join-%s",
	crm_element_value(stored_msg, PCMK__XA_JOIN_ID));
	return I_JOIN_OFFER;

	} else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
	crm_debug("Raising I_JOIN_RESULT: join-%s",
	crm_element_value(stored_msg, PCMK__XA_JOIN_ID));
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_LRM_DELETE) == 0) {
	return handle_lrm_delete(stored_msg);

	} else if ((strcmp(op, CRM_OP_LRM_FAIL) == 0)
	\|\| (strcmp(op, CRM_OP_LRM_REFRESH) == 0) // @COMPAT
	\|\| (strcmp(op, CRM_OP_REPROBE) == 0)) {

	crm_xml_add(stored_msg, PCMK__XA_CRM_SYS_TO, CRM_SYSTEM_LRMD);
	return I_ROUTER;

	} else if (strcmp(op, CRM_OP_NOOP) == 0) {
	return I_NULL;

	} else if (strcmp(op, CRM_OP_PING) == 0) {
	return handle_ping(stored_msg);

	} else if (strcmp(op, CRM_OP_NODE_INFO) == 0) {
	return handle_node_info_request(stored_msg);

	} else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
	int id = 0;
	const char *name = NULL;

	crm_element_value_int(stored_msg, PCMK_XA_ID, &id);
	name = crm_element_value(stored_msg, PCMK_XA_UNAME);

	if(cause == C_IPC_MESSAGE) {
	msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
	if (!pcmk__cluster_send_message(NULL, crm_msg_crmd, msg)) {
	crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
	} else {
	crm_notice("Instructing peers to remove references to node %s/%u", name, id);
	}
	pcmk__xml_free(msg);

	} else {
	pcmk__cluster_forget_cluster_node(id, name);

	/* If we're forgetting this node, also forget any failures to fence
	* it, so we don't carry that over to any node added later with the
	* same name.
	*/
	st_fail_count_reset(name);
	}

	} else if (strcmp(op, CRM_OP_MAINTENANCE_NODES) == 0) {
	xmlNode *wrapper = pcmk__xe_first_child(stored_msg, PCMK__XE_CRM_XML,
	NULL, NULL);
	xmlNode *xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	remote_ra_process_maintenance_nodes(xml);

	} else if (strcmp(op, PCMK__CONTROLD_CMD_NODES) == 0) {
	return handle_node_list(stored_msg);

	/========== (NOT_DC)-Only Actions ==========/
	} else if (!AM_I_DC) {

	if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	return handle_shutdown_ack(stored_msg);
	}

	} else {
	crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
	crm_log_xml_err(stored_msg, "Unexpected");
	}

	return I_NULL;
	}

	static void
	handle_response(xmlNode *stored_msg)
	{
	const char *op = crm_element_value(stored_msg, PCMK__XA_CRM_TASK);

	crm_log_xml_trace(stored_msg, "reply");
	if (op == NULL) {
	crm_warn("Ignoring reply without " PCMK__XA_CRM_TASK);

	} else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
	// Check whether scheduler answer been superseded by subsequent request
	const char *msg_ref = crm_element_value(stored_msg, PCMK_XA_REFERENCE);

	if (msg_ref == NULL) {
	crm_err("%s - Ignoring calculation with no reference", op);

	} else if (pcmk__str_eq(msg_ref, controld_globals.fsa_pe_ref,
	pcmk__str_none)) {
	ha_msg_input_t fsa_input;

	controld_stop_sched_timer();
	fsa_input.msg = stored_msg;
	register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);

	} else {
	crm_info("%s calculation %s is obsolete", op, msg_ref);
	}

	} else if (strcmp(op, CRM_OP_VOTE) == 0
	\|\| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 \|\| strcmp(op, CRM_OP_SHUTDOWN) == 0) {

	} else {
	const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);

	crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
	op, host_from, AM_I_DC ? "DC" : "controller");
	}
	}

	static enum crmd_fsa_input
	handle_shutdown_request(xmlNode * stored_msg)
	{
	/* handle here to avoid potential version issues
	* where the shutdown message/procedure may have
	* been changed in later versions.
	*
	* This way the DC is always in control of the shutdown
	*/

	char *now_s = NULL;
	const char *host_from = crm_element_value(stored_msg, PCMK__XA_SRC);

	if (host_from == NULL) {
	/* we're shutting down and the DC */
	host_from = controld_globals.our_nodename;
	}

	crm_info("Creating shutdown request for %s (state=%s)", host_from,
	fsa_state2string(controld_globals.fsa_state));
	crm_log_xml_trace(stored_msg, "message");

	now_s = pcmk__ttoa(time(NULL));
	update_attrd(host_from, PCMK__NODE_ATTR_SHUTDOWN, now_s, NULL, FALSE);
	free(now_s);

	/* will be picked up by the TE as long as its running */
	return I_NULL;
	}

	static void
	send_msg_via_ipc(xmlNode * msg, const char *sys)
	{
	pcmk__client_t *client_channel = NULL;

	CRM_CHECK(sys != NULL, return);

	client_channel = pcmk__find_client_by_id(sys);

	if (crm_element_value(msg, PCMK__XA_SRC) == NULL) {
	crm_xml_add(msg, PCMK__XA_SRC, controld_globals.our_nodename);
	}

	if (client_channel != NULL) {
	/* Transient clients such as crmadmin */
	pcmk__ipc_send_xml(client_channel, 0, msg, crm_ipc_server_event);

	} else if (pcmk__str_eq(sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
	xmlNode *wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL,
	NULL);
	xmlNode *data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	process_te_message(msg, data);

	} else if (pcmk__str_eq(sys, CRM_SYSTEM_LRMD, pcmk__str_none)) {
	fsa_data_t fsa_data;
	ha_msg_input_t fsa_input;
	xmlNode *wrapper = NULL;

	fsa_input.msg = msg;

	wrapper = pcmk__xe_first_child(msg, PCMK__XE_CRM_XML, NULL, NULL);
	fsa_input.xml = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	fsa_data.id = 0;
	fsa_data.actions = 0;
	fsa_data.data = &fsa_input;
	fsa_data.fsa_input = I_MESSAGE;
	fsa_data.fsa_cause = C_IPC_MESSAGE;
	fsa_data.origin = __func__;
	fsa_data.data_type = fsa_dt_ha_msg;

	do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, controld_globals.fsa_state,
	I_MESSAGE, &fsa_data);

	} else if (crmd_is_proxy_session(sys)) {
	crmd_proxy_send(sys, msg);

	} else {
	crm_info("Received invalid request: unknown subsystem '%s'", sys);
	}
	}

	void
	delete_ha_msg_input(ha_msg_input_t * orig)
	{
	if (orig == NULL) {
	return;
	}
	pcmk__xml_free(orig->msg);
	free(orig);
	}

	/*!
	* \internal
	* \brief Notify the cluster of a remote node state change
	*
	* \param[in] node_name Node's name
	* \param[in] node_up true if node is up, false if down
	*/
	void
	broadcast_remote_state_message(const char *node_name, bool node_up)
	{
	xmlNode *msg = create_request(CRM_OP_REMOTE_STATE, NULL, NULL,
	CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	crm_info("Notifying cluster of Pacemaker Remote node %s %s",
	node_name, node_up? "coming up" : "going down");

	crm_xml_add(msg, PCMK_XA_ID, node_name);
	pcmk__xe_set_bool_attr(msg, PCMK__XA_IN_CCM, node_up);

	if (node_up) {
	crm_xml_add(msg, PCMK__XA_CONNECTION_HOST,
	controld_globals.our_nodename);
	}

	pcmk__cluster_send_message(NULL, crm_msg_crmd, msg);
	pcmk__xml_free(msg);
	}

	diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
	index cf817612f5..8e765bada8 100644
	--- a/daemons/fenced/fenced_commands.c
	+++ b/daemons/fenced/fenced_commands.c
	@@ -1,3643 +1,3643 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>

	#include <crm/crm.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <pacemaker-fenced.h>

	GHashTable *device_list = NULL;
	GHashTable *topology = NULL;
	static GList *cmd_list = NULL;

	static GHashTable *fenced_handlers = NULL;

	struct device_search_s {
	/* target of fence action */
	char *host;
	/* requested fence action */
	char *action;
	/* timeout to use if a device is queried dynamically for possible targets */
	int per_device_timeout;
	/* number of registered fencing devices at time of request */
	int replies_needed;
	/* number of device replies received so far */
	int replies_received;
	/* whether the target is eligible to perform requested action (or off) */
	bool allow_suicide;

	/* private data to pass to search callback function */
	void *user_data;
	/* function to call when all replies have been received */
	void (callback) (GList devices, void *user_data);
	/* devices capable of performing requested action (or off if remapping) */
	GList *capable;
	/* Whether to perform searches that support the action */
	uint32_t support_action_only;
	};

	static gboolean stonith_device_dispatch(gpointer user_data);
	static void st_child_done(int pid, const pcmk__action_result_t *result,
	void *user_data);

	static void search_devices_record_result(struct device_search_s search, const char device,
	gboolean can_fence);

	static int get_agent_metadata(const char agent, xmlNode *metadata);
	static void read_action_metadata(stonith_device_t *device);
	static enum fenced_target_by unpack_level_kind(const xmlNode *level);

	typedef struct async_command_s {

	int id;
	int pid;
	int fd_stdout;
	int options;
	int default_timeout; /* seconds */
	int timeout; /* seconds */

	int start_delay; // seconds (-1 means disable static/random fencing delays)
	int delay_id;

	char *op;
	char *origin;
	char *client;
	char *client_name;
	char *remote_op_id;

	char *target;
	uint32_t target_nodeid;
	char *action;
	char *device;

	GList *device_list;
	GList *next_device_iter; // device_list entry for next device to execute

	void *internal_user_data;
	void (done_cb) (int pid, const pcmk__action_result_t result,
	void *user_data);
	guint timer_sigterm;
	guint timer_sigkill;
	/*! If the operation timed out, this is the last signal
	* we sent to the process to get it to terminate */
	int last_timeout_signo;

	stonith_device_t *active_on;
	stonith_device_t *activating_on;
	} async_command_t;

	static xmlNode construct_async_reply(const async_command_t cmd,
	const pcmk__action_result_t *result);

	static gboolean
	is_action_required(const char action, const stonith_device_t device)
	{
	return (device != NULL) && device->automatic_unfencing
	&& pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none);
	}

	static int
	get_action_delay_max(const stonith_device_t device, const char action)
	{
	const char *value = NULL;
	guint delay_max = 0U;

	if (!pcmk__is_fencing_action(action)) {
	return 0;
	}

	value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
	if (value) {
	pcmk_parse_interval_spec(value, &delay_max);
	delay_max /= 1000;
	}

	return (int) delay_max;
	}

	static int
	get_action_delay_base(const stonith_device_t device, const char action,
	const char *target)
	{
	char *hash_value = NULL;
	guint delay_base = 0U;

	if (!pcmk__is_fencing_action(action)) {
	return 0;
	}

	hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);

	if (hash_value) {
	char *value = pcmk__str_copy(hash_value);
	char *valptr = value;

	if (target != NULL) {
	for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
	char *mapval = strchr(val, ':');

	if (mapval == NULL \|\| mapval[1] == 0) {
	crm_err("pcmk_delay_base: empty value in mapping", val);
	continue;
	}

	if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
	value = mapval + 1;
	crm_debug("pcmk_delay_base mapped to %s for %s",
	value, target);
	break;
	}
	}
	}

	if (strchr(value, ':') == 0) {
	pcmk_parse_interval_spec(value, &delay_base);
	delay_base /= 1000;
	}

	free(valptr);
	}

	return (int) delay_base;
	}

	/*!
	* \internal
	* \brief Override STONITH timeout with pcmk_*_timeout if available
	*
	* \param[in] device STONITH device to use
	* \param[in] action STONITH action name
	* \param[in] default_timeout Timeout to use if device does not have
	* a pcmk_*_timeout parameter for action
	*
	* \return Value of pcmk_(action)_timeout if available, otherwise default_timeout
	* \note For consistency, it would be nice if reboot/off/on timeouts could be
	* set the same way as start/stop/monitor timeouts, i.e. with an
	* <operation> entry in the fencing resource configuration. However that
	* is insufficient because fencing devices may be registered directly via
	* the fencer's register_device() API instead of going through the CIB
	* (e.g. stonith_admin uses it for its -R option, and the executor uses it
	* to ensure a device is registered when a command is issued). As device
	* properties, pcmk_*_timeout parameters can be grabbed by the fencer when
	* the device is registered, whether by CIB change or API call.
	*/
	static int
	get_action_timeout(const stonith_device_t device, const char action,
	int default_timeout)
	{
	if (action && device && device->params) {
	char buffer[64] = { 0, };
	const char *value = NULL;

	/* If "reboot" was requested but the device does not support it,
	* we will remap to "off", so check timeout for "off" instead
	*/
	if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)
	&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {
	crm_trace("%s doesn't support reboot, using timeout for off instead",
	device->id);
	action = PCMK_ACTION_OFF;
	}

	/* If the device config specified an action-specific timeout, use it */
	snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
	value = g_hash_table_lookup(device->params, buffer);
	if (value) {
	long long timeout_ms = crm_get_msec(value);
	return (int) QB_MIN(timeout_ms / 1000, INT_MAX);
	}
	}
	return default_timeout;
	}

	/*!
	* \internal
	* \brief Get the currently executing device for a fencing operation
	*
	* \param[in] cmd Fencing operation to check
	*
	* \return Currently executing device for \p cmd if any, otherwise NULL
	*/
	static stonith_device_t *
	cmd_device(const async_command_t *cmd)
	{
	if ((cmd == NULL) \|\| (cmd->device == NULL) \|\| (device_list == NULL)) {
	return NULL;
	}
	return g_hash_table_lookup(device_list, cmd->device);
	}

	/*!
	* \internal
	* \brief Return the configured reboot action for a given device
	*
	* \param[in] device_id Device ID
	*
	* \return Configured reboot action for \p device_id
	*/
	const char *
	fenced_device_reboot_action(const char *device_id)
	{
	const char *action = NULL;

	if ((device_list != NULL) && (device_id != NULL)) {
	stonith_device_t *device = g_hash_table_lookup(device_list, device_id);

	if ((device != NULL) && (device->params != NULL)) {
	action = g_hash_table_lookup(device->params, "pcmk_reboot_action");
	}
	}
	return pcmk__s(action, PCMK_ACTION_REBOOT);
	}

	/*!
	* \internal
	* \brief Check whether a given device supports the "on" action
	*
	* \param[in] device_id Device ID
	*
	* \return true if \p device_id supports "on", otherwise false
	*/
	bool
	fenced_device_supports_on(const char *device_id)
	{
	if ((device_list != NULL) && (device_id != NULL)) {
	stonith_device_t *device = g_hash_table_lookup(device_list, device_id);

	if (device != NULL) {
	return pcmk_is_set(device->flags, st_device_supports_on);
	}
	}
	return false;
	}

	static void
	free_async_command(async_command_t * cmd)
	{
	if (!cmd) {
	return;
	}

	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}

	cmd_list = g_list_remove(cmd_list, cmd);

	g_list_free_full(cmd->device_list, free);
	free(cmd->device);
	free(cmd->action);
	free(cmd->target);
	free(cmd->remote_op_id);
	free(cmd->client);
	free(cmd->client_name);
	free(cmd->origin);
	free(cmd->op);
	free(cmd);
	}

	/*!
	* \internal
	* \brief Create a new asynchronous fencing operation from request XML
	*
	* \param[in] msg Fencing request XML (from IPC or CPG)
	*
	* \return Newly allocated fencing operation on success, otherwise NULL
	*
	* \note This asserts on memory errors, so a NULL return indicates an
	* unparseable message.
	*/
	static async_command_t *
	create_async_command(xmlNode *msg)
	{
	xmlNode *op = NULL;
	async_command_t *cmd = NULL;

	if (msg == NULL) {
	return NULL;
	}

	op = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, msg, LOG_ERR);
	if (op == NULL) {
	return NULL;
	}

	cmd = pcmk__assert_alloc(1, sizeof(async_command_t));

	// All messages must include these
	cmd->action = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ACTION);
	cmd->op = crm_element_value_copy(msg, PCMK__XA_ST_OP);
	cmd->client = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTID);
	if ((cmd->action == NULL) \|\| (cmd->op == NULL) \|\| (cmd->client == NULL)) {
	free_async_command(cmd);
	return NULL;
	}

	crm_element_value_int(msg, PCMK__XA_ST_CALLID, &(cmd->id));
	crm_element_value_int(msg, PCMK__XA_ST_CALLOPT, &(cmd->options));
	crm_element_value_int(msg, PCMK__XA_ST_DELAY, &(cmd->start_delay));
	crm_element_value_int(msg, PCMK__XA_ST_TIMEOUT, &(cmd->default_timeout));
	cmd->timeout = cmd->default_timeout;

	cmd->origin = crm_element_value_copy(msg, PCMK__XA_SRC);
	cmd->remote_op_id = crm_element_value_copy(msg, PCMK__XA_ST_REMOTE_OP);
	cmd->client_name = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTNAME);
	cmd->target = crm_element_value_copy(op, PCMK__XA_ST_TARGET);
	cmd->device = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ID);

	cmd->done_cb = st_child_done;

	// Track in global command list
	cmd_list = g_list_append(cmd_list, cmd);

	return cmd;
	}

	static int
	get_action_limit(stonith_device_t * device)
	{
	const char *value = NULL;
	int action_limit = 1;

	value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
	if ((value == NULL)
	\|\| (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
	\|\| (action_limit == 0)) {
	action_limit = 1;
	}
	return action_limit;
	}

	static int
	get_active_cmds(stonith_device_t * device)
	{
	int counter = 0;
	GList *gIter = NULL;
	GList *gIterNext = NULL;

	CRM_CHECK(device != NULL, return 0);

	for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
	async_command_t *cmd = gIter->data;

	gIterNext = gIter->next;

	if (cmd->active_on == device) {
	counter++;
	}
	}

	return counter;
	}

	static void
	fork_cb(int pid, void *user_data)
	{
	async_command_t cmd = (async_command_t ) user_data;
	stonith_device_t * device =
	/* in case of a retry we've done the move from
	activating_on to active_on already
	*/
	cmd->activating_on?cmd->activating_on:cmd->active_on;

	CRM_ASSERT(device);
	crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
	cmd->action, pid,
	((cmd->target == NULL)? "" : " targeting "),
	pcmk__s(cmd->target, ""), device->id, cmd->timeout);
	cmd->active_on = device;
	cmd->activating_on = NULL;
	}

	static int
	get_agent_metadata_cb(gpointer data) {
	stonith_device_t *device = data;
	guint period_ms;

	switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
	case pcmk_rc_ok:
	if (device->agent_metadata) {
	read_action_metadata(device);
	stonith__device_parameter_flags(&(device->flags), device->id,
	device->agent_metadata);
	}
	return G_SOURCE_REMOVE;

	case EAGAIN:
	period_ms = pcmk__mainloop_timer_get_period(device->timer);
	if (period_ms < 160 * 1000) {
	mainloop_timer_set_period(device->timer, 2 * period_ms);
	}
	return G_SOURCE_CONTINUE;

	default:
	return G_SOURCE_REMOVE;
	}
	}

	/*!
	* \internal
	* \brief Call a command's action callback for an internal (not library) result
	*
	* \param[in,out] cmd Command to report result for
	* \param[in] execution_status Execution status to use for result
	* \param[in] exit_status Exit status to use for result
	* \param[in] exit_reason Exit reason to use for result
	*/
	static void
	report_internal_result(async_command_t *cmd, int exit_status,
	int execution_status, const char *exit_reason)
	{
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	pcmk__set_result(&result, exit_status, execution_status, exit_reason);
	cmd->done_cb(0, &result, cmd);
	pcmk__reset_result(&result);
	}

	static gboolean
	stonith_device_execute(stonith_device_t * device)
	{
	int exec_rc = 0;
	const char *action_str = NULL;
	const char *host_arg = NULL;
	async_command_t *cmd = NULL;
	stonith_action_t *action = NULL;
	int active_cmds = 0;
	int action_limit = 0;
	GList *gIter = NULL;
	GList *gIterNext = NULL;

	CRM_CHECK(device != NULL, return FALSE);

	active_cmds = get_active_cmds(device);
	action_limit = get_action_limit(device);
	if (action_limit > -1 && active_cmds >= action_limit) {
	crm_trace("%s is over its action limit of %d (%u active action%s)",
	device->id, action_limit, active_cmds,
	pcmk__plural_s(active_cmds));
	return TRUE;
	}

	for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
	async_command_t *pending_op = gIter->data;

	gIterNext = gIter->next;

	if (pending_op && pending_op->delay_id) {
	crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
	"waiting for start delay of %ds",
	pending_op->action,
	((pending_op->target == NULL)? "" : " targeting "),
	pcmk__s(pending_op->target, ""),
	device->id, pending_op->start_delay);
	continue;
	}

	device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
	g_list_free_1(gIter);

	cmd = pending_op;
	break;
	}

	if (cmd == NULL) {
	crm_trace("No actions using %s are needed", device->id);
	return TRUE;
	}

	if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
	STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
	if (pcmk__is_fencing_action(cmd->action)) {
	if (node_does_watchdog_fencing(stonith_our_uname)) {
	pcmk__panic(__func__);
	goto done;
	}
	} else {
	crm_info("Faking success for %s watchdog operation", cmd->action);
	report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	goto done;
	}
	}

	#if SUPPORT_CIBSECRETS
	exec_rc = pcmk__substitute_secrets(device->id, device->params);
	if (exec_rc != pcmk_rc_ok) {
	if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_none)) {
	crm_info("Proceeding with stop operation for %s "
	"despite being unable to load CIB secrets (%s)",
	device->id, pcmk_rc_str(exec_rc));
	} else {
	crm_err("Considering %s unconfigured "
	"because unable to load CIB secrets: %s",
	device->id, pcmk_rc_str(exec_rc));
	report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
	"Failed to get CIB secrets");
	goto done;
	}
	}
	#endif

	action_str = cmd->action;
	if (pcmk__str_eq(cmd->action, PCMK_ACTION_REBOOT, pcmk__str_none)
	&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {

	crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
	"because agent '%s' does not support reboot",
	((cmd->target == NULL)? "" : " targeting "),
	pcmk__s(cmd->target, ""), device->id, device->agent);
	action_str = PCMK_ACTION_OFF;
	}

	if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
	host_arg = "port";

	} else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
	host_arg = "plug";
	}

	action = stonith__action_create(device->agent, action_str, cmd->target,
	cmd->target_nodeid, cmd->timeout,
	device->params, device->aliases, host_arg);

	/* for async exec, exec_rc is negative for early error exit
	otherwise handling of success/errors is done via callbacks */
	cmd->activating_on = device;
	exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
	fork_cb);
	if (exec_rc < 0) {
	cmd->activating_on = NULL;
	cmd->done_cb(0, stonith__action_result(action), cmd);
	stonith__destroy_action(action);
	}

	done:
	/* Device might get triggered to work by multiple fencing commands
	* simultaneously. Trigger the device again to make sure any
	* remaining concurrent commands get executed. */
	if (device->pending_ops) {
	mainloop_set_trigger(device->work);
	}
	return TRUE;
	}

	static gboolean
	stonith_device_dispatch(gpointer user_data)
	{
	return stonith_device_execute(user_data);
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	async_command_t *cmd = data;
	stonith_device_t *device = cmd_device(cmd);

	cmd->delay_id = 0;
	if (device) {
	mainloop_set_trigger(device->work);
	}

	return FALSE;
	}

	static void
	schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
	{
	int delay_max = 0;
	int delay_base = 0;
	int requested_delay = cmd->start_delay;

	CRM_CHECK(cmd != NULL, return);
	CRM_CHECK(device != NULL, return);

	if (cmd->device) {
	free(cmd->device);
	}

	if (device->include_nodeid && (cmd->target != NULL)) {
	pcmk__node_status_t *node =
	pcmk__get_node(0, cmd->target, NULL,
	pcmk__node_search_cluster_member);

	- cmd->target_nodeid = node->id;
	+ cmd->target_nodeid = node->cluster_layer_id;
	}

	cmd->device = pcmk__str_copy(device->id);
	cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);

	if (cmd->remote_op_id) {
	crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
	"with op id %.8s and timeout %ds",
	cmd->action,
	(cmd->target == NULL)? "" : " targeting ",
	pcmk__s(cmd->target, ""),
	device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
	} else {
	crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
	cmd->action,
	(cmd->target == NULL)? "" : " targeting ",
	pcmk__s(cmd->target, ""),
	device->id, cmd->client, cmd->timeout);
	}

	device->pending_ops = g_list_append(device->pending_ops, cmd);
	mainloop_set_trigger(device->work);

	// Value -1 means disable any static/random fencing delays
	if (requested_delay < 0) {
	return;
	}

	delay_max = get_action_delay_max(device, cmd->action);
	delay_base = get_action_delay_base(device, cmd->action, cmd->target);
	if (delay_max == 0) {
	delay_max = delay_base;
	}
	if (delay_max < delay_base) {
	crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
	PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
	"(limiting to maximum delay)",
	delay_base, delay_max, cmd->action, device->id);
	delay_base = delay_max;
	}
	if (delay_max > 0) {
	// coverity[dontcall] It doesn't matter here if rand() is predictable
	cmd->start_delay +=
	((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
	+ delay_base;
	}

	if (cmd->start_delay > 0) {
	crm_notice("Delaying '%s' action%s%s using %s for %ds " QB_XS
	" timeout=%ds requested_delay=%ds base=%ds max=%ds",
	cmd->action,
	(cmd->target == NULL)? "" : " targeting ",
	pcmk__s(cmd->target, ""),
	device->id, cmd->start_delay, cmd->timeout,
	requested_delay, delay_base, delay_max);
	cmd->delay_id =
	g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
	}
	}

	static void
	free_device(gpointer data)
	{
	GList *gIter = NULL;
	stonith_device_t *device = data;

	g_hash_table_destroy(device->params);
	g_hash_table_destroy(device->aliases);

	for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
	async_command_t *cmd = gIter->data;

	crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
	report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"Device was removed before action could be executed");
	}
	g_list_free(device->pending_ops);

	g_list_free_full(device->targets, free);

	if (device->timer) {
	mainloop_timer_stop(device->timer);
	mainloop_timer_del(device->timer);
	}

	mainloop_destroy_trigger(device->work);

	pcmk__xml_free(device->agent_metadata);
	free(device->namespace);
	if (device->on_target_actions != NULL) {
	g_string_free(device->on_target_actions, TRUE);
	}
	free(device->agent);
	free(device->id);
	free(device);
	}

	void free_device_list(void)
	{
	if (device_list != NULL) {
	g_hash_table_destroy(device_list);
	device_list = NULL;
	}
	}

	void
	init_device_list(void)
	{
	if (device_list == NULL) {
	device_list = pcmk__strkey_table(NULL, free_device);
	}
	}

	static GHashTable *
	build_port_aliases(const char hostmap, GList * targets)
	{
	char *name = NULL;
	int last = 0, lpc = 0, max = 0, added = 0;
	GHashTable *aliases = pcmk__strikey_table(free, free);

	if (hostmap == NULL) {
	return aliases;
	}

	max = strlen(hostmap);
	for (; lpc <= max; lpc++) {
	switch (hostmap[lpc]) {
	/* Skip escaped chars */
	case '\\':
	lpc++;
	break;

	/* Assignment chars */
	case '=':
	case ':':
	if (lpc > last) {
	free(name);
	name = pcmk__assert_alloc(1, 1 + lpc - last);
	memcpy(name, hostmap + last, lpc - last);
	}
	last = lpc + 1;
	break;

	/* Delimeter chars */
	/* case ',': Potentially used to specify multiple ports */
	case 0:
	case ';':
	case ' ':
	case '\t':
	if (name) {
	char *value = NULL;
	int k = 0;

	value = pcmk__assert_alloc(1, 1 + lpc - last);
	memcpy(value, hostmap + last, lpc - last);

	for (int i = 0; value[i] != '\0'; i++) {
	if (value[i] != '\\') {
	value[k++] = value[i];
	}
	}
	value[k] = '\0';

	crm_debug("Adding alias '%s'='%s'", name, value);
	g_hash_table_replace(aliases, name, value);
	if (targets) {
	targets = g_list_append(targets, pcmk__str_copy(value));
	}
	value = NULL;
	name = NULL;
	added++;

	} else if (lpc > last) {
	crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
	}

	last = lpc + 1;
	break;
	}

	if (hostmap[lpc] == 0) {
	break;
	}
	}

	if (added == 0) {
	crm_info("No host mappings detected in '%s'", hostmap);
	}

	free(name);
	return aliases;
	}

	GHashTable *metadata_cache = NULL;

	void
	free_metadata_cache(void) {
	if (metadata_cache != NULL) {
	g_hash_table_destroy(metadata_cache);
	metadata_cache = NULL;
	}
	}

	static void
	init_metadata_cache(void) {
	if (metadata_cache == NULL) {
	metadata_cache = pcmk__strkey_table(free, free);
	}
	}

	int
	get_agent_metadata(const char agent, xmlNode * metadata)
	{
	char *buffer = NULL;

	if (metadata == NULL) {
	return EINVAL;
	}
	*metadata = NULL;
	if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
	return pcmk_rc_ok;
	}
	init_metadata_cache();
	buffer = g_hash_table_lookup(metadata_cache, agent);
	if (buffer == NULL) {
	stonith_t *st = stonith_api_new();
	int rc;

	if (st == NULL) {
	crm_warn("Could not get agent meta-data: "
	"API memory allocation failed");
	return EAGAIN;
	}
	rc = st->cmds->metadata(st, st_opt_sync_call, agent,
	NULL, &buffer, 10);
	stonith_api_delete(st);
	if (rc \|\| !buffer) {
	crm_err("Could not retrieve metadata for fencing agent %s", agent);
	return EAGAIN;
	}
	g_hash_table_replace(metadata_cache, pcmk__str_copy(agent), buffer);
	}

	*metadata = pcmk__xml_parse(buffer);
	return pcmk_rc_ok;
	}

	static gboolean
	is_nodeid_required(xmlNode * xml)
	{
	xmlXPathObjectPtr xpath = NULL;

	if (stand_alone) {
	return FALSE;
	}

	if (!xml) {
	return FALSE;
	}

	xpath = xpath_search(xml,
	"//" PCMK_XE_PARAMETER "[@" PCMK_XA_NAME "='nodeid']");
	if (numXpathResults(xpath) <= 0) {
	freeXpathObject(xpath);
	return FALSE;
	}

	freeXpathObject(xpath);
	return TRUE;
	}

	static void
	read_action_metadata(stonith_device_t *device)
	{
	xmlXPathObjectPtr xpath = NULL;
	int max = 0;
	int lpc = 0;

	if (device->agent_metadata == NULL) {
	return;
	}

	xpath = xpath_search(device->agent_metadata, "//action");
	max = numXpathResults(xpath);

	if (max <= 0) {
	freeXpathObject(xpath);
	return;
	}

	for (lpc = 0; lpc < max; lpc++) {
	const char *action = NULL;
	xmlNode *match = getXpathResult(xpath, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match == NULL) { continue; };

	action = crm_element_value(match, PCMK_XA_NAME);

	if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_list);
	} else if (pcmk__str_eq(action, PCMK_ACTION_STATUS, pcmk__str_none)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_status);
	} else if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_reboot);
	} else if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)) {
	/* PCMK_XA_AUTOMATIC means the cluster will unfence a node when it
	* joins.
	*
	* @COMPAT PCMK__XA_REQUIRED is a deprecated synonym for
	* PCMK_XA_AUTOMATIC.
	*/
	if (pcmk__xe_attr_is_true(match, PCMK_XA_AUTOMATIC)
	\|\| pcmk__xe_attr_is_true(match, PCMK__XA_REQUIRED)) {
	device->automatic_unfencing = TRUE;
	}
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_on);
	}

	if ((action != NULL)
	&& pcmk__xe_attr_is_true(match, PCMK_XA_ON_TARGET)) {

	pcmk__add_word(&(device->on_target_actions), 64, action);
	}
	}

	freeXpathObject(xpath);
	}

	static const char *
	target_list_type(stonith_device_t * dev)
	{
	const char *check_type = NULL;

	check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);

	if (check_type == NULL) {

	if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
	check_type = PCMK_VALUE_STATIC_LIST;
	} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
	check_type = PCMK_VALUE_STATIC_LIST;
	} else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
	check_type = PCMK_VALUE_DYNAMIC_LIST;
	} else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
	check_type = PCMK_VALUE_STATUS;
	} else {
	check_type = PCMK_VALUE_NONE;
	}
	}

	return check_type;
	}

	static stonith_device_t *
	build_device_from_xml(xmlNode *dev)
	{
	const char *value;
	stonith_device_t *device = NULL;
	char *agent = crm_element_value_copy(dev, PCMK_XA_AGENT);

	CRM_CHECK(agent != NULL, return device);

	device = pcmk__assert_alloc(1, sizeof(stonith_device_t));

	device->id = crm_element_value_copy(dev, PCMK_XA_ID);
	device->agent = agent;
	device->namespace = crm_element_value_copy(dev, PCMK__XA_NAMESPACE);
	device->params = xml2list(dev);

	value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
	if (value) {
	device->targets = stonith__parse_targets(value);
	}

	value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
	device->aliases = build_port_aliases(value, &(device->targets));

	value = target_list_type(device);
	if (!pcmk__str_eq(value, PCMK_VALUE_STATIC_LIST, pcmk__str_casei)
	&& (device->targets != NULL)) {

	// device->targets is necessary only with PCMK_VALUE_STATIC_LIST
	g_list_free_full(device->targets, free);
	device->targets = NULL;
	}
	switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
	case pcmk_rc_ok:
	if (device->agent_metadata) {
	read_action_metadata(device);
	stonith__device_parameter_flags(&(device->flags), device->id,
	device->agent_metadata);
	}
	break;

	case EAGAIN:
	if (device->timer == NULL) {
	device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
	TRUE, get_agent_metadata_cb, device);
	}
	if (!mainloop_timer_running(device->timer)) {
	mainloop_timer_start(device->timer);
	}
	break;

	default:
	break;
	}

	value = g_hash_table_lookup(device->params, "nodeid");
	if (!value) {
	device->include_nodeid = is_nodeid_required(device->agent_metadata);
	}

	value = crm_element_value(dev, PCMK__XA_RSC_PROVIDES);
	if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
	device->automatic_unfencing = TRUE;
	}

	if (is_action_required(PCMK_ACTION_ON, device)) {
	crm_info("Fencing device '%s' requires unfencing", device->id);
	}

	if (device->on_target_actions != NULL) {
	crm_info("Fencing device '%s' requires actions (%s) to be executed "
	"on target", device->id,
	(const char *) device->on_target_actions->str);
	}

	device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
	/* TODO: Hook up priority */

	return device;
	}

	static void
	schedule_internal_command(const char *origin,
	stonith_device_t * device,
	const char *action,
	const char *target,
	int timeout,
	void *internal_user_data,
	void (*done_cb) (int pid,
	const pcmk__action_result_t *result,
	void *user_data))
	{
	async_command_t *cmd = NULL;

	cmd = pcmk__assert_alloc(1, sizeof(async_command_t));

	cmd->id = -1;
	cmd->default_timeout = timeout ? timeout : 60;
	cmd->timeout = cmd->default_timeout;
	cmd->action = pcmk__str_copy(action);
	cmd->target = pcmk__str_copy(target);
	cmd->device = pcmk__str_copy(device->id);
	cmd->origin = pcmk__str_copy(origin);
	cmd->client = pcmk__str_copy(crm_system_name);
	cmd->client_name = pcmk__str_copy(crm_system_name);

	cmd->internal_user_data = internal_user_data;
	cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */

	schedule_stonith_command(cmd, device);
	}

	// Fence agent status commands use custom exit status codes
	enum fence_status_code {
	fence_status_invalid = -1,
	fence_status_active = 0,
	fence_status_unknown = 1,
	fence_status_inactive = 2,
	};

	static void
	status_search_cb(int pid, const pcmk__action_result_t result, void user_data)
	{
	async_command_t *cmd = user_data;
	struct device_search_s *search = cmd->internal_user_data;
	stonith_device_t *dev = cmd_device(cmd);
	gboolean can = FALSE;

	free_async_command(cmd);

	if (!dev) {
	search_devices_record_result(search, NULL, FALSE);
	return;
	}

	mainloop_set_trigger(dev->work);

	if (result->execution_status != PCMK_EXEC_DONE) {
	crm_warn("Assuming %s cannot fence %s "
	"because status could not be executed: %s%s%s%s",
	dev->id, search->host,
	pcmk_exec_status_str(result->execution_status),
	((result->exit_reason == NULL)? "" : " ("),
	((result->exit_reason == NULL)? "" : result->exit_reason),
	((result->exit_reason == NULL)? "" : ")"));
	search_devices_record_result(search, dev->id, FALSE);
	return;
	}

	switch (result->exit_status) {
	case fence_status_unknown:
	crm_trace("%s reported it cannot fence %s", dev->id, search->host);
	break;

	case fence_status_active:
	case fence_status_inactive:
	crm_trace("%s reported it can fence %s", dev->id, search->host);
	can = TRUE;
	break;

	default:
	crm_warn("Assuming %s cannot fence %s "
	"(status returned unknown code %d)",
	dev->id, search->host, result->exit_status);
	break;
	}
	search_devices_record_result(search, dev->id, can);
	}

	static void
	dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
	void *user_data)
	{
	async_command_t *cmd = user_data;
	struct device_search_s *search = cmd->internal_user_data;
	stonith_device_t *dev = cmd_device(cmd);
	gboolean can_fence = FALSE;

	free_async_command(cmd);

	/* Host/alias must be in the list output to be eligible to be fenced
	*
	* Will cause problems if down'd nodes aren't listed or (for virtual nodes)
	* if the guest is still listed despite being moved to another machine
	*/
	if (!dev) {
	search_devices_record_result(search, NULL, FALSE);
	return;
	}

	mainloop_set_trigger(dev->work);

	if (pcmk__result_ok(result)) {
	crm_info("Refreshing target list for %s", dev->id);
	g_list_free_full(dev->targets, free);
	dev->targets = stonith__parse_targets(result->action_stdout);
	dev->targets_age = time(NULL);

	} else if (dev->targets != NULL) {
	if (result->execution_status == PCMK_EXEC_DONE) {
	crm_info("Reusing most recent target list for %s "
	"because list returned error code %d",
	dev->id, result->exit_status);
	} else {
	crm_info("Reusing most recent target list for %s "
	"because list could not be executed: %s%s%s%s",
	dev->id, pcmk_exec_status_str(result->execution_status),
	((result->exit_reason == NULL)? "" : " ("),
	((result->exit_reason == NULL)? "" : result->exit_reason),
	((result->exit_reason == NULL)? "" : ")"));
	}

	} else { // We have never successfully executed list
	if (result->execution_status == PCMK_EXEC_DONE) {
	crm_warn("Assuming %s cannot fence %s "
	"because list returned error code %d",
	dev->id, search->host, result->exit_status);
	} else {
	crm_warn("Assuming %s cannot fence %s "
	"because list could not be executed: %s%s%s%s",
	dev->id, search->host,
	pcmk_exec_status_str(result->execution_status),
	((result->exit_reason == NULL)? "" : " ("),
	((result->exit_reason == NULL)? "" : result->exit_reason),
	((result->exit_reason == NULL)? "" : ")"));
	}

	/* Fall back to pcmk_host_check=PCMK_VALUE_STATUS if the user didn't
	* explicitly specify PCMK_VALUE_DYNAMIC_LIST
	*/
	if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
	crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
	pcmk__insert_dup(dev->params, PCMK_STONITH_HOST_CHECK,
	PCMK_VALUE_STATUS);
	}
	}

	if (dev->targets) {
	const char *alias = g_hash_table_lookup(dev->aliases, search->host);

	if (!alias) {
	alias = search->host;
	}
	if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
	can_fence = TRUE;
	}
	}
	search_devices_record_result(search, dev->id, can_fence);
	}

	/*!
	* \internal
	* \brief Returns true if any key in first is not in second or second has a different value for key
	*/
	static int
	device_params_diff(GHashTable first, GHashTable second) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter gIter;

	g_hash_table_iter_init(&gIter, first);
	while (g_hash_table_iter_next(&gIter, (void )&key, (void )&value)) {

	if(strstr(key, "CRM_meta") == key) {
	continue;
	} else if (strcmp(key, PCMK_XA_CRM_FEATURE_SET) == 0) {
	continue;
	} else {
	char *other_value = g_hash_table_lookup(second, key);

	if (!other_value \|\| !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
	crm_trace("Different value for %s: %s != %s", key, other_value, value);
	return 1;
	}
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Checks to see if an identical device already exists in the device_list
	*/
	static stonith_device_t *
	device_has_duplicate(const stonith_device_t *device)
	{
	stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);

	if (!dup) {
	crm_trace("No match for %s", device->id);
	return NULL;

	} else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
	crm_trace("Different agent: %s != %s", dup->agent, device->agent);
	return NULL;
	}

	// Use pcmk__digest_operation() here?
	if (device_params_diff(device->params, dup->params) \|\|
	device_params_diff(dup->params, device->params)) {
	return NULL;
	}

	crm_trace("Match");
	return dup;
	}

	int
	stonith_device_register(xmlNode *dev, gboolean from_cib)
	{
	stonith_device_t *dup = NULL;
	stonith_device_t *device = build_device_from_xml(dev);
	guint ndevices = 0;
	int rv = pcmk_ok;

	CRM_CHECK(device != NULL, return -ENOMEM);

	/* do we have a watchdog-device? */
	if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) \|\|
	pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
	STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do {
	if (stonith_watchdog_timeout_ms <= 0) {
	crm_err("Ignoring watchdog fence device without "
	PCMK_OPT_STONITH_WATCHDOG_TIMEOUT " set.");
	rv = -ENODEV;
	/* fall through to cleanup & return */
	} else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
	STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
	crm_err("Ignoring watchdog fence device with unknown "
	"agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.",
	device->agent?device->agent:"");
	rv = -ENODEV;
	/* fall through to cleanup & return */
	} else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID,
	pcmk__str_none)) {
	crm_err("Ignoring watchdog fence device "
	"named %s !='"STONITH_WATCHDOG_ID"'.",
	device->id?device->id:"");
	rv = -ENODEV;
	/* fall through to cleanup & return */
	} else {
	if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
	pcmk__str_none)) {
	/* this either has an empty list or the targets
	configured for watchdog-fencing
	*/
	g_list_free_full(stonith_watchdog_targets, free);
	stonith_watchdog_targets = device->targets;
	device->targets = NULL;
	}
	if (node_does_watchdog_fencing(stonith_our_uname)) {
	g_list_free_full(device->targets, free);
	device->targets = stonith__parse_targets(stonith_our_uname);
	pcmk__insert_dup(device->params,
	PCMK_STONITH_HOST_LIST, stonith_our_uname);
	/* proceed as with any other stonith-device */
	break;
	}

	crm_debug("Skip registration of watchdog fence device on node not in host-list.");
	/* cleanup and fall through to more cleanup and return */
	device->targets = NULL;
	stonith_device_remove(device->id, from_cib);
	}
	free_device(device);
	return rv;
	} while (0);

	dup = device_has_duplicate(device);
	if (dup) {
	ndevices = g_hash_table_size(device_list);
	crm_debug("Device '%s' already in device list (%d active device%s)",
	device->id, ndevices, pcmk__plural_s(ndevices));
	free_device(device);
	device = dup;
	dup = g_hash_table_lookup(device_list, device->id);
	dup->dirty = FALSE;

	} else {
	stonith_device_t *old = g_hash_table_lookup(device_list, device->id);

	if (from_cib && old && old->api_registered) {
	/* If the cib is writing over an entry that is shared with a stonith client,
	* copy any pending ops that currently exist on the old entry to the new one.
	* Otherwise the pending ops will be reported as failures
	*/
	crm_info("Overwriting existing entry for %s from CIB", device->id);
	device->pending_ops = old->pending_ops;
	device->api_registered = TRUE;
	old->pending_ops = NULL;
	if (device->pending_ops) {
	mainloop_set_trigger(device->work);
	}
	}
	g_hash_table_replace(device_list, device->id, device);

	ndevices = g_hash_table_size(device_list);
	crm_notice("Added '%s' to device list (%d active device%s)",
	device->id, ndevices, pcmk__plural_s(ndevices));
	}

	if (from_cib) {
	device->cib_registered = TRUE;
	} else {
	device->api_registered = TRUE;
	}

	return pcmk_ok;
	}

	void
	stonith_device_remove(const char *id, bool from_cib)
	{
	stonith_device_t *device = g_hash_table_lookup(device_list, id);
	guint ndevices = 0;

	if (!device) {
	ndevices = g_hash_table_size(device_list);
	crm_info("Device '%s' not found (%d active device%s)",
	id, ndevices, pcmk__plural_s(ndevices));
	return;
	}

	if (from_cib) {
	device->cib_registered = FALSE;
	} else {
	device->verified = FALSE;
	device->api_registered = FALSE;
	}

	if (!device->cib_registered && !device->api_registered) {
	g_hash_table_remove(device_list, id);
	ndevices = g_hash_table_size(device_list);
	crm_info("Removed '%s' from device list (%d active device%s)",
	id, ndevices, pcmk__plural_s(ndevices));
	} else {
	crm_trace("Not removing '%s' from device list (%d active) because "
	"still registered via:%s%s",
	id, g_hash_table_size(device_list),
	(device->cib_registered? " cib" : ""),
	(device->api_registered? " api" : ""));
	}
	}

	/*!
	* \internal
	* \brief Return the number of stonith levels registered for a node
	*
	* \param[in] tp Node's topology table entry
	*
	* \return Number of non-NULL levels in topology entry
	* \note This function is used only for log messages.
	*/
	static int
	count_active_levels(const stonith_topology_t *tp)
	{
	int lpc = 0;
	int count = 0;

	for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if (tp->levels[lpc] != NULL) {
	count++;
	}
	}
	return count;
	}

	static void
	free_topology_entry(gpointer data)
	{
	stonith_topology_t *tp = data;

	int lpc = 0;

	for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if (tp->levels[lpc] != NULL) {
	g_list_free_full(tp->levels[lpc], free);
	}
	}
	free(tp->target);
	free(tp->target_value);
	free(tp->target_pattern);
	free(tp->target_attribute);
	free(tp);
	}

	void
	free_topology_list(void)
	{
	if (topology != NULL) {
	g_hash_table_destroy(topology);
	topology = NULL;
	}
	}

	void
	init_topology_list(void)
	{
	if (topology == NULL) {
	topology = pcmk__strkey_table(NULL, free_topology_entry);
	}
	}

	char *
	stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
	{
	if (mode == fenced_target_by_unknown) {
	mode = unpack_level_kind(level);
	}
	switch (mode) {
	case fenced_target_by_name:
	return crm_element_value_copy(level, PCMK_XA_TARGET);

	case fenced_target_by_pattern:
	return crm_element_value_copy(level, PCMK_XA_TARGET_PATTERN);

	case fenced_target_by_attribute:
	return crm_strdup_printf("%s=%s",
	crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE),
	crm_element_value(level, PCMK_XA_TARGET_VALUE));

	default:
	return crm_strdup_printf("unknown-%s", pcmk__xe_id(level));
	}
	}

	/*!
	* \internal
	* \brief Parse target identification from topology level XML
	*
	* \param[in] level Topology level XML to parse
	*
	* \return How to identify target of \p level
	*/
	static enum fenced_target_by
	unpack_level_kind(const xmlNode *level)
	{
	if (crm_element_value(level, PCMK_XA_TARGET) != NULL) {
	return fenced_target_by_name;
	}
	if (crm_element_value(level, PCMK_XA_TARGET_PATTERN) != NULL) {
	return fenced_target_by_pattern;
	}
	if (!stand_alone /* if standalone, there's no attribute manager */
	&& (crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE) != NULL)
	&& (crm_element_value(level, PCMK_XA_TARGET_VALUE) != NULL)) {
	return fenced_target_by_attribute;
	}
	return fenced_target_by_unknown;
	}

	static stonith_key_value_t *
	parse_device_list(const char *devices)
	{
	int lpc = 0;
	int max = 0;
	int last = 0;
	stonith_key_value_t *output = NULL;

	if (devices == NULL) {
	return output;
	}

	max = strlen(devices);
	for (lpc = 0; lpc <= max; lpc++) {
	if (devices[lpc] == ',' \|\| devices[lpc] == 0) {
	char *line = strndup(devices + last, lpc - last);

	output = stonith_key_value_add(output, NULL, line);
	free(line);

	last = lpc + 1;
	}
	}

	return output;
	}

	/*!
	* \internal
	* \brief Unpack essential information from topology request XML
	*
	* \param[in] xml Request XML to search
	* \param[out] mode If not NULL, where to store level kind
	* \param[out] target If not NULL, where to store representation of target
	* \param[out] id If not NULL, where to store level number
	* \param[out] desc If not NULL, where to store log-friendly level description
	*
	* \return Topology level XML from within \p xml, or NULL if not found
	* \note The caller is responsible for freeing \p target and \p desc if set.
	*/
	static xmlNode *
	unpack_level_request(xmlNode xml, enum fenced_target_by mode, char **target,
	int id, char *desc)
	{
	enum fenced_target_by local_mode = fenced_target_by_unknown;
	char *local_target = NULL;
	int local_id = 0;

	/* The level element can be the top element or lower. If top level, don't
	* search by xpath, because it might give multiple hits if the XML is the
	* CIB.
	*/
	if ((xml != NULL) && !pcmk__xe_is(xml, PCMK_XE_FENCING_LEVEL)) {
	xml = get_xpath_object("//" PCMK_XE_FENCING_LEVEL, xml, LOG_WARNING);
	}

	if (xml == NULL) {
	if (desc != NULL) {
	*desc = crm_strdup_printf("missing");
	}
	} else {
	local_mode = unpack_level_kind(xml);
	local_target = stonith_level_key(xml, local_mode);
	crm_element_value_int(xml, PCMK_XA_INDEX, &local_id);
	if (desc != NULL) {
	*desc = crm_strdup_printf("%s[%d]", local_target, local_id);
	}
	}

	if (mode != NULL) {
	*mode = local_mode;
	}
	if (id != NULL) {
	*id = local_id;
	}

	if (target != NULL) {
	*target = local_target;
	} else {
	free(local_target);
	}

	return xml;
	}

	/*!
	* \internal
	* \brief Register a fencing topology level for a target
	*
	* Given an XML request specifying the target name, level index, and device IDs
	* for the level, this will create an entry for the target in the global topology
	* table if one does not already exist, then append the specified device IDs to
	* the entry's device list for the specified level.
	*
	* \param[in] msg XML request for STONITH level registration
	* \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]"
	* \param[out] result Where to set result of registration
	*/
	void
	fenced_register_level(xmlNode msg, char desc, pcmk__action_result_t result)
	{
	int id = 0;
	xmlNode *level;
	enum fenced_target_by mode;
	char *target;

	stonith_topology_t *tp;
	stonith_key_value_t *dIter = NULL;
	stonith_key_value_t *devices = NULL;

	CRM_CHECK((msg != NULL) && (result != NULL), return);

	level = unpack_level_request(msg, &mode, &target, &id, desc);
	if (level == NULL) {
	fenced_set_protocol_error(result);
	return;
	}

	// Ensure an ID was given (even the client API adds an ID)
	if (pcmk__str_empty(pcmk__xe_id(level))) {
	crm_warn("Ignoring registration for topology level without ID");
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Topology level is invalid without ID");
	return;
	}

	// Ensure a valid target was specified
	if (mode == fenced_target_by_unknown) {
	crm_warn("Ignoring registration for topology level '%s' "
	"without valid target", pcmk__xe_id(level));
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Invalid target for topology level '%s'",
	pcmk__xe_id(level));
	return;
	}

	// Ensure level ID is in allowed range
	if ((id <= 0) \|\| (id >= ST_LEVEL_MAX)) {
	crm_warn("Ignoring topology registration for %s with invalid level %d",
	target, id);
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Invalid level number '%s' for topology level '%s'",
	pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
	""),
	pcmk__xe_id(level));
	return;
	}

	/* Find or create topology table entry */
	tp = g_hash_table_lookup(topology, target);
	if (tp == NULL) {
	tp = pcmk__assert_alloc(1, sizeof(stonith_topology_t));

	tp->kind = mode;
	tp->target = target;
	tp->target_value = crm_element_value_copy(level, PCMK_XA_TARGET_VALUE);
	tp->target_pattern = crm_element_value_copy(level,
	PCMK_XA_TARGET_PATTERN);
	tp->target_attribute = crm_element_value_copy(level,
	PCMK_XA_TARGET_ATTRIBUTE);

	g_hash_table_replace(topology, tp->target, tp);
	crm_trace("Added %s (%d) to the topology (%d active entries)",
	target, (int) mode, g_hash_table_size(topology));
	} else {
	free(target);
	}

	if (tp->levels[id] != NULL) {
	crm_info("Adding to the existing %s[%d] topology entry",
	tp->target, id);
	}

	devices = parse_device_list(crm_element_value(level, PCMK_XA_DEVICES));
	for (dIter = devices; dIter; dIter = dIter->next) {
	const char *device = dIter->value;

	crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
	tp->levels[id] = g_list_append(tp->levels[id], pcmk__str_copy(device));
	}
	stonith_key_value_freeall(devices, 1, 1);

	{
	int nlevels = count_active_levels(tp);

	crm_info("Target %s has %d active fencing level%s",
	tp->target, nlevels, pcmk__plural_s(nlevels));
	}

	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	}

	/*!
	* \internal
	* \brief Unregister a fencing topology level for a target
	*
	* Given an XML request specifying the target name and level index (or 0 for all
	* levels), this will remove any corresponding entry for the target from the
	* global topology table.
	*
	* \param[in] msg XML request for STONITH level registration
	* \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]"
	* \param[out] result Where to set result of unregistration
	*/
	void
	fenced_unregister_level(xmlNode msg, char *desc,
	pcmk__action_result_t *result)
	{
	int id = -1;
	stonith_topology_t *tp;
	char *target;
	xmlNode *level = NULL;

	CRM_CHECK(result != NULL, return);

	level = unpack_level_request(msg, NULL, &target, &id, desc);
	if (level == NULL) {
	fenced_set_protocol_error(result);
	return;
	}

	// Ensure level ID is in allowed range
	if ((id < 0) \|\| (id >= ST_LEVEL_MAX)) {
	crm_warn("Ignoring topology unregistration for %s with invalid level %d",
	target, id);
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Invalid level number '%s' for topology level %s",
	pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
	"<null>"),

	// Client API doesn't add ID to unregistration XML
	pcmk__s(pcmk__xe_id(level), ""));
	return;
	}

	tp = g_hash_table_lookup(topology, target);
	if (tp == NULL) {
	guint nentries = g_hash_table_size(topology);

	crm_info("No fencing topology found for %s (%d active %s)",
	target, nentries,
	pcmk__plural_alt(nentries, "entry", "entries"));

	} else if (id == 0 && g_hash_table_remove(topology, target)) {
	guint nentries = g_hash_table_size(topology);

	crm_info("Removed all fencing topology entries related to %s "
	"(%d active %s remaining)", target, nentries,
	pcmk__plural_alt(nentries, "entry", "entries"));

	} else if (tp->levels[id] != NULL) {
	guint nlevels;

	g_list_free_full(tp->levels[id], free);
	tp->levels[id] = NULL;

	nlevels = count_active_levels(tp);
	crm_info("Removed level %d from fencing topology for %s "
	"(%d active level%s remaining)",
	id, target, nlevels, pcmk__plural_s(nlevels));
	}

	free(target);
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	}

	static char *
	list_to_string(GList list, const char delim, gboolean terminate_with_delim)
	{
	int max = g_list_length(list);
	size_t delim_len = delim?strlen(delim):0;
	size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
	char *rv;
	GList *gIter;

	char *pos = NULL;
	const char *lead_delim = "";

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	const char value = (const char ) gIter->data;

	alloc_size += strlen(value);
	}

	rv = pcmk__assert_alloc(alloc_size, sizeof(char));
	pos = rv;

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	const char value = (const char ) gIter->data;

	pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
	lead_delim = delim;
	}

	if (max && terminate_with_delim) {
	sprintf(pos, "%s", delim);
	}

	return rv;
	}

	/*!
	* \internal
	* \brief Execute a fence agent action directly (and asynchronously)
	*
	* Handle a STONITH_OP_EXEC API message by scheduling a requested agent action
	* directly on a specified device. Only list, monitor, and status actions are
	* expected to use this call, though it should work with any agent command.
	*
	* \param[in] msg Request XML specifying action
	* \param[out] result Where to store result of action
	*
	* \note If the action is monitor, the device must be registered via the API
	* (CIB registration is not sufficient), because monitor should not be
	* possible unless the device is "started" (API registered).
	*/
	static void
	execute_agent_action(xmlNode msg, pcmk__action_result_t result)
	{
	xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, msg, LOG_ERR);
	xmlNode *op = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, msg,
	LOG_ERR);
	const char *id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
	const char *action = crm_element_value(op, PCMK__XA_ST_DEVICE_ACTION);
	async_command_t *cmd = NULL;
	stonith_device_t *device = NULL;

	if ((id == NULL) \|\| (action == NULL)) {
	crm_info("Malformed API action request: device %s, action %s",
	(id? id : "not specified"),
	(action? action : "not specified"));
	fenced_set_protocol_error(result);
	return;
	}

	if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
	// Watchdog agent actions are implemented internally
	if (stonith_watchdog_timeout_ms <= 0) {
	pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"Watchdog fence device not configured");
	return;

	} else if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	pcmk__set_result_output(result,
	list_to_string(stonith_watchdog_targets,
	"\n", TRUE),
	NULL);
	return;

	} else if (pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return;
	}
	}

	device = g_hash_table_lookup(device_list, id);
	if (device == NULL) {
	crm_info("Ignoring API '%s' action request because device %s not found",
	action, id);
	pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"'%s' not found", id);
	return;

	} else if (!device->api_registered
	&& (strcmp(action, PCMK_ACTION_MONITOR) == 0)) {
	// Monitors may run only on "started" (API-registered) devices
	crm_info("Ignoring API '%s' action request because device %s not active",
	action, id);
	pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"'%s' not active", id);
	return;
	}

	cmd = create_async_command(msg);
	if (cmd == NULL) {
	crm_log_xml_warn(msg, "invalid");
	fenced_set_protocol_error(result);
	return;
	}

	schedule_stonith_command(cmd, device);
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
	}

	static void
	search_devices_record_result(struct device_search_s search, const char device, gboolean can_fence)
	{
	search->replies_received++;
	if (can_fence && device) {
	if (search->support_action_only != st_device_supports_none) {
	stonith_device_t *dev = g_hash_table_lookup(device_list, device);
	if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
	return;
	}
	}
	search->capable = g_list_append(search->capable,
	pcmk__str_copy(device));
	}

	if (search->replies_needed == search->replies_received) {

	guint ndevices = g_list_length(search->capable);

	crm_debug("Search found %d device%s that can perform '%s' targeting %s",
	ndevices, pcmk__plural_s(ndevices),
	(search->action? search->action : "unknown action"),
	(search->host? search->host : "any node"));

	search->callback(search->capable, search->user_data);
	free(search->host);
	free(search->action);
	free(search);
	}
	}

	/*!
	* \internal
	* \brief Check whether the local host is allowed to execute a fencing action
	*
	* \param[in] device Fence device to check
	* \param[in] action Fence action to check
	* \param[in] target Hostname of fence target
	* \param[in] allow_suicide Whether self-fencing is allowed for this operation
	*
	* \return TRUE if local host is allowed to execute action, FALSE otherwise
	*/
	static gboolean
	localhost_is_eligible(const stonith_device_t device, const char action,
	const char *target, gboolean allow_suicide)
	{
	gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname,
	pcmk__str_casei);

	if ((device != NULL) && (action != NULL)
	&& (device->on_target_actions != NULL)
	&& (strstr((const char*) device->on_target_actions->str,
	action) != NULL)) {

	if (!localhost_is_target) {
	crm_trace("Operation '%s' using %s can only be executed for local "
	"host, not %s", action, device->id, target);
	return FALSE;
	}

	} else if (localhost_is_target && !allow_suicide) {
	crm_trace("'%s' operation does not support self-fencing", action);
	return FALSE;
	}
	return TRUE;
	}

	/*!
	* \internal
	* \brief Check if local node is allowed to execute (possibly remapped) action
	*
	* \param[in] device Fence device to check
	* \param[in] action Fence action to check
	* \param[in] target Node name of fence target
	* \param[in] allow_self Whether self-fencing is allowed for this operation
	*
	* \return true if local node is allowed to execute \p action or any actions it
	* might be remapped to, otherwise false
	*/
	static bool
	localhost_is_eligible_with_remap(const stonith_device_t *device,
	const char action, const char target,
	gboolean allow_self)
	{
	// Check exact action
	if (localhost_is_eligible(device, action, target, allow_self)) {
	return true;
	}

	// Check potential remaps

	if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
	/* "reboot" might get remapped to "off" then "on", so even if reboot is
	* disallowed, return true if either of those is allowed. We'll report
	* the disallowed actions with the results. We never allow self-fencing
	* for remapped "on" actions because the target is off at that point.
	*/
	if (localhost_is_eligible(device, PCMK_ACTION_OFF, target, allow_self)
	\|\| localhost_is_eligible(device, PCMK_ACTION_ON, target, FALSE)) {
	return true;
	}
	}

	return false;
	}

	static void
	can_fence_host_with_device(stonith_device_t *dev,
	struct device_search_s *search)
	{
	gboolean can = FALSE;
	const char *check_type = "Internal bug";
	const char *target = NULL;
	const char *alias = NULL;
	const char *dev_id = "Unspecified device";
	const char *action = (search == NULL)? NULL : search->action;

	CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results);

	if (dev->id != NULL) {
	dev_id = dev->id;
	}

	target = search->host;
	if (target == NULL) {
	can = TRUE;
	check_type = "No target";
	goto search_report_results;
	}

	/* Answer immediately if the device does not support the action
	* or the local node is not allowed to perform it
	*/
	if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)
	&& !pcmk_is_set(dev->flags, st_device_supports_on)) {
	check_type = "Agent does not support 'on'";
	goto search_report_results;

	} else if (!localhost_is_eligible_with_remap(dev, action, target,
	search->allow_suicide)) {
	check_type = "This node is not allowed to execute action";
	goto search_report_results;
	}

	// Check eligibility as specified by pcmk_host_check
	check_type = target_list_type(dev);
	alias = g_hash_table_lookup(dev->aliases, target);
	if (pcmk__str_eq(check_type, PCMK_VALUE_NONE, pcmk__str_casei)) {
	can = TRUE;

	} else if (pcmk__str_eq(check_type, PCMK_VALUE_STATIC_LIST,
	pcmk__str_casei)) {

	if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) {
	can = TRUE;
	} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
	&& g_hash_table_lookup(dev->aliases, target)) {
	can = TRUE;
	}

	} else if (pcmk__str_eq(check_type, PCMK_VALUE_DYNAMIC_LIST,
	pcmk__str_casei)) {
	time_t now = time(NULL);

	if (dev->targets == NULL \|\| dev->targets_age + 60 < now) {
	int device_timeout = get_action_timeout(dev, PCMK_ACTION_LIST,
	search->per_device_timeout);

	if (device_timeout > search->per_device_timeout) {
	crm_notice("Since the pcmk_list_timeout (%ds) parameter of %s "
	"is larger than " PCMK_OPT_STONITH_TIMEOUT
	" (%ds), timeout may occur",
	device_timeout, dev_id, search->per_device_timeout);
	}

	crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
	check_type, dev_id, target, action);

	schedule_internal_command(__func__, dev, PCMK_ACTION_LIST, NULL,
	search->per_device_timeout, search, dynamic_list_search_cb);

	/* we'll respond to this search request async in the cb */
	return;
	}

	if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets,
	pcmk__str_casei)) {
	can = TRUE;
	}

	} else if (pcmk__str_eq(check_type, PCMK_VALUE_STATUS, pcmk__str_casei)) {
	int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);

	if (device_timeout > search->per_device_timeout) {
	crm_notice("Since the pcmk_status_timeout (%ds) parameter of %s is "
	"larger than " PCMK_OPT_STONITH_TIMEOUT " (%ds), "
	"timeout may occur",
	device_timeout, dev_id, search->per_device_timeout);
	}

	crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
	check_type, dev_id, target, action);
	schedule_internal_command(__func__, dev, PCMK_ACTION_STATUS, target,
	search->per_device_timeout, search, status_search_cb);
	/* we'll respond to this search request async in the cb */
	return;
	} else {
	crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
	check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
	}

	search_report_results:
	crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s",
	dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"),
	pcmk__s(target, "unspecified target"),
	(alias == NULL)? "" : " (as '", pcmk__s(alias, ""),
	(alias == NULL)? "" : "')", check_type);
	search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can);
	}

	static void
	search_devices(gpointer key, gpointer value, gpointer user_data)
	{
	stonith_device_t *dev = value;
	struct device_search_s *search = user_data;

	can_fence_host_with_device(dev, search);
	}

	#define DEFAULT_QUERY_TIMEOUT 20
	static void
	get_capable_devices(const char host, const char action, int timeout, bool suicide, void *user_data,
	void (callback) (GList devices, void *user_data), uint32_t support_action_only)
	{
	struct device_search_s *search;
	guint ndevices = g_hash_table_size(device_list);

	if (ndevices == 0) {
	callback(NULL, user_data);
	return;
	}

	search = pcmk__assert_alloc(1, sizeof(struct device_search_s));

	search->host = pcmk__str_copy(host);
	search->action = pcmk__str_copy(action);
	search->per_device_timeout = timeout;
	search->allow_suicide = suicide;
	search->callback = callback;
	search->user_data = user_data;
	search->support_action_only = support_action_only;

	/* We are guaranteed this many replies, even if a device is
	* unregistered while the search is in progress.
	*/
	search->replies_needed = ndevices;

	crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
	ndevices, pcmk__plural_s(ndevices),
	(search->action? search->action : "unknown action"),
	(search->host? search->host : "any node"));
	g_hash_table_foreach(device_list, search_devices, search);
	}

	struct st_query_data {
	xmlNode *reply;
	char *remote_peer;
	char *client_id;
	char *target;
	char *action;
	int call_options;
	};

	/*!
	* \internal
	* \brief Add action-specific attributes to query reply XML
	*
	* \param[in,out] xml XML to add attributes to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	*/
	static void
	add_action_specific_attributes(xmlNode xml, const char action,
	const stonith_device_t *device,
	const char *target)
	{
	int action_specific_timeout;
	int delay_max;
	int delay_base;

	CRM_CHECK(xml && action && device, return);

	// PCMK__XA_ST_REQUIRED is currently used only for unfencing
	if (is_action_required(action, device)) {
	crm_trace("Action '%s' is required using %s", action, device->id);
	crm_xml_add_int(xml, PCMK__XA_ST_REQUIRED, 1);
	}

	// pcmk_<action>_timeout if configured
	action_specific_timeout = get_action_timeout(device, action, 0);
	if (action_specific_timeout) {
	crm_trace("Action '%s' has timeout %ds using %s",
	action, action_specific_timeout, device->id);
	crm_xml_add_int(xml, PCMK__XA_ST_ACTION_TIMEOUT,
	action_specific_timeout);
	}

	delay_max = get_action_delay_max(device, action);
	if (delay_max > 0) {
	crm_trace("Action '%s' has maximum random delay %ds using %s",
	action, delay_max, device->id);
	crm_xml_add_int(xml, PCMK__XA_ST_DELAY_MAX, delay_max);
	}

	delay_base = get_action_delay_base(device, action, target);
	if (delay_base > 0) {
	crm_xml_add_int(xml, PCMK__XA_ST_DELAY_BASE, delay_base);
	}

	if ((delay_max > 0) && (delay_base == 0)) {
	crm_trace("Action '%s' has maximum random delay %ds using %s",
	action, delay_max, device->id);
	} else if ((delay_max == 0) && (delay_base > 0)) {
	crm_trace("Action '%s' has a static delay of %ds using %s",
	action, delay_base, device->id);
	} else if ((delay_max > 0) && (delay_base > 0)) {
	crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen "
	"maximum delay of %ds using %s",
	action, delay_base, delay_max, device->id);
	}
	}

	/*!
	* \internal
	* \brief Add "disallowed" attribute to query reply XML if appropriate
	*
	* \param[in,out] xml XML to add attribute to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	* \param[in] allow_suicide Whether self-fencing is allowed
	*/
	static void
	add_disallowed(xmlNode xml, const char action, const stonith_device_t *device,
	const char *target, gboolean allow_suicide)
	{
	if (!localhost_is_eligible(device, action, target, allow_suicide)) {
	crm_trace("Action '%s' using %s is disallowed for local host",
	action, device->id);
	pcmk__xe_set_bool_attr(xml, PCMK__XA_ST_ACTION_DISALLOWED, true);
	}
	}

	/*!
	* \internal
	* \brief Add child element with action-specific values to query reply XML
	*
	* \param[in,out] xml XML to add attribute to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	* \param[in] allow_suicide Whether self-fencing is allowed
	*/
	static void
	add_action_reply(xmlNode xml, const char action,
	const stonith_device_t device, const char target,
	gboolean allow_suicide)
	{
	xmlNode *child = pcmk__xe_create(xml, PCMK__XE_ST_DEVICE_ACTION);

	crm_xml_add(child, PCMK_XA_ID, action);
	add_action_specific_attributes(child, action, device, target);
	add_disallowed(child, action, device, target, allow_suicide);
	}

	/*!
	* \internal
	* \brief Send a reply to a CPG peer or IPC client
	*
	* \param[in] reply XML reply to send
	* \param[in] call_options Send synchronously if st_opt_sync_call is set
	* \param[in] remote_peer If not NULL, name of peer node to send CPG reply
	* \param[in,out] client If not NULL, client to send IPC reply
	*/
	static void
	stonith_send_reply(const xmlNode *reply, int call_options,
	const char remote_peer, pcmk__client_t client)
	{
	CRM_CHECK((reply != NULL) && ((remote_peer != NULL) \|\| (client != NULL)),
	return);

	if (remote_peer == NULL) {
	do_local_reply(reply, client, call_options);
	} else {
	const pcmk__node_status_t *node =
	pcmk__get_node(0, remote_peer, NULL,
	pcmk__node_search_cluster_member);

	pcmk__cluster_send_message(node, crm_msg_stonith_ng, reply);
	}
	}

	static void
	stonith_query_capable_device_cb(GList * devices, void *user_data)
	{
	struct st_query_data *query = user_data;
	int available_devices = 0;
	xmlNode *wrapper = NULL;
	xmlNode *list = NULL;
	GList *lpc = NULL;
	pcmk__client_t *client = NULL;

	if (query->client_id != NULL) {
	client = pcmk__find_client_by_id(query->client_id);
	if ((client == NULL) && (query->remote_peer == NULL)) {
	crm_trace("Skipping reply to %s: no longer a client",
	query->client_id);
	goto done;
	}
	}

	// Pack the results into XML
	wrapper = pcmk__xe_create(query->reply, PCMK__XE_ST_CALLDATA);
	list = pcmk__xe_create(wrapper, __func__);
	crm_xml_add(list, PCMK__XA_ST_TARGET, query->target);

	for (lpc = devices; lpc != NULL; lpc = lpc->next) {
	stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
	const char *action = query->action;
	xmlNode *dev = NULL;

	if (!device) {
	/* It is possible the device got unregistered while
	* determining who can fence the target */
	continue;
	}

	available_devices++;

	dev = pcmk__xe_create(list, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(dev, PCMK_XA_ID, device->id);
	crm_xml_add(dev, PCMK__XA_NAMESPACE, device->namespace);
	crm_xml_add(dev, PCMK_XA_AGENT, device->agent);

	// Has had successful monitor, list, or status on this node
	crm_xml_add_int(dev, PCMK__XA_ST_MONITOR_VERIFIED, device->verified);

	crm_xml_add_int(dev, PCMK__XA_ST_DEVICE_SUPPORT_FLAGS, device->flags);

	/* If the originating fencer wants to reboot the node, and we have a
	* capable device that doesn't support "reboot", remap to "off" instead.
	*/
	if (!pcmk_is_set(device->flags, st_device_supports_reboot)
	&& pcmk__str_eq(query->action, PCMK_ACTION_REBOOT,
	pcmk__str_none)) {
	crm_trace("%s doesn't support reboot, using values for off instead",
	device->id);
	action = PCMK_ACTION_OFF;
	}

	/* Add action-specific values if available */
	add_action_specific_attributes(dev, action, device, query->target);
	if (pcmk__str_eq(query->action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
	/* A "reboot" might get remapped to "off" then "on", so after
	* sending the "reboot"-specific values in the main element, we add
	* sub-elements for "off" and "on" values.
	*
	* We short-circuited earlier if "reboot", "off" and "on" are all
	* disallowed for the local host. However if only one or two are
	* disallowed, we send back the results and mark which ones are
	* disallowed. If "reboot" is disallowed, this might cause problems
	* with older fencer versions, which won't check for it. Older
	* versions will ignore "off" and "on", so they are not a problem.
	*/
	add_disallowed(dev, action, device, query->target,
	pcmk_is_set(query->call_options, st_opt_allow_suicide));
	add_action_reply(dev, PCMK_ACTION_OFF, device, query->target,
	pcmk_is_set(query->call_options, st_opt_allow_suicide));
	add_action_reply(dev, PCMK_ACTION_ON, device, query->target, FALSE);
	}

	/* A query without a target wants device parameters */
	if (query->target == NULL) {
	xmlNode *attrs = pcmk__xe_create(dev, PCMK__XE_ATTRIBUTES);

	g_hash_table_foreach(device->params, hash2field, attrs);
	}
	}

	crm_xml_add_int(list, PCMK__XA_ST_AVAILABLE_DEVICES, available_devices);
	if (query->target) {
	crm_debug("Found %d matching device%s for target '%s'",
	available_devices, pcmk__plural_s(available_devices),
	query->target);
	} else {
	crm_debug("%d device%s installed",
	available_devices, pcmk__plural_s(available_devices));
	}

	crm_log_xml_trace(list, "query-result");

	stonith_send_reply(query->reply, query->call_options, query->remote_peer,
	client);

	done:
	pcmk__xml_free(query->reply);
	free(query->remote_peer);
	free(query->client_id);
	free(query->target);
	free(query->action);
	free(query);
	g_list_free_full(devices, free);
	}

	/*!
	* \internal
	* \brief Log the result of an asynchronous command
	*
	* \param[in] cmd Command the result is for
	* \param[in] result Result of command
	* \param[in] pid Process ID of command, if available
	* \param[in] next Alternate device that will be tried if command failed
	* \param[in] op_merged Whether this command was merged with an earlier one
	*/
	static void
	log_async_result(const async_command_t *cmd,
	const pcmk__action_result_t *result,
	int pid, const char *next, bool op_merged)
	{
	int log_level = LOG_ERR;
	int output_log_level = LOG_NEVER;
	guint devices_remaining = g_list_length(cmd->next_device_iter);

	GString *msg = g_string_sized_new(80); // Reasonable starting size

	// Choose log levels appropriately if we have a result
	if (pcmk__result_ok(result)) {
	log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
	if ((result->action_stdout != NULL)
	&& !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
	pcmk__str_none)) {
	output_log_level = LOG_DEBUG;
	}
	next = NULL;
	} else {
	log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
	if ((result->action_stdout != NULL)
	&& !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
	pcmk__str_none)) {
	output_log_level = LOG_WARNING;
	}
	}

	// Build the log message piece by piece
	pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL);
	if (pid != 0) {
	g_string_append_printf(msg, "[%d] ", pid);
	}
	if (cmd->target != NULL) {
	pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL);
	}
	if (cmd->device != NULL) {
	pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL);
	}

	// Add exit status or execution status as appropriate
	if (result->execution_status == PCMK_EXEC_DONE) {
	g_string_append_printf(msg, "returned %d", result->exit_status);
	} else {
	pcmk__g_strcat(msg, "could not be executed: ",
	pcmk_exec_status_str(result->execution_status), NULL);
	}

	// Add exit reason and next device if appropriate
	if (result->exit_reason != NULL) {
	pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL);
	}
	if (next != NULL) {
	pcmk__g_strcat(msg, ", retrying with ", next, NULL);
	}
	if (devices_remaining > 0) {
	g_string_append_printf(msg, " (%u device%s remaining)",
	(unsigned int) devices_remaining,
	pcmk__plural_s(devices_remaining));
	}
	g_string_append_printf(msg, " " QB_XS " %scall %d from %s",
	(op_merged? "merged " : ""), cmd->id,
	cmd->client_name);

	// Log the result
	do_crm_log(log_level, "%s", msg->str);
	g_string_free(msg, TRUE);

	// Log the output (which may have multiple lines), if appropriate
	if (output_log_level != LOG_NEVER) {
	char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);

	crm_log_output(output_log_level, prefix, result->action_stdout);
	free(prefix);
	}
	}

	/*!
	* \internal
	* \brief Reply to requester after asynchronous command completion
	*
	* \param[in] cmd Command that completed
	* \param[in] result Result of command
	* \param[in] pid Process ID of command, if available
	* \param[in] merged If true, command was merged with another, not executed
	*/
	static void
	send_async_reply(const async_command_t cmd, const pcmk__action_result_t result,
	int pid, bool merged)
	{
	xmlNode *reply = NULL;
	pcmk__client_t *client = NULL;

	CRM_CHECK((cmd != NULL) && (result != NULL), return);

	log_async_result(cmd, result, pid, NULL, merged);

	if (cmd->client != NULL) {
	client = pcmk__find_client_by_id(cmd->client);
	if ((client == NULL) && (cmd->origin == NULL)) {
	crm_trace("Skipping reply to %s: no longer a client", cmd->client);
	return;
	}
	}

	reply = construct_async_reply(cmd, result);
	if (merged) {
	pcmk__xe_set_bool_attr(reply, PCMK__XA_ST_OP_MERGED, true);
	}

	if (!stand_alone && pcmk__is_fencing_action(cmd->action)
	&& pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
	/* The target was also the originator, so broadcast the result on its
	* behalf (since it will be unable to).
	*/
	crm_trace("Broadcast '%s' result for %s (target was also originator)",
	cmd->action, cmd->target);
	crm_xml_add(reply, PCMK__XA_SUBT, PCMK__VALUE_BROADCAST);
	crm_xml_add(reply, PCMK__XA_ST_OP, STONITH_OP_NOTIFY);
	pcmk__cluster_send_message(NULL, crm_msg_stonith_ng, reply);
	} else {
	// Reply only to the originator
	stonith_send_reply(reply, cmd->options, cmd->origin, client);
	}

	crm_log_xml_trace(reply, "Reply");
	pcmk__xml_free(reply);

	if (stand_alone) {
	/* Do notification with a clean data object */
	xmlNode *notify_data = pcmk__xe_create(NULL, PCMK__XE_ST_NOTIFY_FENCE);

	stonith__xe_set_result(notify_data, result);
	crm_xml_add(notify_data, PCMK__XA_ST_TARGET, cmd->target);
	crm_xml_add(notify_data, PCMK__XA_ST_OP, cmd->op);
	crm_xml_add(notify_data, PCMK__XA_ST_DELEGATE, "localhost");
	crm_xml_add(notify_data, PCMK__XA_ST_DEVICE_ID, cmd->device);
	crm_xml_add(notify_data, PCMK__XA_ST_REMOTE_OP, cmd->remote_op_id);
	crm_xml_add(notify_data, PCMK__XA_ST_ORIGIN, cmd->client);

	fenced_send_notification(PCMK__VALUE_ST_NOTIFY_FENCE, result,
	notify_data);
	fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY, NULL, NULL);
	}
	}

	static void
	cancel_stonith_command(async_command_t * cmd)
	{
	stonith_device_t *device = cmd_device(cmd);

	if (device) {
	crm_trace("Cancel scheduled '%s' action using %s",
	cmd->action, device->id);
	device->pending_ops = g_list_remove(device->pending_ops, cmd);
	}
	}

	/*!
	* \internal
	* \brief Cancel and reply to any duplicates of a just-completed operation
	*
	* Check whether any fencing operations are scheduled to do the same thing as
	* one that just succeeded. If so, rather than performing the same operation
	* twice, return the result of this operation for all matching pending commands.
	*
	* \param[in,out] cmd Fencing operation that just succeeded
	* \param[in] result Result of \p cmd
	* \param[in] pid If nonzero, process ID of agent invocation (for logs)
	*
	* \note Duplicate merging will do the right thing for either type of remapped
	* reboot. If the executing fencer remapped an unsupported reboot to off,
	* then cmd->action will be "reboot" and will be merged with any other
	* reboot requests. If the originating fencer remapped a topology reboot
	* to off then on, we will get here once with cmd->action "off" and once
	* with "on", and they will be merged separately with similar requests.
	*/
	static void
	reply_to_duplicates(async_command_t cmd, const pcmk__action_result_t result,
	int pid)
	{
	GList *next = NULL;

	for (GList *iter = cmd_list; iter != NULL; iter = next) {
	async_command_t *cmd_other = iter->data;

	next = iter->next; // We might delete this entry, so grab next now

	if (cmd == cmd_other) {
	continue;
	}

	/* A pending operation matches if:
	* 1. The client connections are different.
	* 2. The target is the same.
	* 3. The fencing action is the same.
	* 4. The device scheduled to execute the action is the same.
	*/
	if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) \|\|
	!pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) \|\|
	!pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) \|\|
	!pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {

	continue;
	}

	crm_notice("Merging fencing action '%s'%s%s originating from "
	"client %s with identical fencing request from client %s",
	cmd_other->action,
	(cmd_other->target == NULL)? "" : " targeting ",
	pcmk__s(cmd_other->target, ""), cmd_other->client_name,
	cmd->client_name);

	// Stop tracking the duplicate, send its result, and cancel it
	cmd_list = g_list_remove_link(cmd_list, iter);
	send_async_reply(cmd_other, result, pid, true);
	cancel_stonith_command(cmd_other);

	free_async_command(cmd_other);
	g_list_free_1(iter);
	}
	}

	/*!
	* \internal
	* \brief Return the next required device (if any) for an operation
	*
	* \param[in,out] cmd Fencing operation that just succeeded
	*
	* \return Next device required for action if any, otherwise NULL
	*/
	static stonith_device_t *
	next_required_device(async_command_t *cmd)
	{
	for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
	stonith_device_t *next_device = g_hash_table_lookup(device_list,
	iter->data);

	if (is_action_required(cmd->action, next_device)) {
	/* This is only called for successful actions, so it's OK to skip
	* non-required devices.
	*/
	cmd->next_device_iter = iter->next;
	return next_device;
	}
	}
	return NULL;
	}

	static void
	st_child_done(int pid, const pcmk__action_result_t result, void user_data)
	{
	async_command_t *cmd = user_data;

	stonith_device_t *device = NULL;
	stonith_device_t *next_device = NULL;

	CRM_CHECK(cmd != NULL, return);

	device = cmd_device(cmd);
	cmd->active_on = NULL;

	/* The device is ready to do something else now */
	if (device) {
	if (!device->verified && pcmk__result_ok(result)
	&& pcmk__strcase_any_of(cmd->action, PCMK_ACTION_LIST,
	PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS,
	NULL)) {

	device->verified = TRUE;
	}

	mainloop_set_trigger(device->work);
	}

	if (pcmk__result_ok(result)) {
	next_device = next_required_device(cmd);

	} else if ((cmd->next_device_iter != NULL)
	&& !is_action_required(cmd->action, device)) {
	/* if this device didn't work out, see if there are any others we can try.
	* if the failed device was 'required', we can't pick another device. */
	next_device = g_hash_table_lookup(device_list,
	cmd->next_device_iter->data);
	cmd->next_device_iter = cmd->next_device_iter->next;
	}

	if (next_device == NULL) {
	send_async_reply(cmd, result, pid, false);
	if (pcmk__result_ok(result)) {
	reply_to_duplicates(cmd, result, pid);
	}
	free_async_command(cmd);

	} else { // This operation requires more fencing
	log_async_result(cmd, result, pid, next_device->id, false);
	schedule_stonith_command(cmd, next_device);
	}
	}

	static gint
	sort_device_priority(gconstpointer a, gconstpointer b)
	{
	const stonith_device_t *dev_a = a;
	const stonith_device_t *dev_b = b;

	if (dev_a->priority > dev_b->priority) {
	return -1;
	} else if (dev_a->priority < dev_b->priority) {
	return 1;
	}
	return 0;
	}

	static void
	stonith_fence_get_devices_cb(GList * devices, void *user_data)
	{
	async_command_t *cmd = user_data;
	stonith_device_t *device = NULL;
	guint ndevices = g_list_length(devices);

	crm_info("Found %d matching device%s for target '%s'",
	ndevices, pcmk__plural_s(ndevices), cmd->target);

	if (devices != NULL) {
	/* Order based on priority */
	devices = g_list_sort(devices, sort_device_priority);
	device = g_hash_table_lookup(device_list, devices->data);
	}

	if (device == NULL) { // No device found
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"No device configured for target '%s'",
	cmd->target);
	send_async_reply(cmd, &result, 0, false);
	pcmk__reset_result(&result);
	free_async_command(cmd);
	g_list_free_full(devices, free);

	} else { // Device found, schedule it for fencing
	cmd->device_list = devices;
	cmd->next_device_iter = devices->next;
	schedule_stonith_command(cmd, device);
	}
	}

	/*!
	* \internal
	* \brief Execute a fence action via the local node
	*
	* \param[in] msg Fencing request
	* \param[out] result Where to store result of fence action
	*/
	static void
	fence_locally(xmlNode msg, pcmk__action_result_t result)
	{
	const char *device_id = NULL;
	stonith_device_t *device = NULL;
	async_command_t *cmd = NULL;
	xmlNode *dev = NULL;

	CRM_CHECK((msg != NULL) && (result != NULL), return);

	dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, msg, LOG_ERR);

	cmd = create_async_command(msg);
	if (cmd == NULL) {
	crm_log_xml_warn(msg, "invalid");
	fenced_set_protocol_error(result);
	return;
	}

	device_id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
	if (device_id != NULL) {
	device = g_hash_table_lookup(device_list, device_id);
	if (device == NULL) {
	crm_err("Requested device '%s' is not available", device_id);
	pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"Requested device '%s' not found", device_id);
	return;
	}
	schedule_stonith_command(cmd, device);

	} else {
	const char *host = crm_element_value(dev, PCMK__XA_ST_TARGET);

	if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
	int nodeid = 0;
	pcmk__node_status_t *node = NULL;

	pcmk__scan_min_int(host, &nodeid, 0);
	node = pcmk__search_node_caches(nodeid, NULL,
	pcmk__node_search_any
	\|pcmk__node_search_cluster_cib);
	if (node != NULL) {
	host = node->uname;
	}
	}

	/* If we get to here, then self-fencing is implicitly allowed */
	get_capable_devices(host, cmd->action, cmd->default_timeout,
	TRUE, cmd, stonith_fence_get_devices_cb,
	fenced_support_flag(cmd->action));
	}

	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
	}

	/*!
	* \internal
	* \brief Build an XML reply for a fencing operation
	*
	* \param[in] request Request that reply is for
	* \param[in] data If not NULL, add to reply as call data
	* \param[in] result Full result of fencing operation
	*
	* \return Newly created XML reply
	* \note The caller is responsible for freeing the result.
	* \note This has some overlap with construct_async_reply(), but that copies
	* values from an async_command_t, whereas this one copies them from the
	* request.
	*/
	xmlNode *
	fenced_construct_reply(const xmlNode request, xmlNode data,
	const pcmk__action_result_t *result)
	{
	xmlNode *reply = NULL;

	reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY);

	crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	stonith__xe_set_result(reply, result);

	if (request == NULL) {
	/* Most likely, this is the result of a stonith operation that was
	* initiated before we came up. Unfortunately that means we lack enough
	* information to provide clients with a full result.
	*
	* @TODO Maybe synchronize this information at start-up?
	*/
	crm_warn("Missing request information for client notifications for "
	"operation with result '%s' (initiated before we came up?)",
	pcmk_exec_status_str(result->execution_status));

	} else {
	const char *name = NULL;
	const char *value = NULL;

	// Attributes to copy from request to reply
	const char *names[] = {
	PCMK__XA_ST_OP,
	PCMK__XA_ST_CALLID,
	PCMK__XA_ST_CLIENTID,
	PCMK__XA_ST_CLIENTNAME,
	PCMK__XA_ST_REMOTE_OP,
	PCMK__XA_ST_CALLOPT,
	};

	for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
	name = names[lpc];
	value = crm_element_value(request, name);
	crm_xml_add(reply, name, value);
	}
	if (data != NULL) {
	xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_ST_CALLDATA);

	pcmk__xml_copy(wrapper, data);
	}
	}
	return reply;
	}

	/*!
	* \internal
	* \brief Build an XML reply to an asynchronous fencing command
	*
	* \param[in] cmd Fencing command that reply is for
	* \param[in] result Command result
	*/
	static xmlNode *
	construct_async_reply(const async_command_t *cmd,
	const pcmk__action_result_t *result)
	{
	xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_ST_REPLY);

	crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(reply, PCMK__XA_ST_OP, cmd->op);
	crm_xml_add(reply, PCMK__XA_ST_DEVICE_ID, cmd->device);
	crm_xml_add(reply, PCMK__XA_ST_REMOTE_OP, cmd->remote_op_id);
	crm_xml_add(reply, PCMK__XA_ST_CLIENTID, cmd->client);
	crm_xml_add(reply, PCMK__XA_ST_CLIENTNAME, cmd->client_name);
	crm_xml_add(reply, PCMK__XA_ST_TARGET, cmd->target);
	crm_xml_add(reply, PCMK__XA_ST_DEVICE_ACTION, cmd->op);
	crm_xml_add(reply, PCMK__XA_ST_ORIGIN, cmd->origin);
	crm_xml_add_int(reply, PCMK__XA_ST_CALLID, cmd->id);
	crm_xml_add_int(reply, PCMK__XA_ST_CALLOPT, cmd->options);

	stonith__xe_set_result(reply, result);
	return reply;
	}

	bool fencing_peer_active(pcmk__node_status_t *peer)
	{
	if (peer == NULL) {
	return FALSE;
	} else if (peer->uname == NULL) {
	return FALSE;
	} else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
	return TRUE;
	}
	return FALSE;
	}

	void
	set_fencing_completed(remote_fencing_op_t *op)
	{
	struct timespec tv;

	qb_util_timespec_from_epoch_get(&tv);
	op->completed = tv.tv_sec;
	op->completed_nsec = tv.tv_nsec;
	}

	/*!
	* \internal
	* \brief Look for alternate node needed if local node shouldn't fence target
	*
	* \param[in] target Node that must be fenced
	*
	* \return Name of an alternate node that should fence \p target if any,
	* or NULL otherwise
	*/
	static const char *
	check_alternate_host(const char *target)
	{
	if (pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
	GHashTableIter gIter;
	pcmk__node_status_t *entry = NULL;

	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	if (fencing_peer_active(entry)
	&& !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) {
	crm_notice("Forwarding self-fencing request to %s",
	entry->uname);
	return entry->uname;
	}
	}
	crm_warn("Will handle own fencing because no peer can");
	}
	return NULL;
	}

	static void
	remove_relay_op(xmlNode * request)
	{
	xmlNode *dev = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, request,
	LOG_TRACE);
	const char *relay_op_id = NULL;
	const char *op_id = NULL;
	const char *client_name = NULL;
	const char *target = NULL;
	remote_fencing_op_t *relay_op = NULL;

	if (dev) {
	target = crm_element_value(dev, PCMK__XA_ST_TARGET);
	}

	relay_op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP_RELAY);
	op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP);
	client_name = crm_element_value(request, PCMK__XA_ST_CLIENTNAME);

	/* Delete RELAY operation. */
	if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
	relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);

	if (relay_op) {
	GHashTableIter iter;
	remote_fencing_op_t *list_op = NULL;
	g_hash_table_iter_init(&iter, stonith_remote_op_list);

	/* If the operation to be deleted is registered as a duplicate, delete the registration. */
	while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
	GList *dup_iter = NULL;
	if (list_op != relay_op) {
	for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
	remote_fencing_op_t *other = dup_iter->data;
	if (other == relay_op) {
	other->duplicates = g_list_remove(other->duplicates, relay_op);
	break;
	}
	}
	}
	}
	crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
	"replaced by op %s ('%s'%s%s for %s)",
	relay_op->id, relay_op->action,
	(relay_op->target == NULL)? "" : " targeting ",
	pcmk__s(relay_op->target, ""),
	relay_op->client_name, op_id, relay_op->action,
	(target == NULL)? "" : " targeting ", pcmk__s(target, ""),
	client_name);

	g_hash_table_remove(stonith_remote_op_list, relay_op_id);
	}
	}
	}

	/*!
	* \internal
	* \brief Check whether an API request was sent by a privileged user
	*
	* API commands related to fencing configuration may be done only by privileged
	* IPC users (i.e. root or hacluster), because all other users should go through
	* the CIB to have ACLs applied. If no client was given, this is a peer request,
	* which is always allowed.
	*
	* \param[in] c IPC client that sent request (or NULL if sent by CPG peer)
	* \param[in] op Requested API operation (for logging only)
	*
	* \return true if sender is peer or privileged client, otherwise false
	*/
	static inline bool
	is_privileged(const pcmk__client_t c, const char op)
	{
	if ((c == NULL) \|\| pcmk_is_set(c->flags, pcmk__client_privileged)) {
	return true;
	} else {
	crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
	pcmk__s(op, ""), pcmk__client_name(c));
	return false;
	}
	}

	// CRM_OP_REGISTER
	static xmlNode *
	handle_register_request(pcmk__request_t *request)
	{
	xmlNode *reply = pcmk__xe_create(NULL, "reply");

	CRM_ASSERT(request->ipc_client != NULL);
	crm_xml_add(reply, PCMK__XA_ST_OP, CRM_OP_REGISTER);
	crm_xml_add(reply, PCMK__XA_ST_CLIENTID, request->ipc_client->id);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	pcmk__set_request_flags(request, pcmk__request_reuse_options);
	return reply;
	}

	// STONITH_OP_EXEC
	static xmlNode *
	handle_agent_request(pcmk__request_t *request)
	{
	execute_agent_action(request->xml, &request->result);
	if (request->result.execution_status == PCMK_EXEC_PENDING) {
	return NULL;
	}
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_TIMEOUT_UPDATE
	static xmlNode *
	handle_update_timeout_request(pcmk__request_t *request)
	{
	const char *call_id = crm_element_value(request->xml, PCMK__XA_ST_CALLID);
	const char *client_id = crm_element_value(request->xml,
	PCMK__XA_ST_CLIENTID);
	int op_timeout = 0;

	crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &op_timeout);
	do_stonith_async_timeout_update(client_id, call_id, op_timeout);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return NULL;
	}

	// STONITH_OP_QUERY
	static xmlNode *
	handle_query_request(pcmk__request_t *request)
	{
	int timeout = 0;
	xmlNode *dev = NULL;
	const char *action = NULL;
	const char *target = NULL;
	const char *client_id = crm_element_value(request->xml,
	PCMK__XA_ST_CLIENTID);
	struct st_query_data *query = NULL;

	if (request->peer != NULL) {
	// Record it for the future notification
	create_remote_stonith_op(client_id, request->xml, TRUE);
	}

	/* Delete the DC node RELAY operation. */
	remove_relay_op(request->xml);

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);

	dev = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, request->xml,
	LOG_NEVER);
	if (dev != NULL) {
	const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);

	if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
	return NULL; // No query or reply necessary
	}
	target = crm_element_value(dev, PCMK__XA_ST_TARGET);
	action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
	}

	crm_log_xml_trace(request->xml, "Query");

	query = pcmk__assert_alloc(1, sizeof(struct st_query_data));

	query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
	query->remote_peer = pcmk__str_copy(request->peer);
	query->client_id = pcmk__str_copy(client_id);
	query->target = pcmk__str_copy(target);
	query->action = pcmk__str_copy(action);
	query->call_options = request->call_options;

	crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &timeout);
	get_capable_devices(target, action, timeout,
	pcmk_is_set(query->call_options, st_opt_allow_suicide),
	query, stonith_query_capable_device_cb, st_device_supports_none);
	return NULL;
	}

	// STONITH_OP_NOTIFY
	static xmlNode *
	handle_notify_request(pcmk__request_t *request)
	{
	const char *flag_name = NULL;

	CRM_ASSERT(request->ipc_client != NULL);
	flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_ACTIVATE);
	if (flag_name != NULL) {
	crm_debug("Enabling %s callbacks for client %s",
	flag_name, pcmk__request_origin(request));
	pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name));
	}

	flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_DEACTIVATE);
	if (flag_name != NULL) {
	crm_debug("Disabling %s callbacks for client %s",
	flag_name, pcmk__request_origin(request));
	pcmk__clear_client_flags(request->ipc_client,
	get_stonith_flag(flag_name));
	}

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	pcmk__set_request_flags(request, pcmk__request_reuse_options);

	return pcmk__ipc_create_ack(request->ipc_flags, PCMK__XE_ACK, NULL,
	CRM_EX_OK);
	}

	// STONITH_OP_RELAY
	static xmlNode *
	handle_relay_request(pcmk__request_t *request)
	{
	xmlNode *dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, request->xml,
	LOG_TRACE);

	crm_notice("Received forwarded fencing request from "
	"%s %s to fence (%s) peer %s",
	pcmk__request_origin_type(request),
	pcmk__request_origin(request),
	crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION),
	crm_element_value(dev, PCMK__XA_ST_TARGET));

	if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
	fenced_set_protocol_error(&request->result);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
	return NULL;
	}

	// STONITH_OP_FENCE
	static xmlNode *
	handle_fence_request(pcmk__request_t *request)
	{
	if ((request->peer != NULL) \|\| stand_alone) {
	fence_locally(request->xml, &request->result);

	} else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
	switch (fenced_handle_manual_confirmation(request->ipc_client,
	request->xml)) {
	case pcmk_rc_ok:
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
	NULL);
	break;
	case EINPROGRESS:
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
	NULL);
	break;
	default:
	fenced_set_protocol_error(&request->result);
	break;
	}

	} else {
	const char *alternate_host = NULL;
	xmlNode *dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, request->xml,
	LOG_TRACE);
	const char *target = crm_element_value(dev, PCMK__XA_ST_TARGET);
	const char *action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
	const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);

	if (request->ipc_client != NULL) {
	int tolerance = 0;

	crm_notice("Client %s wants to fence (%s) %s using %s",
	pcmk__request_origin(request), action,
	target, (device? device : "any device"));
	crm_element_value_int(dev, PCMK__XA_ST_TOLERANCE, &tolerance);
	if (stonith_check_fence_tolerance(tolerance, target, action)) {
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
	NULL);
	return fenced_construct_reply(request->xml, NULL,
	&request->result);
	}
	alternate_host = check_alternate_host(target);

	} else {
	crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
	request->peer, action, target,
	(device == NULL)? "(any)" : device);
	}

	if (alternate_host != NULL) {
	const char *client_id = NULL;
	remote_fencing_op_t *op = NULL;
	pcmk__node_status_t *node =
	pcmk__get_node(0, alternate_host, NULL,
	pcmk__node_search_cluster_member);

	if (request->ipc_client->id == 0) {
	client_id = crm_element_value(request->xml,
	PCMK__XA_ST_CLIENTID);
	} else {
	client_id = request->ipc_client->id;
	}

	/* Create a duplicate fencing operation to relay with the client ID.
	* When a query response is received, this operation should be
	* deleted to avoid keeping the duplicate around.
	*/
	op = create_remote_stonith_op(client_id, request->xml, FALSE);

	crm_xml_add(request->xml, PCMK__XA_ST_OP, STONITH_OP_RELAY);
	crm_xml_add(request->xml, PCMK__XA_ST_CLIENTID,
	request->ipc_client->id);
	crm_xml_add(request->xml, PCMK__XA_ST_REMOTE_OP, op->id);
	pcmk__cluster_send_message(node, crm_msg_stonith_ng, request->xml);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
	NULL);

	} else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
	FALSE) == NULL) {
	fenced_set_protocol_error(&request->result);

	} else {
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
	NULL);
	}
	}

	if (request->result.execution_status == PCMK_EXEC_PENDING) {
	return NULL;
	}
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_FENCE_HISTORY
	static xmlNode *
	handle_history_request(pcmk__request_t *request)
	{
	xmlNode *reply = NULL;
	xmlNode *data = NULL;

	stonith_fence_history(request->xml, &data, request->peer,
	request->call_options);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
	/* When the local node broadcasts its history, it sets
	* st_opt_discard_reply and doesn't need a reply.
	*/
	reply = fenced_construct_reply(request->xml, data, &request->result);
	}
	pcmk__xml_free(data);
	return reply;
	}

	// STONITH_OP_DEVICE_ADD
	static xmlNode *
	handle_device_add_request(pcmk__request_t *request)
	{
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
	xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, request->xml,
	LOG_ERR);

	if (is_privileged(request->ipc_client, op)) {
	int rc = stonith_device_register(dev, FALSE);

	pcmk__set_result(&request->result,
	((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
	stonith__legacy2status(rc),
	((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
	} else {
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must register device via CIB");
	}
	fenced_send_config_notification(op, &request->result,
	(dev == NULL)? NULL : pcmk__xe_id(dev));
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_DEVICE_DEL
	static xmlNode *
	handle_device_delete_request(pcmk__request_t *request)
	{
	xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, request->xml,
	LOG_ERR);
	const char *device_id = crm_element_value(dev, PCMK_XA_ID);
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);

	if (is_privileged(request->ipc_client, op)) {
	stonith_device_remove(device_id, false);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	} else {
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must delete device via CIB");
	}
	fenced_send_config_notification(op, &request->result, device_id);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_LEVEL_ADD
	static xmlNode *
	handle_level_add_request(pcmk__request_t *request)
	{
	char *desc = NULL;
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);

	if (is_privileged(request->ipc_client, op)) {
	fenced_register_level(request->xml, &desc, &request->result);
	} else {
	unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must add level via CIB");
	}
	fenced_send_config_notification(op, &request->result, desc);
	free(desc);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_LEVEL_DEL
	static xmlNode *
	handle_level_delete_request(pcmk__request_t *request)
	{
	char *desc = NULL;
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);

	if (is_privileged(request->ipc_client, op)) {
	fenced_unregister_level(request->xml, &desc, &request->result);
	} else {
	unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must delete level via CIB");
	}
	fenced_send_config_notification(op, &request->result, desc);
	free(desc);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// CRM_OP_RM_NODE_CACHE
	static xmlNode *
	handle_cache_request(pcmk__request_t *request)
	{
	int node_id = 0;
	const char *name = NULL;

	crm_element_value_int(request->xml, PCMK_XA_ID, &node_id);
	name = crm_element_value(request->xml, PCMK_XA_UNAME);
	pcmk__cluster_forget_cluster_node(node_id, name);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return NULL;
	}

	static xmlNode *
	handle_unknown_request(pcmk__request_t *request)
	{
	crm_err("Unknown IPC request %s from %s %s",
	request->op, pcmk__request_origin_type(request),
	pcmk__request_origin(request));
	pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
	"Unknown IPC request type '%s' (bug?)", request->op);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	static void
	fenced_register_handlers(void)
	{
	pcmk__server_command_t handlers[] = {
	{ CRM_OP_REGISTER, handle_register_request },
	{ STONITH_OP_EXEC, handle_agent_request },
	{ STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
	{ STONITH_OP_QUERY, handle_query_request },
	{ STONITH_OP_NOTIFY, handle_notify_request },
	{ STONITH_OP_RELAY, handle_relay_request },
	{ STONITH_OP_FENCE, handle_fence_request },
	{ STONITH_OP_FENCE_HISTORY, handle_history_request },
	{ STONITH_OP_DEVICE_ADD, handle_device_add_request },
	{ STONITH_OP_DEVICE_DEL, handle_device_delete_request },
	{ STONITH_OP_LEVEL_ADD, handle_level_add_request },
	{ STONITH_OP_LEVEL_DEL, handle_level_delete_request },
	{ CRM_OP_RM_NODE_CACHE, handle_cache_request },
	{ NULL, handle_unknown_request },
	};

	fenced_handlers = pcmk__register_handlers(handlers);
	}

	void
	fenced_unregister_handlers(void)
	{
	if (fenced_handlers != NULL) {
	g_hash_table_destroy(fenced_handlers);
	fenced_handlers = NULL;
	}
	}

	static void
	handle_request(pcmk__request_t *request)
	{
	xmlNode *reply = NULL;
	const char *reason = NULL;

	if (fenced_handlers == NULL) {
	fenced_register_handlers();
	}
	reply = pcmk__process_request(request, fenced_handlers);
	if (reply != NULL) {
	if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
	&& (request->ipc_client != NULL)) {
	/* Certain IPC-only commands must reuse the call options from the
	* original request rather than the ones set by stonith_send_reply()
	* -> do_local_reply().
	*/
	pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
	request->ipc_flags);
	request->ipc_client->request_id = 0;
	} else {
	stonith_send_reply(reply, request->call_options,
	request->peer, request->ipc_client);
	}
	pcmk__xml_free(reply);
	}

	reason = request->result.exit_reason;
	crm_debug("Processed %s request from %s %s: %s%s%s%s",
	request->op, pcmk__request_origin_type(request),
	pcmk__request_origin(request),
	pcmk_exec_status_str(request->result.execution_status),
	(reason == NULL)? "" : " (",
	(reason == NULL)? "" : reason,
	(reason == NULL)? "" : ")");
	}

	static void
	handle_reply(pcmk__client_t client, xmlNode request, const char *remote_peer)
	{
	// Copy, because request might be freed before we want to log this
	char *op = crm_element_value_copy(request, PCMK__XA_ST_OP);

	if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
	process_remote_stonith_query(request);

	} else if (pcmk__str_any_of(op, STONITH_OP_NOTIFY, STONITH_OP_FENCE,
	NULL)) {
	fenced_process_fencing_reply(request);

	} else {
	crm_err("Ignoring unknown %s reply from %s %s",
	pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));
	crm_log_xml_warn(request, "UnknownOp");
	free(op);
	return;
	}
	crm_debug("Processed %s reply from %s %s",
	op, ((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));
	free(op);
	}

	/*!
	* \internal
	* \brief Handle a message from an IPC client or CPG peer
	*
	* \param[in,out] client If not NULL, IPC client that sent message
	* \param[in] id If from IPC client, IPC message ID
	* \param[in] flags Message flags
	* \param[in,out] message Message XML
	* \param[in] remote_peer If not NULL, CPG peer that sent message
	*/
	void
	stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
	xmlNode message, const char remote_peer)
	{
	int call_options = st_opt_none;
	bool is_reply = false;

	CRM_CHECK(message != NULL, return);

	if (get_xpath_object("//" PCMK__XE_ST_REPLY, message, LOG_NEVER) != NULL) {
	is_reply = true;
	}
	crm_element_value_int(message, PCMK__XA_ST_CALLOPT, &call_options);
	crm_debug("Processing %ssynchronous %s %s %u from %s %s",
	pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
	crm_element_value(message, PCMK__XA_ST_OP),
	(is_reply? "reply" : "request"), id,
	((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(client == NULL \|\| client->request_id == id);
	}

	if (is_reply) {
	handle_reply(client, message, remote_peer);
	} else {
	pcmk__request_t request = {
	.ipc_client = client,
	.ipc_id = id,
	.ipc_flags = flags,
	.peer = remote_peer,
	.xml = message,
	.call_options = call_options,
	.result = PCMK__UNKNOWN_RESULT,
	};

	request.op = crm_element_value_copy(request.xml, PCMK__XA_ST_OP);
	CRM_CHECK(request.op != NULL, return);

	if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
	pcmk__set_request_flags(&request, pcmk__request_sync);
	}

	handle_request(&request);
	pcmk__reset_request(&request);
	}
	}
	diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
	index 364fb2687a..ede07a0f72 100644
	--- a/daemons/fenced/pacemaker-fenced.c
	+++ b/daemons/fenced/pacemaker-fenced.c
	@@ -1,680 +1,681 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <inttypes.h> // PRIu32, PRIx32

	#include <crm/crm.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/output_internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib/internal.h>

	#include <pacemaker-fenced.h>

	#define SUMMARY "daemon for executing fencing devices in a Pacemaker cluster"

	char *stonith_our_uname = NULL;
	long long stonith_watchdog_timeout_ms = 0;
	GList *stonith_watchdog_targets = NULL;

	static GMainLoop *mainloop = NULL;

	gboolean stand_alone = FALSE;
	gboolean stonith_shutdown_flag = FALSE;

	static qb_ipcs_service_t *ipcs = NULL;
	static pcmk__output_t *out = NULL;

	pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	static struct {
	bool no_cib_connect;
	gchar **log_files;
	} options;

	crm_exit_t exit_code = CRM_EX_OK;

	static void stonith_cleanup(void);

	static int32_t
	st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (stonith_shutdown_flag) {
	crm_info("Ignoring new client [%d] during shutdown",
	pcmk__client_pid(c));
	return -ECONNREFUSED;
	}

	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -ENOMEM;
	}
	return 0;
	}

	/* Exit code means? */
	static int32_t
	st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	xmlNode *request = NULL;
	pcmk__client_t *c = pcmk__find_client(qbc);
	const char *op = NULL;

	if (c == NULL) {
	crm_info("Invalid client: %p", qbc);
	return 0;
	}

	request = pcmk__client_data2xml(c, data, &id, &flags);
	if (request == NULL) {
	pcmk__ipc_send_ack(c, id, flags, PCMK__XE_NACK, NULL, CRM_EX_PROTOCOL);
	return 0;
	}


	op = crm_element_value(request, PCMK__XA_CRM_TASK);
	if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
	crm_xml_add(request, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(request, PCMK__XA_ST_OP, op);
	crm_xml_add(request, PCMK__XA_ST_CLIENTID, c->id);
	crm_xml_add(request, PCMK__XA_ST_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, PCMK__XA_ST_CLIENTNODE, stonith_our_uname);

	pcmk__cluster_send_message(NULL, crm_msg_stonith_ng, request);
	pcmk__xml_free(request);
	return 0;
	}

	if (c->name == NULL) {
	const char *value = crm_element_value(request, PCMK__XA_ST_CLIENTNAME);

	c->name = crm_strdup_printf("%s.%u", pcmk__s(value, "unknown"), c->pid);
	}

	crm_element_value_int(request, PCMK__XA_ST_CALLOPT, &call_options);
	crm_trace("Flags %#08" PRIx32 "/%#08x for command %" PRIu32
	" from client %s", flags, call_options, id, pcmk__client_name(c));

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(c->request_id == 0); /* This means the client has two synchronous events in-flight */
	c->request_id = id; /* Reply only to the last one */
	}

	crm_xml_add(request, PCMK__XA_ST_CLIENTID, c->id);
	crm_xml_add(request, PCMK__XA_ST_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, PCMK__XA_ST_CLIENTNODE, stonith_our_uname);

	crm_log_xml_trace(request, "ipc-received");
	stonith_command(c, id, flags, request, NULL);

	pcmk__xml_free(request);
	return 0;
	}

	/* Error code means? */
	static int32_t
	st_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p closed", c);
	pcmk__free_client(client);

	/* 0 means: yes, go ahead and destroy the connection */
	return 0;
	}

	static void
	st_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p destroyed", c);
	st_ipc_closed(c);
	}

	static void
	stonith_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *remote_peer = crm_element_value(msg, PCMK__XA_SRC);
	const char *op = crm_element_value(msg, PCMK__XA_ST_OP);

	if (pcmk__str_eq(op, STONITH_OP_POKE, pcmk__str_none)) {
	return;
	}

	crm_log_xml_trace(msg, "Peer[inbound]");
	stonith_command(NULL, 0, 0, msg, remote_peer);
	}

	#if SUPPORT_COROSYNC
	static void
	stonith_peer_ais_callback(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	xmlNode *xml = NULL;
	const char *from = NULL;
	char *data = pcmk__cpg_message_data(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	xml = pcmk__xml_parse(data);
	if (xml == NULL) {
	crm_err("Invalid XML: '%.120s'", data);
	free(data);
	return;
	}
	crm_xml_add(xml, PCMK__XA_SRC, from);
	stonith_peer_callback(xml, NULL);
	}

	pcmk__xml_free(xml);
	free(data);
	return;
	}

	static void
	stonith_peer_cs_destroy(gpointer user_data)
	{
	crm_crit("Lost connection to cluster layer, shutting down");
	stonith_shutdown(0);
	}
	#endif

	void
	do_local_reply(const xmlNode notify_src, pcmk__client_t client,
	int call_options)
	{
	/* send callback to originating child */
	int local_rc = pcmk_rc_ok;
	int rid = 0;
	uint32_t ipc_flags = crm_ipc_server_event;

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_LOG_ASSERT(client->request_id);
	rid = client->request_id;
	client->request_id = 0;
	ipc_flags = crm_ipc_flags_none;
	}

	local_rc = pcmk__ipc_send_xml(client, rid, notify_src, ipc_flags);
	if (local_rc == pcmk_rc_ok) {
	crm_trace("Sent response %d to client %s",
	rid, pcmk__client_name(client));
	} else {
	crm_warn("%synchronous reply to client %s failed: %s",
	(pcmk_is_set(call_options, st_opt_sync_call)? "S" : "As"),
	pcmk__client_name(client), pcmk_rc_str(local_rc));
	}
	}

	uint64_t
	get_stonith_flag(const char *name)
	{
	if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_FENCE, pcmk__str_none)) {
	return st_callback_notify_fence;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) {
	return st_callback_device_add;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) {
	return st_callback_device_del;

	} else if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_HISTORY,
	pcmk__str_none)) {
	return st_callback_notify_history;

	} else if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED,
	pcmk__str_none)) {
	return st_callback_notify_history_synced;

	}
	return st_callback_unknown;
	}

	static void
	stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
	{

	const xmlNode *update_msg = user_data;
	pcmk__client_t *client = value;
	const char *type = NULL;

	CRM_CHECK(client != NULL, return);
	CRM_CHECK(update_msg != NULL, return);

	type = crm_element_value(update_msg, PCMK__XA_SUBT);
	CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);

	if (client->ipcs == NULL) {
	crm_trace("Skipping client with NULL channel");
	return;
	}

	if (pcmk_is_set(client->flags, get_stonith_flag(type))) {
	int rc = pcmk__ipc_send_xml(client, 0, update_msg,
	crm_ipc_server_event);

	if (rc != pcmk_rc_ok) {
	crm_warn("%s notification of client %s failed: %s "
	QB_XS " id=%.8s rc=%d", type, pcmk__client_name(client),
	pcmk_rc_str(rc), client->id, rc);
	} else {
	crm_trace("Sent %s notification to client %s",
	type, pcmk__client_name(client));
	}
	}
	}

	void
	do_stonith_async_timeout_update(const char client_id, const char call_id, int timeout)
	{
	pcmk__client_t *client = NULL;
	xmlNode *notify_data = NULL;

	if (!timeout \|\| !call_id \|\| !client_id) {
	return;
	}

	client = pcmk__find_client_by_id(client_id);
	if (!client) {
	return;
	}

	notify_data = pcmk__xe_create(NULL, PCMK__XE_ST_ASYNC_TIMEOUT_VALUE);
	crm_xml_add(notify_data, PCMK__XA_T, PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE);
	crm_xml_add(notify_data, PCMK__XA_ST_CALLID, call_id);
	crm_xml_add_int(notify_data, PCMK__XA_ST_TIMEOUT, timeout);

	crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);

	if (client) {
	pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event);
	}

	pcmk__xml_free(notify_data);
	}

	/*!
	* \internal
	* \brief Notify relevant IPC clients of a fencing operation result
	*
	* \param[in] type Notification type
	* \param[in] result Result of fencing operation (assume success if NULL)
	* \param[in] data If not NULL, add to notification as call data
	*/
	void
	fenced_send_notification(const char type, const pcmk__action_result_t result,
	xmlNode *data)
	{
	/* TODO: Standardize the contents of data */
	xmlNode *update_msg = pcmk__xe_create(NULL, PCMK__XE_NOTIFY);

	CRM_LOG_ASSERT(type != NULL);

	crm_xml_add(update_msg, PCMK__XA_T, PCMK__VALUE_ST_NOTIFY);
	crm_xml_add(update_msg, PCMK__XA_SUBT, type);
	crm_xml_add(update_msg, PCMK__XA_ST_OP, type);
	stonith__xe_set_result(update_msg, result);

	if (data != NULL) {
	xmlNode *wrapper = pcmk__xe_create(update_msg, PCMK__XE_ST_CALLDATA);

	pcmk__xml_copy(wrapper, data);
	}

	crm_trace("Notifying clients");
	pcmk__foreach_ipc_client(stonith_notify_client, update_msg);
	pcmk__xml_free(update_msg);
	crm_trace("Notify complete");
	}

	/*!
	* \internal
	* \brief Send notifications for a configuration change to subscribed clients
	*
	* \param[in] op Notification type (\c STONITH_OP_DEVICE_ADD,
	* \c STONITH_OP_DEVICE_DEL, \c STONITH_OP_LEVEL_ADD, or
	* \c STONITH_OP_LEVEL_DEL)
	* \param[in] result Operation result
	* \param[in] desc Description of what changed (either device ID or string
	* representation of level
	* (<tt><target>[<level_index>]</tt>))
	*/
	void
	fenced_send_config_notification(const char *op,
	const pcmk__action_result_t *result,
	const char *desc)
	{
	xmlNode *notify_data = pcmk__xe_create(NULL, op);

	crm_xml_add(notify_data, PCMK__XA_ST_DEVICE_ID, desc);

	fenced_send_notification(op, result, notify_data);
	pcmk__xml_free(notify_data);
	}

	/*!
	* \internal
	* \brief Check whether a node does watchdog-fencing
	*
	* \param[in] node Name of node to check
	*
	* \return TRUE if node found in stonith_watchdog_targets
	* or stonith_watchdog_targets is empty indicating
	* all nodes are doing watchdog-fencing
	*/
	gboolean
	node_does_watchdog_fencing(const char *node)
	{
	return ((stonith_watchdog_targets == NULL) \|\|
	pcmk__str_in_list(node, stonith_watchdog_targets, pcmk__str_casei));
	}

	void
	stonith_shutdown(int nsig)
	{
	crm_info("Terminating with %d clients", pcmk__ipc_client_count());
	stonith_shutdown_flag = TRUE;
	if (mainloop != NULL && g_main_loop_is_running(mainloop)) {
	g_main_loop_quit(mainloop);
	}
	}

	static void
	stonith_cleanup(void)
	{
	fenced_cib_cleanup();
	if (ipcs) {
	qb_ipcs_destroy(ipcs);
	}

	pcmk__cluster_destroy_node_caches();
	pcmk__client_cleanup();
	free_stonith_remote_op_list();
	free_topology_list();
	free_device_list();
	free_metadata_cache();
	fenced_unregister_handlers();

	free(stonith_our_uname);
	stonith_our_uname = NULL;
	}

	static gboolean
	stand_alone_cpg_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	stand_alone = FALSE;
	options.no_cib_connect = true;
	return TRUE;
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = st_ipc_accept,
	.connection_created = NULL,
	.msg_process = st_ipc_dispatch,
	.connection_closed = st_ipc_closed,
	.connection_destroyed = st_ipc_destroy
	};

	/*!
	* \internal
	* \brief Callback for peer status changes
	*
	* \param[in] type What changed
	* \param[in] node What peer had the change
	* \param[in] data Previous value of what changed
	*/
	static void
	st_peer_update_callback(enum crm_status_type type, pcmk__node_status_t *node,
	const void *data)
	{
	if ((type != crm_status_processes)
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/*
	* This is a hack until we can send to a nodeid and/or we fix node name lookups
	* These messages are ignored in stonith_peer_callback()
	*/
	xmlNode *query = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND);

	crm_xml_add(query, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(query, PCMK__XA_ST_OP, STONITH_OP_POKE);

	- crm_debug("Broadcasting our uname because of node %u", node->id);
	+ crm_debug("Broadcasting our uname because of node %" PRIu32,
	+ node->cluster_layer_id);
	pcmk__cluster_send_message(NULL, crm_msg_stonith_ng, query);

	pcmk__xml_free(query);
	}
	}

	/* @COMPAT Deprecated since 2.1.8. Use pcmk_list_fence_attrs() or
	* crm_resource --list-options=fencing instead of querying daemon metadata.
	*/
	static int
	fencer_metadata(void)
	{
	const char *name = "pacemaker-fenced";
	const char *desc_short = N_("Instance attributes available for all "
	"\"stonith\"-class resources");
	const char *desc_long = N_("Instance attributes available for all "
	"\"stonith\"-class resources and used by "
	"Pacemaker's fence daemon");

	return pcmk__daemon_metadata(out, name, desc_short, desc_long,
	pcmk__opt_fencing);
	}

	static GOptionEntry entries[] = {
	{ "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone,
	N_("Deprecated (will be removed in a future release)"), NULL },

	{ "stand-alone-w-cpg", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	stand_alone_cpg_cb, N_("Intended for use in regression testing only"), NULL },

	{ "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
	&options.log_files, N_("Send logs to the additional named logfile"), NULL },

	{ NULL }
	};

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group)
	{
	GOptionContext *context = NULL;

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
	pcmk__add_main_args(context, entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	pcmk_cluster_t *cluster = NULL;
	crm_ipc_t *old_instance = NULL;

	GError *error = NULL;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "l");
	GOptionContext *context = build_arg_context(args, &output_group);

	crm_log_preinit(NULL, argc, argv);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_ERROR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error creating output format %s: %s",
	args->output_ty, pcmk_rc_str(rc));
	goto done;
	}

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if ((g_strv_length(processed_args) >= 2)
	&& pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) {

	rc = fencer_metadata();
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_FATAL;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Unable to display metadata: %s", pcmk_rc_str(rc));
	}
	goto done;
	}

	// Open additional log files
	pcmk__add_logfiles(options.log_files, out);

	crm_log_init(NULL, LOG_INFO + args->verbosity, TRUE,
	(args->verbosity > 0), argc, argv, FALSE);

	crm_notice("Starting Pacemaker fencer");

	old_instance = crm_ipc_new("stonith-ng", 0);
	if (old_instance == NULL) {
	/* crm_ipc_new() will have already logged an error message with
	* crm_err()
	*/
	exit_code = CRM_EX_FATAL;
	goto done;
	}

	if (pcmk__connect_generic_ipc(old_instance) == pcmk_rc_ok) {
	// IPC endpoint already up
	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);
	crm_err("pacemaker-fenced is already active, aborting startup");
	goto done;
	} else {
	// Not up or not authentic, we'll proceed either way
	crm_ipc_destroy(old_instance);
	old_instance = NULL;
	}

	mainloop_add_signal(SIGTERM, stonith_shutdown);

	pcmk__cluster_init_node_caches();

	rc = fenced_scheduler_init();
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_FATAL;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error initializing scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}

	cluster = pcmk_cluster_new();

	if (!stand_alone) {
	#if SUPPORT_COROSYNC
	if (pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync) {
	pcmk_cluster_set_destroy_fn(cluster, stonith_peer_cs_destroy);
	pcmk_cpg_set_deliver_fn(cluster, stonith_peer_ais_callback);
	pcmk_cpg_set_confchg_fn(cluster, pcmk__cpg_confchg_cb);
	}
	#endif // SUPPORT_COROSYNC

	pcmk__cluster_set_status_callback(&st_peer_update_callback);

	if (pcmk_cluster_connect(cluster) != pcmk_rc_ok) {
	exit_code = CRM_EX_FATAL;
	crm_crit("Cannot sign in to the cluster... terminating");
	goto done;
	}
	pcmk__str_update(&stonith_our_uname, cluster->uname);

	if (!options.no_cib_connect) {
	setup_cib();
	}

	} else {
	pcmk__str_update(&stonith_our_uname, "localhost");
	crm_warn("Stand-alone mode is deprecated and will be removed "
	"in a future release");
	}

	init_device_list();
	init_topology_list();

	pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks);

	// Create the mainloop and run it...
	mainloop = g_main_loop_new(NULL, FALSE);
	crm_notice("Pacemaker fencer successfully started and accepting connections");
	g_main_loop_run(mainloop);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	g_strfreev(options.log_files);

	stonith_cleanup();
	pcmk_cluster_free(cluster);
	fenced_scheduler_cleanup();

	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}

	pcmk__unregister_formats();
	crm_exit(exit_code);
	}
	diff --git a/include/crm/cluster/internal.h b/include/crm/cluster/internal.h
	index aeb67f4cba..94d95a79fd 100644
	--- a/include/crm/cluster/internal.h
	+++ b/include/crm/cluster/internal.h
	@@ -1,247 +1,249 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_CLUSTER_INTERNAL__H
	# define PCMK__CRM_CLUSTER_INTERNAL__H

	# include <stdbool.h>
	# include <stdint.h> // uint32_t, uint64_t

	# include <glib.h> // gboolean

	# include <crm/cluster.h>

	enum crm_proc_flag {
	/* @COMPAT When pcmk__node_status_t:processes is made internal, we can merge
	* this into node flags or turn it into a boolean. Until then, in theory
	* something could depend on these particular numeric values.
	*/
	crm_proc_none = 0x00000001,

	// Cluster layers
	crm_proc_cpg = 0x04000000,
	};

	// Used with node cache search functions
	enum pcmk__node_search_flags {
	//! Does not affect search
	pcmk__node_search_none = 0,

	//! Search for cluster nodes from membership cache
	pcmk__node_search_cluster_member = (1 << 0),

	//! Search for remote nodes
	pcmk__node_search_remote = (1 << 1),

	//! Search for cluster member nodes and remote nodes
	pcmk__node_search_any = pcmk__node_search_cluster_member
	\|pcmk__node_search_remote,

	//! Search for cluster nodes from CIB (as of last cache refresh)
	pcmk__node_search_cluster_cib = (1 << 2),
	};

	-//! Cluster member node data (may be a cluster node or a Pacemaker Remote node)
	+//! Node status data (may be a cluster node or a Pacemaker Remote node)
	typedef struct pcmk__node_status {
	char *uname; // Node name as known to cluster

	/* @COMPAT This is less than ideal since the value is not a valid XML ID
	* (for Corosync, it's the string equivalent of the node's numeric node ID,
	* but XML IDs can't start with a number) and the three elements should have
	* different IDs.
	*
	* Ideally, we would use something like node-NODEID, node_state-NODEID, and
	* transient_attributes-NODEID as the element IDs. Unfortunately changing it
	* would be impractical due to backward compatibility; older nodes in a
	* rolling upgrade will always write and expect the value in the old format.
	*
	* This is also named poorly, since the value is not a UUID, but at least
	* that can be changed at an API compatibility break.
	*/
	/*! Value of the PCMK_XA_ID XML attribute to use with the node's
	* PCMK_XE_NODE, PCMK_XE_NODE_STATE, and PCMK_XE_TRANSIENT_ATTRIBUTES
	* XML elements in the CIB
	*/
	char *uuid;

	char *state; // @TODO change to enum
	uint64_t flags; // Bitmask of crm_node_flags
	uint64_t last_seen; // Only needed by cluster nodes
	uint32_t processes; // @TODO most not needed, merge into flags

	/* @TODO When we can break public API compatibility, we can make the rest of
	* these members separate structs and use void *cluster_data and
	* void *user_data here instead, to abstract the cluster layer further.
	*/

	- // Currently only needed by corosync stack
	- uint32_t id; // Node ID
	- time_t when_lost; // When CPG membership was last lost
	-
	// Only used by controller
	enum crm_join_phase join;
	char *expected;

	time_t peer_lost;
	char *conn_host;

	time_t when_member; // Since when node has been a cluster member
	time_t when_online; // Since when peer has been online in CPG
	+
	+ /* @TODO The following are currently needed only by the Corosync stack.
	+ * Eventually consider moving them to a cluster-layer-specific data object.
	+ */
	+ uint32_t cluster_layer_id; //! Cluster-layer numeric node ID
	+ time_t when_lost; //! When CPG membership was last lost
	} pcmk__node_status_t;

	/*!
	* \internal
	* \brief Return the process bit corresponding to the current cluster stack
	*
	* \return Process flag if detectable, otherwise 0
	*/
	static inline uint32_t
	crm_get_cluster_proc(void)
	{
	switch (pcmk_get_cluster_layer()) {
	case pcmk_cluster_layer_corosync:
	return crm_proc_cpg;

	default:
	break;
	}
	return crm_proc_none;
	}

	/*!
	* \internal
	* \brief Get log-friendly string description of a Corosync return code
	*
	* \param[in] error Corosync return code
	*
	* \return Log-friendly string description corresponding to \p error
	*/
	static inline const char *
	pcmk__cs_err_str(int error)
	{
	# if SUPPORT_COROSYNC
	switch (error) {
	case CS_OK: return "OK";
	case CS_ERR_LIBRARY: return "Library error";
	case CS_ERR_VERSION: return "Version error";
	case CS_ERR_INIT: return "Initialization error";
	case CS_ERR_TIMEOUT: return "Timeout";
	case CS_ERR_TRY_AGAIN: return "Try again";
	case CS_ERR_INVALID_PARAM: return "Invalid parameter";
	case CS_ERR_NO_MEMORY: return "No memory";
	case CS_ERR_BAD_HANDLE: return "Bad handle";
	case CS_ERR_BUSY: return "Busy";
	case CS_ERR_ACCESS: return "Access error";
	case CS_ERR_NOT_EXIST: return "Doesn't exist";
	case CS_ERR_NAME_TOO_LONG: return "Name too long";
	case CS_ERR_EXIST: return "Exists";
	case CS_ERR_NO_SPACE: return "No space";
	case CS_ERR_INTERRUPT: return "Interrupt";
	case CS_ERR_NAME_NOT_FOUND: return "Name not found";
	case CS_ERR_NO_RESOURCES: return "No resources";
	case CS_ERR_NOT_SUPPORTED: return "Not supported";
	case CS_ERR_BAD_OPERATION: return "Bad operation";
	case CS_ERR_FAILED_OPERATION: return "Failed operation";
	case CS_ERR_MESSAGE_ERROR: return "Message error";
	case CS_ERR_QUEUE_FULL: return "Queue full";
	case CS_ERR_QUEUE_NOT_AVAILABLE: return "Queue not available";
	case CS_ERR_BAD_FLAGS: return "Bad flags";
	case CS_ERR_TOO_BIG: return "Too big";
	case CS_ERR_NO_SECTIONS: return "No sections";
	}
	# endif
	return "Corosync error";
	}

	# if SUPPORT_COROSYNC

	#if 0
	/* This is the new way to do it, but we still support all Corosync 2 versions,
	* and this isn't always available. A better alternative here would be to check
	* for support in the configure script and enable this conditionally.
	*/
	#define pcmk__init_cmap(handle) cmap_initialize_map((handle), CMAP_MAP_ICMAP)
	#else
	#define pcmk__init_cmap(handle) cmap_initialize(handle)
	#endif

	char *pcmk__corosync_cluster_name(void);
	bool pcmk__corosync_add_nodes(xmlNode *xml_parent);

	void pcmk__cpg_confchg_cb(cpg_handle_t handle,
	const struct cpg_name *group_name,
	const struct cpg_address *member_list,
	size_t member_list_entries,
	const struct cpg_address *left_list,
	size_t left_list_entries,
	const struct cpg_address *joined_list,
	size_t joined_list_entries);

	char *pcmk__cpg_message_data(cpg_handle_t handle, uint32_t sender_id,
	uint32_t pid, void content, uint32_t kind,
	const char **from);

	# endif

	const char pcmk__cluster_node_uuid(pcmk__node_status_t node);
	char *pcmk__cluster_node_name(uint32_t nodeid);
	const char *pcmk__cluster_local_node_name(void);
	const char pcmk__node_name_from_uuid(const char uuid);

	pcmk__node_status_t crm_update_peer_proc(const char source,
	pcmk__node_status_t *peer,
	uint32_t flag, const char *status);
	pcmk__node_status_t pcmk__update_peer_state(const char source,
	pcmk__node_status_t *node,
	const char *state,
	uint64_t membership);

	void pcmk__update_peer_expected(const char source, pcmk__node_status_t node,
	const char *expected);
	void pcmk__reap_unseen_nodes(uint64_t ring_id);

	void pcmk__corosync_quorum_connect(gboolean (*dispatch)(unsigned long long,
	gboolean),
	void (*destroy) (gpointer));

	enum crm_ais_msg_types pcmk__cluster_parse_msg_type(const char *text);
	bool pcmk__cluster_send_message(const pcmk__node_status_t *node,
	enum crm_ais_msg_types service,
	const xmlNode *data);

	// Membership

	bool pcmk__cluster_has_quorum(void);

	void pcmk__cluster_init_node_caches(void);
	void pcmk__cluster_destroy_node_caches(void);

	void pcmk__cluster_set_autoreap(bool enable);
	void pcmk__cluster_set_status_callback(void (*dispatch)(enum crm_status_type,
	pcmk__node_status_t *,
	const void *));

	bool pcmk__cluster_is_node_active(const pcmk__node_status_t *node);
	unsigned int pcmk__cluster_num_active_nodes(void);
	unsigned int pcmk__cluster_num_remote_nodes(void);

	pcmk__node_status_t pcmk__cluster_lookup_remote_node(const char node_name);
	void pcmk__cluster_forget_cluster_node(uint32_t id, const char *node_name);
	void pcmk__cluster_forget_remote_node(const char *node_name);
	pcmk__node_status_t *pcmk__search_node_caches(unsigned int id,
	const char *uname,
	uint32_t flags);
	void pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id);

	void pcmk__refresh_node_caches_from_cib(xmlNode *cib);

	pcmk__node_status_t pcmk__get_node(unsigned int id, const char uname,
	const char *uuid, uint32_t flags);

	#endif // PCMK__CRM_CLUSTER_INTERNAL__H
	diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c
	index 2fd1a9a72e..9db3bde06a 100644
	--- a/lib/cluster/corosync.c
	+++ b/lib/cluster/corosync.c
	@@ -1,815 +1,817 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <arpa/inet.h>
	-#include <inttypes.h> // PRIu64, PRIx32
	+#include <inttypes.h> // PRIu64, etc.
	#include <netdb.h>
	#include <netinet/in.h>
	#include <stdbool.h>
	#include <sys/socket.h>
	#include <sys/utsname.h>

	#include <bzlib.h>
	#include <corosync/cfg.h>
	#include <corosync/cmap.h>
	#include <corosync/corodefs.h>
	#include <corosync/corotypes.h>
	#include <corosync/hdb.h>
	#include <corosync/quorum.h>
	#include <qb/qbipcc.h>
	#include <qb/qbutil.h>

	#include <crm/cluster/internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h> // PCMK__SPECIAL_PID
	#include <crm/common/mainloop.h>
	#include <crm/common/xml.h>

	#include "crmcluster_private.h"

	static quorum_handle_t pcmk_quorum_handle = 0;

	static gboolean (*quorum_app_callback)(unsigned long long seq,
	gboolean quorate) = NULL;

	/*!
	* \internal
	* \brief Get the Corosync UUID associated with a Pacemaker node
	*
	* \param[in] node Pacemaker node
	*
	* \return Newly allocated string with node's Corosync UUID, or NULL if unknown
	* \note It is the caller's responsibility to free the result with free().
	*/
	char *
	pcmk__corosync_uuid(const pcmk__node_status_t *node)
	{
	CRM_ASSERT(pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync);

	if (node != NULL) {
	- if (node->id > 0) {
	- return crm_strdup_printf("%u", node->id);
	+ if (node->cluster_layer_id > 0) {
	+ return crm_strdup_printf("%" PRIu32, node->cluster_layer_id);
	} else {
	crm_info("Node %s is not yet known by Corosync", node->uname);
	}
	}
	return NULL;
	}

	static bool
	node_name_is_valid(const char key, const char name)
	{
	int octet;

	if (name == NULL) {
	crm_trace("%s is empty", key);
	return false;

	} else if (sscanf(name, "%d.%d.%d.%d", &octet, &octet, &octet, &octet) == 4) {
	crm_trace("%s contains an IPv4 address (%s), ignoring", key, name);
	return false;

	} else if (strstr(name, ":") != NULL) {
	crm_trace("%s contains an IPv6 address (%s), ignoring", key, name);
	return false;
	}
	crm_trace("'%s: %s' is valid", key, name);
	return true;
	}

	/*
	* \internal
	* \brief Get Corosync node name corresponding to a node ID
	*
	* \param[in] cmap_handle Connection to Corosync CMAP
	* \param[in] nodeid Node ID to check
	*
	* \return Newly allocated string with name or (if no name) IP address
	* associated with first address assigned to a Corosync node ID (or NULL
	* if unknown)
	* \note It is the caller's responsibility to free the result with free().
	*/
	char *
	pcmk__corosync_name(uint64_t /cmap_handle_t / cmap_handle, uint32_t nodeid)
	{
	// Originally based on corosync-quorumtool.c:node_name()

	int lpc = 0;
	cs_error_t rc = CS_OK;
	int retries = 0;
	char *name = NULL;
	cmap_handle_t local_handle = 0;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	if (nodeid == 0) {
	nodeid = pcmk__cpg_local_nodeid(0);
	}

	if (cmap_handle == 0 && local_handle == 0) {
	retries = 0;
	crm_trace("Initializing CMAP connection");
	do {
	rc = pcmk__init_cmap(&local_handle);
	if (rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
	retries);
	sleep(retries);
	}

	} while (retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %s",
	cs_strerror(rc));
	local_handle = 0;
	}
	}

	if (cmap_handle == 0) {
	cmap_handle = local_handle;

	rc = cmap_fd_get(cmap_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}
	}

	while (name == NULL && cmap_handle != 0) {
	uint32_t id = 0;
	char *key = NULL;

	key = crm_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &id);
	crm_trace("Checking %u vs %u from %s", nodeid, id, key);
	free(key);

	if (rc != CS_OK) {
	break;
	}

	if (nodeid == id) {
	crm_trace("Searching for node name for %u in nodelist.node.%d %s",
	nodeid, lpc, pcmk__s(name, "<null>"));
	if (name == NULL) {
	key = crm_strdup_printf("nodelist.node.%d.name", lpc);
	cmap_get_string(cmap_handle, key, &name);
	crm_trace("%s = %s", key, pcmk__s(name, "<null>"));
	free(key);
	}
	if (name == NULL) {
	key = crm_strdup_printf("nodelist.node.%d.ring0_addr", lpc);
	cmap_get_string(cmap_handle, key, &name);
	crm_trace("%s = %s", key, pcmk__s(name, "<null>"));

	if (!node_name_is_valid(key, name)) {
	free(name);
	name = NULL;
	}
	free(key);
	}
	break;
	}

	lpc++;
	}

	bail:
	if(local_handle) {
	cmap_finalize(local_handle);
	}

	if (name == NULL) {
	crm_info("Unable to get node name for nodeid %u", nodeid);
	}
	return name;
	}

	/*!
	* \internal
	* \brief Disconnect from Corosync cluster
	*
	* \param[in,out] cluster Cluster object to disconnect
	*/
	void
	pcmk__corosync_disconnect(pcmk_cluster_t *cluster)
	{
	pcmk__cpg_disconnect(cluster);

	if (pcmk_quorum_handle != 0) {
	quorum_finalize(pcmk_quorum_handle);
	pcmk_quorum_handle = 0;
	}
	crm_notice("Disconnected from Corosync");
	}

	/*!
	* \internal
	* \brief Dispatch function for quorum connection file descriptor
	*
	* \param[in] user_data Ignored
	*
	* \return 0 on success, -1 on error (per mainloop_io_t interface)
	*/
	static int
	quorum_dispatch_cb(gpointer user_data)
	{
	int rc = quorum_dispatch(pcmk_quorum_handle, CS_DISPATCH_ALL);

	if (rc < 0) {
	crm_err("Connection to the Quorum API failed: %d", rc);
	quorum_finalize(pcmk_quorum_handle);
	pcmk_quorum_handle = 0;
	return -1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Notification callback for Corosync quorum connection
	*
	* \param[in] handle Corosync quorum connection
	* \param[in] quorate Whether cluster is quorate
	* \param[in] ring_id Corosync ring ID
	* \param[in] view_list_entries Number of entries in \p view_list
	* \param[in] view_list Corosync node IDs in membership
	*/
	static void
	quorum_notification_cb(quorum_handle_t handle, uint32_t quorate,
	uint64_t ring_id, uint32_t view_list_entries,
	uint32_t *view_list)
	{
	int i;
	GHashTableIter iter;
	pcmk__node_status_t *node = NULL;
	static gboolean init_phase = TRUE;

	bool is_quorate = (quorate != 0);
	bool was_quorate = pcmk__cluster_has_quorum();

	if (is_quorate && !was_quorate) {
	crm_notice("Quorum acquired " QB_XS " membership=%" PRIu64
	" members=%" PRIu32,
	ring_id, view_list_entries);
	pcmk__cluster_set_quorum(true);

	} else if (!is_quorate && was_quorate) {
	crm_warn("Quorum lost " QB_XS " membership=%" PRIu64 " members=" PRIu32,
	ring_id, view_list_entries);
	pcmk__cluster_set_quorum(false);

	} else {
	crm_info("Quorum %s " QB_XS " membership=%" PRIu64 " members=%" PRIu32,
	(is_quorate? "retained" : "still lost"), ring_id,
	view_list_entries);
	}

	if (view_list_entries == 0 && init_phase) {
	crm_info("Corosync membership is still forming, ignoring");
	return;
	}

	init_phase = FALSE;

	/* Reset last_seen for all cached nodes so we can tell which ones aren't
	* in the view list */
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	node->last_seen = 0;
	}

	/* Update the peer cache for each node in view list */
	for (i = 0; i < view_list_entries; i++) {
	uint32_t id = view_list[i];

	crm_debug("Member[%d] %u ", i, id);

	/* Get this node's peer cache entry (adding one if not already there) */
	node = pcmk__get_node(id, NULL, NULL, pcmk__node_search_cluster_member);
	if (node->uname == NULL) {
	char *name = pcmk__corosync_name(0, id);

	crm_info("Obtaining name for new node %u", id);
	node = pcmk__get_node(id, name, NULL,
	pcmk__node_search_cluster_member);
	free(name);
	}

	/* Update the node state (including updating last_seen to ring_id) */
	pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, ring_id);
	}

	/* Remove any peer cache entries we didn't update */
	pcmk__reap_unseen_nodes(ring_id);

	if (quorum_app_callback) {
	quorum_app_callback(ring_id, is_quorate);
	}
	}

	/*!
	* \internal
	* \brief Connect to Corosync quorum service
	*
	* \param[in] dispatch Connection dispatch callback
	* \param[in] destroy Connection destroy callback
	*/
	void
	pcmk__corosync_quorum_connect(gboolean (*dispatch)(unsigned long long,
	gboolean),
	void (*destroy)(gpointer))
	{
	cs_error_t rc;
	int fd = 0;
	int quorate = 0;
	uint32_t quorum_type = 0;
	struct mainloop_fd_callbacks quorum_fd_callbacks;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	quorum_fd_callbacks.dispatch = quorum_dispatch_cb;
	quorum_fd_callbacks.destroy = destroy;

	crm_debug("Configuring Pacemaker to obtain quorum from Corosync");

	{
	#if 0
	// New way but not supported by all Corosync 2 versions
	quorum_model_v0_data_t quorum_model_data = {
	.model = QUORUM_MODEL_V0,
	.quorum_notify_fn = quorum_notification_cb,
	};

	rc = quorum_model_initialize(&pcmk_quorum_handle, QUORUM_MODEL_V0,
	(quorum_model_data_t *) &quorum_model_data,
	&quorum_type, NULL);
	#else
	quorum_callbacks_t quorum_callbacks = {
	.quorum_notify_fn = quorum_notification_cb,
	};

	rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks,
	&quorum_type);
	#endif
	}

	if (rc != CS_OK) {
	crm_err("Could not connect to the Quorum API: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;

	} else if (quorum_type != QUORUM_SET) {
	crm_err("Corosync quorum is not configured");
	goto bail;
	}

	rc = quorum_fd_get(pcmk_quorum_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the Quorum API connection: %s (%d)",
	strerror(rc), rc);
	goto bail;
	}

	/* Quorum provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("Quorum provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	rc = CS_ERR_ACCESS;
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of Quorum provider: %s (%d)",
	strerror(-rv), -rv);
	rc = CS_ERR_ACCESS;
	goto bail;
	}

	rc = quorum_getquorate(pcmk_quorum_handle, &quorate);
	if (rc != CS_OK) {
	crm_err("Could not obtain the current Quorum API state: %d", rc);
	goto bail;
	}

	if (quorate) {
	crm_notice("Quorum acquired");
	} else {
	crm_warn("No quorum");
	}
	quorum_app_callback = dispatch;
	pcmk__cluster_set_quorum(quorate != 0);

	rc = quorum_trackstart(pcmk_quorum_handle, CS_TRACK_CHANGES \| CS_TRACK_CURRENT);
	if (rc != CS_OK) {
	crm_err("Could not setup Quorum API notifications: %d", rc);
	goto bail;
	}

	mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks);

	pcmk__corosync_add_nodes(NULL);

	bail:
	if (rc != CS_OK) {
	quorum_finalize(pcmk_quorum_handle);
	}
	}

	/*!
	* \internal
	* \brief Connect to Corosync cluster layer
	*
	* \param[in,out] cluster Initialized cluster object to connect
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__corosync_connect(pcmk_cluster_t *cluster)
	{
	pcmk__node_status_t *peer = NULL;
	const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer();
	const char *cluster_layer_s = pcmk_cluster_layer_text(cluster_layer);
	int rc = pcmk_rc_ok;

	pcmk__cluster_init_node_caches();

	if (cluster_layer != pcmk_cluster_layer_corosync) {
	crm_err("Invalid cluster layer: %s " QB_XS " cluster_layer=%d",
	cluster_layer_s, cluster_layer);
	return EINVAL;
	}

	rc = pcmk__cpg_connect(cluster);
	if (rc != pcmk_rc_ok) {
	// Error message was logged by pcmk__cpg_connect()
	return rc;
	}
	crm_info("Connection to %s established", cluster_layer_s);

	cluster->nodeid = pcmk__cpg_local_nodeid(0);
	if (cluster->nodeid == 0) {
	crm_err("Could not determine local node ID");
	return ENXIO;
	}

	cluster->uname = pcmk__cluster_node_name(0);
	if (cluster->uname == NULL) {
	crm_err("Could not determine local node name");
	return ENXIO;
	}

	// Ensure local node always exists in peer cache
	peer = pcmk__get_node(cluster->nodeid, cluster->uname, NULL,
	pcmk__node_search_cluster_member);
	cluster->uuid = pcmk__corosync_uuid(peer);

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Check whether a Corosync cluster is active
	*
	* \return \c true if Corosync is found active, or \c false otherwise
	*/
	bool
	pcmk__corosync_is_active(void)
	{
	cmap_handle_t handle;
	int rc = pcmk__init_cmap(&handle);

	if (rc == CS_OK) {
	cmap_finalize(handle);
	return true;
	}

	crm_info("Failed to initialize the cmap API: %s (%d)",
	pcmk__cs_err_str(rc), rc);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a Corosync cluster peer is active
	*
	* \param[in] node Node to check
	*
	* \return \c true if \p node is an active Corosync peer, or \c false otherwise
	*/
	bool
	pcmk__corosync_is_peer_active(const pcmk__node_status_t *node)
	{
	if (node == NULL) {
	crm_trace("Corosync peer inactive: NULL");
	return false;
	}
	if (!pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_none)) {
	crm_trace("Corosync peer %s inactive: state=%s",
	node->uname, node->state);
	return false;
	}
	if (!pcmk_is_set(node->processes, crm_proc_cpg)) {
	crm_trace("Corosync peer %s inactive " QB_XS " processes=%.16" PRIx32,
	node->uname, node->processes);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Load Corosync node list (via CMAP) into peer cache and optionally XML
	*
	* \param[in,out] xml_parent If not NULL, add <node> entry here for each node
	*
	* \return true if any nodes were found, false otherwise
	*/
	bool
	pcmk__corosync_add_nodes(xmlNode *xml_parent)
	{
	int lpc = 0;
	cs_error_t rc = CS_OK;
	int retries = 0;
	bool any = false;
	cmap_handle_t cmap_handle;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	do {
	rc = pcmk__init_cmap(&cmap_handle);
	if (rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
	retries);
	sleep(retries);
	}

	} while (retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %d", rc);
	return false;
	}

	rc = cmap_fd_get(cmap_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}

	pcmk__cluster_init_node_caches();
	crm_trace("Initializing Corosync node list");
	for (lpc = 0; TRUE; lpc++) {
	uint32_t nodeid = 0;
	char *name = NULL;
	char *key = NULL;

	key = crm_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &nodeid);
	free(key);

	if (rc != CS_OK) {
	break;
	}

	name = pcmk__corosync_name(cmap_handle, nodeid);
	if (name != NULL) {
	GHashTableIter iter;
	pcmk__node_status_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node && node->uname && strcasecmp(node->uname, name) == 0) {
	- if (node->id && node->id != nodeid) {
	- crm_crit("Nodes %u and %u share the same name '%s': shutting down", node->id,
	- nodeid, name);
	+ if ((node->cluster_layer_id > 0)
	+ && (node->cluster_layer_id != nodeid)) {
	+ crm_crit("Nodes %" PRIu32 " and %" PRIu32 " share the "
	+ "same name '%s': shutting down",
	+ node->cluster_layer_id, nodeid, name);
	crm_exit(CRM_EX_FATAL);
	}
	}
	}
	}

	if (nodeid > 0 \|\| name != NULL) {
	crm_trace("Initializing node[%d] %u = %s", lpc, nodeid, name);
	pcmk__get_node(nodeid, name, NULL, pcmk__node_search_cluster_member);
	}

	if (nodeid > 0 && name != NULL) {
	any = true;

	if (xml_parent) {
	xmlNode *node = pcmk__xe_create(xml_parent, PCMK_XE_NODE);

	pcmk__xe_set_id(node, "%u", nodeid);
	crm_xml_add(node, PCMK_XA_UNAME, name);
	}
	}

	free(name);
	}
	bail:
	cmap_finalize(cmap_handle);
	return any;
	}

	/*!
	* \internal
	* \brief Get cluster name from Corosync configuration (via CMAP)
	*
	* \return Newly allocated string with cluster name if configured, or NULL
	*/
	char *
	pcmk__corosync_cluster_name(void)
	{
	cmap_handle_t handle;
	char *cluster_name = NULL;
	cs_error_t rc = CS_OK;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	rc = pcmk__init_cmap(&handle);
	if (rc != CS_OK) {
	crm_info("Failed to initialize the cmap API: %s (%d)",
	cs_strerror(rc), rc);
	return NULL;
	}

	rc = cmap_fd_get(handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}

	rc = cmap_get_string(handle, "totem.cluster_name", &cluster_name);
	if (rc != CS_OK) {
	crm_info("Cannot get totem.cluster_name: %s (%d)", cs_strerror(rc), rc);

	} else {
	crm_debug("cmap totem.cluster_name = '%s'", cluster_name);
	}

	bail:
	cmap_finalize(handle);
	return cluster_name;
	}

	/*!
	* \internal
	* \brief Check (via CMAP) whether Corosync configuration has a node list
	*
	* \return true if Corosync has node list, otherwise false
	*/
	bool
	pcmk__corosync_has_nodelist(void)
	{
	cs_error_t cs_rc = CS_OK;
	int retries = 0;
	cmap_handle_t cmap_handle;
	cmap_iter_handle_t iter_handle;
	char key_name[CMAP_KEYNAME_MAXLEN + 1];
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rc = pcmk_ok;

	static bool got_result = false;
	static bool result = false;

	if (got_result) {
	return result;
	}

	// Connect to CMAP
	do {
	cs_rc = pcmk__init_cmap(&cmap_handle);
	if (cs_rc != CS_OK) {
	retries++;
	crm_debug("CMAP connection failed: %s (rc=%d, retrying in %ds)",
	cs_strerror(cs_rc), cs_rc, retries);
	sleep(retries);
	}
	} while ((retries < 5) && (cs_rc != CS_OK));
	if (cs_rc != CS_OK) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP connection failed (%s) " QB_XS " rc=%d",
	cs_strerror(cs_rc), cs_rc);
	return false;
	}

	// Get CMAP connection file descriptor
	cs_rc = cmap_fd_get(cmap_handle, &fd);
	if (cs_rc != CS_OK) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP unusable (%s) " QB_XS " rc=%d",
	cs_strerror(cs_rc), cs_rc);
	goto bail;
	}

	// Check whether CMAP connection is authentic (i.e. provided by root)
	rc = crm_ipc_is_authentic_process(fd, (uid_t) 0, (gid_t) 0,
	&found_pid, &found_uid, &found_gid);
	if (rc == 0) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP provider is inauthentic "
	QB_XS " pid=%lld uid=%lld gid=%lld",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rc < 0) {
	crm_warn("Assuming Corosync does not have node list: "
	"Could not verify CMAP authenticity (%s) " QB_XS " rc=%d",
	pcmk_strerror(rc), rc);
	goto bail;
	}

	// Check whether nodelist section is presetn
	cs_rc = cmap_iter_init(cmap_handle, "nodelist", &iter_handle);
	if (cs_rc != CS_OK) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP not readable (%s) " QB_XS " rc=%d",
	cs_strerror(cs_rc), cs_rc);
	goto bail;
	}

	cs_rc = cmap_iter_next(cmap_handle, iter_handle, key_name, NULL, NULL);
	if (cs_rc == CS_OK) {
	result = true;
	}

	cmap_iter_finalize(cmap_handle, iter_handle);
	got_result = true;
	crm_debug("Corosync %s node list", (result? "has" : "does not have"));

	bail:
	cmap_finalize(cmap_handle);
	return result;
	}
	diff --git a/lib/cluster/cpg.c b/lib/cluster/cpg.c
	index 4f6d644b94..6a36947386 100644
	--- a/lib/cluster/cpg.c
	+++ b/lib/cluster/cpg.c
	@@ -1,1103 +1,1093 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <arpa/inet.h>
	#include <inttypes.h> // PRIu32
	#include <netdb.h>
	#include <netinet/in.h>
	#include <stdbool.h>
	#include <stdint.h> // uint32_t
	#include <sys/socket.h>
	#include <sys/types.h> // size_t
	#include <sys/utsname.h>

	#include <bzlib.h>
	#include <corosync/corodefs.h>
	#include <corosync/corotypes.h>
	#include <corosync/hdb.h>
	#include <corosync/cpg.h>
	#include <qb/qbipc_common.h>
	#include <qb/qbipcc.h>
	#include <qb/qbutil.h>

	#include <crm/cluster/internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h> // PCMK__SPECIAL_PID
	#include <crm/common/mainloop.h>
	#include <crm/common/xml.h>

	#include "crmcluster_private.h"

	/* @TODO Once we can update the public API to require pcmk_cluster_t* in more
	* functions, we can ditch this in favor of cluster->cpg_handle.
	*/
	static cpg_handle_t pcmk_cpg_handle = 0;

	// @TODO These could be moved to pcmk_cluster_t* at that time as well
	static bool cpg_evicted = false;
	static GList *cs_message_queue = NULL;
	static int cs_message_timer = 0;

	struct pcmk__cpg_host_s {
	uint32_t id;
	uint32_t pid;
	- gboolean local;
	enum crm_ais_msg_types type;
	uint32_t size;
	char uname[MAX_NAME];
	} __attribute__ ((packed));

	typedef struct pcmk__cpg_host_s pcmk__cpg_host_t;

	struct pcmk__cpg_msg_s {
	struct qb_ipc_response_header header __attribute__ ((aligned(8)));
	uint32_t id;
	gboolean is_compressed;

	pcmk__cpg_host_t host;
	pcmk__cpg_host_t sender;

	uint32_t size;
	uint32_t compressed_size;
	/* 584 bytes */
	char data[0];

	} __attribute__ ((packed));

	typedef struct pcmk__cpg_msg_s pcmk__cpg_msg_t;

	static void crm_cs_flush(gpointer data);

	#define msg_data_len(msg) (msg->is_compressed?msg->compressed_size:msg->size)

	#define cs_repeat(rc, counter, max, code) do { \
	rc = code; \
	if ((rc == CS_ERR_TRY_AGAIN) \|\| (rc == CS_ERR_QUEUE_FULL)) { \
	counter++; \
	crm_debug("Retrying operation after %ds", counter); \
	sleep(counter); \
	} else { \
	break; \
	} \
	} while (counter < max)

	/*!
	* \internal
	* \brief Get the local Corosync node ID (via CPG)
	*
	* \param[in] handle CPG connection to use (or 0 to use new connection)
	*
	* \return Corosync ID of local node (or 0 if not known)
	*/
	uint32_t
	pcmk__cpg_local_nodeid(cpg_handle_t handle)
	{
	cs_error_t rc = CS_OK;
	int retries = 0;
	static uint32_t local_nodeid = 0;
	cpg_handle_t local_handle = handle;
	cpg_model_v1_data_t cpg_model_info = {CPG_MODEL_V1, NULL, NULL, NULL, 0};
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv = 0;

	if (local_nodeid != 0) {
	return local_nodeid;
	}

	if (handle == 0) {
	crm_trace("Creating connection");
	cs_repeat(rc, retries, 5,
	cpg_model_initialize(&local_handle, CPG_MODEL_V1,
	(cpg_model_data_t *) &cpg_model_info,
	NULL));
	if (rc != CS_OK) {
	crm_err("Could not connect to the CPG API: %s (%d)",
	cs_strerror(rc), rc);
	return 0;
	}

	rc = cpg_fd_get(local_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CPG API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	// CPG provider run as root (at least in given user namespace)?
	rv = crm_ipc_is_authentic_process(fd, (uid_t) 0, (gid_t) 0, &found_pid,
	&found_uid, &found_gid);
	if (rv == 0) {
	crm_err("CPG provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;

	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CPG provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}
	}

	if (rc == CS_OK) {
	retries = 0;
	crm_trace("Performing lookup");
	cs_repeat(rc, retries, 5, cpg_local_get(local_handle, &local_nodeid));
	}

	if (rc != CS_OK) {
	crm_err("Could not get local node id from the CPG API: %s (%d)",
	pcmk__cs_err_str(rc), rc);
	}

	bail:
	if (handle == 0) {
	crm_trace("Closing connection");
	cpg_finalize(local_handle);
	}
	crm_debug("Local nodeid is %u", local_nodeid);
	return local_nodeid;
	}

	/*!
	* \internal
	* \brief Callback function for Corosync message queue timer
	*
	* \param[in] data CPG handle
	*
	* \return FALSE (to indicate to glib that timer should not be removed)
	*/
	static gboolean
	crm_cs_flush_cb(gpointer data)
	{
	cs_message_timer = 0;
	crm_cs_flush(data);
	return FALSE;
	}

	// Send no more than this many CPG messages in one flush
	#define CS_SEND_MAX 200

	/*!
	* \internal
	* \brief Send messages in Corosync CPG message queue
	*
	* \param[in] data CPG handle
	*/
	static void
	crm_cs_flush(gpointer data)
	{
	unsigned int sent = 0;
	guint queue_len = 0;
	cs_error_t rc = 0;
	cpg_handle_t handle = (cpg_handle_t ) data;

	if (*handle == 0) {
	crm_trace("Connection is dead");
	return;
	}

	queue_len = g_list_length(cs_message_queue);
	if (((queue_len % 1000) == 0) && (queue_len > 1)) {
	crm_err("CPG queue has grown to %d", queue_len);

	} else if (queue_len == CS_SEND_MAX) {
	crm_warn("CPG queue has grown to %d", queue_len);
	}

	if (cs_message_timer != 0) {
	/* There is already a timer, wait until it goes off */
	crm_trace("Timer active %d", cs_message_timer);
	return;
	}

	while ((cs_message_queue != NULL) && (sent < CS_SEND_MAX)) {
	struct iovec *iov = cs_message_queue->data;

	rc = cpg_mcast_joined(*handle, CPG_TYPE_AGREED, iov, 1);
	if (rc != CS_OK) {
	break;
	}

	sent++;
	crm_trace("CPG message sent, size=%llu",
	(unsigned long long) iov->iov_len);

	cs_message_queue = g_list_remove(cs_message_queue, iov);
	free(iov->iov_base);
	free(iov);
	}

	queue_len -= sent;
	do_crm_log((queue_len > 5)? LOG_INFO : LOG_TRACE,
	"Sent %u CPG message%s (%d still queued): %s (rc=%d)",
	sent, pcmk__plural_s(sent), queue_len, pcmk__cs_err_str(rc),
	(int) rc);

	if (cs_message_queue) {
	uint32_t delay_ms = 100;
	if (rc != CS_OK) {
	/* Proportionally more if sending failed but cap at 1s */
	delay_ms = QB_MIN(1000, CS_SEND_MAX + (10 * queue_len));
	}
	cs_message_timer = g_timeout_add(delay_ms, crm_cs_flush_cb, data);
	}
	}

	/*!
	* \internal
	* \brief Dispatch function for CPG handle
	*
	* \param[in,out] user_data Cluster object
	*
	* \return 0 on success, -1 on error (per mainloop_io_t interface)
	*/
	static int
	pcmk_cpg_dispatch(gpointer user_data)
	{
	cs_error_t rc = CS_OK;
	pcmk_cluster_t cluster = (pcmk_cluster_t ) user_data;

	rc = cpg_dispatch(cluster->cpg_handle, CS_DISPATCH_ONE);
	if (rc != CS_OK) {
	crm_err("Connection to the CPG API failed: %s (%d)",
	pcmk__cs_err_str(rc), rc);
	cpg_finalize(cluster->cpg_handle);
	cluster->cpg_handle = 0;
	return -1;

	} else if (cpg_evicted) {
	crm_err("Evicted from CPG membership");
	return -1;
	}
	return 0;
	}

	static inline const char *
	ais_dest(const pcmk__cpg_host_t *host)
	{
	- if (host->local) {
	- return "local";
	- } else if (host->size > 0) {
	- return host->uname;
	- } else {
	- return "<all>";
	- }
	+ return (host->size > 0)? host->uname : "<all>";
	}

	static inline const char *
	msg_type2text(enum crm_ais_msg_types type)
	{
	const char *text = "unknown";

	switch (type) {
	case crm_msg_none:
	text = "unknown";
	break;
	case crm_msg_ais:
	text = "ais";
	break;
	case crm_msg_cib:
	text = "cib";
	break;
	case crm_msg_crmd:
	text = "crmd";
	break;
	case crm_msg_pe:
	text = "pengine";
	break;
	case crm_msg_te:
	text = "tengine";
	break;
	case crm_msg_lrmd:
	text = "lrmd";
	break;
	case crm_msg_attrd:
	text = "attrd";
	break;
	case crm_msg_stonithd:
	text = "stonithd";
	break;
	case crm_msg_stonith_ng:
	text = "stonith-ng";
	break;
	}
	return text;
	}

	/*!
	* \internal
	* \brief Check whether a Corosync CPG message is valid
	*
	* \param[in] msg Corosync CPG message to check
	*
	* \return true if \p msg is valid, otherwise false
	*/
	static bool
	check_message_sanity(const pcmk__cpg_msg_t *msg)
	{
	int32_t payload_size = msg->header.size - sizeof(pcmk__cpg_msg_t);

	if (payload_size < 1) {
	crm_err("%sCPG message %d from %s invalid: "
	"Claimed size of %d bytes is too small "
	QB_XS " from %s[%u] to %s@%s",
	(msg->is_compressed? "Compressed " : ""),
	msg->id, ais_dest(&(msg->sender)),
	(int) msg->header.size,
	msg_type2text(msg->sender.type), msg->sender.pid,
	msg_type2text(msg->host.type), ais_dest(&(msg->host)));
	return false;
	}

	if (msg->header.error != CS_OK) {
	crm_err("%sCPG message %d from %s invalid: "
	"Sender indicated error %d "
	QB_XS " from %s[%u] to %s@%s",
	(msg->is_compressed? "Compressed " : ""),
	msg->id, ais_dest(&(msg->sender)),
	msg->header.error,
	msg_type2text(msg->sender.type), msg->sender.pid,
	msg_type2text(msg->host.type), ais_dest(&(msg->host)));
	return false;
	}

	if (msg_data_len(msg) != payload_size) {
	crm_err("%sCPG message %d from %s invalid: "
	"Total size %d inconsistent with payload size %d "
	QB_XS " from %s[%u] to %s@%s",
	(msg->is_compressed? "Compressed " : ""),
	msg->id, ais_dest(&(msg->sender)),
	(int) msg->header.size, (int) msg_data_len(msg),
	msg_type2text(msg->sender.type), msg->sender.pid,
	msg_type2text(msg->host.type), ais_dest(&(msg->host)));
	return false;
	}

	if (!msg->is_compressed &&
	/* msg->size != (strlen(msg->data) + 1) would be a stronger check,
	* but checking the last byte or two should be quick
	*/
	(((msg->size > 1) && (msg->data[msg->size - 2] == '\0'))
	\|\| (msg->data[msg->size - 1] != '\0'))) {
	crm_err("CPG message %d from %s invalid: "
	"Payload does not end at byte %llu "
	QB_XS " from %s[%u] to %s@%s",
	msg->id, ais_dest(&(msg->sender)),
	(unsigned long long) msg->size,
	msg_type2text(msg->sender.type), msg->sender.pid,
	msg_type2text(msg->host.type), ais_dest(&(msg->host)));
	return false;
	}

	crm_trace("Verified %d-byte %sCPG message %d from %s[%u]@%s to %s@%s",
	(int) msg->header.size, (msg->is_compressed? "compressed " : ""),
	msg->id, msg_type2text(msg->sender.type), msg->sender.pid,
	ais_dest(&(msg->sender)),
	msg_type2text(msg->host.type), ais_dest(&(msg->host)));
	return true;
	}

	/*!
	* \internal
	* \brief Extract text data from a Corosync CPG message
	*
	* \param[in] handle CPG connection (to get local node ID if not known)
	* \param[in] sender_id Corosync ID of node that sent message
	* \param[in] pid Process ID of message sender (for logging only)
	* \param[in,out] content CPG message
	* \param[out] kind If not \c NULL, will be set to CPG header ID
	* (which should be an <tt>enum crm_ais_msg_class</tt>
	* value, currently always \c crm_class_cluster)
	* \param[out] from If not \c NULL, will be set to sender uname
	* (valid for the lifetime of \p content)
	*
	* \return Newly allocated string with message data
	*
	* \note The caller is responsible for freeing the return value using \c free().
	*/
	char *
	pcmk__cpg_message_data(cpg_handle_t handle, uint32_t sender_id, uint32_t pid,
	void content, uint32_t kind, const char **from)
	{
	char *data = NULL;
	pcmk__cpg_msg_t *msg = content;

	if (handle != 0) {
	// Do filtering and field massaging
	uint32_t local_nodeid = pcmk__cpg_local_nodeid(handle);
	const char *local_name = pcmk__cluster_local_node_name();

	if ((msg->sender.id != 0) && (msg->sender.id != sender_id)) {
	crm_err("Nodeid mismatch from %" PRIu32 ".%" PRIu32
	": claimed nodeid=%" PRIu32,
	sender_id, pid, msg->sender.id);
	return NULL;
	}
	if ((msg->host.id != 0) && (local_nodeid != msg->host.id)) {
	crm_trace("Not for us: %" PRIu32" != %" PRIu32,
	msg->host.id, local_nodeid);
	return NULL;
	}
	if ((msg->host.size > 0)
	&& !pcmk__str_eq(msg->host.uname, local_name, pcmk__str_casei)) {

	crm_trace("Not for us: %s != %s", msg->host.uname, local_name);
	return NULL;
	}

	msg->sender.id = sender_id;
	if (msg->sender.size == 0) {
	const pcmk__node_status_t *peer =
	pcmk__get_node(sender_id, NULL, NULL,
	pcmk__node_search_cluster_member);

	if (peer->uname == NULL) {
	crm_err("No uname for peer with nodeid=%u", sender_id);

	} else {
	crm_notice("Fixing uname for peer with nodeid=%u", sender_id);
	msg->sender.size = strlen(peer->uname);
	memset(msg->sender.uname, 0, MAX_NAME);
	memcpy(msg->sender.uname, peer->uname, msg->sender.size);
	}
	}
	}

	crm_trace("Got new%s message (size=%d, %d, %d)",
	msg->is_compressed ? " compressed" : "",
	msg_data_len(msg), msg->size, msg->compressed_size);

	if (kind != NULL) {
	*kind = msg->header.id;
	}
	if (from != NULL) {
	*from = msg->sender.uname;
	}

	if (msg->is_compressed && (msg->size > 0)) {
	int rc = BZ_OK;
	char *uncompressed = NULL;
	unsigned int new_size = msg->size + 1;

	if (!check_message_sanity(msg)) {
	goto badmsg;
	}

	crm_trace("Decompressing message data");
	uncompressed = pcmk__assert_alloc(1, new_size);
	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &new_size, msg->data,
	msg->compressed_size, 1, 0);

	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_err("Decompression failed: %s " QB_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	free(uncompressed);
	goto badmsg;
	}

	CRM_ASSERT(new_size == msg->size);

	data = uncompressed;

	} else if (!check_message_sanity(msg)) {
	goto badmsg;

	} else {
	data = strdup(msg->data);
	}

	// Is this necessary?
	pcmk__get_node(msg->sender.id, msg->sender.uname, NULL,
	pcmk__node_search_cluster_member);

	crm_trace("Payload: %.200s", data);
	return data;

	badmsg:
	crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
	" min=%d, total=%d, size=%d, bz2_size=%d",
	msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
	ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
	msg->sender.pid, (int)sizeof(pcmk__cpg_msg_t),
	msg->header.size, msg->size, msg->compressed_size);

	free(data);
	return NULL;
	}

	/*!
	* \internal
	* \brief Compare cpg_address objects by node ID
	*
	* \param[in] first First cpg_address structure to compare
	* \param[in] second Second cpg_address structure to compare
	*
	* \return Negative number if first's node ID is lower,
	* positive number if first's node ID is greater,
	* or 0 if both node IDs are equal
	*/
	static int
	cmp_member_list_nodeid(const void first, const void second)
	{
	const struct cpg_address const a = ((const struct cpg_address **) first),
	const b = ((const struct cpg_address **) second);
	if (a->nodeid < b->nodeid) {
	return -1;
	} else if (a->nodeid > b->nodeid) {
	return 1;
	}
	/* don't bother with "reason" nor "pid" */
	return 0;
	}

	/*!
	* \internal
	* \brief Get a readable string equivalent of a cpg_reason_t value
	*
	* \param[in] reason CPG reason value
	*
	* \return Readable string suitable for logging
	*/
	static const char *
	cpgreason2str(cpg_reason_t reason)
	{
	switch (reason) {
	case CPG_REASON_JOIN: return " via cpg_join";
	case CPG_REASON_LEAVE: return " via cpg_leave";
	case CPG_REASON_NODEDOWN: return " via cluster exit";
	case CPG_REASON_NODEUP: return " via cluster join";
	case CPG_REASON_PROCDOWN: return " for unknown reason";
	default: break;
	}
	return "";
	}

	/*!
	* \internal
	* \brief Get a log-friendly node name
	*
	* \param[in] peer Node to check
	*
	* \return Node's uname, or readable string if not known
	*/
	static inline const char *
	peer_name(const pcmk__node_status_t *peer)
	{
	if (peer == NULL) {
	return "unknown node";
	} else if (peer->uname == NULL) {
	return "peer node";
	} else {
	return peer->uname;
	}
	}

	/*!
	* \internal
	* \brief Process a CPG peer's leaving the cluster
	*
	* \param[in] cpg_group_name CPG group name (for logging)
	* \param[in] event_counter Event number (for logging)
	* \param[in] local_nodeid Node ID of local node
	* \param[in] cpg_peer CPG peer that left
	* \param[in] sorted_member_list List of remaining members, qsort()-ed by ID
	* \param[in] member_list_entries Number of entries in \p sorted_member_list
	*/
	static void
	node_left(const char *cpg_group_name, int event_counter,
	uint32_t local_nodeid, const struct cpg_address *cpg_peer,
	const struct cpg_address **sorted_member_list,
	size_t member_list_entries)
	{
	pcmk__node_status_t *peer =
	pcmk__search_node_caches(cpg_peer->nodeid, NULL,
	pcmk__node_search_cluster_member);
	const struct cpg_address **rival = NULL;

	/* Most CPG-related Pacemaker code assumes that only one process on a node
	* can be in the process group, but Corosync does not impose this
	* limitation, and more than one can be a member in practice due to a
	* daemon attempting to start while another instance is already running.
	*
	* Check for any such duplicate instances, because we don't want to process
	* their leaving as if our actual peer left. If the peer that left still has
	* an entry in sorted_member_list (with a different PID), we will ignore the
	* leaving.
	*
	* @TODO Track CPG members' PIDs so we can tell exactly who left.
	*/
	if (peer != NULL) {
	rival = bsearch(&cpg_peer, sorted_member_list, member_list_entries,
	sizeof(const struct cpg_address *),
	cmp_member_list_nodeid);
	}

	if (rival == NULL) {
	crm_info("Group %s event %d: %s (node %u pid %u) left%s",
	cpg_group_name, event_counter, peer_name(peer),
	cpg_peer->nodeid, cpg_peer->pid,
	cpgreason2str(cpg_peer->reason));
	if (peer != NULL) {
	crm_update_peer_proc(__func__, peer, crm_proc_cpg,
	PCMK_VALUE_OFFLINE);
	}
	} else if (cpg_peer->nodeid == local_nodeid) {
	crm_warn("Group %s event %d: duplicate local pid %u left%s",
	cpg_group_name, event_counter,
	cpg_peer->pid, cpgreason2str(cpg_peer->reason));
	} else {
	crm_warn("Group %s event %d: "
	"%s (node %u) duplicate pid %u left%s (%u remains)",
	cpg_group_name, event_counter, peer_name(peer),
	cpg_peer->nodeid, cpg_peer->pid,
	cpgreason2str(cpg_peer->reason), (*rival)->pid);
	}
	}

	/*!
	* \internal
	* \brief Handle a CPG configuration change event
	*
	* \param[in] handle CPG connection
	* \param[in] group_name CPG group name
	* \param[in] member_list List of current CPG members
	* \param[in] member_list_entries Number of entries in \p member_list
	* \param[in] left_list List of CPG members that left
	* \param[in] left_list_entries Number of entries in \p left_list
	* \param[in] joined_list List of CPG members that joined
	* \param[in] joined_list_entries Number of entries in \p joined_list
	*
	* \note This is of type \c cpg_confchg_fn_t, intended to be used in a
	* \c cpg_callbacks_t object.
	*/
	void
	pcmk__cpg_confchg_cb(cpg_handle_t handle,
	const struct cpg_name *group_name,
	const struct cpg_address *member_list,
	size_t member_list_entries,
	const struct cpg_address *left_list,
	size_t left_list_entries,
	const struct cpg_address *joined_list,
	size_t joined_list_entries)
	{
	static int counter = 0;

	bool found = false;
	uint32_t local_nodeid = pcmk__cpg_local_nodeid(handle);
	const struct cpg_address **sorted = NULL;

	sorted = pcmk__assert_alloc(member_list_entries,
	sizeof(const struct cpg_address *));

	for (size_t iter = 0; iter < member_list_entries; iter++) {
	sorted[iter] = member_list + iter;
	}

	// So that the cross-matching of multiply-subscribed nodes is then cheap
	qsort(sorted, member_list_entries, sizeof(const struct cpg_address *),
	cmp_member_list_nodeid);

	for (int i = 0; i < left_list_entries; i++) {
	node_left(group_name->value, counter, local_nodeid, &left_list[i],
	sorted, member_list_entries);
	}
	free(sorted);
	sorted = NULL;

	for (int i = 0; i < joined_list_entries; i++) {
	crm_info("Group %s event %d: node %u pid %u joined%s",
	group_name->value, counter, joined_list[i].nodeid,
	joined_list[i].pid, cpgreason2str(joined_list[i].reason));
	}

	for (int i = 0; i < member_list_entries; i++) {
	pcmk__node_status_t *peer =
	pcmk__get_node(member_list[i].nodeid, NULL, NULL,
	pcmk__node_search_cluster_member);

	if (member_list[i].nodeid == local_nodeid
	&& member_list[i].pid != getpid()) {
	// See the note in node_left()
	crm_warn("Group %s event %d: detected duplicate local pid %u",
	group_name->value, counter, member_list[i].pid);
	continue;
	}
	crm_info("Group %s event %d: %s (node %u pid %u) is member",
	group_name->value, counter, peer_name(peer),
	member_list[i].nodeid, member_list[i].pid);

	/* If the caller left auto-reaping enabled, this will also update the
	* state to member.
	*/
	peer = crm_update_peer_proc(__func__, peer, crm_proc_cpg,
	PCMK_VALUE_ONLINE);

	if (peer && peer->state && strcmp(peer->state, CRM_NODE_MEMBER)) {
	/* The node is a CPG member, but we currently think it's not a
	* cluster member. This is possible only if auto-reaping was
	* disabled. The node may be joining, and we happened to get the CPG
	* notification before the quorum notification; or the node may have
	* just died, and we are processing its final messages; or a bug
	* has affected the peer cache.
	*/
	time_t now = time(NULL);

	if (peer->when_lost == 0) {
	// Track when we first got into this contradictory state
	peer->when_lost = now;

	} else if (now > (peer->when_lost + 60)) {
	// If it persists for more than a minute, update the state
	crm_warn("Node %u is member of group %s but was believed "
	"offline",
	member_list[i].nodeid, group_name->value);
	pcmk__update_peer_state(__func__, peer, CRM_NODE_MEMBER, 0);
	}
	}

	if (local_nodeid == member_list[i].nodeid) {
	found = true;
	}
	}

	if (!found) {
	crm_err("Local node was evicted from group %s", group_name->value);
	cpg_evicted = true;
	}

	counter++;
	}

	/*!
	* \brief Set the CPG deliver callback function for a cluster object
	*
	* \param[in,out] cluster Cluster object
	* \param[in] fn Deliver callback function to set
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk_cpg_set_deliver_fn(pcmk_cluster_t *cluster, cpg_deliver_fn_t fn)
	{
	if (cluster == NULL) {
	return EINVAL;
	}
	cluster->cpg.cpg_deliver_fn = fn;
	return pcmk_rc_ok;
	}

	/*!
	* \brief Set the CPG config change callback function for a cluster object
	*
	* \param[in,out] cluster Cluster object
	* \param[in] fn Configuration change callback function to set
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk_cpg_set_confchg_fn(pcmk_cluster_t *cluster, cpg_confchg_fn_t fn)
	{
	if (cluster == NULL) {
	return EINVAL;
	}
	cluster->cpg.cpg_confchg_fn = fn;
	return pcmk_rc_ok;
	}

	/*!
	* \brief Connect to Corosync CPG
	*
	* \param[in,out] cluster Initialized cluster object to connect
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__cpg_connect(pcmk_cluster_t *cluster)
	{
	cs_error_t rc;
	int fd = -1;
	int retries = 0;
	uint32_t id = 0;
	pcmk__node_status_t *peer = NULL;
	cpg_handle_t handle = 0;
	const char *message_name = pcmk__message_name(crm_system_name);
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	struct mainloop_fd_callbacks cpg_fd_callbacks = {
	.dispatch = pcmk_cpg_dispatch,
	.destroy = cluster->destroy,
	};

	cpg_model_v1_data_t cpg_model_info = {
	.model = CPG_MODEL_V1,
	.cpg_deliver_fn = cluster->cpg.cpg_deliver_fn,
	.cpg_confchg_fn = cluster->cpg.cpg_confchg_fn,
	.cpg_totem_confchg_fn = NULL,
	.flags = 0,
	};

	cpg_evicted = false;
	cluster->group.length = 0;
	cluster->group.value[0] = 0;

	/* group.value is char[128] */
	strncpy(cluster->group.value, message_name, 127);
	cluster->group.value[127] = 0;
	cluster->group.length = 1 + QB_MIN(127, strlen(cluster->group.value));

	cs_repeat(rc, retries, 30, cpg_model_initialize(&handle, CPG_MODEL_V1, (cpg_model_data_t *)&cpg_model_info, NULL));
	if (rc != CS_OK) {
	crm_err("Could not connect to the CPG API: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	rc = cpg_fd_get(handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CPG API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CPG provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CPG provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	rc = CS_ERR_ACCESS;
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CPG provider: %s (%d)",
	strerror(-rv), -rv);
	rc = CS_ERR_ACCESS;
	goto bail;
	}

	id = pcmk__cpg_local_nodeid(handle);
	if (id == 0) {
	crm_err("Could not get local node id from the CPG API");
	goto bail;

	}
	cluster->nodeid = id;

	retries = 0;
	cs_repeat(rc, retries, 30, cpg_join(handle, &cluster->group));
	if (rc != CS_OK) {
	crm_err("Could not join the CPG group '%s': %d", message_name, rc);
	goto bail;
	}

	pcmk_cpg_handle = handle;
	cluster->cpg_handle = handle;
	mainloop_add_fd("corosync-cpg", G_PRIORITY_MEDIUM, fd, cluster, &cpg_fd_callbacks);

	bail:
	if (rc != CS_OK) {
	cpg_finalize(handle);
	// @TODO Map rc to more specific Pacemaker return code
	return ENOTCONN;
	}

	peer = pcmk__get_node(id, NULL, NULL, pcmk__node_search_cluster_member);
	crm_update_peer_proc(__func__, peer, crm_proc_cpg, PCMK_VALUE_ONLINE);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Disconnect from Corosync CPG
	*
	* \param[in,out] cluster Cluster object to disconnect
	*/
	void
	pcmk__cpg_disconnect(pcmk_cluster_t *cluster)
	{
	pcmk_cpg_handle = 0;
	if (cluster->cpg_handle != 0) {
	crm_trace("Disconnecting CPG");
	cpg_leave(cluster->cpg_handle, &cluster->group);
	cpg_finalize(cluster->cpg_handle);
	cluster->cpg_handle = 0;

	} else {
	crm_info("No CPG connection");
	}
	}

	/*!
	* \internal
	* \brief Send string data via Corosync CPG
	*
	* \param[in] data Data to send
	- * \param[in] local What to set as host "local" value (which is never used)
	* \param[in] node Cluster node to send message to
	* \param[in] dest Type of message to send
	*
	* \return \c true on success, or \c false otherwise
	*/
	static bool
	-send_cpg_text(const char data, bool local, const pcmk__node_status_t node,
	+send_cpg_text(const char data, const pcmk__node_status_t node,
	enum crm_ais_msg_types dest)
	{
	- // @COMPAT Drop local argument when send_cluster_text is dropped
	static int msg_id = 0;
	static int local_pid = 0;
	static int local_name_len = 0;
	static const char *local_name = NULL;

	char *target = NULL;
	struct iovec *iov;
	pcmk__cpg_msg_t *msg = NULL;

	CRM_CHECK(dest != crm_msg_ais, return false);

	if (local_name == NULL) {
	local_name = pcmk__cluster_local_node_name();
	}
	if ((local_name_len == 0) && (local_name != NULL)) {
	local_name_len = strlen(local_name);
	}

	if (data == NULL) {
	data = "";
	}

	if (local_pid == 0) {
	local_pid = getpid();
	}

	msg = pcmk__assert_alloc(1, sizeof(pcmk__cpg_msg_t));

	msg_id++;
	msg->id = msg_id;
	msg->header.id = crm_class_cluster;
	msg->header.error = CS_OK;

	msg->host.type = dest;
	- msg->host.local = local;

	if (node != NULL) {
	if (node->uname != NULL) {
	target = pcmk__str_copy(node->uname);
	msg->host.size = strlen(node->uname);
	memset(msg->host.uname, 0, MAX_NAME);
	memcpy(msg->host.uname, node->uname, msg->host.size);

	} else {
	- target = crm_strdup_printf("%u", node->id);
	+ target = crm_strdup_printf("%" PRIu32, node->cluster_layer_id);
	}
	- msg->host.id = node->id;
	+ msg->host.id = node->cluster_layer_id;

	} else {
	target = pcmk__str_copy("all");
	}

	msg->sender.id = 0;
	msg->sender.type = pcmk__cluster_parse_msg_type(crm_system_name);
	msg->sender.pid = local_pid;
	msg->sender.size = local_name_len;
	memset(msg->sender.uname, 0, MAX_NAME);

	if ((local_name != NULL) && (msg->sender.size != 0)) {
	memcpy(msg->sender.uname, local_name, msg->sender.size);
	}

	msg->size = 1 + strlen(data);
	msg->header.size = sizeof(pcmk__cpg_msg_t) + msg->size;

	if (msg->size < CRM_BZ2_THRESHOLD) {
	msg = pcmk__realloc(msg, msg->header.size);
	memcpy(msg->data, data, msg->size);

	} else {
	char *compressed = NULL;
	unsigned int new_size = 0;

	if (pcmk__compress(data, (unsigned int) msg->size, 0, &compressed,
	&new_size) == pcmk_rc_ok) {

	msg->header.size = sizeof(pcmk__cpg_msg_t) + new_size;
	msg = pcmk__realloc(msg, msg->header.size);
	memcpy(msg->data, compressed, new_size);

	msg->is_compressed = TRUE;
	msg->compressed_size = new_size;

	} else {
	// cppcheck seems not to understand the abort logic in pcmk__realloc
	// cppcheck-suppress memleak
	msg = pcmk__realloc(msg, msg->header.size);
	memcpy(msg->data, data, msg->size);
	}

	free(compressed);
	}

	iov = pcmk__assert_alloc(1, sizeof(struct iovec));
	iov->iov_base = msg;
	iov->iov_len = msg->header.size;

	if (msg->compressed_size > 0) {
	crm_trace("Queueing CPG message %u to %s "
	"(%llu bytes, %d bytes compressed payload): %.200s",
	msg->id, target, (unsigned long long) iov->iov_len,
	msg->compressed_size, data);
	} else {
	crm_trace("Queueing CPG message %u to %s "
	"(%llu bytes, %d bytes payload): %.200s",
	msg->id, target, (unsigned long long) iov->iov_len,
	msg->size, data);
	}

	free(target);

	cs_message_queue = g_list_append(cs_message_queue, iov);
	crm_cs_flush(&pcmk_cpg_handle);

	return true;
	}

	/*!
	* \internal
	* \brief Send an XML message via Corosync CPG
	*
	* \param[in] msg XML message to send
	* \param[in] node Cluster node to send message to
	* \param[in] dest Type of message to send
	*
	* \return TRUE on success, otherwise FALSE
	*/
	bool
	pcmk__cpg_send_xml(const xmlNode msg, const pcmk__node_status_t node,
	enum crm_ais_msg_types dest)
	{
	bool rc = true;
	GString *data = g_string_sized_new(1024);

	pcmk__xml_string(msg, 0, data, 0);

	- rc = send_cpg_text(data->str, false, node, dest);
	+ rc = send_cpg_text(data->str, node, dest);
	g_string_free(data, TRUE);
	return rc;
	}
	diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c
	index 447871b6c9..cd5d90a3f8 100644
	--- a/lib/cluster/membership.c
	+++ b/lib/cluster/membership.c
	@@ -1,1486 +1,1501 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <inttypes.h> // PRIu32
	#include <stdbool.h> // bool
	#include <stdio.h>
	#include <string.h>
	#include <sys/param.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <glib.h>

	#include <crm/common/ipc.h>
	#include <crm/common/xml_internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/xml.h>
	#include <crm/stonith-ng.h>
	#include "crmcluster_private.h"

	/* The peer cache remembers cluster nodes that have been seen.
	* This is managed mostly automatically by libcluster, based on
	* cluster membership events.
	*
	* Because cluster nodes can have conflicting names or UUIDs,
	* the hash table key is a uniquely generated ID.
	*
	* @COMPAT When this is internal, rename to cluster_node_member_cache and make
	* static.
	*/
	GHashTable *crm_peer_cache = NULL;

	/*
	* The remote peer cache tracks pacemaker_remote nodes. While the
	* value has the same type as the peer cache's, it is tracked separately for
	* three reasons: pacemaker_remote nodes can't have conflicting names or UUIDs,
	* so the name (which is also the UUID) is used as the hash table key; there
	* is no equivalent of membership events, so management is not automatic; and
	* most users of the peer cache need to exclude pacemaker_remote nodes.
	*
	* That said, using a single cache would be more logical and less error-prone,
	* so it would be a good idea to merge them one day.
	*
	* libcluster provides two avenues for populating the cache:
	* pcmk__cluster_lookup_remote_node() and pcmk__cluster_forget_remote_node()
	* directly manage it, while refresh_remote_nodes() populates it via the CIB.
	*/
	GHashTable *crm_remote_peer_cache = NULL;

	/*
	* The CIB cluster node cache tracks cluster nodes that have been seen in
	* the CIB. It is useful mainly when a caller needs to know about a node that
	* may no longer be in the membership, but doesn't want to add the node to the
	* main peer cache tables.
	*/
	static GHashTable *cluster_node_cib_cache = NULL;

	unsigned long long crm_peer_seq = 0;
	static bool autoreap = true;
	static bool has_quorum = false;

	// Flag setting and clearing for pcmk__node_status_t:flags

	#define set_peer_flags(peer, flags_to_set) do { \
	(peer)->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Peer", (peer)->uname, \
	(peer)->flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define clear_peer_flags(peer, flags_to_clear) do { \
	(peer)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Peer", (peer)->uname, \
	(peer)->flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	static void update_peer_uname(pcmk__node_status_t node, const char uname);
	static pcmk__node_status_t find_cib_cluster_node(const char id,
	const char *uname);

	/*!
	* \internal
	* \brief Check whether the cluster currently has quorum
	*
	* \return \c true if the cluster has quorum, or \c false otherwise
	*/
	bool
	pcmk__cluster_has_quorum(void)
	{
	return has_quorum;
	}

	/*!
	* \internal
	* \brief Set whether the cluster currently has quorum
	*
	* \param[in] quorate \c true if the cluster has quorum, or \c false otherwise
	*/
	void
	pcmk__cluster_set_quorum(bool quorate)
	{
	has_quorum = quorate;
	}

	/*!
	* \internal
	* \brief Get the number of Pacemaker Remote nodes that have been seen
	*
	* \return Number of cached Pacemaker Remote nodes
	*/
	unsigned int
	pcmk__cluster_num_remote_nodes(void)
	{
	if (crm_remote_peer_cache == NULL) {
	return 0U;
	}
	return g_hash_table_size(crm_remote_peer_cache);
	}

	/*!
	* \internal
	* \brief Get a remote node cache entry, creating it if necessary
	*
	* \param[in] node_name Name of remote node
	*
	* \return Cache entry for node on success, or \c NULL (and set \c errno)
	* otherwise
	*
	* \note When creating a new entry, this will leave the node state undetermined.
	* The caller should also call \c pcmk__update_peer_state() if the state
	* is known.
	* \note Because this can add and remove cache entries, callers should not
	* assume any previously obtained cache entry pointers remain valid.
	*/
	pcmk__node_status_t *
	pcmk__cluster_lookup_remote_node(const char *node_name)
	{
	pcmk__node_status_t *node = NULL;
	char *node_name_copy = NULL;

	if (node_name == NULL) {
	errno = EINVAL;
	return NULL;
	}

	/* It's theoretically possible that the node was added to the cluster peer
	* cache before it was known to be a Pacemaker Remote node. Remove that
	* entry unless it has a node ID, which means the name actually is
	* associated with a cluster node. (@TODO return an error in that case?)
	*/
	node = pcmk__search_node_caches(0, node_name,
	pcmk__node_search_cluster_member);
	if ((node != NULL) && (node->uuid == NULL)) {
	/* node_name could be a pointer into the cache entry being removed, so
	* reassign it to a copy before the original gets freed
	*/
	node_name_copy = strdup(node_name);
	if (node_name_copy == NULL) {
	errno = ENOMEM;
	return NULL;
	}
	node_name = node_name_copy;
	pcmk__cluster_forget_cluster_node(0, node_name);
	}

	/* Return existing cache entry if one exists */
	node = g_hash_table_lookup(crm_remote_peer_cache, node_name);
	if (node) {
	free(node_name_copy);
	return node;
	}

	/* Allocate a new entry */
	node = calloc(1, sizeof(pcmk__node_status_t));
	if (node == NULL) {
	free(node_name_copy);
	return NULL;
	}

	/* Populate the essential information */
	set_peer_flags(node, crm_remote_node);
	node->uuid = strdup(node_name);
	if (node->uuid == NULL) {
	free(node);
	errno = ENOMEM;
	free(node_name_copy);
	return NULL;
	}

	/* Add the new entry to the cache */
	g_hash_table_replace(crm_remote_peer_cache, node->uuid, node);
	crm_trace("added %s to remote cache", node_name);

	/* Update the entry's uname, ensuring peer status callbacks are called */
	update_peer_uname(node, node_name);
	free(node_name_copy);
	return node;
	}

	/*!
	* \internal
	* \brief Remove a node from the Pacemaker Remote node cache
	*
	* \param[in] node_name Name of node to remove from cache
	*
	* \note The caller must be careful not to use \p node_name after calling this
	* function if it might be a pointer into the cache entry being removed.
	*/
	void
	pcmk__cluster_forget_remote_node(const char *node_name)
	{
	/* Do a lookup first, because node_name could be a pointer within the entry
	* being removed -- we can't log it after removing it.
	*/
	if (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL) {
	crm_trace("Removing %s from Pacemaker Remote node cache", node_name);
	g_hash_table_remove(crm_remote_peer_cache, node_name);
	}
	}

	/*!
	* \internal
	* \brief Return node status based on a CIB status entry
	*
	* \param[in] node_state XML of node state
	*
	* \return \c CRM_NODE_MEMBER if \c PCMK__XA_IN_CCM is true in
	* \c PCMK__XE_NODE_STATE, or \c CRM_NODE_LOST otherwise
	*/
	static const char *
	remote_state_from_cib(const xmlNode *node_state)
	{
	bool in_ccm = false;

	if ((pcmk__xe_get_bool_attr(node_state, PCMK__XA_IN_CCM,
	&in_ccm) == pcmk_rc_ok) && in_ccm) {
	return CRM_NODE_MEMBER;
	}
	return CRM_NODE_LOST;
	}

	/* user data for looping through remote node xpath searches */
	struct refresh_data {
	const char field; / XML attribute to check for node name */
	gboolean has_state; /* whether to update node state based on XML */
	};

	/*!
	* \internal
	* \brief Process one pacemaker_remote node xpath search result
	*
	* \param[in] result XML search result
	* \param[in] user_data what to look for in the XML
	*/
	static void
	remote_cache_refresh_helper(xmlNode result, void user_data)
	{
	const struct refresh_data *data = user_data;
	const char *remote = crm_element_value(result, data->field);
	const char *state = NULL;
	pcmk__node_status_t *node;

	CRM_CHECK(remote != NULL, return);

	/* Determine node's state, if the result has it */
	if (data->has_state) {
	state = remote_state_from_cib(result);
	}

	/* Check whether cache already has entry for node */
	node = g_hash_table_lookup(crm_remote_peer_cache, remote);

	if (node == NULL) {
	/* Node is not in cache, so add a new entry for it */
	node = pcmk__cluster_lookup_remote_node(remote);
	CRM_ASSERT(node);
	if (state) {
	pcmk__update_peer_state(__func__, node, state, 0);
	}

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	/* Node is in cache and hasn't been updated already, so mark it clean */
	clear_peer_flags(node, crm_node_dirty);
	if (state) {
	pcmk__update_peer_state(__func__, node, state, 0);
	}
	}
	}

	static void
	mark_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	set_peer_flags((pcmk__node_status_t *) value, crm_node_dirty);
	}

	static gboolean
	is_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	return pcmk_is_set(((pcmk__node_status_t*)value)->flags, crm_node_dirty);
	}

	/*!
	* \internal
	* \brief Repopulate the remote node cache based on CIB XML
	*
	* \param[in] cib CIB XML to parse
	*/
	static void
	refresh_remote_nodes(xmlNode *cib)
	{
	struct refresh_data data;

	pcmk__cluster_init_node_caches();

	/* First, we mark all existing cache entries as dirty,
	* so that later we can remove any that weren't in the CIB.
	* We don't empty the cache, because we need to detect changes in state.
	*/
	g_hash_table_foreach(crm_remote_peer_cache, mark_dirty, NULL);

	/* Look for guest nodes and remote nodes in the status section */
	data.field = PCMK_XA_ID;
	data.has_state = TRUE;
	crm_foreach_xpath_result(cib, PCMK__XP_REMOTE_NODE_STATUS,
	remote_cache_refresh_helper, &data);

	/* Look for guest nodes and remote nodes in the configuration section,
	* because they may have just been added and not have a status entry yet.
	* In that case, the cached node state will be left NULL, so that the
	* peer status callback isn't called until we're sure the node started
	* successfully.
	*/
	data.field = PCMK_XA_VALUE;
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, PCMK__XP_GUEST_NODE_CONFIG,
	remote_cache_refresh_helper, &data);
	data.field = PCMK_XA_ID;
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, PCMK__XP_REMOTE_NODE_CONFIG,
	remote_cache_refresh_helper, &data);

	/* Remove all old cache entries that weren't seen in the CIB */
	g_hash_table_foreach_remove(crm_remote_peer_cache, is_dirty, NULL);
	}

	/*!
	* \internal
	* \brief Check whether a node is an active cluster node
	*
	* Remote nodes are never considered active. This guarantees that they can never
	* become DC.
	*
	* \param[in] node Node to check
	*
	* \return \c true if the node is an active cluster node, or \c false otherwise
	*/
	bool
	pcmk__cluster_is_node_active(const pcmk__node_status_t *node)
	{
	const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer();

	if ((node == NULL) \|\| pcmk_is_set(node->flags, crm_remote_node)) {
	return false;
	}

	switch (cluster_layer) {
	case pcmk_cluster_layer_corosync:
	#if SUPPORT_COROSYNC
	return pcmk__corosync_is_peer_active(node);
	#else
	break;
	#endif // SUPPORT_COROSYNC
	default:
	break;
	}

	crm_err("Unhandled cluster layer: %s",
	pcmk_cluster_layer_text(cluster_layer));
	return false;
	}

	/*!
	* \internal
	* \brief Check if a node's entry should be removed from the cluster node cache
	*
	* A node should be removed from the cache if it's inactive and matches another
	* \c pcmk__node_status_t (the search object). The node is considered a
	* mismatch if any of the following are true:
	* * The search object is \c NULL.
	* * The search object has an ID set and the cached node's ID does not match it.
	* * The search object does not have an ID set, and the cached node's name does
	* not match the search node's name. (If both names are \c NULL, it's a
	* match.)
	*
	* Otherwise, the node is considered a match.
	*
	* Note that if the search object has both an ID and a name set, the name is
	* ignored for matching purposes.
	*
	* \param[in] key Ignored
	* \param[in] value \c pcmk__node_status_t object from cluster node cache
	* \param[in] user_data \c pcmk__node_status_t object to match against (search
	* object)
	*
	* \return \c TRUE if the node entry should be removed from \c crm_peer_cache,
	* or \c FALSE otherwise
	*/
	static gboolean
	should_forget_cluster_node(gpointer key, gpointer value, gpointer user_data)
	{
	pcmk__node_status_t *node = value;
	pcmk__node_status_t *search = user_data;

	if (search == NULL) {
	return FALSE;
	}
	- if ((search->id != 0) && (node->id != search->id)) {
	+ if ((search->cluster_layer_id != 0)
	+ && (node->cluster_layer_id != search->cluster_layer_id)) {
	return FALSE;
	}
	- if ((search->id == 0)
	+ if ((search->cluster_layer_id == 0)
	&& !pcmk__str_eq(node->uname, search->uname, pcmk__str_casei)) {
	// @TODO Consider name even if ID is set?
	return FALSE;
	}
	if (pcmk__cluster_is_node_active(value)) {
	return FALSE;
	}

	- crm_info("Removing node with name %s and " PCMK_XA_ID " %u from membership "
	- "cache",
	- pcmk__s(node->uname, "(unknown)"), node->id);
	+ crm_info("Removing node with name %s and cluster layer ID " PRIu32
	+ " from membership cache",
	+ pcmk__s(node->uname, "(unknown)"), node->cluster_layer_id);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Remove one or more inactive nodes from the cluster node cache
	*
	* All inactive nodes matching \p id and \p node_name as described in
	* \c should_forget_cluster_node documentation are removed from the cache.
	*
	* If \p id is 0 and \p node_name is \c NULL, all inactive nodes are removed
	* from the cache regardless of ID and name. This differs from clearing the
	* cache, in that entries for active nodes are preserved.
	*
	* \param[in] id ID of node to remove from cache (0 to ignore)
	* \param[in] node_name Name of node to remove from cache (ignored if \p id is
	* nonzero)
	*
	* \note \p node_name is not modified directly, but it will be freed if it's a
	* pointer into a cache entry that is removed.
	*/
	void
	pcmk__cluster_forget_cluster_node(uint32_t id, const char *node_name)
	{
	pcmk__node_status_t search = { 0, };
	char *criterion = NULL; // For logging
	guint matches = 0;

	if (crm_peer_cache == NULL) {
	crm_trace("Membership cache not initialized, ignoring removal request");
	return;
	}

	- search.id = id;
	+ search.cluster_layer_id = id;
	search.uname = pcmk__str_copy(node_name); // May log after original freed

	if (id > 0) {
	- criterion = crm_strdup_printf(PCMK_XA_ID "=%" PRIu32, id);
	+ criterion = crm_strdup_printf("cluster layer ID %" PRIu32, id);

	} else if (node_name != NULL) {
	- criterion = crm_strdup_printf(PCMK_XA_UNAME "=%s", node_name);
	+ criterion = crm_strdup_printf("name %s", node_name);
	}

	matches = g_hash_table_foreach_remove(crm_peer_cache,
	should_forget_cluster_node, &search);
	if (matches > 0) {
	if (criterion != NULL) {
	crm_notice("Removed %u inactive node%s with %s from the membership "
	"cache",
	matches, pcmk__plural_s(matches), criterion);
	} else {
	crm_notice("Removed all (%u) inactive cluster nodes from the "
	"membership cache",
	matches);
	}

	} else {
	crm_info("No inactive cluster nodes%s%s to remove from the membership "
	"cache",
	((criterion != NULL)? " with " : ""), pcmk__s(criterion, ""));
	}

	free(search.uname);
	free(criterion);
	}

	static void
	count_peer(gpointer key, gpointer value, gpointer user_data)
	{
	unsigned int *count = user_data;
	pcmk__node_status_t *node = value;

	if (pcmk__cluster_is_node_active(node)) {
	count = count + 1;
	}
	}

	/*!
	* \internal
	* \brief Get the number of active cluster nodes that have been seen
	*
	* Remote nodes are never considered active. This guarantees that they can never
	* become DC.
	*
	* \return Number of active nodes in the cluster node cache
	*/
	unsigned int
	pcmk__cluster_num_active_nodes(void)
	{
	unsigned int count = 0;

	if (crm_peer_cache != NULL) {
	g_hash_table_foreach(crm_peer_cache, count_peer, &count);
	}
	return count;
	}

	static void
	destroy_crm_node(gpointer data)
	{
	pcmk__node_status_t *node = data;

	- crm_trace("Destroying entry for node %u: %s", node->id, node->uname);
	+ crm_trace("Destroying entry for node %" PRIu32 ": %s",
	+ node->cluster_layer_id, node->uname);

	free(node->uname);
	free(node->state);
	free(node->uuid);
	free(node->expected);
	free(node->conn_host);
	free(node);
	}

	/*!
	* \internal
	* \brief Initialize node caches
	*/
	void
	pcmk__cluster_init_node_caches(void)
	{
	if (crm_peer_cache == NULL) {
	crm_peer_cache = pcmk__strikey_table(free, destroy_crm_node);
	}

	if (crm_remote_peer_cache == NULL) {
	crm_remote_peer_cache = pcmk__strikey_table(NULL, destroy_crm_node);
	}

	if (cluster_node_cib_cache == NULL) {
	cluster_node_cib_cache = pcmk__strikey_table(free, destroy_crm_node);
	}
	}

	/*!
	* \internal
	* \brief Initialize node caches
	*/
	void
	pcmk__cluster_destroy_node_caches(void)
	{
	if (crm_peer_cache != NULL) {
	crm_trace("Destroying peer cache with %d members",
	g_hash_table_size(crm_peer_cache));
	g_hash_table_destroy(crm_peer_cache);
	crm_peer_cache = NULL;
	}

	if (crm_remote_peer_cache != NULL) {
	crm_trace("Destroying remote peer cache with %d members",
	pcmk__cluster_num_remote_nodes());
	g_hash_table_destroy(crm_remote_peer_cache);
	crm_remote_peer_cache = NULL;
	}

	if (cluster_node_cib_cache != NULL) {
	crm_trace("Destroying configured cluster node cache with %d members",
	g_hash_table_size(cluster_node_cib_cache));
	g_hash_table_destroy(cluster_node_cib_cache);
	cluster_node_cib_cache = NULL;
	}
	}

	static void (peer_status_callback)(enum crm_status_type, pcmk__node_status_t ,
	const void *) = NULL;

	/*!
	* \internal
	* \brief Set a client function that will be called after peer status changes
	*
	* \param[in] dispatch Pointer to function to use as callback
	*
	* \note Client callbacks should do only client-specific handling. Callbacks
	* must not add or remove entries in the peer caches.
	*/
	void
	pcmk__cluster_set_status_callback(void (*dispatch)(enum crm_status_type,
	pcmk__node_status_t *,
	const void *))
	{
	// @TODO Improve documentation of peer_status_callback
	peer_status_callback = dispatch;
	}

	/*!
	* \internal
	* \brief Tell the library whether to automatically reap lost nodes
	*
	* If \c true (the default), calling \c crm_update_peer_proc() will also update
	* the peer state to \c CRM_NODE_MEMBER or \c CRM_NODE_LOST, and updating the
	* peer state will reap peers whose state changes to anything other than
	* \c CRM_NODE_MEMBER.
	*
	* Callers should leave this enabled unless they plan to manage the cache
	* separately on their own.
	*
	* \param[in] enable \c true to enable automatic reaping, \c false to disable
	*/
	void
	pcmk__cluster_set_autoreap(bool enable)
	{
	autoreap = enable;
	}

	static void
	dump_peer_hash(int level, const char *caller)
	{
	GHashTableIter iter;
	const char *id = NULL;
	pcmk__node_status_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, (gpointer ) &id, (gpointer ) &node)) {
	- do_crm_log(level, "%s: Node %u/%s = %p - %s", caller, node->id, node->uname, node, id);
	+ do_crm_log(level, "%s: Node %" PRIu32 "/%s = %p - %s",
	+ caller, node->cluster_layer_id, node->uname, node, id);
	}
	}

	static gboolean
	hash_find_by_data(gpointer key, gpointer value, gpointer user_data)
	{
	return value == user_data;
	}

	/*!
	* \internal
	* \brief Search cluster member node cache
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] uuid If not NULL while id is 0, node UUID instead of cluster
	* node ID to search for
	*
	* \return Cluster node cache entry if found, otherwise NULL
	*/
	static pcmk__node_status_t *
	search_cluster_member_cache(unsigned int id, const char *uname,
	const char *uuid)
	{
	GHashTableIter iter;
	pcmk__node_status_t *node = NULL;
	pcmk__node_status_t *by_id = NULL;
	pcmk__node_status_t *by_name = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	pcmk__cluster_init_node_caches();

	if (uname != NULL) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id > 0) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	- if(node->id == id) {
	- crm_trace("ID match: %u = %p", node->id, node);
	+ if (node->cluster_layer_id == id) {
	+ crm_trace("ID match: %" PRIu32, node->cluster_layer_id);
	by_id = node;
	break;
	}
	}

	} else if (uuid != NULL) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (pcmk__str_eq(node->uuid, uuid, pcmk__str_casei)) {
	crm_trace("UUID match: %s = %p", node->uuid, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if(by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %u/%s", by_id, id, uname);

	} else if(by_id == NULL && by_name) {
	crm_trace("Only one: %p for %u/%s", by_name, id, uname);

	- if(id && by_name->id) {
	+ if (id && by_name->cluster_layer_id) {
	dump_peer_hash(LOG_WARNING, __func__);
	- crm_crit("Node %u and %u share the same name '%s'",
	- id, by_name->id, uname);
	+ crm_crit("Nodes %u and %" PRIu32 " share the same name '%s'",
	+ id, by_name->cluster_layer_id, uname);
	node = NULL; /* Create a new one */

	} else {
	node = by_name;
	}

	} else if(by_name == NULL && by_id) {
	crm_trace("Only one: %p for %u/%s", by_id, id, uname);

	if(uname && by_id->uname) {
	dump_peer_hash(LOG_WARNING, __func__);
	crm_crit("Node '%s' and '%s' share the same cluster nodeid %u: assuming '%s' is correct",
	uname, by_id->uname, id, uname);
	}

	} else if(uname && by_id->uname) {
	if(pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	- crm_notice("Node '%s' has changed its ID from %u to %u", by_id->uname, by_name->id, by_id->id);
	+ crm_notice("Node '%s' has changed its cluster layer ID "
	+ "from %" PRIu32 " to %" PRIu32,
	+ by_id->uname, by_name->cluster_layer_id,
	+ by_id->cluster_layer_id);
	g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);

	} else {
	crm_warn("Node '%s' and '%s' share the same cluster nodeid: %u %s", by_id->uname, by_name->uname, id, uname);
	dump_peer_hash(LOG_INFO, __func__);
	crm_abort(__FILE__, __func__, __LINE__, "member weirdness", TRUE,
	TRUE);
	}

	- } else if(id && by_name->id) {
	- crm_warn("Node %u and %u share the same name: '%s'", by_id->id, by_name->id, uname);
	+ } else if ((id > 0) && (by_name->cluster_layer_id > 0)) {
	+ crm_warn("Nodes %" PRIu32 " and %" PRIu32 " share the same name: '%s'",
	+ by_id->cluster_layer_id, by_name->cluster_layer_id, uname);

	} else {
	/* Simple merge */

	/* Only corosync-based clusters use node IDs. The functions that call
	* pcmk__update_peer_state() and crm_update_peer_proc() only know
	* nodeid, so 'by_id' is authoritative when merging.
	*/
	dump_peer_hash(LOG_DEBUG, __func__);

	crm_info("Merging %p into %p", by_name, by_id);
	g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);
	}

	return node;
	}

	/*!
	* \internal
	* \brief Search caches for a node (cluster or Pacemaker Remote)
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] flags Group of enum pcmk__node_search_flags
	*
	* \return Node cache entry if found, otherwise NULL
	*/
	pcmk__node_status_t *
	pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags)
	{
	pcmk__node_status_t *node = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	pcmk__cluster_init_node_caches();

	if ((uname != NULL) && pcmk_is_set(flags, pcmk__node_search_remote)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	}

	if ((node == NULL)
	&& pcmk_is_set(flags, pcmk__node_search_cluster_member)) {

	node = search_cluster_member_cache(id, uname, NULL);
	}

	if ((node == NULL) && pcmk_is_set(flags, pcmk__node_search_cluster_cib)) {
	char *id_str = (id == 0)? NULL : crm_strdup_printf("%u", id);

	node = find_cib_cluster_node(id_str, uname);
	free(id_str);
	}

	return node;
	}

	/*!
	* \internal
	* \brief Purge a node from cache (both cluster and Pacemaker Remote)
	*
	* \param[in] node_name If not NULL, purge only nodes with this name
	* \param[in] node_id If not 0, purge cluster nodes only if they have this ID
	*
	* \note If \p node_name is NULL and \p node_id is 0, no nodes will be purged.
	* If \p node_name is not NULL and \p node_id is not 0, Pacemaker Remote
	* nodes that match \p node_name will be purged, and cluster nodes that
	* match both \p node_name and \p node_id will be purged.
	* \note The caller must be careful not to use \p node_name after calling this
	* function if it might be a pointer into a cache entry being removed.
	*/
	void
	pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id)
	{
	char *node_name_copy = NULL;

	if ((node_name == NULL) && (node_id == 0U)) {
	return;
	}

	// Purge from Pacemaker Remote node cache
	if ((node_name != NULL)
	&& (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL)) {
	/* node_name could be a pointer into the cache entry being purged,
	* so reassign it to a copy before the original gets freed
	*/
	node_name_copy = pcmk__str_copy(node_name);
	node_name = node_name_copy;

	crm_trace("Purging %s from Pacemaker Remote node cache", node_name);
	g_hash_table_remove(crm_remote_peer_cache, node_name);
	}

	pcmk__cluster_forget_cluster_node(node_id, node_name);
	free(node_name_copy);
	}

	#if SUPPORT_COROSYNC
	static guint
	remove_conflicting_peer(pcmk__node_status_t *node)
	{
	int matches = 0;
	GHashTableIter iter;
	pcmk__node_status_t *existing_node = NULL;

	- if (node->id == 0 \|\| node->uname == NULL) {
	+ if ((node->cluster_layer_id == 0) \|\| (node->uname == NULL)) {
	return 0;
	}

	if (!pcmk__corosync_has_nodelist()) {
	return 0;
	}

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &existing_node)) {
	- if (existing_node->id > 0
	- && existing_node->id != node->id
	+ if ((existing_node->cluster_layer_id > 0)
	+ && (existing_node->cluster_layer_id != node->cluster_layer_id)
	&& existing_node->uname != NULL
	&& strcasecmp(existing_node->uname, node->uname) == 0) {

	if (pcmk__cluster_is_node_active(existing_node)) {
	continue;
	}

	- crm_warn("Removing cached offline node %u/%s which has conflicting uname with %u",
	- existing_node->id, existing_node->uname, node->id);
	+ crm_warn("Removing cached offline node %" PRIu32 "/%s which has "
	+ "conflicting uname with %" PRIu32,
	+ existing_node->cluster_layer_id, existing_node->uname,
	+ node->cluster_layer_id);

	g_hash_table_iter_remove(&iter);
	matches++;
	}
	}

	return matches;
	}
	#endif

	/*!
	* \internal
	* \brief Get a cluster node cache entry, possibly creating one if not found
	*
	* If \c pcmk__node_search_cluster_member is set in \p flags, the return value
	* is guaranteed not to be \c NULL. A new cache entry is created if one does not
	* already exist.
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] uuid If not NULL while id is 0, node UUID instead of cluster
	* node ID to search for
	* \param[in] flags Group of enum pcmk__node_search_flags
	*
	* \return (Possibly newly created) cluster node cache entry
	*/
	/* coverity[-alloc] Memory is referenced in one or both hashtables */
	pcmk__node_status_t *
	pcmk__get_node(unsigned int id, const char uname, const char uuid,
	uint32_t flags)
	{
	pcmk__node_status_t *node = NULL;
	char *uname_lookup = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	pcmk__cluster_init_node_caches();

	// Check the Pacemaker Remote node cache first
	if (pcmk_is_set(flags, pcmk__node_search_remote)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	if (node != NULL) {
	return node;
	}
	}

	if (!pcmk_is_set(flags, pcmk__node_search_cluster_member)) {
	return NULL;
	}

	node = search_cluster_member_cache(id, uname, uuid);

	/* if uname wasn't provided, and find_peer did not turn up a uname based on id.
	* we need to do a lookup of the node name using the id in the cluster membership. */
	if ((node == NULL \|\| node->uname == NULL) && (uname == NULL)) {
	uname_lookup = pcmk__cluster_node_name(id);
	}

	if (uname_lookup) {
	uname = uname_lookup;
	crm_trace("Inferred a name of '%s' for node %u", uname, id);

	/* try to turn up the node one more time now that we know the uname. */
	if (node == NULL) {
	node = search_cluster_member_cache(id, uname, uuid);
	}
	}

	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = pcmk__assert_alloc(1, sizeof(pcmk__node_status_t));

	crm_info("Created entry %s/%p for node %s/%u (%d total)",
	uniqueid, node, uname, id, 1 + g_hash_table_size(crm_peer_cache));
	g_hash_table_replace(crm_peer_cache, uniqueid, node);
	}

	- if(id > 0 && uname && (node->id == 0 \|\| node->uname == NULL)) {
	+ if ((id > 0) && (uname != NULL)
	+ && ((node->cluster_layer_id == 0) \|\| (node->uname == NULL))) {
	crm_info("Node %u is now known as %s", id, uname);
	}

	- if(id > 0 && node->id == 0) {
	- node->id = id;
	+ if ((id > 0) && (node->cluster_layer_id == 0)) {
	+ node->cluster_layer_id = id;
	}

	if (uname && (node->uname == NULL)) {
	update_peer_uname(node, uname);
	}

	if(node->uuid == NULL) {
	if (uuid == NULL) {
	uuid = pcmk__cluster_node_uuid(node);
	}

	if (uuid) {
	crm_info("Node %u has uuid %s", id, uuid);

	} else {
	crm_info("Cannot obtain a UUID for node %u/%s", id, node->uname);
	}
	}

	free(uname_lookup);

	return node;
	}

	/*!
	* \internal
	* \brief Update a node's uname
	*
	* \param[in,out] node Node object to update
	* \param[in] uname New name to set
	*
	* \note This function should not be called within a peer cache iteration,
	* because in some cases it can remove conflicting cache entries,
	* which would invalidate the iterator.
	*/
	static void
	update_peer_uname(pcmk__node_status_t node, const char uname)
	{
	CRM_CHECK(uname != NULL,
	crm_err("Bug: can't update node name without name"); return);
	CRM_CHECK(node != NULL,
	crm_err("Bug: can't update node name to %s without node", uname);
	return);

	if (pcmk__str_eq(uname, node->uname, pcmk__str_casei)) {
	crm_debug("Node uname '%s' did not change", uname);
	return;
	}

	for (const char c = uname; c; ++c) {
	if ((c >= 'A') && (c <= 'Z')) {
	crm_warn("Node names with capitals are discouraged, consider changing '%s'",
	uname);
	break;
	}
	}

	pcmk__str_update(&node->uname, uname);

	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_uname, node, NULL);
	}

	#if SUPPORT_COROSYNC
	if ((pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync)
	&& !pcmk_is_set(node->flags, crm_remote_node)) {

	remove_conflicting_peer(node);
	}
	#endif
	}

	/*!
	* \internal
	* \brief Get log-friendly string equivalent of a process flag
	*
	* \param[in] proc Process flag
	*
	* \return Log-friendly string equivalent of \p proc
	*/
	static inline const char *
	proc2text(enum crm_proc_flag proc)
	{
	const char *text = "unknown";

	switch (proc) {
	case crm_proc_none:
	text = "none";
	break;
	case crm_proc_cpg:
	text = "corosync-cpg";
	break;
	}
	return text;
	}

	/*!
	* \internal
	* \brief Update a node's process information (and potentially state)
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] flag Bitmask of new process information
	* \param[in] status node status (online, offline, etc.)
	*
	* \return NULL if any node was reaped from peer caches, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible, otherwise
	* reaping could invalidate the iterator.
	*/
	pcmk__node_status_t *
	crm_update_peer_proc(const char source, pcmk__node_status_t node,
	uint32_t flag, const char *status)
	{
	uint32_t last = 0;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set %s to %s for NULL",
	source, proc2text(flag), status);
	return NULL);

	/* Pacemaker doesn't spawn processes on remote nodes */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return node;
	}

	last = node->processes;
	if (status == NULL) {
	node->processes = flag;
	if (node->processes != last) {
	changed = TRUE;
	}

	} else if (pcmk__str_eq(status, PCMK_VALUE_ONLINE, pcmk__str_casei)) {
	if ((node->processes & flag) != flag) {
	node->processes = pcmk__set_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	} else if (node->processes & flag) {
	node->processes = pcmk__clear_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	if (changed) {
	if (status == NULL && flag <= crm_proc_none) {
	- crm_info("%s: Node %s[%u] - all processes are now offline", source, node->uname,
	- node->id);
	+ crm_info("%s: Node %s[%" PRIu32 "] - all processes are now offline",
	+ source, node->uname, node->cluster_layer_id);
	} else {
	- crm_info("%s: Node %s[%u] - %s is now %s", source, node->uname, node->id,
	+ crm_info("%s: Node %s[%" PRIu32 "] - %s is now %s",
	+ source, node->uname, node->cluster_layer_id,
	proc2text(flag), status);
	}

	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	node->when_online = time(NULL);

	} else {
	node->when_online = 0;
	}

	/* Call the client callback first, then update the peer state,
	* in case the node will be reaped
	*/
	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_processes, node, &last);
	}

	/* The client callback shouldn't touch the peer caches,
	* but as a safety net, bail if the peer cache was destroyed.
	*/
	if (crm_peer_cache == NULL) {
	return NULL;
	}

	if (autoreap) {
	const char *peer_state = NULL;

	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	peer_state = CRM_NODE_MEMBER;
	} else {
	peer_state = CRM_NODE_LOST;
	}
	node = pcmk__update_peer_state(__func__, node, peer_state, 0);
	}
	} else {
	- crm_trace("%s: Node %s[%u] - %s is unchanged (%s)", source, node->uname, node->id,
	- proc2text(flag), status);
	+ crm_trace("%s: Node %s[%" PRIu32 "] - %s is unchanged (%s)",
	+ source, node->uname, node->cluster_layer_id, proc2text(flag),
	+ status);
	}
	return node;
	}

	/*!
	* \internal
	* \brief Update a cluster node cache entry's expected join state
	*
	* \param[in] source Caller's function name (for logging)
	* \param[in,out] node Node to update
	* \param[in] expected Node's new join state
	*/
	void
	pcmk__update_peer_expected(const char source, pcmk__node_status_t node,
	const char *expected)
	{
	char *last = NULL;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set 'expected' to %s", source, expected);
	return);

	/* Remote nodes don't participate in joins */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return;
	}

	last = node->expected;
	if (expected != NULL && !pcmk__str_eq(node->expected, expected, pcmk__str_casei)) {
	node->expected = strdup(expected);
	changed = TRUE;
	}

	if (changed) {
	- crm_info("%s: Node %s[%u] - expected state is now %s (was %s)", source, node->uname, node->id,
	- expected, last);
	+ crm_info("%s: Node %s[%" PRIu32 "] - expected state is now %s (was %s)",
	+ source, node->uname, node->cluster_layer_id, expected, last);
	free(last);
	} else {
	- crm_trace("%s: Node %s[%u] - expected state is unchanged (%s)", source, node->uname,
	- node->id, expected);
	+ crm_trace("%s: Node %s[%" PRIu32 "] - expected state is unchanged (%s)",
	+ source, node->uname, node->cluster_layer_id, expected);
	}
	}

	/*!
	* \internal
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	* \param[in,out] iter If not NULL, pointer to node's peer cache iterator
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function may be called from
	* within a peer cache iteration if the iterator is supplied.
	*/
	static pcmk__node_status_t *
	update_peer_state_iter(const char source, pcmk__node_status_t node,
	const char *state, uint64_t membership,
	GHashTableIter *iter)
	{
	gboolean is_member;

	CRM_CHECK(node != NULL,
	crm_err("Could not set state for unknown host to %s"
	QB_XS " source=%s", state, source);
	return NULL);

	is_member = pcmk__str_eq(state, CRM_NODE_MEMBER, pcmk__str_casei);
	if (is_member) {
	node->when_lost = 0;
	if (membership) {
	node->last_seen = membership;
	}
	}

	if (state && !pcmk__str_eq(node->state, state, pcmk__str_casei)) {
	char *last = node->state;

	if (is_member) {
	node->when_member = time(NULL);

	} else {
	node->when_member = 0;
	}

	node->state = strdup(state);
	crm_notice("Node %s state is now %s " QB_XS
	- " nodeid=%u previous=%s source=%s", node->uname, state,
	- node->id, (last? last : "unknown"), source);
	+ " nodeid=%" PRIu32 " previous=%s source=%s",
	+ node->uname, state, node->cluster_layer_id,
	+ pcmk__s(last, "unknown"), source);
	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_nstate, node, last);
	}
	free(last);

	if (autoreap && !is_member
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/* We only autoreap from the peer cache, not the remote peer cache,
	* because the latter should be managed only by
	* refresh_remote_nodes().
	*/
	if(iter) {
	- crm_notice("Purged 1 peer with " PCMK_XA_ID
	- "=%u and/or uname=%s from the membership cache",
	- node->id, node->uname);
	+ crm_notice("Purged 1 peer with cluster layer ID=" PRIu32
	+ "and/or name=%s from the membership cache",
	+ node->cluster_layer_id, node->uname);
	g_hash_table_iter_remove(iter);

	} else {
	- pcmk__cluster_forget_cluster_node(node->id, node->uname);
	+ pcmk__cluster_forget_cluster_node(node->cluster_layer_id,
	+ node->uname);
	}
	node = NULL;
	}

	} else {
	crm_trace("Node %s state is unchanged (%s) " QB_XS
	- " nodeid=%u source=%s", node->uname, state, node->id, source);
	+ " nodeid=%" PRIu32 " source=%s",
	+ node->uname, state, node->cluster_layer_id, source);
	}
	return node;
	}

	/*!
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible,
	* otherwise reaping could invalidate the iterator.
	*/
	pcmk__node_status_t *
	pcmk__update_peer_state(const char source, pcmk__node_status_t node,
	const char *state, uint64_t membership)
	{
	return update_peer_state_iter(source, node, state, membership, NULL);
	}

	/*!
	* \internal
	* \brief Reap all nodes from cache whose membership information does not match
	*
	* \param[in] membership Membership ID of nodes to keep
	*/
	void
	pcmk__reap_unseen_nodes(uint64_t membership)
	{
	GHashTableIter iter;
	pcmk__node_status_t *node = NULL;

	crm_trace("Reaping unseen nodes...");
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&node)) {
	if (node->last_seen != membership) {
	if (node->state) {
	/*
	* Calling update_peer_state_iter() allows us to
	* remove the node from crm_peer_cache without
	* invalidating our iterator
	*/
	update_peer_state_iter(__func__, node, CRM_NODE_LOST,
	membership, &iter);

	} else {
	- crm_info("State of node %s[%u] is still unknown",
	- node->uname, node->id);
	+ crm_info("State of node %s[%" PRIu32 "] is still unknown",
	+ node->uname, node->cluster_layer_id);
	}
	}
	}
	}

	static pcmk__node_status_t *
	find_cib_cluster_node(const char id, const char uname)
	{
	GHashTableIter iter;
	pcmk__node_status_t *node = NULL;
	pcmk__node_status_t *by_id = NULL;
	pcmk__node_status_t *by_name = NULL;

	if (uname) {
	g_hash_table_iter_init(&iter, cluster_node_cib_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id) {
	g_hash_table_iter_init(&iter, cluster_node_cib_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(strcasecmp(node->uuid, id) == 0) {
	crm_trace("ID match: %s= %p", id, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if (by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %s/%s", by_id, id, uname);

	} else if (by_id == NULL && by_name) {
	crm_trace("Only one: %p for %s/%s", by_name, id, uname);

	if (id) {
	node = NULL;

	} else {
	node = by_name;
	}

	} else if (by_name == NULL && by_id) {
	crm_trace("Only one: %p for %s/%s", by_id, id, uname);

	if (uname) {
	node = NULL;
	}

	} else if (uname && by_id->uname
	&& pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	/* Multiple nodes have the same uname in the CIB.
	* Return by_id. */

	} else if (id && by_name->uuid
	&& pcmk__str_eq(id, by_name->uuid, pcmk__str_casei)) {
	/* Multiple nodes have the same id in the CIB.
	* Return by_name. */
	node = by_name;

	} else {
	node = NULL;
	}

	if (node == NULL) {
	crm_debug("Couldn't find node%s%s%s%s",
	id? " " : "",
	id? id : "",
	uname? " with name " : "",
	uname? uname : "");
	}

	return node;
	}

	static void
	cluster_node_cib_cache_refresh_helper(xmlNode xml_node, void user_data)
	{
	const char *id = crm_element_value(xml_node, PCMK_XA_ID);
	const char *uname = crm_element_value(xml_node, PCMK_XA_UNAME);
	pcmk__node_status_t * node = NULL;

	CRM_CHECK(id != NULL && uname !=NULL, return);
	node = find_cib_cluster_node(id, uname);

	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = pcmk__assert_alloc(1, sizeof(pcmk__node_status_t));

	node->uname = pcmk__str_copy(uname);
	node->uuid = pcmk__str_copy(id);

	g_hash_table_replace(cluster_node_cib_cache, uniqueid, node);

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	pcmk__str_update(&node->uname, uname);

	/* Node is in cache and hasn't been updated already, so mark it clean */
	clear_peer_flags(node, crm_node_dirty);
	}

	}

	static void
	refresh_cluster_node_cib_cache(xmlNode *cib)
	{
	pcmk__cluster_init_node_caches();

	g_hash_table_foreach(cluster_node_cib_cache, mark_dirty, NULL);

	crm_foreach_xpath_result(cib, PCMK__XP_MEMBER_NODE_CONFIG,
	cluster_node_cib_cache_refresh_helper, NULL);

	// Remove all old cache entries that weren't seen in the CIB
	g_hash_table_foreach_remove(cluster_node_cib_cache, is_dirty, NULL);
	}

	void
	pcmk__refresh_node_caches_from_cib(xmlNode *cib)
	{
	refresh_remote_nodes(cib);
	refresh_cluster_node_cib_cache(cib);
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/cluster/compat.h>

	void
	crm_peer_init(void)
	{
	pcmk__cluster_init_node_caches();
	}

	// LCOV_EXCL_STOP
	// End deprecated API

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 6:32 PM (7 h, 12 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002677
Default Alt Text: (400 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions