No OneTemporary
Actions

Size

521 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/attrd/attrd_attributes.c b/daemons/attrd/attrd_attributes.c
	index 974de89e3c..e049793ece 100644
	--- a/daemons/attrd/attrd_attributes.c
	+++ b/daemons/attrd/attrd_attributes.c
	@@ -1,275 +1,275 @@
	/*
	* Copyright 2013-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <errno.h>
	#include <stdbool.h>
	#include <stdlib.h>
	#include <glib.h>

	#include <crm/common/logging.h>
	#include <crm/common/results.h>
	#include <crm/common/strings_internal.h>
	#include <crm/common/xml.h>

	#include "pacemaker-attrd.h"

	static attribute_t *
	attrd_create_attribute(xmlNode *xml)
	{
	int is_private = 0;
	long long dampen = 0;
	const char *name = crm_element_value(xml, PCMK__XA_ATTR_NAME);
	const char *set_type = crm_element_value(xml, PCMK__XA_ATTR_SET_TYPE);
	const char *dampen_s = crm_element_value(xml, PCMK__XA_ATTR_DAMPENING);
	attribute_t *a = NULL;

	if (set_type == NULL) {
	set_type = PCMK_XE_INSTANCE_ATTRIBUTES;
	}

	/* Set type is meaningful only when writing to the CIB. Private
	* attributes are not written.
	*/
	crm_element_value_int(xml, PCMK__XA_ATTR_IS_PRIVATE, &is_private);
	if (!is_private && !pcmk__str_any_of(set_type,
	PCMK_XE_INSTANCE_ATTRIBUTES,
	PCMK_XE_UTILIZATION, NULL)) {
	crm_warn("Ignoring attribute %s with invalid set type %s",
	pcmk__s(name, "(unidentified)"), set_type);
	return NULL;
	}

	a = pcmk__assert_alloc(1, sizeof(attribute_t));

	a->id = pcmk__str_copy(name);
	a->set_type = pcmk__str_copy(set_type);
	a->set_id = crm_element_value_copy(xml, PCMK__XA_ATTR_SET);
	a->user = crm_element_value_copy(xml, PCMK__XA_ATTR_USER);
	a->values = pcmk__strikey_table(NULL, attrd_free_attribute_value);

	if (is_private) {
	attrd_set_attr_flags(a, attrd_attr_is_private);
	}

	if (dampen_s != NULL) {
	dampen = crm_get_msec(dampen_s);
	}

	if (dampen > 0) {
	a->timeout_ms = (int) QB_MIN(dampen, INT_MAX);
	a->timer = attrd_add_timer(a->id, a->timeout_ms, a);
	} else if (dampen < 0) {
	crm_warn("Ignoring invalid delay %s for attribute %s", dampen_s, a->id);
	}

	crm_trace("Created attribute %s with %s write delay and %s CIB user",
	a->id,
	((dampen > 0)? pcmk__readable_interval(a->timeout_ms) : "no"),
	pcmk__s(a->user, "default"));

	g_hash_table_replace(attributes, a->id, a);
	return a;
	}

	static int
	attrd_update_dampening(attribute_t a, xmlNode xml, const char *attr)
	{
	const char *dvalue = crm_element_value(xml, PCMK__XA_ATTR_DAMPENING);
	long long dampen = 0;

	if (dvalue == NULL) {
	crm_warn("Could not update %s: peer did not specify value for delay",
	attr);
	return EINVAL;
	}

	dampen = crm_get_msec(dvalue);
	if (dampen < 0) {
	crm_warn("Could not update %s: invalid delay value %dms (%s)",
	attr, dampen, dvalue);
	return EINVAL;
	}

	if (a->timeout_ms != dampen) {
	mainloop_timer_del(a->timer);
	a->timeout_ms = (int) QB_MIN(dampen, INT_MAX);
	if (dampen > 0) {
	a->timer = attrd_add_timer(attr, a->timeout_ms, a);
	crm_info("Update attribute %s delay to %dms (%s)",
	attr, dampen, dvalue);
	} else {
	a->timer = NULL;
	crm_info("Update attribute %s to remove delay", attr);
	}

	/* If dampening changed, do an immediate write-out,
	* otherwise repeated dampening changes would prevent write-outs
	*/
	attrd_write_or_elect_attribute(a);
	}

	return pcmk_rc_ok;
	}

	GHashTable *attributes = NULL;

	/*!
	* \internal
	* \brief Create an XML representation of an attribute for use in peer messages
	*
	* \param[in,out] parent Create attribute XML as child element of this
	* \param[in] a Attribute to represent
	* \param[in] v Attribute value to represent
	* \param[in] force_write If true, value should be written even if unchanged
	*
	* \return XML representation of attribute
	*/
	xmlNode *
	attrd_add_value_xml(xmlNode parent, const attribute_t a,
	const attribute_value_t *v, bool force_write)
	{
	xmlNode *xml = pcmk__xe_create(parent, __func__);

	crm_xml_add(xml, PCMK__XA_ATTR_NAME, a->id);
	crm_xml_add(xml, PCMK__XA_ATTR_SET_TYPE, a->set_type);
	crm_xml_add(xml, PCMK__XA_ATTR_SET, a->set_id);
	crm_xml_add(xml, PCMK__XA_ATTR_USER, a->user);
	pcmk__xe_add_node(xml, v->nodename, v->nodeid);
	if (pcmk_is_set(v->flags, attrd_value_remote)) {
	crm_xml_add_int(xml, PCMK__XA_ATTR_IS_REMOTE, 1);
	}
	crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current);
	crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000);
	crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE,
	pcmk_is_set(a->flags, attrd_attr_is_private));
	crm_xml_add_int(xml, PCMK__XA_ATTRD_IS_FORCE_WRITE, force_write);

	return xml;
	}

	void
	attrd_clear_value_seen(void)
	{
	GHashTableIter aIter;
	GHashTableIter vIter;
	attribute_t *a;
	attribute_value_t *v = NULL;

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
	g_hash_table_iter_init(&vIter, a->values);
	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
	attrd_clear_value_flags(v, attrd_value_from_peer);
	}
	}
	}

	attribute_t *
	attrd_populate_attribute(xmlNode xml, const char attr)
	{
	attribute_t *a = NULL;
	bool update_both = false;

	const char *op = crm_element_value(xml, PCMK_XA_TASK);

	// NULL because PCMK__ATTRD_CMD_SYNC_RESPONSE has no PCMK_XA_TASK
	update_both = pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_BOTH,
	pcmk__str_null_matches);

	// Look up or create attribute entry
	a = g_hash_table_lookup(attributes, attr);
	if (a == NULL) {
	if (update_both \|\| pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE, pcmk__str_none)) {
	a = attrd_create_attribute(xml);
	if (a == NULL) {
	return NULL;
	}

	} else {
	crm_warn("Could not update %s: attribute not found", attr);
	return NULL;
	}
	}

	// Update attribute dampening
	if (update_both \|\| pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_DELAY, pcmk__str_none)) {
	int rc = attrd_update_dampening(a, xml, attr);

	if (rc != pcmk_rc_ok \|\| !update_both) {
	return NULL;
	}
	}

	return a;
	}

	/*!
	* \internal
	* \brief Get the XML ID used to write out an attribute set
	*
	* \param[in] attr Attribute to get set ID for
	* \param[in] node_state_id XML ID of node state that attribute value is for
	*
	* \return Newly allocated string with XML ID to use for \p attr set
	*/
	char *
	attrd_set_id(const attribute_t attr, const char node_state_id)
	{
	char *set_id = NULL;

	CRM_ASSERT((attr != NULL) && (node_state_id != NULL));

	- if (attr->set_id == NULL) {
	+ if (pcmk__str_empty(attr->set_id)) {
	/* @COMPAT This should really take the set type into account. Currently
	* we use the same XML ID for transient attributes and utilization
	* attributes. It doesn't cause problems because the status section is
	* not limited by the schema in any way, but it's still unfortunate.
	* For backward compatibility reasons, we can't change this.
	*/
	set_id = crm_strdup_printf("%s-%s", PCMK_XE_STATUS, node_state_id);
	} else {
	/* @COMPAT When the user specifies a set ID for an attribute, it is the
	* same for every node. That is less than ideal, but again, the schema
	* doesn't enforce anything for the status section. We couldn't change
	* it without allowing the set ID to vary per value rather than per
	* attribute, which would break backward compatibility, pose design
	* challenges, and potentially cause problems in rolling upgrades.
	*/
	set_id = pcmk__str_copy(attr->set_id);
	}
	- crm_xml_sanitize_id(set_id);
	+ pcmk__xml_sanitize_id(set_id);
	return set_id;
	}

	/*!
	* \internal
	* \brief Get the XML ID used to write out an attribute value
	*
	* \param[in] attr Attribute to get value XML ID for
	* \param[in] node_state_id UUID of node that attribute value is for
	*
	* \return Newly allocated string with XML ID of \p attr value
	*/
	char *
	attrd_nvpair_id(const attribute_t attr, const char node_state_id)
	{
	char *nvpair_id = NULL;

	if (attr->set_id != NULL) {
	nvpair_id = crm_strdup_printf("%s-%s", attr->set_id, attr->id);

	} else {
	nvpair_id = crm_strdup_printf(PCMK_XE_STATUS "-%s-%s",
	node_state_id, attr->id);
	}
	- crm_xml_sanitize_id(nvpair_id);
	+ pcmk__xml_sanitize_id(nvpair_id);
	return nvpair_id;
	}
	diff --git a/daemons/controld/controld_schedulerd.c b/daemons/controld/controld_schedulerd.c
	index 26f8329a77..2993933641 100644
	--- a/daemons/controld/controld_schedulerd.c
	+++ b/daemons/controld/controld_schedulerd.c
	@@ -1,511 +1,511 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h> /* pid_t, sleep, ssize_t */

	#include <crm/cib.h>
	#include <crm/cluster.h>
	#include <crm/common/xml.h>
	#include <crm/crm.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_schedulerd.h>

	#include <pacemaker-controld.h>

	static void handle_disconnect(void);

	static pcmk_ipc_api_t *schedulerd_api = NULL;

	/*!
	* \internal
	* \brief Close any scheduler connection and free associated memory
	*/
	void
	controld_shutdown_schedulerd_ipc(void)
	{
	controld_clear_fsa_input_flags(R_PE_REQUIRED);
	pcmk_disconnect_ipc(schedulerd_api);
	handle_disconnect();

	pcmk_free_ipc_api(schedulerd_api);
	schedulerd_api = NULL;
	}

	/*!
	* \internal
	* \brief Save CIB query result to file, raising FSA error
	*
	* \param[in] msg Ignored
	* \param[in] call_id Call ID of CIB query
	* \param[in] rc Return code of CIB query
	* \param[in] output Result of CIB query
	* \param[in] user_data Unique identifier for filename
	*
	* \note This is intended to be called after a scheduler connection fails.
	*/
	static void
	save_cib_contents(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data)
	{
	const char *id = user_data;

	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
	CRM_CHECK(id != NULL, return);

	if (rc == pcmk_ok) {
	char *filename = crm_strdup_printf(PE_STATE_DIR "/pe-core-%s.bz2", id);

	if (pcmk__xml_write_file(output, filename, true, NULL) != pcmk_rc_ok) {
	crm_err("Could not save Cluster Information Base to %s after scheduler crash",
	filename);
	} else {
	crm_notice("Saved Cluster Information Base to %s after scheduler crash",
	filename);
	}
	free(filename);
	}
	}

	/*!
	* \internal
	* \brief Respond to scheduler connection failure
	*/
	static void
	handle_disconnect(void)
	{
	// If we aren't connected to the scheduler, we can't expect a reply
	controld_expect_sched_reply(NULL);

	if (pcmk_is_set(controld_globals.fsa_input_register, R_PE_REQUIRED)) {
	int rc = pcmk_ok;
	char *uuid_str = crm_generate_uuid();

	crm_crit("Lost connection to the scheduler "
	CRM_XS " CIB will be saved to " PE_STATE_DIR "/pe-core-%s.bz2",
	uuid_str);

	/*
	* The scheduler died...
	*
	* Save the current CIB so that we have a chance of
	* figuring out what killed it.
	*
	* Delay raising the I_ERROR until the query below completes or
	* 5s is up, whichever comes first.
	*
	*/
	rc = controld_globals.cib_conn->cmds->query(controld_globals.cib_conn,
	NULL, NULL,
	cib_scope_local);
	fsa_register_cib_callback(rc, uuid_str, save_cib_contents);
	}

	controld_clear_fsa_input_flags(R_PE_CONNECTED);
	controld_trigger_fsa();
	return;
	}

	static void
	handle_reply(pcmk_schedulerd_api_reply_t *reply)
	{
	const char *msg_ref = NULL;

	if (!AM_I_DC) {
	return;
	}

	msg_ref = reply->data.graph.reference;

	if (msg_ref == NULL) {
	crm_err("%s - Ignoring calculation with no reference", CRM_OP_PECALC);

	} else if (pcmk__str_eq(msg_ref, controld_globals.fsa_pe_ref,
	pcmk__str_none)) {
	ha_msg_input_t fsa_input;
	xmlNode *crm_data_node;

	controld_stop_sched_timer();

	/* do_te_invoke (which will eventually process the fsa_input we are constructing
	* here) requires that fsa_input.xml be non-NULL. That will only happen if
	* copy_ha_msg_input (which is called by register_fsa_input_adv) sees the
	* fsa_input.msg that it is expecting. The scheduler's IPC dispatch function
	* gave us the values we need, we just need to put them into XML.
	*
	* The name of the top level element here is irrelevant. Nothing checks it.
	*/
	fsa_input.msg = pcmk__xe_create(NULL, "dummy-reply");
	crm_xml_add(fsa_input.msg, PCMK_XA_REFERENCE, msg_ref);
	crm_xml_add(fsa_input.msg, PCMK__XA_CRM_TGRAPH_IN,
	reply->data.graph.input);

	crm_data_node = pcmk__xe_create(fsa_input.msg, PCMK__XE_CRM_XML);
	pcmk__xml_copy(crm_data_node, reply->data.graph.tgraph);
	register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);

	pcmk__xml_free(fsa_input.msg);

	} else {
	crm_info("%s calculation %s is obsolete", CRM_OP_PECALC, msg_ref);
	}
	}

	static void
	scheduler_event_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
	crm_exit_t status, void event_data, void user_data)
	{
	pcmk_schedulerd_api_reply_t *reply = event_data;

	switch (event_type) {
	case pcmk_ipc_event_disconnect:
	handle_disconnect();
	break;

	case pcmk_ipc_event_reply:
	handle_reply(reply);
	break;

	default:
	break;
	}
	}

	static bool
	new_schedulerd_ipc_connection(void)
	{
	int rc;

	controld_set_fsa_input_flags(R_PE_REQUIRED);

	if (schedulerd_api == NULL) {
	rc = pcmk_new_ipc_api(&schedulerd_api, pcmk_ipc_schedulerd);

	if (rc != pcmk_rc_ok) {
	crm_err("Error connecting to the scheduler: %s", pcmk_rc_str(rc));
	return false;
	}
	}

	pcmk_register_ipc_callback(schedulerd_api, scheduler_event_callback, NULL);

	rc = pcmk__connect_ipc(schedulerd_api, pcmk_ipc_dispatch_main, 3);
	if (rc != pcmk_rc_ok) {
	crm_err("Error connecting to %s: %s",
	pcmk_ipc_name(schedulerd_api, true), pcmk_rc_str(rc));
	return false;
	}

	controld_set_fsa_input_flags(R_PE_CONNECTED);
	return true;
	}

	static void do_pe_invoke_callback(xmlNode *msg, int call_id, int rc,
	xmlNode output, void user_data);

	/* A_PE_START, A_PE_STOP, O_PE_RESTART */
	void
	do_pe_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (pcmk_is_set(action, A_PE_STOP)) {
	controld_clear_fsa_input_flags(R_PE_REQUIRED);
	pcmk_disconnect_ipc(schedulerd_api);
	handle_disconnect();
	}
	if (pcmk_is_set(action, A_PE_START)
	&& !pcmk_is_set(controld_globals.fsa_input_register, R_PE_CONNECTED)) {

	if (cur_state == S_STOPPING) {
	crm_info("Ignoring request to connect to scheduler while shutting down");

	} else if (!new_schedulerd_ipc_connection()) {
	crm_warn("Could not connect to scheduler");
	register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
	}
	}
	}

	static int fsa_pe_query = 0;
	static mainloop_timer_t *controld_sched_timer = NULL;

	// @TODO Make this a configurable cluster option if there's demand for it
	#define SCHED_TIMEOUT_MS (120000)

	/*!
	* \internal
	* \brief Handle a timeout waiting for scheduler reply
	*
	* \param[in] user_data Ignored
	*
	* \return FALSE (indicating that timer should not be restarted)
	*/
	static gboolean
	controld_sched_timeout(gpointer user_data)
	{
	if (AM_I_DC) {
	/* If this node is the DC but can't communicate with the scheduler, just
	* exit (and likely get fenced) so this node doesn't interfere with any
	* further DC elections.
	*
	* @TODO We could try something less drastic first, like disconnecting
	* and reconnecting to the scheduler, but something is likely going
	* seriously wrong, so perhaps it's better to just fail as quickly as
	* possible.
	*/
	crmd_exit(CRM_EX_FATAL);
	}
	return FALSE;
	}

	void
	controld_stop_sched_timer(void)
	{
	if ((controld_sched_timer != NULL)
	&& (controld_globals.fsa_pe_ref != NULL)) {
	crm_trace("Stopping timer for scheduler reply %s",
	controld_globals.fsa_pe_ref);
	}
	mainloop_timer_stop(controld_sched_timer);
	}

	/*!
	* \internal
	* \brief Set the scheduler request currently being waited on
	*
	* \param[in] ref Request to expect reply to (or NULL for none)
	*
	* \note This function takes ownership of \p ref.
	*/
	void
	controld_expect_sched_reply(char *ref)
	{
	if (ref) {
	if (controld_sched_timer == NULL) {
	controld_sched_timer = mainloop_timer_add("scheduler_reply_timer",
	SCHED_TIMEOUT_MS, FALSE,
	controld_sched_timeout,
	NULL);
	}
	mainloop_timer_start(controld_sched_timer);
	} else {
	controld_stop_sched_timer();
	}
	free(controld_globals.fsa_pe_ref);
	controld_globals.fsa_pe_ref = ref;
	}

	/*!
	* \internal
	* \brief Free the scheduler reply timer
	*/
	void
	controld_free_sched_timer(void)
	{
	if (controld_sched_timer != NULL) {
	mainloop_timer_del(controld_sched_timer);
	controld_sched_timer = NULL;
	}
	}

	/* A_PE_INVOKE */
	void
	do_pe_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	cib_t *cib_conn = controld_globals.cib_conn;

	if (AM_I_DC == FALSE) {
	crm_err("Not invoking scheduler because not DC: %s",
	fsa_action2string(action));
	return;
	}

	if (!pcmk_is_set(controld_globals.fsa_input_register, R_PE_CONNECTED)) {
	if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
	crm_err("Cannot shut down gracefully without the scheduler");
	register_fsa_input_before(C_FSA_INTERNAL, I_TERMINATE, NULL);

	} else {
	crm_info("Waiting for the scheduler to connect");
	crmd_fsa_stall(FALSE);
	controld_set_fsa_action_flags(A_PE_START);
	controld_trigger_fsa();
	}
	return;
	}

	if (cur_state != S_POLICY_ENGINE) {
	crm_notice("Not invoking scheduler because in state %s",
	fsa_state2string(cur_state));
	return;
	}
	if (!pcmk_is_set(controld_globals.fsa_input_register, R_HAVE_CIB)) {
	crm_err("Attempted to invoke scheduler without consistent Cluster Information Base!");

	/* start the join from scratch */
	register_fsa_input_before(C_FSA_INTERNAL, I_ELECTION, NULL);
	return;
	}

	fsa_pe_query = cib_conn->cmds->query(cib_conn, NULL, NULL, cib_scope_local);

	crm_debug("Query %d: Requesting the current CIB: %s", fsa_pe_query,
	fsa_state2string(controld_globals.fsa_state));

	controld_expect_sched_reply(NULL);
	fsa_register_cib_callback(fsa_pe_query, NULL, do_pe_invoke_callback);
	}

	static void
	force_local_option(xmlNode xml, const char attr_name, const char *attr_value)
	{
	int max = 0;
	int lpc = 0;
	const char *xpath_base = NULL;
	char *xpath_string = NULL;
	xmlXPathObjectPtr xpathObj = NULL;

	xpath_base = pcmk_cib_xpath_for(PCMK_XE_CRM_CONFIG);
	if (xpath_base == NULL) {
	crm_err(PCMK_XE_CRM_CONFIG " CIB element not known (bug?)");
	return;
	}

	xpath_string = crm_strdup_printf("%s//%s//nvpair[@name='%s']",
	xpath_base, PCMK_XE_CLUSTER_PROPERTY_SET,
	attr_name);
	xpathObj = xpath_search(xml, xpath_string);
	max = numXpathResults(xpathObj);
	free(xpath_string);

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);
	crm_trace("Forcing %s/%s = %s",
	pcmk__xe_id(match), attr_name, attr_value);
	crm_xml_add(match, PCMK_XA_VALUE, attr_value);
	}

	if(max == 0) {
	xmlNode *configuration = NULL;
	xmlNode *crm_config = NULL;
	xmlNode *cluster_property_set = NULL;

	crm_trace("Creating %s-%s for %s=%s",
	PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, attr_name, attr_name,
	attr_value);

	configuration = pcmk__xe_first_child(xml, PCMK_XE_CONFIGURATION, NULL,
	NULL);
	if (configuration == NULL) {
	configuration = pcmk__xe_create(xml, PCMK_XE_CONFIGURATION);
	}

	crm_config = pcmk__xe_first_child(configuration, PCMK_XE_CRM_CONFIG,
	NULL, NULL);
	if (crm_config == NULL) {
	crm_config = pcmk__xe_create(configuration, PCMK_XE_CRM_CONFIG);
	}

	cluster_property_set =
	pcmk__xe_first_child(crm_config, PCMK_XE_CLUSTER_PROPERTY_SET, NULL,
	NULL);
	if (cluster_property_set == NULL) {
	cluster_property_set =
	pcmk__xe_create(crm_config, PCMK_XE_CLUSTER_PROPERTY_SET);
	crm_xml_add(cluster_property_set, PCMK_XA_ID,
	PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS);
	}

	xml = pcmk__xe_create(cluster_property_set, PCMK_XE_NVPAIR);

	- crm_xml_set_id(xml, "%s-%s",
	- PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, attr_name);
	+ pcmk__xe_set_id(xml, "%s-%s",
	+ PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, attr_name);
	crm_xml_add(xml, PCMK_XA_NAME, attr_name);
	crm_xml_add(xml, PCMK_XA_VALUE, attr_value);
	}
	freeXpathObject(xpathObj);
	}

	static void
	do_pe_invoke_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	char *ref = NULL;
	pid_t watchdog = pcmk__locate_sbd();

	if (rc != pcmk_ok) {
	crm_err("Could not retrieve the Cluster Information Base: %s "
	CRM_XS " rc=%d call=%d", pcmk_strerror(rc), rc, call_id);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
	return;

	} else if (call_id != fsa_pe_query) {
	crm_trace("Skipping superseded CIB query: %d (current=%d)", call_id, fsa_pe_query);
	return;

	} else if (!AM_I_DC
	\|\| !pcmk_is_set(controld_globals.fsa_input_register,
	R_PE_CONNECTED)) {
	crm_debug("No need to invoke the scheduler anymore");
	return;

	} else if (controld_globals.fsa_state != S_POLICY_ENGINE) {
	crm_debug("Discarding scheduler request in state: %s",
	fsa_state2string(controld_globals.fsa_state));
	return;

	/* this callback counts as 1 */
	} else if (num_cib_op_callbacks() > 1) {
	crm_debug("Re-asking for the CIB: %d other peer updates still pending",
	(num_cib_op_callbacks() - 1));
	sleep(1);
	controld_set_fsa_action_flags(A_PE_INVOKE);
	controld_trigger_fsa();
	return;
	}

	CRM_LOG_ASSERT(output != NULL);

	/* Refresh the remote node cache and the known node cache when the
	* scheduler is invoked */
	pcmk__refresh_node_caches_from_cib(output);

	crm_xml_add(output, PCMK_XA_DC_UUID, controld_globals.our_uuid);
	pcmk__xe_set_bool_attr(output, PCMK_XA_HAVE_QUORUM,
	pcmk_is_set(controld_globals.flags,
	controld_has_quorum));

	force_local_option(output, PCMK_OPT_HAVE_WATCHDOG, pcmk__btoa(watchdog));

	if (pcmk_is_set(controld_globals.flags, controld_ever_had_quorum)
	&& !crm_have_quorum) {
	crm_xml_add_int(output, PCMK_XA_NO_QUORUM_PANIC, 1);
	}

	rc = pcmk_rc2legacy(pcmk_schedulerd_api_graph(schedulerd_api, output, &ref));

	if (rc < 0) {
	crm_err("Could not contact the scheduler: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __func__);
	} else {
	CRM_ASSERT(ref != NULL);
	controld_expect_sched_reply(ref);
	crm_debug("Invoking the scheduler: query=%d, ref=%s, seq=%llu, "
	"quorate=%s",
	fsa_pe_query, controld_globals.fsa_pe_ref, crm_peer_seq,
	pcmk__flag_text(controld_globals.flags, controld_has_quorum));
	}
	}
	diff --git a/include/crm/cluster/compat.h b/include/crm/cluster/compat.h
	index 79ea846156..df90c74349 100644
	--- a/include/crm/cluster/compat.h
	+++ b/include/crm/cluster/compat.h
	@@ -1,62 +1,66 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_CLUSTER_COMPAT__H
	#define PCMK__CRM_CLUSTER_COMPAT__H

	#include <crm/cluster.h> // pcmk_cluster_t, enum pcmk_cluster_layer

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Deprecated Pacemaker cluster API
	* \ingroup cluster
	* \deprecated Do not include this header directly. The cluster APIs in this
	* header, and the header itself, will be removed in a future
	* release.
	*/

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use \c pcmk_cluster_t instead
	typedef pcmk_cluster_t crm_cluster_t;

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use \c pcmk_cluster_connect() instead
	gboolean crm_cluster_connect(pcmk_cluster_t *cluster);

	// NOTE: sbd (as of at least 1.5.2) uses this enum
	//!@{
	//! \deprecated Use <tt>enum pcmk_cluster_layer</tt> instead
	enum cluster_type_e {
	// NOTE: sbd (as of at least 1.5.2) uses this value
	pcmk_cluster_unknown = pcmk_cluster_layer_unknown,

	pcmk_cluster_invalid = pcmk_cluster_layer_invalid,

	// NOTE: sbd (as of at least 1.5.2) uses this value
	pcmk_cluster_corosync = pcmk_cluster_layer_corosync,
	};
	//!@}

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use \c pcmk_cluster_layer_text() instead
	const char *name_for_cluster_type(enum cluster_type_e type);

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use \c pcmk_get_cluster_layer() instead
	enum cluster_type_e get_cluster_type(void);

	+// NOTE: sbd (as of at least 1.5.2) uses this
	+//! \deprecated Do not use
	+void crm_peer_init(void);
	+
	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK_CLUSTER_COMPAT__H
	diff --git a/include/crm/common/xml.h b/include/crm/common/xml.h
	index f3eaaaf5aa..3b85276d37 100644
	--- a/include/crm/common/xml.h
	+++ b/include/crm/common/xml.h
	@@ -1,98 +1,95 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_XML__H
	#define PCMK__CRM_COMMON_XML__H


	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <libxml/tree.h>
	#include <libxml/xpath.h>

	#include <crm/crm.h>
	#include <crm/common/nvpair.h>
	#include <crm/common/schemas.h>
	#include <crm/common/xml_io.h>
	#include <crm/common/xml_names.h>

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Wrappers for and extensions to libxml2
	* \ingroup core
	*/

	typedef const xmlChar *pcmkXmlStr;


	/*
	* Searching & Modifying
	*/

	// NOTE: sbd (as of at least 1.5.2) uses this
	xmlNode get_xpath_object(const char xpath, xmlNode * xml_obj, int error_level);

	char calculate_on_disk_digest(xmlNode local_cib);
	char calculate_operation_digest(xmlNode local_cib, const char *version);
	char calculate_xml_versioned_digest(xmlNode input, gboolean sort, gboolean do_filter,
	const char *version);

	xmlNode sorted_xml(xmlNode input, xmlNode * parent, gboolean recursive);
	xmlXPathObjectPtr xpath_search(const xmlNode xml_top, const char path);
	void crm_foreach_xpath_result(xmlNode xml, const char xpath,
	void (helper)(xmlNode, void), void user_data);

	void freeXpathObject(xmlXPathObjectPtr xpathObj);
	xmlNode *getXpathResult(xmlXPathObjectPtr xpathObj, int index);
	void dedupXpathResults(xmlXPathObjectPtr xpathObj);

	static inline int numXpathResults(xmlXPathObjectPtr xpathObj)
	{
	if(xpathObj == NULL \|\| xpathObj->nodesetval == NULL) {
	return 0;
	}
	return xpathObj->nodesetval->nodeNr;
	}

	bool xml_tracking_changes(xmlNode * xml);
	bool xml_document_dirty(xmlNode *xml);
	void xml_track_changes(xmlNode * xml, const char user, xmlNode acl_source, bool enforce_acls);
	void xml_calculate_changes(xmlNode old_xml, xmlNode new_xml);
	void xml_calculate_significant_changes(xmlNode old_xml, xmlNode new_xml);
	void xml_accept_changes(xmlNode * xml);
	bool xml_patch_versions(const xmlNode *patchset, int add[3], int del[3]);

	xmlNode *xml_create_patchset(
	int format, xmlNode source, xmlNode target, bool *config, bool manage_version);
	int xml_apply_patchset(xmlNode xml, xmlNode patchset, bool check_version);

	void patchset_process_digest(xmlNode patch, xmlNode source, xmlNode *target, bool with_digest);

	-void crm_xml_sanitize_id(char *id);
	-void crm_xml_set_id(xmlNode xml, const char format, ...) G_GNUC_PRINTF(2, 3);
	-
	#if !defined(PCMK_ALLOW_DEPRECATED) \|\| (PCMK_ALLOW_DEPRECATED == 1)
	#include <crm/common/xml_compat.h>
	#endif

	#ifdef __cplusplus
	}
	#endif

	#endif
	diff --git a/include/crm/common/xml_compat.h b/include/crm/common/xml_compat.h
	index 54423eb3f6..6b7499c8ef 100644
	--- a/include/crm/common/xml_compat.h
	+++ b/include/crm/common/xml_compat.h
	@@ -1,69 +1,75 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_COMMON_XML_COMPAT__H
	#define PCMK__CRM_COMMON_XML_COMPAT__H

	#include <glib.h> // gboolean
	#include <libxml/tree.h> // xmlNode

	#include <crm/common/nvpair.h> // crm_xml_add()
	#include <crm/common/xml_names.h> // PCMK_XE_CLONE

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Deprecated Pacemaker XML API
	* \ingroup core
	* \deprecated Do not include this header directly. The XML APIs in this
	* header, and the header itself, will be removed in a future
	* release.
	*/

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use name member directly
	static inline const char *
	crm_element_name(const xmlNode *xml)
	{
	return (xml == NULL)? NULL : (const char *) xml->name;
	}

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Do not use Pacemaker for general-purpose XML manipulation
	xmlNode copy_xml(xmlNode src_node);

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Do not use
	gboolean cli_config_update(xmlNode *xml, int best_version, gboolean to_logs);

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Call \c crm_log_init() or \c crm_log_cli_init() instead
	void crm_xml_init(void);

	//! \deprecated Exit with \c crm_exit() instead
	void crm_xml_cleanup(void);

	//! \deprecated Do not use Pacemaker for general-purpose XML manipulation
	void pcmk_free_xml_subtree(xmlNode *xml);

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Do not use Pacemaker for general-purpose XML manipulation
	void free_xml(xmlNode *child);

	//! \deprecated Do not use Pacemaker for general-purpose XML manipulation
	xmlNode expand_idref(xmlNode input, xmlNode *top);

	+//! \deprecated Do not use Pacemaker for general-purpose XML manipulation
	+void crm_xml_set_id(xmlNode xml, const char format, ...) G_GNUC_PRINTF(2, 3);
	+
	+//! \deprecated Do not use Pacemaker for general-purpose XML manipulation
	+void crm_xml_sanitize_id(char *id);
	+
	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_COMMON_XML_COMPAT__H
	diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h
	index 3e71f0d27e..2091f47b75 100644
	--- a/include/crm/common/xml_internal.h
	+++ b/include/crm/common/xml_internal.h
	@@ -1,600 +1,604 @@
	/*
	* Copyright 2017-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__XML_INTERNAL__H
	#define PCMK__XML_INTERNAL__H

	/*
	* Internal-only wrappers for and extensions to libxml2 (libxslt)
	*/

	#include <stdlib.h>
	#include <stdint.h> // uint32_t
	#include <stdio.h>
	#include <string.h>

	#include <crm/crm.h> /* transitively imports qblog.h */
	#include <crm/common/output_internal.h>
	#include <crm/common/xml_io_internal.h>
	#include <crm/common/xml_names_internal.h> // PCMK__XE_PROMOTABLE_LEGACY

	#include <libxml/relaxng.h>

	/*!
	* \brief Base for directing lib{xml2,xslt} log into standard libqb backend
	*
	* This macro implements the core of what can be needed for directing
	* libxml2 or libxslt error messaging into standard, preconfigured
	* libqb-backed log stream.
	*
	* It's a bit unfortunate that libxml2 (and more sparsely, also libxslt)
	* emits a single message by chunks (location is emitted separatedly from
	* the message itself), so we have to take the effort to combine these
	* chunks back to single message. Whether to do this or not is driven
	* with \p dechunk toggle.
	*
	* The form of a macro was chosen for implicit deriving of __FILE__, etc.
	* and also because static dechunking buffer should be differentiated per
	* library (here we assume different functions referring to this macro
	* will not ever be using both at once), preferably also per-library
	* context of use to avoid clashes altogether.
	*
	* Note that we cannot use qb_logt, because callsite data have to be known
	* at the moment of compilation, which it is not always the case -- xml_log
	* (and unfortunately there's no clear explanation of the fail to compile).
	*
	* Also note that there's no explicit guard against said libraries producing
	* never-newline-terminated chunks (which would just keep consuming memory),
	* as it's quite improbable. Termination of the program in between the
	* same-message chunks will raise a flag with valgrind and the likes, though.
	*
	* And lastly, regarding how dechunking combines with other non-message
	* parameters -- for \p priority, most important running specification
	* wins (possibly elevated to LOG_ERR in case of nonconformance with the
	* newline-termination "protocol"), \p dechunk is expected to always be
	* on once it was at the start, and the rest (\p postemit and \p prefix)
	* are picked directly from the last chunk entry finalizing the message
	* (also reasonable to always have it the same with all related entries).
	*
	* \param[in] priority Syslog priority for the message to be logged
	* \param[in] dechunk Whether to dechunk new-line terminated message
	* \param[in] postemit Code to be executed once message is sent out
	* \param[in] prefix How to prefix the message or NULL for raw passing
	* \param[in] fmt Format string as with printf-like functions
	* \param[in] ap Variable argument list to supplement \p fmt format string
	*/
	#define PCMK__XML_LOG_BASE(priority, dechunk, postemit, prefix, fmt, ap) \
	do { \
	if (!(dechunk) && (prefix) == NULL) { /* quick pass */ \
	qb_log_from_external_source_va(__func__, __FILE__, (fmt), \
	(priority), __LINE__, 0, (ap)); \
	(void) (postemit); \
	} else { \
	int CXLB_len = 0; \
	char *CXLB_buf = NULL; \
	static int CXLB_buffer_len = 0; \
	static char *CXLB_buffer = NULL; \
	static uint8_t CXLB_priority = 0; \
	\
	CXLB_len = vasprintf(&CXLB_buf, (fmt), (ap)); \
	\
	if (CXLB_len <= 0 \|\| CXLB_buf[CXLB_len - 1] == '\n' \|\| !(dechunk)) { \
	if (CXLB_len < 0) { \
	CXLB_buf = (char ) "LOG CORRUPTION HAZARD"; /we don't modify*/\
	CXLB_priority = QB_MIN(CXLB_priority, LOG_ERR); \
	} else if (CXLB_len > 0 /* && (dechunk) */ \
	&& CXLB_buf[CXLB_len - 1] == '\n') { \
	CXLB_buf[CXLB_len - 1] = '\0'; \
	} \
	if (CXLB_buffer) { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s%s", \
	CXLB_priority, __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buffer, CXLB_buf); \
	free(CXLB_buffer); \
	} else { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s", \
	(priority), __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buf); \
	} \
	if (CXLB_len < 0) { \
	CXLB_buf = NULL; /* restore temporary override */ \
	} \
	CXLB_buffer = NULL; \
	CXLB_buffer_len = 0; \
	(void) (postemit); \
	\
	} else if (CXLB_buffer == NULL) { \
	CXLB_buffer_len = CXLB_len; \
	CXLB_buffer = CXLB_buf; \
	CXLB_buf = NULL; \
	CXLB_priority = (priority); /* remember as a running severest */ \
	\
	} else { \
	CXLB_buffer = realloc(CXLB_buffer, 1 + CXLB_buffer_len + CXLB_len); \
	memcpy(CXLB_buffer + CXLB_buffer_len, CXLB_buf, CXLB_len); \
	CXLB_buffer_len += CXLB_len; \
	CXLB_buffer[CXLB_buffer_len] = '\0'; \
	CXLB_priority = QB_MIN(CXLB_priority, (priority)); /* severest? */ \
	} \
	free(CXLB_buf); \
	} \
	} while (0)

	/*
	* \enum pcmk__xml_fmt_options
	* \brief Bit flags to control format in XML logs and dumps
	*/
	enum pcmk__xml_fmt_options {
	//! Exclude certain XML attributes (for calculating digests)
	pcmk__xml_fmt_filtered = (1 << 0),

	//! Include indentation and newlines
	pcmk__xml_fmt_pretty = (1 << 1),

	//! Include the opening tag of an XML element, and include XML comments
	pcmk__xml_fmt_open = (1 << 3),

	//! Include the children of an XML element
	pcmk__xml_fmt_children = (1 << 4),

	//! Include the closing tag of an XML element
	pcmk__xml_fmt_close = (1 << 5),

	// @COMPAT Can we start including text nodes unconditionally?
	//! Include XML text nodes
	pcmk__xml_fmt_text = (1 << 6),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a created XML subtree
	pcmk__xml_fmt_diff_plus = (1 << 7),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a removed XML subtree
	pcmk__xml_fmt_diff_minus = (1 << 8),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a minimal version of an XML diff (only showing the changes)
	pcmk__xml_fmt_diff_short = (1 << 9),
	};

	void pcmk__xml_init(void);
	void pcmk__xml_cleanup(void);

	int pcmk__xml_show(pcmk__output_t out, const char prefix, const xmlNode *data,
	int depth, uint32_t options);
	int pcmk__xml_show_changes(pcmk__output_t out, const xmlNode xml);

	/* XML search strings for guest, remote and pacemaker_remote nodes */

	/* search string to find CIB resources entries for cluster nodes */
	#define PCMK__XP_MEMBER_NODE_CONFIG \
	"//" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_NODES \
	"/" PCMK_XE_NODE \
	"[not(@" PCMK_XA_TYPE ") or @" PCMK_XA_TYPE "='" PCMK_VALUE_MEMBER "']"

	/* search string to find CIB resources entries for guest nodes */
	#define PCMK__XP_GUEST_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" PCMK_XE_PRIMITIVE \
	"//" PCMK_XE_META_ATTRIBUTES "//" PCMK_XE_NVPAIR \
	"[@" PCMK_XA_NAME "='" PCMK_META_REMOTE_NODE "']"

	/* search string to find CIB resources entries for remote nodes */
	#define PCMK__XP_REMOTE_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" PCMK_XE_PRIMITIVE \
	"[@" PCMK_XA_TYPE "='" PCMK_VALUE_REMOTE "']" \
	"[@" PCMK_XA_PROVIDER "='pacemaker']"

	/* search string to find CIB node status entries for pacemaker_remote nodes */
	#define PCMK__XP_REMOTE_NODE_STATUS \
	"//" PCMK_XE_CIB "//" PCMK_XE_STATUS "//" PCMK__XE_NODE_STATE \
	"[@" PCMK_XA_REMOTE_NODE "='" PCMK_VALUE_TRUE "']"
	/*!
	* \internal
	* \brief Serialize XML (using libxml) into provided descriptor
	*
	* \param[in] fd File descriptor to (piece-wise) write to
	* \param[in] cur XML subtree to proceed
	*
	* \return a standard Pacemaker return code
	*/
	int pcmk__xml2fd(int fd, xmlNode *cur);

	enum pcmk__xml_artefact_ns {
	pcmk__xml_artefact_ns_legacy_rng = 1,
	pcmk__xml_artefact_ns_legacy_xslt,
	pcmk__xml_artefact_ns_base_rng,
	pcmk__xml_artefact_ns_base_xslt,
	};

	void pcmk__strip_xml_text(xmlNode *xml);
	const char pcmk__xe_add_last_written(xmlNode xe);

	xmlNode pcmk__xe_first_child(const xmlNode parent, const char *node_name,
	const char attr_n, const char attr_v);


	xmlNode pcmk__xe_resolve_idref(xmlNode xml, xmlNode *search);

	void pcmk__xe_remove_attr(xmlNode element, const char name);
	bool pcmk__xe_remove_attr_cb(xmlNode xml, void user_data);
	void pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data);
	int pcmk__xe_delete_match(xmlNode xml, xmlNode search);
	int pcmk__xe_replace_match(xmlNode xml, xmlNode replace);
	int pcmk__xe_update_match(xmlNode xml, xmlNode update, uint32_t flags);

	GString pcmk__element_xpath(const xmlNode xml);

	/*!
	* \internal
	* \enum pcmk__xml_escape_type
	* \brief Indicators of which XML characters to escape
	*
	* XML allows the escaping of special characters by replacing them with entity
	* references (for example, <tt>"""</tt>) or character references (for
	* example, <tt>" "</tt>).
	*
	* The special characters <tt>'&'</tt> (except as the beginning of an entity
	* reference) and <tt>'<'</tt> are not allowed in their literal forms in XML
	* character data. Character data is non-markup text (for example, the content
	* of a text node). <tt>'>'</tt> is allowed under most circumstances; we escape
	* it for safety and symmetry.
	*
	* For more details, see the "Character Data and Markup" section of the XML
	* spec, currently section 2.4:
	* https://www.w3.org/TR/xml/#dt-markup
	*
	* Attribute values are handled specially.
	* * If an attribute value is delimited by single quotes, then single quotes
	* must be escaped within the value.
	* * Similarly, if an attribute value is delimited by double quotes, then double
	* quotes must be escaped within the value.
	* * A conformant XML processor replaces a literal whitespace character (tab,
	* newline, carriage return, space) in an attribute value with a space
	* (\c '#x20') character. However, a reference to a whitespace character (for
	* example, \c " " for \c '\n') does not get replaced.
	* * For more details, see the "Attribute-Value Normalization" section of the
	* XML spec, currently section 3.3.3. Note that the default attribute type
	* is CDATA; we don't deal with NMTOKENS, etc.:
	* https://www.w3.org/TR/xml/#AVNormalize
	*
	* Pacemaker always delimits attribute values with double quotes, so there's no
	* need to escape single quotes.
	*
	* Newlines and tabs should be escaped in attribute values when XML is
	* serialized to text, so that future parsing preserves them rather than
	* normalizing them to spaces.
	*
	* We always escape carriage returns, so that they're not converted to spaces
	* during attribute-value normalization and because displaying them as literals
	* is messy.
	*/
	enum pcmk__xml_escape_type {
	/*!
	* For text nodes.
	* * Escape \c '<', \c '>', and \c '&' using entity references.
	* * Do not escape \c '\n' and \c '\t'.
	* * Escape other non-printing characters using character references.
	*/
	pcmk__xml_escape_text,

	/*!
	* For attribute values.
	* * Escape \c '<', \c '>', \c '&', and \c '"' using entity references.
	* * Escape \c '\n', \c '\t', and other non-printing characters using
	* character references.
	*/
	pcmk__xml_escape_attr,

	/* @COMPAT Drop escaping of at least '\n' and '\t' for
	* pcmk__xml_escape_attr_pretty when openstack-info, openstack-floating-ip,
	* and openstack-virtual-ip resource agents no longer depend on it.
	*
	* At time of writing, openstack-info may set a multiline value for the
	* openstack_ports node attribute. The other two agents query the value and
	* require it to be on one line with no spaces.
	*/
	/*!
	* For attribute values displayed in text output delimited by double quotes.
	* * Escape \c '\n' as \c "\\n"
	* * Escape \c '\r' as \c "\\r"
	* * Escape \c '\t' as \c "\\t"
	* * Escape \c '"' as \c "\\""
	*/
	pcmk__xml_escape_attr_pretty,
	};

	bool pcmk__xml_needs_escape(const char *text, enum pcmk__xml_escape_type type);
	char pcmk__xml_escape(const char text, enum pcmk__xml_escape_type type);

	/*!
	* \internal
	* \brief Get the root directory to scan XML artefacts of given kind for
	*
	* \param[in] ns governs the hierarchy nesting against the inherent root dir
	*
	* \return root directory to scan XML artefacts of given kind for
	*/
	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns);

	/*!
	* \internal
	* \brief Get the fully unwrapped path to particular XML artifact (RNG/XSLT)
	*
	* \param[in] ns denotes path forming details (parent dir, suffix)
	* \param[in] filespec symbolic file specification to be combined with
	* #artefact_ns to form the final path
	* \return unwrapped path to particular XML artifact (RNG/XSLT)
	*/
	char *pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns,
	const char *filespec);

	/*!
	* \internal
	* \brief Retrieve the value of the \c PCMK_XA_ID XML attribute
	*
	* \param[in] xml XML element to check
	*
	* \return Value of the \c PCMK_XA_ID attribute (may be \c NULL)
	*/
	static inline const char *
	pcmk__xe_id(const xmlNode *xml)
	{
	return crm_element_value(xml, PCMK_XA_ID);
	}

	/*!
	* \internal
	* \brief Check whether an XML element is of a particular type
	*
	* \param[in] xml XML element to compare
	* \param[in] name XML element name to compare
	*
	* \return \c true if \p xml is of type \p name, otherwise \c false
	*/
	static inline bool
	pcmk__xe_is(const xmlNode xml, const char name)
	{
	return (xml != NULL) && (xml->name != NULL) && (name != NULL)
	&& (strcmp((const char *) xml->name, name) == 0);
	}

	/*!
	* \internal
	* \brief Return first non-text child node of an XML node
	*
	* \param[in] parent XML node to check
	*
	* \return First non-text child node of \p parent (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_first_child(const xmlNode *parent)
	{
	xmlNode *child = (parent? parent->children : NULL);

	while (child && (child->type == XML_TEXT_NODE)) {
	child = child->next;
	}
	return child;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling node of an XML node
	*
	* \param[in] child XML node to check
	*
	* \return Next non-text sibling of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_next(const xmlNode *child)
	{
	xmlNode *next = (child? child->next : NULL);

	while (next && (next->type == XML_TEXT_NODE)) {
	next = next->next;
	}
	return next;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling element of an XML element
	*
	* \param[in] child XML element to check
	*
	* \return Next sibling element of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xe_next(const xmlNode *child)
	{
	xmlNode *next = child? child->next : NULL;

	while (next && (next->type != XML_ELEMENT_NODE)) {
	next = next->next;
	}
	return next;
	}

	xmlNode pcmk__xe_create(xmlNode parent, const char *name);
	void pcmk__xml_free(xmlNode *xml);
	xmlNode pcmk__xml_copy(xmlNode parent, xmlNode *src);
	xmlNode pcmk__xe_next_same(const xmlNode node);

	void pcmk__xe_set_content(xmlNode node, const char format, ...)
	G_GNUC_PRINTF(2, 3);

	/*!
	* \internal
	* \enum pcmk__xa_flags
	* \brief Flags for operations affecting XML attributes
	*/
	enum pcmk__xa_flags {
	//! Flag has no effect
	pcmk__xaf_none = 0U,

	//! Don't overwrite existing values
	pcmk__xaf_no_overwrite = (1U << 0),

	/*!
	* Treat values as score updates where possible (see
	* \c pcmk__xe_set_score())
	*/
	pcmk__xaf_score_update = (1U << 1),
	};

	int pcmk__xe_copy_attrs(xmlNode target, const xmlNode src, uint32_t flags);

	+void pcmk__xml_sanitize_id(char *id);
	+void pcmk__xe_set_id(xmlNode xml, const char format, ...)
	+ G_GNUC_PRINTF(2, 3);
	+
	/*!
	* \internal
	* \brief Like pcmk__xe_set_props, but takes a va_list instead of
	* arguments directly.
	*
	* \param[in,out] node XML to add attributes to
	* \param[in] pairs NULL-terminated list of name/value pairs to add
	*/
	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs);

	/*!
	* \internal
	* \brief Add a NULL-terminated list of name/value pairs to the given
	* XML node as properties.
	*
	* \param[in,out] node XML node to add properties to
	* \param[in] ... NULL-terminated list of name/value pairs
	*
	* \note A NULL name terminates the arguments; a NULL value will be skipped.
	*/
	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	G_GNUC_NULL_TERMINATED;

	/*!
	* \internal
	* \brief Get first attribute of an XML element
	*
	* \param[in] xe XML element to check
	*
	* \return First attribute of \p xe (or NULL if \p xe is NULL or has none)
	*/
	static inline xmlAttr *
	pcmk__xe_first_attr(const xmlNode *xe)
	{
	return (xe == NULL)? NULL : xe->properties;
	}

	/*!
	* \internal
	* \brief Extract the ID attribute from an XML element
	*
	* \param[in] xpath String to search
	* \param[in] node Node to get the ID for
	*
	* \return ID attribute of \p node in xpath string \p xpath
	*/
	char *
	pcmk__xpath_node_id(const char xpath, const char node);

	/*!
	* \internal
	* \brief Print an informational message if an xpath query returned multiple
	* items with the same ID.
	*
	* \param[in,out] out The output object
	* \param[in] search The xpath search result, most typically the result of
	* calling cib->cmds->query().
	* \param[in] name The name searched for
	*/
	void
	pcmk__warn_multiple_name_matches(pcmk__output_t out, xmlNode search,
	const char *name);

	/* internal XML-related utilities */

	enum xml_private_flags {
	pcmk__xf_none = 0x0000,
	pcmk__xf_dirty = 0x0001,
	pcmk__xf_deleted = 0x0002,
	pcmk__xf_created = 0x0004,
	pcmk__xf_modified = 0x0008,

	pcmk__xf_tracking = 0x0010,
	pcmk__xf_processed = 0x0020,
	pcmk__xf_skip = 0x0040,
	pcmk__xf_moved = 0x0080,

	pcmk__xf_acl_enabled = 0x0100,
	pcmk__xf_acl_read = 0x0200,
	pcmk__xf_acl_write = 0x0400,
	pcmk__xf_acl_deny = 0x0800,

	pcmk__xf_acl_create = 0x1000,
	pcmk__xf_acl_denied = 0x2000,
	pcmk__xf_lazy = 0x4000,
	};

	void pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag);

	/*!
	* \internal
	* \brief Iterate over child elements of \p xml
	*
	* This function iterates over the children of \p xml, performing the
	* callback function \p handler on each node. If the callback returns
	* a value other than pcmk_rc_ok, the iteration stops and the value is
	* returned. It is therefore possible that not all children will be
	* visited.
	*
	* \param[in,out] xml The starting XML node. Can be NULL.
	* \param[in] child_element_name The name that the node must match in order
	* for \p handler to be run. If NULL, all
	* child elements will match.
	* \param[in] handler The callback function.
	* \param[in,out] userdata User data to pass to the callback function.
	* Can be NULL.
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__xe_foreach_child(xmlNode xml, const char child_element_name,
	int (handler)(xmlNode xml, void *userdata),
	void *userdata);

	bool pcmk__xml_tree_foreach(xmlNode xml, bool (fn)(xmlNode , void ),
	void *user_data);

	static inline const char *
	pcmk__xml_attr_value(const xmlAttr *attr)
	{
	return ((attr == NULL) \|\| (attr->children == NULL))? NULL
	: (const char *) attr->children->content;
	}

	// @COMPAT Remove when v1 patchsets are removed
	xmlNode pcmk__diff_v1_xml_object(xmlNode left, xmlNode *right, bool suppress);

	// @COMPAT Drop when PCMK__XE_PROMOTABLE_LEGACY is removed
	static inline const char *
	pcmk__map_element_name(const xmlNode *xml)
	{
	if (xml == NULL) {
	return NULL;
	} else if (pcmk__xe_is(xml, PCMK__XE_PROMOTABLE_LEGACY)) {
	return PCMK_XE_CLONE;
	} else {
	return (const char *) xml->name;
	}
	}

	#endif // PCMK__XML_INTERNAL__H
	diff --git a/include/crm/msg_xml_compat.h b/include/crm/msg_xml_compat.h
	index 0571f27765..9a5c3fafe4 100644
	--- a/include/crm/msg_xml_compat.h
	+++ b/include/crm/msg_xml_compat.h
	@@ -1,36 +1,56 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__CRM_MSG_XML_COMPAT__H
	#define PCMK__CRM_MSG_XML_COMPAT__H

	#include <crm/common/xml.h> // PCMK_XE_CIB

	#ifdef __cplusplus
	extern "C" {
	#endif

	/**
	* \file
	* \brief Deprecated Pacemaker XML constants API
	* \ingroup core
	* \deprecated Do not include this header directly. The XML constants in this
	* header, and the header itself, will be removed in a future
	* release.
	*/

	// NOTE: sbd (as of at least 1.5.2) uses this
	//! \deprecated Use \c PCMK_XE_CIB instead
	#define XML_TAG_CIB PCMK_XE_CIB

	+// NOTE: sbd (as of at least 1.5.2) uses this
	+//! \deprecated Do not use
	+#define XML_CIB_TAG_STATE "node_state"
	+
	+// NOTE: sbd (as of at least 1.5.2) uses this
	+//! \deprecated Do not use
	+#define XML_TAG_TRANSIENT_NODEATTRS "transient_attributes"
	+
	+// NOTE: sbd (as of at least 1.5.2) uses this
	+//! \deprecated Use \c PCMK_XE_INSTANCE_ATTRIBUTES instead
	+#define XML_TAG_ATTR_SETS PCMK_XE_INSTANCE_ATTRIBUTES
	+
	+// NOTE: sbd (as of at least 1.5.2) uses this
	+//! \deprecated Use \c PCMK_XE_NVPAIR instead
	+#define XML_CIB_TAG_NVPAIR PCMK_XE_NVPAIR
	+
	+// NOTE: sbd (as of at least 1.5.2) uses this
	+//! \deprecated Do not use
	+#define XML_CIB_ATTR_SHUTDOWN "shutdown"
	+
	#ifdef __cplusplus
	}
	#endif

	#endif // PCMK__CRM_MSG_XML_COMPAT__H
	diff --git a/lib/cib/cib_attrs.c b/lib/cib/cib_attrs.c
	index 75de1b946a..d6f8901085 100644
	--- a/lib/cib/cib_attrs.c
	+++ b/lib/cib/cib_attrs.c
	@@ -1,666 +1,666 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/cib/internal.h>

	static pcmk__output_t *
	new_output_object(const char *ty)
	{
	int rc = pcmk_rc_ok;
	pcmk__output_t *out = NULL;
	const char* argv[] = { "", NULL };
	pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_LOG,
	PCMK__SUPPORTED_FORMAT_TEXT,
	{ NULL, NULL, NULL }
	};

	pcmk__register_formats(NULL, formats);
	rc = pcmk__output_new(&out, ty, NULL, (char**)argv);
	if ((rc != pcmk_rc_ok) \|\| (out == NULL)) {
	crm_err("Can't out due to internal error: %s", pcmk_rc_str(rc));
	return NULL;
	}

	return out;
	}

	static int
	find_attr(cib_t cib, const char section, const char *node_uuid,
	const char attr_set_type, const char set_name, const char *attr_id,
	const char attr_name, const char user_name, xmlNode **result)
	{
	int rc = pcmk_rc_ok;

	const char *xpath_base = NULL;
	GString *xpath = NULL;
	xmlNode *xml_search = NULL;
	const char *set_type = NULL;
	const char *node_type = NULL;

	if (attr_set_type) {
	set_type = attr_set_type;
	} else {
	set_type = PCMK_XE_INSTANCE_ATTRIBUTES;
	}

	if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) {
	node_uuid = NULL;
	set_type = PCMK_XE_CLUSTER_PROPERTY_SET;

	} else if (pcmk__strcase_any_of(section,
	PCMK_XE_OP_DEFAULTS, PCMK_XE_RSC_DEFAULTS,
	NULL)) {
	node_uuid = NULL;
	set_type = PCMK_XE_META_ATTRIBUTES;

	} else if (pcmk__str_eq(section, PCMK_XE_TICKETS, pcmk__str_casei)) {
	node_uuid = NULL;
	section = PCMK_XE_STATUS;
	node_type = PCMK_XE_TICKETS;

	} else if (node_uuid == NULL) {
	return EINVAL;
	}

	xpath_base = pcmk_cib_xpath_for(section);
	if (xpath_base == NULL) {
	crm_warn("%s CIB section not known", section);
	return ENOMSG;
	}

	xpath = g_string_sized_new(1024);
	g_string_append(xpath, xpath_base);

	if (pcmk__str_eq(node_type, PCMK_XE_TICKETS, pcmk__str_casei)) {
	pcmk__g_strcat(xpath, "//", node_type, NULL);

	} else if (node_uuid) {
	const char *node_type = PCMK_XE_NODE;

	if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) {
	node_type = PCMK__XE_NODE_STATE;
	set_type = PCMK__XE_TRANSIENT_ATTRIBUTES;
	}
	pcmk__g_strcat(xpath,
	"//", node_type, "[@" PCMK_XA_ID "='", node_uuid, "']",
	NULL);
	}

	pcmk__g_strcat(xpath, "//", set_type, NULL);
	if (set_name) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", set_name, "']", NULL);
	}

	g_string_append(xpath, "//nvpair");

	if (attr_id && attr_name) {
	pcmk__g_strcat(xpath,
	"[@" PCMK_XA_ID "='", attr_id, "' "
	"and @" PCMK_XA_NAME "='", attr_name, "']", NULL);

	} else if (attr_id) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "']", NULL);

	} else if (attr_name) {
	pcmk__g_strcat(xpath, "[@" PCMK_XA_NAME "='", attr_name, "']", NULL);
	}

	rc = cib_internal_op(cib, PCMK__CIB_REQUEST_QUERY, NULL,
	(const char *) xpath->str, NULL, &xml_search,
	cib_sync_call\|cib_scope_local\|cib_xpath, user_name);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_trace("Query failed for attribute %s (section=%s, node=%s, set=%s, xpath=%s): %s",
	attr_name, section, pcmk__s(node_uuid, "<null>"),
	pcmk__s(set_name, "<null>"), (const char *) xpath->str,
	pcmk_rc_str(rc));
	} else {
	crm_log_xml_debug(xml_search, "Match");
	}

	g_string_free(xpath, TRUE);
	*result = xml_search;
	return rc;
	}

	static int
	handle_multiples(pcmk__output_t out, xmlNode search, const char *attr_name)
	{
	if ((search != NULL) && (search->children != NULL)) {
	pcmk__warn_multiple_name_matches(out, search, attr_name);
	return ENOTUNIQ;
	} else {
	return pcmk_rc_ok;
	}
	}

	int
	cib__update_node_attr(pcmk__output_t out, cib_t cib, int call_options, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *attr_value,
	const char user_name, const char node_type)
	{
	const char *tag = NULL;
	int rc = pcmk_rc_ok;
	xmlNode *xml_top = NULL;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_search = NULL;

	char *local_attr_id = NULL;
	char *local_set_name = NULL;

	CRM_CHECK((out != NULL) && (cib != NULL) && (section != NULL)
	&& ((attr_id != NULL) \|\| (attr_name != NULL))
	&& (attr_value != NULL), return EINVAL);

	rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id,
	attr_name, user_name, &xml_search);

	if (rc == pcmk_rc_ok) {
	if (handle_multiples(out, xml_search, attr_name) == ENOTUNIQ) {
	pcmk__xml_free(xml_search);
	return ENOTUNIQ;
	} else {
	local_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
	attr_id = local_attr_id;
	pcmk__xml_free(xml_search);
	goto do_modify;
	}

	} else if (rc != ENXIO) {
	pcmk__xml_free(xml_search);
	return rc;

	/* } else if(attr_id == NULL) { */
	/* return EINVAL; */

	} else {
	pcmk__xml_free(xml_search);
	crm_trace("%s does not exist, create it", attr_name);
	if (pcmk__str_eq(section, PCMK_XE_TICKETS, pcmk__str_casei)) {
	node_uuid = NULL;
	section = PCMK_XE_STATUS;
	node_type = PCMK_XE_TICKETS;

	xml_top = pcmk__xe_create(xml_obj, PCMK_XE_STATUS);
	xml_obj = pcmk__xe_create(xml_top, PCMK_XE_TICKETS);

	} else if (pcmk__str_eq(section, PCMK_XE_NODES, pcmk__str_casei)) {

	if (node_uuid == NULL) {
	return EINVAL;
	}

	if (pcmk__str_eq(node_type, PCMK_VALUE_REMOTE, pcmk__str_casei)) {
	xml_top = pcmk__xe_create(xml_obj, PCMK_XE_NODES);
	xml_obj = pcmk__xe_create(xml_top, PCMK_XE_NODE);
	crm_xml_add(xml_obj, PCMK_XA_TYPE, PCMK_VALUE_REMOTE);
	crm_xml_add(xml_obj, PCMK_XA_ID, node_uuid);
	crm_xml_add(xml_obj, PCMK_XA_UNAME, node_uuid);
	} else {
	tag = PCMK_XE_NODE;
	}

	} else if (pcmk__str_eq(section, PCMK_XE_STATUS, pcmk__str_casei)) {
	tag = PCMK__XE_TRANSIENT_ATTRIBUTES;
	if (node_uuid == NULL) {
	return EINVAL;
	}

	xml_top = pcmk__xe_create(xml_obj, PCMK__XE_NODE_STATE);
	crm_xml_add(xml_top, PCMK_XA_ID, node_uuid);
	xml_obj = xml_top;

	} else {
	tag = section;
	node_uuid = NULL;
	}

	if (set_name == NULL) {
	if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) {
	local_set_name =
	pcmk__str_copy(PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS);

	} else if (pcmk__str_eq(node_type, PCMK_XE_TICKETS,
	pcmk__str_casei)) {
	local_set_name = crm_strdup_printf("%s-%s", section,
	PCMK_XE_TICKETS);

	} else if (node_uuid) {
	local_set_name = crm_strdup_printf("%s-%s", section, node_uuid);

	if (set_type) {
	char *tmp_set_name = local_set_name;

	local_set_name = crm_strdup_printf("%s-%s", tmp_set_name,
	set_type);
	free(tmp_set_name);
	}
	} else {
	local_set_name = crm_strdup_printf("%s-options", section);
	}
	set_name = local_set_name;
	}

	if (attr_id == NULL) {
	local_attr_id = crm_strdup_printf("%s-%s", set_name, attr_name);
	- crm_xml_sanitize_id(local_attr_id);
	+ pcmk__xml_sanitize_id(local_attr_id);
	attr_id = local_attr_id;

	} else if (attr_name == NULL) {
	attr_name = attr_id;
	}

	crm_trace("Creating %s/%s", section, tag);
	if (tag != NULL) {
	xml_obj = pcmk__xe_create(xml_obj, tag);
	crm_xml_add(xml_obj, PCMK_XA_ID, node_uuid);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}
	}

	if ((node_uuid == NULL)
	&& !pcmk__str_eq(node_type, PCMK_XE_TICKETS, pcmk__str_casei)) {

	if (pcmk__str_eq(section, PCMK_XE_CRM_CONFIG, pcmk__str_casei)) {
	xml_obj = pcmk__xe_create(xml_obj,
	PCMK_XE_CLUSTER_PROPERTY_SET);
	} else {
	xml_obj = pcmk__xe_create(xml_obj, PCMK_XE_META_ATTRIBUTES);
	}

	} else if (set_type) {
	xml_obj = pcmk__xe_create(xml_obj, set_type);

	} else {
	xml_obj = pcmk__xe_create(xml_obj, PCMK_XE_INSTANCE_ATTRIBUTES);
	}
	crm_xml_add(xml_obj, PCMK_XA_ID, set_name);

	if (xml_top == NULL) {
	xml_top = xml_obj;
	}
	}

	do_modify:
	xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
	if (xml_top == NULL) {
	xml_top = xml_obj;
	}

	crm_log_xml_trace(xml_top, "update_attr");
	rc = cib_internal_op(cib, PCMK__CIB_REQUEST_MODIFY, NULL, section, xml_top,
	NULL, call_options, user_name);

	if (!pcmk_is_set(call_options, cib_sync_call) && (cib->variant != cib_file)
	&& (rc >= 0)) {
	// For async call, positive rc is the call ID (file always synchronous)
	rc = pcmk_rc_ok;
	} else {
	rc = pcmk_legacy2rc(rc);
	}

	if (rc != pcmk_rc_ok) {
	out->err(out, "Error setting %s=%s (section=%s, set=%s): %s",
	attr_name, attr_value, section, pcmk__s(set_name, "<null>"),
	pcmk_rc_str(rc));
	crm_log_xml_info(xml_top, "Update");
	}

	free(local_set_name);
	free(local_attr_id);
	pcmk__xml_free(xml_top);

	return rc;
	}

	int
	cib__get_node_attrs(pcmk__output_t out, cib_t cib, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *user_name,
	xmlNode **result)
	{
	int rc = pcmk_rc_ok;

	CRM_ASSERT(result != NULL);
	CRM_CHECK(section != NULL, return EINVAL);

	*result = NULL;

	rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id, attr_name,
	user_name, result);

	if (rc != pcmk_rc_ok) {
	crm_trace("Query failed for attribute %s (section=%s node=%s set=%s): %s",
	pcmk__s(attr_name, "with unspecified name"),
	section, pcmk__s(set_name, "<null>"),
	pcmk__s(node_uuid, "<null>"), pcmk_rc_str(rc));
	}

	return rc;
	}

	int
	cib__delete_node_attr(pcmk__output_t out, cib_t cib, int options, const char *section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *attr_value,
	const char *user_name)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml_obj = NULL;
	xmlNode *xml_search = NULL;
	char *local_attr_id = NULL;

	CRM_CHECK(section != NULL, return EINVAL);
	CRM_CHECK(attr_name != NULL \|\| attr_id != NULL, return EINVAL);

	if (attr_id == NULL) {
	rc = find_attr(cib, section, node_uuid, set_type, set_name, attr_id,
	attr_name, user_name, &xml_search);

	if (rc != pcmk_rc_ok \|\| handle_multiples(out, xml_search, attr_name) == ENOTUNIQ) {
	pcmk__xml_free(xml_search);
	return rc;
	} else {
	local_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
	attr_id = local_attr_id;
	pcmk__xml_free(xml_search);
	}
	}

	xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, attr_value);

	rc = cib_internal_op(cib, PCMK__CIB_REQUEST_DELETE, NULL, section, xml_obj,
	NULL, options, user_name);

	if (!pcmk_is_set(options, cib_sync_call) && (cib->variant != cib_file)
	&& (rc >= 0)) {
	// For async call, positive rc is the call ID (file always synchronous)
	rc = pcmk_rc_ok;
	} else {
	rc = pcmk_legacy2rc(rc);
	}

	if (rc == pcmk_rc_ok) {
	out->info(out, "Deleted %s %s: id=%s%s%s%s%s",
	section, node_uuid ? "attribute" : "option", local_attr_id,
	set_name ? " set=" : "", set_name ? set_name : "",
	attr_name ? " name=" : "", attr_name ? attr_name : "");
	}
	free(local_attr_id);
	pcmk__xml_free(xml_obj);
	return rc;
	}

	int
	find_nvpair_attr_delegate(cib_t cib, const char attr, const char *section,
	const char node_uuid, const char attr_set_type, const char *set_name,
	const char attr_id, const char attr_name, gboolean to_console,
	char *value, const char user_name)
	{
	pcmk__output_t *out = NULL;
	xmlNode *xml_search = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = find_attr(cib, section, node_uuid, attr_set_type, set_name, attr_id,
	attr_name, user_name, &xml_search);

	if (rc == pcmk_rc_ok) {
	rc = handle_multiples(out, xml_search, attr_name);

	if (rc == pcmk_rc_ok) {
	pcmk__str_update(value, crm_element_value(xml_search, attr));
	}
	}

	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__xml_free(xml_search);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	int
	update_attr_delegate(cib_t cib, int call_options, const char section,
	const char node_uuid, const char set_type, const char *set_name,
	const char attr_id, const char attr_name, const char *attr_value,
	gboolean to_console, const char user_name, const char node_type)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = cib__update_node_attr(out, cib, call_options, section, node_uuid, set_type,
	set_name, attr_id, attr_name, attr_value, user_name,
	node_type);

	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	int
	read_attr_delegate(cib_t cib, const char section, const char *node_uuid,
	const char set_type, const char set_name, const char *attr_id,
	const char attr_name, char *attr_value, gboolean to_console,
	const char *user_name)
	{
	pcmk__output_t *out = NULL;
	xmlNode *result = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = cib__get_node_attrs(out, cib, section, node_uuid, set_type, set_name,
	attr_id, attr_name, user_name, &result);

	if (rc == pcmk_rc_ok) {
	if (result->children == NULL) {
	pcmk__str_update(attr_value,
	crm_element_value(result, PCMK_XA_VALUE));
	} else {
	rc = ENOTUNIQ;
	}
	}

	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__xml_free(result);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	int
	delete_attr_delegate(cib_t cib, int options, const char section, const char *node_uuid,
	const char set_type, const char set_name, const char *attr_id,
	const char attr_name, const char attr_value, gboolean to_console,
	const char *user_name)
	{
	pcmk__output_t *out = NULL;
	int rc = pcmk_ok;

	out = new_output_object(to_console ? "text" : "log");
	if (out == NULL) {
	return pcmk_err_generic;
	}

	rc = cib__delete_node_attr(out, cib, options, section, node_uuid, set_type,
	set_name, attr_id, attr_name, attr_value, user_name);

	out->finish(out, CRM_EX_OK, true, NULL);
	pcmk__output_free(out);
	return pcmk_rc2legacy(rc);
	}

	/*!
	* \internal
	* \brief Parse node UUID from search result
	*
	* \param[in] result XML search result
	* \param[out] uuid If non-NULL, where to store parsed UUID
	* \param[out] is_remote If non-NULL, set TRUE if result is remote node
	*
	* \return pcmk_ok if UUID was successfully parsed, -ENXIO otherwise
	*/
	static int
	get_uuid_from_result(const xmlNode result, char uuid, int is_remote)
	{
	int rc = -ENXIO;
	const char *parsed_uuid = NULL;
	int parsed_is_remote = FALSE;

	if (result == NULL) {
	return rc;
	}

	/* If there are multiple results, the first is sufficient */
	if (pcmk__xe_is(result, PCMK__XE_XPATH_QUERY)) {
	result = pcmk__xe_first_child(result, NULL, NULL, NULL);
	CRM_CHECK(result != NULL, return rc);
	}

	if (pcmk__xe_is(result, PCMK_XE_NODE)) {
	// Result is PCMK_XE_NODE element from PCMK_XE_NODES section

	if (pcmk__str_eq(crm_element_value(result, PCMK_XA_TYPE),
	PCMK_VALUE_REMOTE, pcmk__str_casei)) {
	parsed_uuid = crm_element_value(result, PCMK_XA_UNAME);
	parsed_is_remote = TRUE;
	} else {
	parsed_uuid = pcmk__xe_id(result);
	parsed_is_remote = FALSE;
	}

	} else if (pcmk__xe_is(result, PCMK_XE_PRIMITIVE)) {
	/* Result is <primitive> for ocf:pacemaker:remote resource */

	parsed_uuid = pcmk__xe_id(result);
	parsed_is_remote = TRUE;

	} else if (pcmk__xe_is(result, PCMK_XE_NVPAIR)) {
	/* Result is PCMK_META_REMOTE_NODE parameter of <primitive> for guest
	* node
	*/

	parsed_uuid = crm_element_value(result, PCMK_XA_VALUE);
	parsed_is_remote = TRUE;

	} else if (pcmk__xe_is(result, PCMK__XE_NODE_STATE)) {
	// Result is PCMK__XE_NODE_STATE element from PCMK_XE_STATUS section

	parsed_uuid = crm_element_value(result, PCMK_XA_UNAME);
	if (pcmk__xe_attr_is_true(result, PCMK_XA_REMOTE_NODE)) {
	parsed_is_remote = TRUE;
	}
	}

	if (parsed_uuid) {
	if (uuid) {
	*uuid = strdup(parsed_uuid);
	}
	if (is_remote) {
	*is_remote = parsed_is_remote;
	}
	rc = pcmk_ok;
	}

	return rc;
	}

	/* Search string to find a node by name, as:
	* - cluster or remote node in nodes section
	* - remote node in resources section
	* - guest node in resources section
	* - orphaned remote node or bundle guest node in status section
	*/
	#define XPATH_UPPER_TRANS "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
	#define XPATH_LOWER_TRANS "abcdefghijklmnopqrstuvwxyz"
	#define XPATH_NODE \
	"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_NODES \
	"/" PCMK_XE_NODE "[translate(@" PCMK_XA_UNAME ",'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \
	"\|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \
	"/" PCMK_XE_PRIMITIVE \
	"[@class='ocf'][@provider='pacemaker'][@type='remote'][translate(@id,'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \
	"\|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES \
	"/" PCMK_XE_PRIMITIVE "/" PCMK_XE_META_ATTRIBUTES "/" PCMK_XE_NVPAIR \
	"[@name='" PCMK_META_REMOTE_NODE "'][translate(@value,'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']" \
	"\|/" PCMK_XE_CIB "/" PCMK_XE_STATUS "/" PCMK__XE_NODE_STATE \
	"[@" PCMK_XA_REMOTE_NODE "='true'][translate(@" PCMK_XA_ID ",'" XPATH_UPPER_TRANS "','" XPATH_LOWER_TRANS "') ='%s']"

	int
	query_node_uuid(cib_t * the_cib, const char uname, char uuid, int is_remote_node)
	{
	int rc = pcmk_ok;
	char *xpath_string;
	xmlNode *xml_search = NULL;
	char *host_lowercase = NULL;

	CRM_ASSERT(uname != NULL);

	host_lowercase = g_ascii_strdown(uname, -1);

	if (uuid) {
	*uuid = NULL;
	}
	if (is_remote_node) {
	*is_remote_node = FALSE;
	}

	xpath_string = crm_strdup_printf(XPATH_NODE, host_lowercase, host_lowercase, host_lowercase, host_lowercase);
	if (cib_internal_op(the_cib, PCMK__CIB_REQUEST_QUERY, NULL, xpath_string,
	NULL, &xml_search,
	cib_sync_call\|cib_scope_local\|cib_xpath,
	NULL) == pcmk_ok) {
	rc = get_uuid_from_result(xml_search, uuid, is_remote_node);
	} else {
	rc = -ENXIO;
	}
	free(xpath_string);
	pcmk__xml_free(xml_search);
	g_free(host_lowercase);

	if (rc != pcmk_ok) {
	crm_debug("Could not map node name '%s' to a UUID: %s",
	uname, pcmk_strerror(rc));
	} else {
	crm_info("Mapped node name '%s' to UUID %s", uname, (uuid? *uuid : ""));
	}
	return rc;
	}
	diff --git a/lib/cluster/corosync.c b/lib/cluster/corosync.c
	index edfea41bfb..39e4ed2b04 100644
	--- a/lib/cluster/corosync.c
	+++ b/lib/cluster/corosync.c
	@@ -1,811 +1,811 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <arpa/inet.h>
	#include <inttypes.h> // PRIu64, PRIx32
	#include <netdb.h>
	#include <netinet/in.h>
	#include <stdbool.h>
	#include <sys/socket.h>
	#include <sys/utsname.h>

	#include <bzlib.h>
	#include <corosync/cfg.h>
	#include <corosync/cmap.h>
	#include <corosync/corodefs.h>
	#include <corosync/corotypes.h>
	#include <corosync/hdb.h>
	#include <corosync/quorum.h>
	#include <qb/qbipcc.h>
	#include <qb/qbutil.h>

	#include <crm/cluster/internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h> // PCMK__SPECIAL_PID
	#include <crm/common/mainloop.h>
	#include <crm/common/xml.h>

	#include "crmcluster_private.h"

	static quorum_handle_t pcmk_quorum_handle = 0;

	static gboolean (*quorum_app_callback)(unsigned long long seq,
	gboolean quorate) = NULL;

	/*!
	* \internal
	* \brief Get the Corosync UUID associated with a Pacemaker node
	*
	* \param[in] node Pacemaker node
	*
	* \return Newly allocated string with node's Corosync UUID, or NULL if unknown
	* \note It is the caller's responsibility to free the result with free().
	*/
	char *
	pcmk__corosync_uuid(const crm_node_t *node)
	{
	CRM_ASSERT(pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync);

	if (node != NULL) {
	if (node->id > 0) {
	return crm_strdup_printf("%u", node->id);
	} else {
	crm_info("Node %s is not yet known by Corosync", node->uname);
	}
	}
	return NULL;
	}

	static bool
	node_name_is_valid(const char key, const char name)
	{
	int octet;

	if (name == NULL) {
	crm_trace("%s is empty", key);
	return false;

	} else if (sscanf(name, "%d.%d.%d.%d", &octet, &octet, &octet, &octet) == 4) {
	crm_trace("%s contains an IPv4 address (%s), ignoring", key, name);
	return false;

	} else if (strstr(name, ":") != NULL) {
	crm_trace("%s contains an IPv6 address (%s), ignoring", key, name);
	return false;
	}
	crm_trace("'%s: %s' is valid", key, name);
	return true;
	}

	/*
	* \internal
	* \brief Get Corosync node name corresponding to a node ID
	*
	* \param[in] cmap_handle Connection to Corosync CMAP
	* \param[in] nodeid Node ID to check
	*
	* \return Newly allocated string with name or (if no name) IP address
	* associated with first address assigned to a Corosync node ID (or NULL
	* if unknown)
	* \note It is the caller's responsibility to free the result with free().
	*/
	char *
	pcmk__corosync_name(uint64_t /cmap_handle_t / cmap_handle, uint32_t nodeid)
	{
	// Originally based on corosync-quorumtool.c:node_name()

	int lpc = 0;
	cs_error_t rc = CS_OK;
	int retries = 0;
	char *name = NULL;
	cmap_handle_t local_handle = 0;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	if (nodeid == 0) {
	nodeid = pcmk__cpg_local_nodeid(0);
	}

	if (cmap_handle == 0 && local_handle == 0) {
	retries = 0;
	crm_trace("Initializing CMAP connection");
	do {
	rc = pcmk__init_cmap(&local_handle);
	if (rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
	retries);
	sleep(retries);
	}

	} while (retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %s",
	cs_strerror(rc));
	local_handle = 0;
	}
	}

	if (cmap_handle == 0) {
	cmap_handle = local_handle;

	rc = cmap_fd_get(cmap_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}
	}

	while (name == NULL && cmap_handle != 0) {
	uint32_t id = 0;
	char *key = NULL;

	key = crm_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &id);
	crm_trace("Checking %u vs %u from %s", nodeid, id, key);
	free(key);

	if (rc != CS_OK) {
	break;
	}

	if (nodeid == id) {
	crm_trace("Searching for node name for %u in nodelist.node.%d %s",
	nodeid, lpc, pcmk__s(name, "<null>"));
	if (name == NULL) {
	key = crm_strdup_printf("nodelist.node.%d.name", lpc);
	cmap_get_string(cmap_handle, key, &name);
	crm_trace("%s = %s", key, pcmk__s(name, "<null>"));
	free(key);
	}
	if (name == NULL) {
	key = crm_strdup_printf("nodelist.node.%d.ring0_addr", lpc);
	cmap_get_string(cmap_handle, key, &name);
	crm_trace("%s = %s", key, pcmk__s(name, "<null>"));

	if (!node_name_is_valid(key, name)) {
	free(name);
	name = NULL;
	}
	free(key);
	}
	break;
	}

	lpc++;
	}

	bail:
	if(local_handle) {
	cmap_finalize(local_handle);
	}

	if (name == NULL) {
	crm_info("Unable to get node name for nodeid %u", nodeid);
	}
	return name;
	}

	/*!
	* \internal
	* \brief Disconnect from Corosync cluster
	*
	* \param[in,out] cluster Cluster object to disconnect
	*/
	void
	pcmk__corosync_disconnect(pcmk_cluster_t *cluster)
	{
	pcmk__cpg_disconnect(cluster);

	if (pcmk_quorum_handle != 0) {
	quorum_finalize(pcmk_quorum_handle);
	pcmk_quorum_handle = 0;
	}
	crm_notice("Disconnected from Corosync");
	}

	/*!
	* \internal
	* \brief Dispatch function for quorum connection file descriptor
	*
	* \param[in] user_data Ignored
	*
	* \return 0 on success, -1 on error (per mainloop_io_t interface)
	*/
	static int
	quorum_dispatch_cb(gpointer user_data)
	{
	int rc = quorum_dispatch(pcmk_quorum_handle, CS_DISPATCH_ALL);

	if (rc < 0) {
	crm_err("Connection to the Quorum API failed: %d", rc);
	quorum_finalize(pcmk_quorum_handle);
	pcmk_quorum_handle = 0;
	return -1;
	}
	return 0;
	}

	/*!
	* \internal
	* \brief Notification callback for Corosync quorum connection
	*
	* \param[in] handle Corosync quorum connection
	* \param[in] quorate Whether cluster is quorate
	* \param[in] ring_id Corosync ring ID
	* \param[in] view_list_entries Number of entries in \p view_list
	* \param[in] view_list Corosync node IDs in membership
	*/
	static void
	quorum_notification_cb(quorum_handle_t handle, uint32_t quorate,
	uint64_t ring_id, uint32_t view_list_entries,
	uint32_t *view_list)
	{
	int i;
	GHashTableIter iter;
	crm_node_t *node = NULL;
	static gboolean init_phase = TRUE;

	if (quorate != crm_have_quorum) {
	if (quorate) {
	crm_notice("Quorum acquired " CRM_XS " membership=%" PRIu64 " members=%lu",
	ring_id, (long unsigned int)view_list_entries);
	} else {
	crm_warn("Quorum lost " CRM_XS " membership=%" PRIu64 " members=%lu",
	ring_id, (long unsigned int)view_list_entries);
	}
	crm_have_quorum = quorate;

	} else {
	crm_info("Quorum %s " CRM_XS " membership=%" PRIu64 " members=%lu",
	(quorate? "retained" : "still lost"), ring_id,
	(long unsigned int)view_list_entries);
	}

	if (view_list_entries == 0 && init_phase) {
	crm_info("Corosync membership is still forming, ignoring");
	return;
	}

	init_phase = FALSE;

	/* Reset last_seen for all cached nodes so we can tell which ones aren't
	* in the view list */
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	node->last_seen = 0;
	}

	/* Update the peer cache for each node in view list */
	for (i = 0; i < view_list_entries; i++) {
	uint32_t id = view_list[i];

	crm_debug("Member[%d] %u ", i, id);

	/* Get this node's peer cache entry (adding one if not already there) */
	node = pcmk__get_node(id, NULL, NULL, pcmk__node_search_cluster_member);
	if (node->uname == NULL) {
	char *name = pcmk__corosync_name(0, id);

	crm_info("Obtaining name for new node %u", id);
	node = pcmk__get_node(id, name, NULL,
	pcmk__node_search_cluster_member);
	free(name);
	}

	/* Update the node state (including updating last_seen to ring_id) */
	pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, ring_id);
	}

	/* Remove any peer cache entries we didn't update */
	pcmk__reap_unseen_nodes(ring_id);

	if (quorum_app_callback) {
	quorum_app_callback(ring_id, quorate);
	}
	}

	/*!
	* \internal
	* \brief Connect to Corosync quorum service
	*
	* \param[in] dispatch Connection dispatch callback
	* \param[in] destroy Connection destroy callback
	*/
	void
	pcmk__corosync_quorum_connect(gboolean (*dispatch)(unsigned long long,
	gboolean),
	void (*destroy)(gpointer))
	{
	cs_error_t rc;
	int fd = 0;
	int quorate = 0;
	uint32_t quorum_type = 0;
	struct mainloop_fd_callbacks quorum_fd_callbacks;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	quorum_fd_callbacks.dispatch = quorum_dispatch_cb;
	quorum_fd_callbacks.destroy = destroy;

	crm_debug("Configuring Pacemaker to obtain quorum from Corosync");

	{
	#if 0
	// New way but not supported by all Corosync 2 versions
	quorum_model_v0_data_t quorum_model_data = {
	.model = QUORUM_MODEL_V0,
	.quorum_notify_fn = quorum_notification_cb,
	};

	rc = quorum_model_initialize(&pcmk_quorum_handle, QUORUM_MODEL_V0,
	(quorum_model_data_t *) &quorum_model_data,
	&quorum_type, NULL);
	#else
	quorum_callbacks_t quorum_callbacks = {
	.quorum_notify_fn = quorum_notification_cb,
	};

	rc = quorum_initialize(&pcmk_quorum_handle, &quorum_callbacks,
	&quorum_type);
	#endif
	}

	if (rc != CS_OK) {
	crm_err("Could not connect to the Quorum API: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;

	} else if (quorum_type != QUORUM_SET) {
	crm_err("Corosync quorum is not configured");
	goto bail;
	}

	rc = quorum_fd_get(pcmk_quorum_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the Quorum API connection: %s (%d)",
	strerror(rc), rc);
	goto bail;
	}

	/* Quorum provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("Quorum provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	rc = CS_ERR_ACCESS;
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of Quorum provider: %s (%d)",
	strerror(-rv), -rv);
	rc = CS_ERR_ACCESS;
	goto bail;
	}

	rc = quorum_getquorate(pcmk_quorum_handle, &quorate);
	if (rc != CS_OK) {
	crm_err("Could not obtain the current Quorum API state: %d", rc);
	goto bail;
	}

	if (quorate) {
	crm_notice("Quorum acquired");
	} else {
	crm_warn("No quorum");
	}
	quorum_app_callback = dispatch;
	crm_have_quorum = quorate;

	rc = quorum_trackstart(pcmk_quorum_handle, CS_TRACK_CHANGES \| CS_TRACK_CURRENT);
	if (rc != CS_OK) {
	crm_err("Could not setup Quorum API notifications: %d", rc);
	goto bail;
	}

	mainloop_add_fd("quorum", G_PRIORITY_HIGH, fd, dispatch, &quorum_fd_callbacks);

	pcmk__corosync_add_nodes(NULL);

	bail:
	if (rc != CS_OK) {
	quorum_finalize(pcmk_quorum_handle);
	}
	}

	/*!
	* \internal
	* \brief Connect to Corosync cluster layer
	*
	* \param[in,out] cluster Initialized cluster object to connect
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__corosync_connect(pcmk_cluster_t *cluster)
	{
	crm_node_t *peer = NULL;
	const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer();
	const char *cluster_layer_s = pcmk_cluster_layer_text(cluster_layer);
	int rc = pcmk_rc_ok;

	pcmk__cluster_init_node_caches();

	if (cluster_layer != pcmk_cluster_layer_corosync) {
	crm_err("Invalid cluster layer: %s " CRM_XS " cluster_layer=%d",
	cluster_layer_s, cluster_layer);
	return EINVAL;
	}

	rc = pcmk__cpg_connect(cluster);
	if (rc != pcmk_rc_ok) {
	// Error message was logged by pcmk__cpg_connect()
	return rc;
	}
	crm_info("Connection to %s established", cluster_layer_s);

	cluster->nodeid = pcmk__cpg_local_nodeid(0);
	if (cluster->nodeid == 0) {
	crm_err("Could not determine local node ID");
	return ENXIO;
	}

	cluster->uname = pcmk__cluster_node_name(0);
	if (cluster->uname == NULL) {
	crm_err("Could not determine local node name");
	return ENXIO;
	}

	// Ensure local node always exists in peer cache
	peer = pcmk__get_node(cluster->nodeid, cluster->uname, NULL,
	pcmk__node_search_cluster_member);
	cluster->uuid = pcmk__corosync_uuid(peer);

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Check whether a Corosync cluster is active
	*
	* \return \c true if Corosync is found active, or \c false otherwise
	*/
	bool
	pcmk__corosync_is_active(void)
	{
	cmap_handle_t handle;
	int rc = pcmk__init_cmap(&handle);

	if (rc == CS_OK) {
	cmap_finalize(handle);
	return true;
	}

	crm_info("Failed to initialize the cmap API: %s (%d)",
	pcmk__cs_err_str(rc), rc);
	return false;
	}

	/*!
	* \internal
	* \brief Check whether a Corosync cluster peer is active
	*
	* \param[in] node Node to check
	*
	* \return \c true if \p node is an active Corosync peer, or \c false otherwise
	*/
	bool
	pcmk__corosync_is_peer_active(const crm_node_t *node)
	{
	if (node == NULL) {
	crm_trace("Corosync peer inactive: NULL");
	return false;
	}
	if (!pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_none)) {
	crm_trace("Corosync peer %s inactive: state=%s",
	node->uname, node->state);
	return false;
	}
	if (!pcmk_is_set(node->processes, crm_proc_cpg)) {
	crm_trace("Corosync peer %s inactive " CRM_XS " processes=%.16" PRIx32,
	node->uname, node->processes);
	return false;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Load Corosync node list (via CMAP) into peer cache and optionally XML
	*
	* \param[in,out] xml_parent If not NULL, add <node> entry here for each node
	*
	* \return true if any nodes were found, false otherwise
	*/
	bool
	pcmk__corosync_add_nodes(xmlNode *xml_parent)
	{
	int lpc = 0;
	cs_error_t rc = CS_OK;
	int retries = 0;
	bool any = false;
	cmap_handle_t cmap_handle;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	do {
	rc = pcmk__init_cmap(&cmap_handle);
	if (rc != CS_OK) {
	retries++;
	crm_debug("API connection setup failed: %s. Retrying in %ds", cs_strerror(rc),
	retries);
	sleep(retries);
	}

	} while (retries < 5 && rc != CS_OK);

	if (rc != CS_OK) {
	crm_warn("Could not connect to Cluster Configuration Database API, error %d", rc);
	return false;
	}

	rc = cmap_fd_get(cmap_handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}

	pcmk__cluster_init_node_caches();
	crm_trace("Initializing Corosync node list");
	for (lpc = 0; TRUE; lpc++) {
	uint32_t nodeid = 0;
	char *name = NULL;
	char *key = NULL;

	key = crm_strdup_printf("nodelist.node.%d.nodeid", lpc);
	rc = cmap_get_uint32(cmap_handle, key, &nodeid);
	free(key);

	if (rc != CS_OK) {
	break;
	}

	name = pcmk__corosync_name(cmap_handle, nodeid);
	if (name != NULL) {
	GHashTableIter iter;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node && node->uname && strcasecmp(node->uname, name) == 0) {
	if (node->id && node->id != nodeid) {
	crm_crit("Nodes %u and %u share the same name '%s': shutting down", node->id,
	nodeid, name);
	crm_exit(CRM_EX_FATAL);
	}
	}
	}
	}

	if (nodeid > 0 \|\| name != NULL) {
	crm_trace("Initializing node[%d] %u = %s", lpc, nodeid, name);
	pcmk__get_node(nodeid, name, NULL, pcmk__node_search_cluster_member);
	}

	if (nodeid > 0 && name != NULL) {
	any = true;

	if (xml_parent) {
	xmlNode *node = pcmk__xe_create(xml_parent, PCMK_XE_NODE);

	- crm_xml_set_id(node, "%u", nodeid);
	+ pcmk__xe_set_id(node, "%u", nodeid);
	crm_xml_add(node, PCMK_XA_UNAME, name);
	}
	}

	free(name);
	}
	bail:
	cmap_finalize(cmap_handle);
	return any;
	}

	/*!
	* \internal
	* \brief Get cluster name from Corosync configuration (via CMAP)
	*
	* \return Newly allocated string with cluster name if configured, or NULL
	*/
	char *
	pcmk__corosync_cluster_name(void)
	{
	cmap_handle_t handle;
	char *cluster_name = NULL;
	cs_error_t rc = CS_OK;
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rv;

	rc = pcmk__init_cmap(&handle);
	if (rc != CS_OK) {
	crm_info("Failed to initialize the cmap API: %s (%d)",
	cs_strerror(rc), rc);
	return NULL;
	}

	rc = cmap_fd_get(handle, &fd);
	if (rc != CS_OK) {
	crm_err("Could not obtain the CMAP API connection: %s (%d)",
	cs_strerror(rc), rc);
	goto bail;
	}

	/* CMAP provider run as root (in given user namespace, anyway)? */
	if (!(rv = crm_ipc_is_authentic_process(fd, (uid_t) 0,(gid_t) 0, &found_pid,
	&found_uid, &found_gid))) {
	crm_err("CMAP provider is not authentic:"
	" process %lld (uid: %lld, gid: %lld)",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rv < 0) {
	crm_err("Could not verify authenticity of CMAP provider: %s (%d)",
	strerror(-rv), -rv);
	goto bail;
	}

	rc = cmap_get_string(handle, "totem.cluster_name", &cluster_name);
	if (rc != CS_OK) {
	crm_info("Cannot get totem.cluster_name: %s (%d)", cs_strerror(rc), rc);

	} else {
	crm_debug("cmap totem.cluster_name = '%s'", cluster_name);
	}

	bail:
	cmap_finalize(handle);
	return cluster_name;
	}

	/*!
	* \internal
	* \brief Check (via CMAP) whether Corosync configuration has a node list
	*
	* \return true if Corosync has node list, otherwise false
	*/
	bool
	pcmk__corosync_has_nodelist(void)
	{
	cs_error_t cs_rc = CS_OK;
	int retries = 0;
	cmap_handle_t cmap_handle;
	cmap_iter_handle_t iter_handle;
	char key_name[CMAP_KEYNAME_MAXLEN + 1];
	int fd = -1;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	pid_t found_pid = 0;
	int rc = pcmk_ok;

	static bool got_result = false;
	static bool result = false;

	if (got_result) {
	return result;
	}

	// Connect to CMAP
	do {
	cs_rc = pcmk__init_cmap(&cmap_handle);
	if (cs_rc != CS_OK) {
	retries++;
	crm_debug("CMAP connection failed: %s (rc=%d, retrying in %ds)",
	cs_strerror(cs_rc), cs_rc, retries);
	sleep(retries);
	}
	} while ((retries < 5) && (cs_rc != CS_OK));
	if (cs_rc != CS_OK) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP connection failed (%s) " CRM_XS " rc=%d",
	cs_strerror(cs_rc), cs_rc);
	return false;
	}

	// Get CMAP connection file descriptor
	cs_rc = cmap_fd_get(cmap_handle, &fd);
	if (cs_rc != CS_OK) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP unusable (%s) " CRM_XS " rc=%d",
	cs_strerror(cs_rc), cs_rc);
	goto bail;
	}

	// Check whether CMAP connection is authentic (i.e. provided by root)
	rc = crm_ipc_is_authentic_process(fd, (uid_t) 0, (gid_t) 0,
	&found_pid, &found_uid, &found_gid);
	if (rc == 0) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP provider is inauthentic "
	CRM_XS " pid=%lld uid=%lld gid=%lld",
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	goto bail;
	} else if (rc < 0) {
	crm_warn("Assuming Corosync does not have node list: "
	"Could not verify CMAP authenticity (%s) " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	goto bail;
	}

	// Check whether nodelist section is presetn
	cs_rc = cmap_iter_init(cmap_handle, "nodelist", &iter_handle);
	if (cs_rc != CS_OK) {
	crm_warn("Assuming Corosync does not have node list: "
	"CMAP not readable (%s) " CRM_XS " rc=%d",
	cs_strerror(cs_rc), cs_rc);
	goto bail;
	}

	cs_rc = cmap_iter_next(cmap_handle, iter_handle, key_name, NULL, NULL);
	if (cs_rc == CS_OK) {
	result = true;
	}

	cmap_iter_finalize(cmap_handle, iter_handle);
	got_result = true;
	crm_debug("Corosync %s node list", (result? "has" : "does not have"));

	bail:
	cmap_finalize(cmap_handle);
	return result;
	}
	diff --git a/lib/cluster/membership.c b/lib/cluster/membership.c
	index 44edcdde56..271fded623 100644
	--- a/lib/cluster/membership.c
	+++ b/lib/cluster/membership.c
	@@ -1,1443 +1,1457 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <inttypes.h> // PRIu32
	#include <sys/param.h>
	#include <sys/types.h>
	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <glib.h>
	#include <crm/common/ipc.h>
	#include <crm/common/xml_internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/xml.h>
	#include <crm/stonith-ng.h>
	#include "crmcluster_private.h"

	/* The peer cache remembers cluster nodes that have been seen.
	* This is managed mostly automatically by libcluster, based on
	* cluster membership events.
	*
	* Because cluster nodes can have conflicting names or UUIDs,
	* the hash table key is a uniquely generated ID.
	*
	* @COMPAT When this is internal, rename to cluster_node_member_cache and make
	* static.
	*/
	GHashTable *crm_peer_cache = NULL;

	/*
	* The remote peer cache tracks pacemaker_remote nodes. While the
	* value has the same type as the peer cache's, it is tracked separately for
	* three reasons: pacemaker_remote nodes can't have conflicting names or UUIDs,
	* so the name (which is also the UUID) is used as the hash table key; there
	* is no equivalent of membership events, so management is not automatic; and
	* most users of the peer cache need to exclude pacemaker_remote nodes.
	*
	* That said, using a single cache would be more logical and less error-prone,
	* so it would be a good idea to merge them one day.
	*
	* libcluster provides two avenues for populating the cache:
	* pcmk__cluster_lookup_remote_node() and pcmk__cluster_forget_remote_node()
	* directly manage it, while refresh_remote_nodes() populates it via the CIB.
	*/
	GHashTable *crm_remote_peer_cache = NULL;

	/*
	* The CIB cluster node cache tracks cluster nodes that have been seen in
	* the CIB. It is useful mainly when a caller needs to know about a node that
	* may no longer be in the membership, but doesn't want to add the node to the
	* main peer cache tables.
	*/
	static GHashTable *cluster_node_cib_cache = NULL;

	unsigned long long crm_peer_seq = 0;
	gboolean crm_have_quorum = FALSE;
	static bool autoreap = true;

	// Flag setting and clearing for crm_node_t:flags

	#define set_peer_flags(peer, flags_to_set) do { \
	(peer)->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Peer", (peer)->uname, \
	(peer)->flags, (flags_to_set), \
	#flags_to_set); \
	} while (0)

	#define clear_peer_flags(peer, flags_to_clear) do { \
	(peer)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_TRACE, \
	"Peer", (peer)->uname, \
	(peer)->flags, (flags_to_clear), \
	#flags_to_clear); \
	} while (0)

	static void update_peer_uname(crm_node_t node, const char uname);
	static crm_node_t find_cib_cluster_node(const char id, const char *uname);

	/*!
	* \internal
	* \brief Get the number of Pacemaker Remote nodes that have been seen
	*
	* \return Number of cached Pacemaker Remote nodes
	*/
	unsigned int
	pcmk__cluster_num_remote_nodes(void)
	{
	if (crm_remote_peer_cache == NULL) {
	return 0U;
	}
	return g_hash_table_size(crm_remote_peer_cache);
	}

	/*!
	* \internal
	* \brief Get a remote node cache entry, creating it if necessary
	*
	* \param[in] node_name Name of remote node
	*
	* \return Cache entry for node on success, or \c NULL (and set \c errno)
	* otherwise
	*
	* \note When creating a new entry, this will leave the node state undetermined.
	* The caller should also call \c pcmk__update_peer_state() if the state
	* is known.
	* \note Because this can add and remove cache entries, callers should not
	* assume any previously obtained cache entry pointers remain valid.
	*/
	crm_node_t *
	pcmk__cluster_lookup_remote_node(const char *node_name)
	{
	crm_node_t *node;
	char *node_name_copy = NULL;

	if (node_name == NULL) {
	errno = EINVAL;
	return NULL;
	}

	/* It's theoretically possible that the node was added to the cluster peer
	* cache before it was known to be a Pacemaker Remote node. Remove that
	* entry unless it has a node ID, which means the name actually is
	* associated with a cluster node. (@TODO return an error in that case?)
	*/
	node = pcmk__search_node_caches(0, node_name,
	pcmk__node_search_cluster_member);
	if ((node != NULL) && (node->uuid == NULL)) {
	/* node_name could be a pointer into the cache entry being removed, so
	* reassign it to a copy before the original gets freed
	*/
	node_name_copy = strdup(node_name);
	if (node_name_copy == NULL) {
	errno = ENOMEM;
	return NULL;
	}
	node_name = node_name_copy;
	pcmk__cluster_forget_cluster_node(0, node_name);
	}

	/* Return existing cache entry if one exists */
	node = g_hash_table_lookup(crm_remote_peer_cache, node_name);
	if (node) {
	free(node_name_copy);
	return node;
	}

	/* Allocate a new entry */
	node = calloc(1, sizeof(crm_node_t));
	if (node == NULL) {
	free(node_name_copy);
	return NULL;
	}

	/* Populate the essential information */
	set_peer_flags(node, crm_remote_node);
	node->uuid = strdup(node_name);
	if (node->uuid == NULL) {
	free(node);
	errno = ENOMEM;
	free(node_name_copy);
	return NULL;
	}

	/* Add the new entry to the cache */
	g_hash_table_replace(crm_remote_peer_cache, node->uuid, node);
	crm_trace("added %s to remote cache", node_name);

	/* Update the entry's uname, ensuring peer status callbacks are called */
	update_peer_uname(node, node_name);
	free(node_name_copy);
	return node;
	}

	/*!
	* \internal
	* \brief Remove a node from the Pacemaker Remote node cache
	*
	* \param[in] node_name Name of node to remove from cache
	*
	* \note The caller must be careful not to use \p node_name after calling this
	* function if it might be a pointer into the cache entry being removed.
	*/
	void
	pcmk__cluster_forget_remote_node(const char *node_name)
	{
	/* Do a lookup first, because node_name could be a pointer within the entry
	* being removed -- we can't log it after removing it.
	*/
	if (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL) {
	crm_trace("Removing %s from Pacemaker Remote node cache", node_name);
	g_hash_table_remove(crm_remote_peer_cache, node_name);
	}
	}

	/*!
	* \internal
	* \brief Return node status based on a CIB status entry
	*
	* \param[in] node_state XML of node state
	*
	* \return \c CRM_NODE_LOST if \c PCMK__XA_IN_CCM is false in
	* \c PCMK__XE_NODE_STATE, \c CRM_NODE_MEMBER otherwise
	* \note Unlike most boolean XML attributes, this one defaults to true, for
	* backward compatibility with older controllers that don't set it.
	*/
	static const char *
	remote_state_from_cib(const xmlNode *node_state)
	{
	bool status = false;

	if ((pcmk__xe_get_bool_attr(node_state, PCMK__XA_IN_CCM,
	&status) == pcmk_rc_ok) && !status) {
	return CRM_NODE_LOST;
	} else {
	return CRM_NODE_MEMBER;
	}
	}

	/* user data for looping through remote node xpath searches */
	struct refresh_data {
	const char field; / XML attribute to check for node name */
	gboolean has_state; /* whether to update node state based on XML */
	};

	/*!
	* \internal
	* \brief Process one pacemaker_remote node xpath search result
	*
	* \param[in] result XML search result
	* \param[in] user_data what to look for in the XML
	*/
	static void
	remote_cache_refresh_helper(xmlNode result, void user_data)
	{
	const struct refresh_data *data = user_data;
	const char *remote = crm_element_value(result, data->field);
	const char *state = NULL;
	crm_node_t *node;

	CRM_CHECK(remote != NULL, return);

	/* Determine node's state, if the result has it */
	if (data->has_state) {
	state = remote_state_from_cib(result);
	}

	/* Check whether cache already has entry for node */
	node = g_hash_table_lookup(crm_remote_peer_cache, remote);

	if (node == NULL) {
	/* Node is not in cache, so add a new entry for it */
	node = pcmk__cluster_lookup_remote_node(remote);
	CRM_ASSERT(node);
	if (state) {
	pcmk__update_peer_state(__func__, node, state, 0);
	}

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	/* Node is in cache and hasn't been updated already, so mark it clean */
	clear_peer_flags(node, crm_node_dirty);
	if (state) {
	pcmk__update_peer_state(__func__, node, state, 0);
	}
	}
	}

	static void
	mark_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	set_peer_flags((crm_node_t *) value, crm_node_dirty);
	}

	static gboolean
	is_dirty(gpointer key, gpointer value, gpointer user_data)
	{
	return pcmk_is_set(((crm_node_t*)value)->flags, crm_node_dirty);
	}

	/*!
	* \internal
	* \brief Repopulate the remote node cache based on CIB XML
	*
	* \param[in] cib CIB XML to parse
	*/
	static void
	refresh_remote_nodes(xmlNode *cib)
	{
	struct refresh_data data;

	pcmk__cluster_init_node_caches();

	/* First, we mark all existing cache entries as dirty,
	* so that later we can remove any that weren't in the CIB.
	* We don't empty the cache, because we need to detect changes in state.
	*/
	g_hash_table_foreach(crm_remote_peer_cache, mark_dirty, NULL);

	/* Look for guest nodes and remote nodes in the status section */
	data.field = PCMK_XA_ID;
	data.has_state = TRUE;
	crm_foreach_xpath_result(cib, PCMK__XP_REMOTE_NODE_STATUS,
	remote_cache_refresh_helper, &data);

	/* Look for guest nodes and remote nodes in the configuration section,
	* because they may have just been added and not have a status entry yet.
	* In that case, the cached node state will be left NULL, so that the
	* peer status callback isn't called until we're sure the node started
	* successfully.
	*/
	data.field = PCMK_XA_VALUE;
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, PCMK__XP_GUEST_NODE_CONFIG,
	remote_cache_refresh_helper, &data);
	data.field = PCMK_XA_ID;
	data.has_state = FALSE;
	crm_foreach_xpath_result(cib, PCMK__XP_REMOTE_NODE_CONFIG,
	remote_cache_refresh_helper, &data);

	/* Remove all old cache entries that weren't seen in the CIB */
	g_hash_table_foreach_remove(crm_remote_peer_cache, is_dirty, NULL);
	}

	/*!
	* \internal
	* \brief Check whether a node is an active cluster node
	*
	* Remote nodes are never considered active. This guarantees that they can never
	* become DC.
	*
	* \param[in] node Node to check
	*
	* \return \c true if the node is an active cluster node, or \c false otherwise
	*/
	bool
	pcmk__cluster_is_node_active(const crm_node_t *node)
	{
	const enum pcmk_cluster_layer cluster_layer = pcmk_get_cluster_layer();

	if ((node == NULL) \|\| pcmk_is_set(node->flags, crm_remote_node)) {
	return false;
	}

	switch (cluster_layer) {
	case pcmk_cluster_layer_corosync:
	#if SUPPORT_COROSYNC
	return pcmk__corosync_is_peer_active(node);
	#else
	break;
	#endif // SUPPORT_COROSYNC
	default:
	break;
	}

	crm_err("Unhandled cluster layer: %s",
	pcmk_cluster_layer_text(cluster_layer));
	return false;
	}

	/*!
	* \internal
	* \brief Check if a node's entry should be removed from the cluster node cache
	*
	* A node should be removed from the cache if it's inactive and matches another
	* \c crm_node_t (the search object). The node is considered a mismatch if any
	* of the following are true:
	* * The search object is \c NULL.
	* * The search object has an ID set and the cached node's ID does not match it.
	* * The search object does not have an ID set, and the cached node's name does
	* not match the search node's name. (If both names are \c NULL, it's a
	* match.)
	*
	* Otherwise, the node is considered a match.
	*
	* Note that if the search object has both an ID and a name set, the name is
	* ignored for matching purposes.
	*
	* \param[in] key Ignored
	* \param[in] value \c crm_node_t object from cluster node cache
	* \param[in] user_data \c crm_node_t object to match against (search object)
	*
	* \return \c TRUE if the node entry should be removed from \c crm_peer_cache,
	* or \c FALSE otherwise
	*/
	static gboolean
	should_forget_cluster_node(gpointer key, gpointer value, gpointer user_data)
	{
	crm_node_t *node = value;
	crm_node_t *search = user_data;

	if (search == NULL) {
	return FALSE;
	}
	if ((search->id != 0) && (node->id != search->id)) {
	return FALSE;
	}
	if ((search->id == 0)
	&& !pcmk__str_eq(node->uname, search->uname, pcmk__str_casei)) {
	// @TODO Consider name even if ID is set?
	return FALSE;
	}
	if (pcmk__cluster_is_node_active(value)) {
	return FALSE;
	}

	crm_info("Removing node with name %s and " PCMK_XA_ID " %u from membership "
	"cache",
	pcmk__s(node->uname, "(unknown)"), node->id);
	return TRUE;
	}

	/*!
	* \internal
	* \brief Remove one or more inactive nodes from the cluster node cache
	*
	* All inactive nodes matching \p id and \p node_name as described in
	* \c should_forget_cluster_node documentation are removed from the cache.
	*
	* If \p id is 0 and \p node_name is \c NULL, all inactive nodes are removed
	* from the cache regardless of ID and name. This differs from clearing the
	* cache, in that entries for active nodes are preserved.
	*
	* \param[in] id ID of node to remove from cache (0 to ignore)
	* \param[in] node_name Name of node to remove from cache (ignored if \p id is
	* nonzero)
	*
	* \note \p node_name is not modified directly, but it will be freed if it's a
	* pointer into a cache entry that is removed.
	*/
	void
	pcmk__cluster_forget_cluster_node(uint32_t id, const char *node_name)
	{
	crm_node_t search = { 0, };
	char *criterion = NULL; // For logging
	guint matches = 0;

	if (crm_peer_cache == NULL) {
	crm_trace("Membership cache not initialized, ignoring removal request");
	return;
	}

	search.id = id;
	search.uname = pcmk__str_copy(node_name); // May log after original freed

	if (id > 0) {
	criterion = crm_strdup_printf(PCMK_XA_ID "=%" PRIu32, id);

	} else if (node_name != NULL) {
	criterion = crm_strdup_printf(PCMK_XA_UNAME "=%s", node_name);
	}

	matches = g_hash_table_foreach_remove(crm_peer_cache,
	should_forget_cluster_node, &search);
	if (matches > 0) {
	if (criterion != NULL) {
	crm_notice("Removed %u inactive node%s with %s from the membership "
	"cache",
	matches, pcmk__plural_s(matches), criterion);
	} else {
	crm_notice("Removed all (%u) inactive cluster nodes from the "
	"membership cache",
	matches);
	}

	} else {
	crm_info("No inactive cluster nodes%s%s to remove from the membership "
	"cache",
	((criterion != NULL)? " with " : ""), pcmk__s(criterion, ""));
	}

	free(search.uname);
	free(criterion);
	}

	static void
	count_peer(gpointer key, gpointer value, gpointer user_data)
	{
	unsigned int *count = user_data;
	crm_node_t *node = value;

	if (pcmk__cluster_is_node_active(node)) {
	count = count + 1;
	}
	}

	/*!
	* \internal
	* \brief Get the number of active cluster nodes that have been seen
	*
	* Remote nodes are never considered active. This guarantees that they can never
	* become DC.
	*
	* \return Number of active nodes in the cluster node cache
	*/
	unsigned int
	pcmk__cluster_num_active_nodes(void)
	{
	unsigned int count = 0;

	if (crm_peer_cache != NULL) {
	g_hash_table_foreach(crm_peer_cache, count_peer, &count);
	}
	return count;
	}

	static void
	destroy_crm_node(gpointer data)
	{
	crm_node_t *node = data;

	crm_trace("Destroying entry for node %u: %s", node->id, node->uname);

	free(node->uname);
	free(node->state);
	free(node->uuid);
	free(node->expected);
	free(node->conn_host);
	free(node);
	}

	/*!
	* \internal
	* \brief Initialize node caches
	*/
	void
	pcmk__cluster_init_node_caches(void)
	{
	if (crm_peer_cache == NULL) {
	crm_peer_cache = pcmk__strikey_table(free, destroy_crm_node);
	}

	if (crm_remote_peer_cache == NULL) {
	crm_remote_peer_cache = pcmk__strikey_table(NULL, destroy_crm_node);
	}

	if (cluster_node_cib_cache == NULL) {
	cluster_node_cib_cache = pcmk__strikey_table(free, destroy_crm_node);
	}
	}

	/*!
	* \internal
	* \brief Initialize node caches
	*/
	void
	pcmk__cluster_destroy_node_caches(void)
	{
	if (crm_peer_cache != NULL) {
	crm_trace("Destroying peer cache with %d members",
	g_hash_table_size(crm_peer_cache));
	g_hash_table_destroy(crm_peer_cache);
	crm_peer_cache = NULL;
	}

	if (crm_remote_peer_cache != NULL) {
	crm_trace("Destroying remote peer cache with %d members",
	pcmk__cluster_num_remote_nodes());
	g_hash_table_destroy(crm_remote_peer_cache);
	crm_remote_peer_cache = NULL;
	}

	if (cluster_node_cib_cache != NULL) {
	crm_trace("Destroying configured cluster node cache with %d members",
	g_hash_table_size(cluster_node_cib_cache));
	g_hash_table_destroy(cluster_node_cib_cache);
	cluster_node_cib_cache = NULL;
	}
	}

	static void (peer_status_callback)(enum crm_status_type, crm_node_t ,
	const void *) = NULL;

	/*!
	* \internal
	* \brief Set a client function that will be called after peer status changes
	*
	* \param[in] dispatch Pointer to function to use as callback
	*
	* \note Client callbacks should do only client-specific handling. Callbacks
	* must not add or remove entries in the peer caches.
	*/
	void
	pcmk__cluster_set_status_callback(void (*dispatch)(enum crm_status_type,
	crm_node_t , const void ))
	{
	// @TODO Improve documentation of peer_status_callback
	peer_status_callback = dispatch;
	}

	/*!
	* \internal
	* \brief Tell the library whether to automatically reap lost nodes
	*
	* If \c true (the default), calling \c crm_update_peer_proc() will also update
	* the peer state to \c CRM_NODE_MEMBER or \c CRM_NODE_LOST, and updating the
	* peer state will reap peers whose state changes to anything other than
	* \c CRM_NODE_MEMBER.
	*
	* Callers should leave this enabled unless they plan to manage the cache
	* separately on their own.
	*
	* \param[in] enable \c true to enable automatic reaping, \c false to disable
	*/
	void
	pcmk__cluster_set_autoreap(bool enable)
	{
	autoreap = enable;
	}

	static void
	dump_peer_hash(int level, const char *caller)
	{
	GHashTableIter iter;
	const char *id = NULL;
	crm_node_t *node = NULL;

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, (gpointer ) &id, (gpointer ) &node)) {
	do_crm_log(level, "%s: Node %u/%s = %p - %s", caller, node->id, node->uname, node, id);
	}
	}

	static gboolean
	hash_find_by_data(gpointer key, gpointer value, gpointer user_data)
	{
	return value == user_data;
	}

	/*!
	* \internal
	* \brief Search cluster member node cache
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] uuid If not NULL while id is 0, node UUID instead of cluster
	* node ID to search for
	*
	* \return Cluster node cache entry if found, otherwise NULL
	*/
	static crm_node_t *
	search_cluster_member_cache(unsigned int id, const char *uname,
	const char *uuid)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	crm_node_t *by_id = NULL;
	crm_node_t *by_name = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	pcmk__cluster_init_node_caches();

	if (uname != NULL) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id > 0) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(node->id == id) {
	crm_trace("ID match: %u = %p", node->id, node);
	by_id = node;
	break;
	}
	}

	} else if (uuid != NULL) {
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (pcmk__str_eq(node->uuid, uuid, pcmk__str_casei)) {
	crm_trace("UUID match: %s = %p", node->uuid, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if(by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %u/%s", by_id, id, uname);

	} else if(by_id == NULL && by_name) {
	crm_trace("Only one: %p for %u/%s", by_name, id, uname);

	if(id && by_name->id) {
	dump_peer_hash(LOG_WARNING, __func__);
	crm_crit("Node %u and %u share the same name '%s'",
	id, by_name->id, uname);
	node = NULL; /* Create a new one */

	} else {
	node = by_name;
	}

	} else if(by_name == NULL && by_id) {
	crm_trace("Only one: %p for %u/%s", by_id, id, uname);

	if(uname && by_id->uname) {
	dump_peer_hash(LOG_WARNING, __func__);
	crm_crit("Node '%s' and '%s' share the same cluster nodeid %u: assuming '%s' is correct",
	uname, by_id->uname, id, uname);
	}

	} else if(uname && by_id->uname) {
	if(pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	crm_notice("Node '%s' has changed its ID from %u to %u", by_id->uname, by_name->id, by_id->id);
	g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);

	} else {
	crm_warn("Node '%s' and '%s' share the same cluster nodeid: %u %s", by_id->uname, by_name->uname, id, uname);
	dump_peer_hash(LOG_INFO, __func__);
	crm_abort(__FILE__, __func__, __LINE__, "member weirdness", TRUE,
	TRUE);
	}

	} else if(id && by_name->id) {
	crm_warn("Node %u and %u share the same name: '%s'", by_id->id, by_name->id, uname);

	} else {
	/* Simple merge */

	/* Only corosync-based clusters use node IDs. The functions that call
	* pcmk__update_peer_state() and crm_update_peer_proc() only know
	* nodeid, so 'by_id' is authoritative when merging.
	*/
	dump_peer_hash(LOG_DEBUG, __func__);

	crm_info("Merging %p into %p", by_name, by_id);
	g_hash_table_foreach_remove(crm_peer_cache, hash_find_by_data, by_name);
	}

	return node;
	}

	/*!
	* \internal
	* \brief Search caches for a node (cluster or Pacemaker Remote)
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] flags Group of enum pcmk__node_search_flags
	*
	* \return Node cache entry if found, otherwise NULL
	*/
	crm_node_t *
	pcmk__search_node_caches(unsigned int id, const char *uname, uint32_t flags)
	{
	crm_node_t *node = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	pcmk__cluster_init_node_caches();

	if ((uname != NULL) && pcmk_is_set(flags, pcmk__node_search_remote)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	}

	if ((node == NULL)
	&& pcmk_is_set(flags, pcmk__node_search_cluster_member)) {

	node = search_cluster_member_cache(id, uname, NULL);
	}

	if ((node == NULL) && pcmk_is_set(flags, pcmk__node_search_cluster_cib)) {
	char *id_str = (id == 0)? NULL : crm_strdup_printf("%u", id);

	node = find_cib_cluster_node(id_str, uname);
	free(id_str);
	}

	return node;
	}

	/*!
	* \internal
	* \brief Purge a node from cache (both cluster and Pacemaker Remote)
	*
	* \param[in] node_name If not NULL, purge only nodes with this name
	* \param[in] node_id If not 0, purge cluster nodes only if they have this ID
	*
	* \note If \p node_name is NULL and \p node_id is 0, no nodes will be purged.
	* If \p node_name is not NULL and \p node_id is not 0, Pacemaker Remote
	* nodes that match \p node_name will be purged, and cluster nodes that
	* match both \p node_name and \p node_id will be purged.
	* \note The caller must be careful not to use \p node_name after calling this
	* function if it might be a pointer into a cache entry being removed.
	*/
	void
	pcmk__purge_node_from_cache(const char *node_name, uint32_t node_id)
	{
	char *node_name_copy = NULL;

	if ((node_name == NULL) && (node_id == 0U)) {
	return;
	}

	// Purge from Pacemaker Remote node cache
	if ((node_name != NULL)
	&& (g_hash_table_lookup(crm_remote_peer_cache, node_name) != NULL)) {
	/* node_name could be a pointer into the cache entry being purged,
	* so reassign it to a copy before the original gets freed
	*/
	node_name_copy = pcmk__str_copy(node_name);
	node_name = node_name_copy;

	crm_trace("Purging %s from Pacemaker Remote node cache", node_name);
	g_hash_table_remove(crm_remote_peer_cache, node_name);
	}

	pcmk__cluster_forget_cluster_node(node_id, node_name);
	free(node_name_copy);
	}

	#if SUPPORT_COROSYNC
	static guint
	remove_conflicting_peer(crm_node_t *node)
	{
	int matches = 0;
	GHashTableIter iter;
	crm_node_t *existing_node = NULL;

	if (node->id == 0 \|\| node->uname == NULL) {
	return 0;
	}

	if (!pcmk__corosync_has_nodelist()) {
	return 0;
	}

	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &existing_node)) {
	if (existing_node->id > 0
	&& existing_node->id != node->id
	&& existing_node->uname != NULL
	&& strcasecmp(existing_node->uname, node->uname) == 0) {

	if (pcmk__cluster_is_node_active(existing_node)) {
	continue;
	}

	crm_warn("Removing cached offline node %u/%s which has conflicting uname with %u",
	existing_node->id, existing_node->uname, node->id);

	g_hash_table_iter_remove(&iter);
	matches++;
	}
	}

	return matches;
	}
	#endif

	/*!
	* \internal
	* \brief Get a cluster node cache entry, possibly creating one if not found
	*
	* If \c pcmk__node_search_cluster_member is set in \p flags, the return value
	* is guaranteed not to be \c NULL. A new cache entry is created if one does not
	* already exist.
	*
	* \param[in] id If not 0, cluster node ID to search for
	* \param[in] uname If not NULL, node name to search for
	* \param[in] uuid If not NULL while id is 0, node UUID instead of cluster
	* node ID to search for
	* \param[in] flags Group of enum pcmk__node_search_flags
	*
	* \return (Possibly newly created) cluster node cache entry
	*/
	/* coverity[-alloc] Memory is referenced in one or both hashtables */
	crm_node_t *
	pcmk__get_node(unsigned int id, const char uname, const char uuid,
	uint32_t flags)
	{
	crm_node_t *node = NULL;
	char *uname_lookup = NULL;

	CRM_ASSERT(id > 0 \|\| uname != NULL);

	pcmk__cluster_init_node_caches();

	// Check the Pacemaker Remote node cache first
	if (pcmk_is_set(flags, pcmk__node_search_remote)) {
	node = g_hash_table_lookup(crm_remote_peer_cache, uname);
	if (node != NULL) {
	return node;
	}
	}

	if (!pcmk_is_set(flags, pcmk__node_search_cluster_member)) {
	return NULL;
	}

	node = search_cluster_member_cache(id, uname, uuid);

	/* if uname wasn't provided, and find_peer did not turn up a uname based on id.
	* we need to do a lookup of the node name using the id in the cluster membership. */
	if ((node == NULL \|\| node->uname == NULL) && (uname == NULL)) {
	uname_lookup = pcmk__cluster_node_name(id);
	}

	if (uname_lookup) {
	uname = uname_lookup;
	crm_trace("Inferred a name of '%s' for node %u", uname, id);

	/* try to turn up the node one more time now that we know the uname. */
	if (node == NULL) {
	node = search_cluster_member_cache(id, uname, uuid);
	}
	}

	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = pcmk__assert_alloc(1, sizeof(crm_node_t));

	crm_info("Created entry %s/%p for node %s/%u (%d total)",
	uniqueid, node, uname, id, 1 + g_hash_table_size(crm_peer_cache));
	g_hash_table_replace(crm_peer_cache, uniqueid, node);
	}

	if(id > 0 && uname && (node->id == 0 \|\| node->uname == NULL)) {
	crm_info("Node %u is now known as %s", id, uname);
	}

	if(id > 0 && node->id == 0) {
	node->id = id;
	}

	if (uname && (node->uname == NULL)) {
	update_peer_uname(node, uname);
	}

	if(node->uuid == NULL) {
	if (uuid == NULL) {
	uuid = pcmk__cluster_node_uuid(node);
	}

	if (uuid) {
	crm_info("Node %u has uuid %s", id, uuid);

	} else {
	crm_info("Cannot obtain a UUID for node %u/%s", id, node->uname);
	}
	}

	free(uname_lookup);

	return node;
	}

	/*!
	* \internal
	* \brief Update a node's uname
	*
	* \param[in,out] node Node object to update
	* \param[in] uname New name to set
	*
	* \note This function should not be called within a peer cache iteration,
	* because in some cases it can remove conflicting cache entries,
	* which would invalidate the iterator.
	*/
	static void
	update_peer_uname(crm_node_t node, const char uname)
	{
	CRM_CHECK(uname != NULL,
	crm_err("Bug: can't update node name without name"); return);
	CRM_CHECK(node != NULL,
	crm_err("Bug: can't update node name to %s without node", uname);
	return);

	if (pcmk__str_eq(uname, node->uname, pcmk__str_casei)) {
	crm_debug("Node uname '%s' did not change", uname);
	return;
	}

	for (const char c = uname; c; ++c) {
	if ((c >= 'A') && (c <= 'Z')) {
	crm_warn("Node names with capitals are discouraged, consider changing '%s'",
	uname);
	break;
	}
	}

	pcmk__str_update(&node->uname, uname);

	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_uname, node, NULL);
	}

	#if SUPPORT_COROSYNC
	if ((pcmk_get_cluster_layer() == pcmk_cluster_layer_corosync)
	&& !pcmk_is_set(node->flags, crm_remote_node)) {

	remove_conflicting_peer(node);
	}
	#endif
	}

	/*!
	* \internal
	* \brief Get log-friendly string equivalent of a process flag
	*
	* \param[in] proc Process flag
	*
	* \return Log-friendly string equivalent of \p proc
	*/
	static inline const char *
	proc2text(enum crm_proc_flag proc)
	{
	const char *text = "unknown";

	switch (proc) {
	case crm_proc_none:
	text = "none";
	break;
	case crm_proc_cpg:
	text = "corosync-cpg";
	break;
	}
	return text;
	}

	/*!
	* \internal
	* \brief Update a node's process information (and potentially state)
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] flag Bitmask of new process information
	* \param[in] status node status (online, offline, etc.)
	*
	* \return NULL if any node was reaped from peer caches, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible, otherwise
	* reaping could invalidate the iterator.
	*/
	crm_node_t *
	crm_update_peer_proc(const char source, crm_node_t node, uint32_t flag, const char *status)
	{
	uint32_t last = 0;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set %s to %s for NULL",
	source, proc2text(flag), status);
	return NULL);

	/* Pacemaker doesn't spawn processes on remote nodes */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return node;
	}

	last = node->processes;
	if (status == NULL) {
	node->processes = flag;
	if (node->processes != last) {
	changed = TRUE;
	}

	} else if (pcmk__str_eq(status, PCMK_VALUE_ONLINE, pcmk__str_casei)) {
	if ((node->processes & flag) != flag) {
	node->processes = pcmk__set_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	} else if (node->processes & flag) {
	node->processes = pcmk__clear_flags_as(__func__, __LINE__,
	LOG_TRACE, "Peer process",
	node->uname, node->processes,
	flag, "processes");
	changed = TRUE;
	}

	if (changed) {
	if (status == NULL && flag <= crm_proc_none) {
	crm_info("%s: Node %s[%u] - all processes are now offline", source, node->uname,
	node->id);
	} else {
	crm_info("%s: Node %s[%u] - %s is now %s", source, node->uname, node->id,
	proc2text(flag), status);
	}

	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	node->when_online = time(NULL);

	} else {
	node->when_online = 0;
	}

	/* Call the client callback first, then update the peer state,
	* in case the node will be reaped
	*/
	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_processes, node, &last);
	}

	/* The client callback shouldn't touch the peer caches,
	* but as a safety net, bail if the peer cache was destroyed.
	*/
	if (crm_peer_cache == NULL) {
	return NULL;
	}

	if (autoreap) {
	const char *peer_state = NULL;

	if (pcmk_is_set(node->processes, crm_get_cluster_proc())) {
	peer_state = CRM_NODE_MEMBER;
	} else {
	peer_state = CRM_NODE_LOST;
	}
	node = pcmk__update_peer_state(__func__, node, peer_state, 0);
	}
	} else {
	crm_trace("%s: Node %s[%u] - %s is unchanged (%s)", source, node->uname, node->id,
	proc2text(flag), status);
	}
	return node;
	}

	/*!
	* \internal
	* \brief Update a cluster node cache entry's expected join state
	*
	* \param[in] source Caller's function name (for logging)
	* \param[in,out] node Node to update
	* \param[in] expected Node's new join state
	*/
	void
	pcmk__update_peer_expected(const char source, crm_node_t node,
	const char *expected)
	{
	char *last = NULL;
	gboolean changed = FALSE;

	CRM_CHECK(node != NULL, crm_err("%s: Could not set 'expected' to %s", source, expected);
	return);

	/* Remote nodes don't participate in joins */
	if (pcmk_is_set(node->flags, crm_remote_node)) {
	return;
	}

	last = node->expected;
	if (expected != NULL && !pcmk__str_eq(node->expected, expected, pcmk__str_casei)) {
	node->expected = strdup(expected);
	changed = TRUE;
	}

	if (changed) {
	crm_info("%s: Node %s[%u] - expected state is now %s (was %s)", source, node->uname, node->id,
	expected, last);
	free(last);
	} else {
	crm_trace("%s: Node %s[%u] - expected state is unchanged (%s)", source, node->uname,
	node->id, expected);
	}
	}

	/*!
	* \internal
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	* \param[in,out] iter If not NULL, pointer to node's peer cache iterator
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function may be called from
	* within a peer cache iteration if the iterator is supplied.
	*/
	static crm_node_t *
	update_peer_state_iter(const char source, crm_node_t node, const char *state,
	uint64_t membership, GHashTableIter *iter)
	{
	gboolean is_member;

	CRM_CHECK(node != NULL,
	crm_err("Could not set state for unknown host to %s"
	CRM_XS " source=%s", state, source);
	return NULL);

	is_member = pcmk__str_eq(state, CRM_NODE_MEMBER, pcmk__str_casei);
	if (is_member) {
	node->when_lost = 0;
	if (membership) {
	node->last_seen = membership;
	}
	}

	if (state && !pcmk__str_eq(node->state, state, pcmk__str_casei)) {
	char *last = node->state;

	if (is_member) {
	node->when_member = time(NULL);

	} else {
	node->when_member = 0;
	}

	node->state = strdup(state);
	crm_notice("Node %s state is now %s " CRM_XS
	" nodeid=%u previous=%s source=%s", node->uname, state,
	node->id, (last? last : "unknown"), source);
	if (peer_status_callback != NULL) {
	peer_status_callback(crm_status_nstate, node, last);
	}
	free(last);

	if (autoreap && !is_member
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/* We only autoreap from the peer cache, not the remote peer cache,
	* because the latter should be managed only by
	* refresh_remote_nodes().
	*/
	if(iter) {
	crm_notice("Purged 1 peer with " PCMK_XA_ID
	"=%u and/or uname=%s from the membership cache",
	node->id, node->uname);
	g_hash_table_iter_remove(iter);

	} else {
	pcmk__cluster_forget_cluster_node(node->id, node->uname);
	}
	node = NULL;
	}

	} else {
	crm_trace("Node %s state is unchanged (%s) " CRM_XS
	" nodeid=%u source=%s", node->uname, state, node->id, source);
	}
	return node;
	}

	/*!
	* \brief Update a node's state and membership information
	*
	* \param[in] source Caller's function name (for log messages)
	* \param[in,out] node Node object to update
	* \param[in] state Node's new state
	* \param[in] membership Node's new membership ID
	*
	* \return NULL if any node was reaped, value of node otherwise
	*
	* \note If this function returns NULL, the supplied node object was likely
	* freed and should not be used again. This function should not be
	* called within a cache iteration if reaping is possible,
	* otherwise reaping could invalidate the iterator.
	*/
	crm_node_t *
	pcmk__update_peer_state(const char source, crm_node_t node,
	const char *state, uint64_t membership)
	{
	return update_peer_state_iter(source, node, state, membership, NULL);
	}

	/*!
	* \internal
	* \brief Reap all nodes from cache whose membership information does not match
	*
	* \param[in] membership Membership ID of nodes to keep
	*/
	void
	pcmk__reap_unseen_nodes(uint64_t membership)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;

	crm_trace("Reaping unseen nodes...");
	g_hash_table_iter_init(&iter, crm_peer_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&node)) {
	if (node->last_seen != membership) {
	if (node->state) {
	/*
	* Calling update_peer_state_iter() allows us to
	* remove the node from crm_peer_cache without
	* invalidating our iterator
	*/
	update_peer_state_iter(__func__, node, CRM_NODE_LOST,
	membership, &iter);

	} else {
	crm_info("State of node %s[%u] is still unknown",
	node->uname, node->id);
	}
	}
	}
	}

	static crm_node_t *
	find_cib_cluster_node(const char id, const char uname)
	{
	GHashTableIter iter;
	crm_node_t *node = NULL;
	crm_node_t *by_id = NULL;
	crm_node_t *by_name = NULL;

	if (uname) {
	g_hash_table_iter_init(&iter, cluster_node_cib_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if (node->uname && strcasecmp(node->uname, uname) == 0) {
	crm_trace("Name match: %s = %p", node->uname, node);
	by_name = node;
	break;
	}
	}
	}

	if (id) {
	g_hash_table_iter_init(&iter, cluster_node_cib_cache);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
	if(strcasecmp(node->uuid, id) == 0) {
	crm_trace("ID match: %s= %p", id, node);
	by_id = node;
	break;
	}
	}
	}

	node = by_id; /* Good default */
	if (by_id == by_name) {
	/* Nothing to do if they match (both NULL counts) */
	crm_trace("Consistent: %p for %s/%s", by_id, id, uname);

	} else if (by_id == NULL && by_name) {
	crm_trace("Only one: %p for %s/%s", by_name, id, uname);

	if (id) {
	node = NULL;

	} else {
	node = by_name;
	}

	} else if (by_name == NULL && by_id) {
	crm_trace("Only one: %p for %s/%s", by_id, id, uname);

	if (uname) {
	node = NULL;
	}

	} else if (uname && by_id->uname
	&& pcmk__str_eq(uname, by_id->uname, pcmk__str_casei)) {
	/* Multiple nodes have the same uname in the CIB.
	* Return by_id. */

	} else if (id && by_name->uuid
	&& pcmk__str_eq(id, by_name->uuid, pcmk__str_casei)) {
	/* Multiple nodes have the same id in the CIB.
	* Return by_name. */
	node = by_name;

	} else {
	node = NULL;
	}

	if (node == NULL) {
	crm_debug("Couldn't find node%s%s%s%s",
	id? " " : "",
	id? id : "",
	uname? " with name " : "",
	uname? uname : "");
	}

	return node;
	}

	static void
	cluster_node_cib_cache_refresh_helper(xmlNode xml_node, void user_data)
	{
	const char *id = crm_element_value(xml_node, PCMK_XA_ID);
	const char *uname = crm_element_value(xml_node, PCMK_XA_UNAME);
	crm_node_t * node = NULL;

	CRM_CHECK(id != NULL && uname !=NULL, return);
	node = find_cib_cluster_node(id, uname);

	if (node == NULL) {
	char *uniqueid = crm_generate_uuid();

	node = pcmk__assert_alloc(1, sizeof(crm_node_t));

	node->uname = pcmk__str_copy(uname);
	node->uuid = pcmk__str_copy(id);

	g_hash_table_replace(cluster_node_cib_cache, uniqueid, node);

	} else if (pcmk_is_set(node->flags, crm_node_dirty)) {
	pcmk__str_update(&node->uname, uname);

	/* Node is in cache and hasn't been updated already, so mark it clean */
	clear_peer_flags(node, crm_node_dirty);
	}

	}

	static void
	refresh_cluster_node_cib_cache(xmlNode *cib)
	{
	pcmk__cluster_init_node_caches();

	g_hash_table_foreach(cluster_node_cib_cache, mark_dirty, NULL);

	crm_foreach_xpath_result(cib, PCMK__XP_MEMBER_NODE_CONFIG,
	cluster_node_cib_cache_refresh_helper, NULL);

	// Remove all old cache entries that weren't seen in the CIB
	g_hash_table_foreach_remove(cluster_node_cib_cache, is_dirty, NULL);
	}

	void
	pcmk__refresh_node_caches_from_cib(xmlNode *cib)
	{
	refresh_remote_nodes(cib);
	refresh_cluster_node_cib_cache(cib);
	}
	+
	+// Deprecated functions kept only for backward API compatibility
	+// LCOV_EXCL_START
	+
	+#include <crm/cluster/compat.h>
	+
	+void
	+crm_peer_init(void)
	+{
	+ pcmk__cluster_init_node_caches();
	+}
	+
	+// LCOV_EXCL_STOP
	+// End deprecated API
	diff --git a/lib/common/actions.c b/lib/common/actions.c
	index ed1056f45f..76874fd3a6 100644
	--- a/lib/common/actions.c
	+++ b/lib/common/actions.c
	@@ -1,587 +1,587 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <stdio.h>
	#include <string.h>
	#include <stdlib.h>
	#include <sys/types.h>
	#include <ctype.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/util.h>
	#include <crm/common/scheduler.h>

	/*!
	* \brief Get string equivalent of an action type
	*
	* \param[in] action Action type
	*
	* \return Static string describing \p action
	*/
	const char *
	pcmk_action_text(enum action_tasks action)
	{
	switch (action) {
	case pcmk_action_stop:
	return PCMK_ACTION_STOP;

	case pcmk_action_stopped:
	return PCMK_ACTION_STOPPED;

	case pcmk_action_start:
	return PCMK_ACTION_START;

	case pcmk_action_started:
	return PCMK_ACTION_RUNNING;

	case pcmk_action_shutdown:
	return PCMK_ACTION_DO_SHUTDOWN;

	case pcmk_action_fence:
	return PCMK_ACTION_STONITH;

	case pcmk_action_monitor:
	return PCMK_ACTION_MONITOR;

	case pcmk_action_notify:
	return PCMK_ACTION_NOTIFY;

	case pcmk_action_notified:
	return PCMK_ACTION_NOTIFIED;

	case pcmk_action_promote:
	return PCMK_ACTION_PROMOTE;

	case pcmk_action_promoted:
	return PCMK_ACTION_PROMOTED;

	case pcmk_action_demote:
	return PCMK_ACTION_DEMOTE;

	case pcmk_action_demoted:
	return PCMK_ACTION_DEMOTED;

	default: // pcmk_action_unspecified or invalid
	return "no_action";
	}
	}

	/*!
	* \brief Parse an action type from an action name
	*
	* \param[in] action_name Action name
	*
	* \return Action type corresponding to \p action_name
	*/
	enum action_tasks
	pcmk_parse_action(const char *action_name)
	{
	if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
	return pcmk_action_stop;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_STOPPED, pcmk__str_none)) {
	return pcmk_action_stopped;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
	return pcmk_action_start;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_RUNNING, pcmk__str_none)) {
	return pcmk_action_started;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_DO_SHUTDOWN,
	pcmk__str_none)) {
	return pcmk_action_shutdown;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_STONITH, pcmk__str_none)) {
	return pcmk_action_fence;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	return pcmk_action_monitor;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
	return pcmk_action_notify;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_NOTIFIED,
	pcmk__str_none)) {
	return pcmk_action_notified;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
	return pcmk_action_promote;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
	return pcmk_action_demote;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTED,
	pcmk__str_none)) {
	return pcmk_action_promoted;

	} else if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
	return pcmk_action_demoted;
	}
	return pcmk_action_unspecified;
	}

	/*!
	* \brief Get string equivalent of a failure handling type
	*
	* \param[in] on_fail Failure handling type
	*
	* \return Static string describing \p on_fail
	*/
	const char *
	pcmk_on_fail_text(enum action_fail_response on_fail)
	{
	switch (on_fail) {
	case pcmk_on_fail_ignore:
	return "ignore";

	case pcmk_on_fail_demote:
	return "demote";

	case pcmk_on_fail_block:
	return "block";

	case pcmk_on_fail_restart:
	return "recover";

	case pcmk_on_fail_ban:
	return "migrate";

	case pcmk_on_fail_stop:
	return "stop";

	case pcmk_on_fail_fence_node:
	return "fence";

	case pcmk_on_fail_standby_node:
	return "standby";

	case pcmk_on_fail_restart_container:
	return "restart-container";

	case pcmk_on_fail_reset_remote:
	return "reset-remote";
	}
	return "<unknown>";
	}

	/*!
	* \brief Generate an operation key (RESOURCE_ACTION_INTERVAL)
	*
	* \param[in] rsc_id ID of resource being operated on
	* \param[in] op_type Operation name
	* \param[in] interval_ms Operation interval
	*
	* \return Newly allocated memory containing operation key as string
	*
	* \note This function asserts on errors, so it will never return NULL.
	* The caller is responsible for freeing the result with free().
	*/
	char *
	pcmk__op_key(const char rsc_id, const char op_type, guint interval_ms)
	{
	CRM_ASSERT(rsc_id != NULL);
	CRM_ASSERT(op_type != NULL);
	return crm_strdup_printf(PCMK__OP_FMT, rsc_id, op_type, interval_ms);
	}

	static inline gboolean
	convert_interval(const char s, guint interval_ms)
	{
	unsigned long l;

	errno = 0;
	l = strtoul(s, NULL, 10);

	if (errno != 0) {
	return FALSE;
	}

	*interval_ms = (guint) l;
	return TRUE;
	}

	/*!
	* \internal
	* \brief Check for underbar-separated substring match
	*
	* \param[in] key Overall string being checked
	* \param[in] position Match before underbar at this \p key index
	* \param[in] matches Substrings to match (may contain underbars)
	*
	* \return \p key index of underbar before any matching substring,
	* or 0 if none
	*/
	static size_t
	match_before(const char key, size_t position, const char *matches)
	{
	for (int i = 0; matches[i] != NULL; ++i) {
	const size_t match_len = strlen(matches[i]);

	// Must have at least X_MATCH before position
	if (position > (match_len + 1)) {
	const size_t possible = position - match_len - 1;

	if ((key[possible] == '_')
	&& (strncmp(key + possible + 1, matches[i], match_len) == 0)) {
	return possible;
	}
	}
	}
	return 0;
	}

	gboolean
	parse_op_key(const char key, char rsc_id, char op_type, guint interval_ms)
	{
	guint local_interval_ms = 0;
	const size_t key_len = (key == NULL)? 0 : strlen(key);

	// Operation keys must be formatted as RSC_ACTION_INTERVAL
	size_t action_underbar = 0; // Index in key of underbar before ACTION
	size_t interval_underbar = 0; // Index in key of underbar before INTERVAL
	size_t possible = 0;

	/* Underbar was a poor choice of separator since both RSC and ACTION can
	* contain underbars. Here, list action names and name prefixes that can.
	*/
	const char *actions_with_underbars[] = {
	PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_MIGRATE_TO,
	NULL
	};
	const char *action_prefixes_with_underbars[] = {
	"pre_" PCMK_ACTION_NOTIFY,
	"post_" PCMK_ACTION_NOTIFY,
	"confirmed-pre_" PCMK_ACTION_NOTIFY,
	"confirmed-post_" PCMK_ACTION_NOTIFY,
	NULL,
	};

	// Initialize output variables in case of early return
	if (rsc_id) {
	*rsc_id = NULL;
	}
	if (op_type) {
	*op_type = NULL;
	}
	if (interval_ms) {
	*interval_ms = 0;
	}

	// RSC_ACTION_INTERVAL implies a minimum of 5 characters
	if (key_len < 5) {
	return FALSE;
	}

	// Find, parse, and validate interval
	interval_underbar = key_len - 2;
	while ((interval_underbar > 2) && (key[interval_underbar] != '_')) {
	--interval_underbar;
	}
	if ((interval_underbar == 2)
	\|\| !convert_interval(key + interval_underbar + 1, &local_interval_ms)) {
	return FALSE;
	}

	// Find the base (OCF) action name, disregarding prefixes
	action_underbar = match_before(key, interval_underbar,
	actions_with_underbars);
	if (action_underbar == 0) {
	action_underbar = interval_underbar - 2;
	while ((action_underbar > 0) && (key[action_underbar] != '_')) {
	--action_underbar;
	}
	if (action_underbar == 0) {
	return FALSE;
	}
	}
	possible = match_before(key, action_underbar,
	action_prefixes_with_underbars);
	if (possible != 0) {
	action_underbar = possible;
	}

	// Set output variables
	if (rsc_id != NULL) {
	*rsc_id = strndup(key, action_underbar);
	pcmk__mem_assert(*rsc_id);
	}
	if (op_type != NULL) {
	*op_type = strndup(key + action_underbar + 1,
	interval_underbar - action_underbar - 1);
	pcmk__mem_assert(*op_type);
	}
	if (interval_ms != NULL) {
	*interval_ms = local_interval_ms;
	}
	return TRUE;
	}

	char *
	pcmk__notify_key(const char rsc_id, const char notify_type,
	const char *op_type)
	{
	CRM_CHECK(rsc_id != NULL, return NULL);
	CRM_CHECK(op_type != NULL, return NULL);
	CRM_CHECK(notify_type != NULL, return NULL);
	return crm_strdup_printf("%s_%s_notify_%s_0",
	rsc_id, notify_type, op_type);
	}

	/*!
	* \brief Parse a transition magic string into its constituent parts
	*
	* \param[in] magic Magic string to parse (must be non-NULL)
	* \param[out] uuid If non-NULL, where to store copy of parsed UUID
	* \param[out] transition_id If non-NULL, where to store parsed transition ID
	* \param[out] action_id If non-NULL, where to store parsed action ID
	* \param[out] op_status If non-NULL, where to store parsed result status
	* \param[out] op_rc If non-NULL, where to store parsed actual rc
	* \param[out] target_rc If non-NULL, where to stored parsed target rc
	*
	* \return TRUE if key was valid, FALSE otherwise
	* \note If uuid is supplied and this returns TRUE, the caller is responsible
	* for freeing the memory for *uuid using free().
	*/
	gboolean
	decode_transition_magic(const char magic, char uuid, int transition_id, int *action_id,
	int op_status, int op_rc, int *target_rc)
	{
	int res = 0;
	char *key = NULL;
	gboolean result = TRUE;
	int local_op_status = -1;
	int local_op_rc = -1;

	CRM_CHECK(magic != NULL, return FALSE);

	#ifdef HAVE_SSCANF_M
	res = sscanf(magic, "%d:%d;%ms", &local_op_status, &local_op_rc, &key);
	#else
	// magic must have >=4 other characters
	key = pcmk__assert_alloc(1, strlen(magic) - 3);
	res = sscanf(magic, "%d:%d;%s", &local_op_status, &local_op_rc, key);
	#endif
	if (res == EOF) {
	crm_err("Could not decode transition information '%s': %s",
	magic, pcmk_rc_str(errno));
	result = FALSE;
	} else if (res < 3) {
	crm_warn("Transition information '%s' incomplete (%d of 3 expected items)",
	magic, res);
	result = FALSE;
	} else {
	if (op_status) {
	*op_status = local_op_status;
	}
	if (op_rc) {
	*op_rc = local_op_rc;
	}
	result = decode_transition_key(key, uuid, transition_id, action_id,
	target_rc);
	}
	free(key);
	return result;
	}

	char *
	pcmk__transition_key(int transition_id, int action_id, int target_rc,
	const char *node)
	{
	CRM_CHECK(node != NULL, return NULL);
	return crm_strdup_printf("%d:%d:%d:%-*s",
	action_id, transition_id, target_rc, 36, node);
	}

	/*!
	* \brief Parse a transition key into its constituent parts
	*
	* \param[in] key Transition key to parse (must be non-NULL)
	* \param[out] uuid If non-NULL, where to store copy of parsed UUID
	* \param[out] transition_id If non-NULL, where to store parsed transition ID
	* \param[out] action_id If non-NULL, where to store parsed action ID
	* \param[out] target_rc If non-NULL, where to stored parsed target rc
	*
	* \return TRUE if key was valid, FALSE otherwise
	* \note If uuid is supplied and this returns TRUE, the caller is responsible
	* for freeing the memory for *uuid using free().
	*/
	gboolean
	decode_transition_key(const char key, char uuid, int transition_id, int *action_id,
	int *target_rc)
	{
	int local_transition_id = -1;
	int local_action_id = -1;
	int local_target_rc = -1;
	char local_uuid[37] = { '\0' };

	// Initialize any supplied output arguments
	if (uuid) {
	*uuid = NULL;
	}
	if (transition_id) {
	*transition_id = -1;
	}
	if (action_id) {
	*action_id = -1;
	}
	if (target_rc) {
	*target_rc = -1;
	}

	CRM_CHECK(key != NULL, return FALSE);
	if (sscanf(key, "%d:%d:%d:%36s", &local_action_id, &local_transition_id,
	&local_target_rc, local_uuid) != 4) {
	crm_err("Invalid transition key '%s'", key);
	return FALSE;
	}
	if (strlen(local_uuid) != 36) {
	crm_warn("Invalid UUID '%s' in transition key '%s'", local_uuid, key);
	}
	if (uuid) {
	*uuid = pcmk__str_copy(local_uuid);
	}
	if (transition_id) {
	*transition_id = local_transition_id;
	}
	if (action_id) {
	*action_id = local_action_id;
	}
	if (target_rc) {
	*target_rc = local_target_rc;
	}
	return TRUE;
	}

	int
	rsc_op_expected_rc(const lrmd_event_data_t *op)
	{
	int rc = 0;

	if (op && op->user_data) {
	decode_transition_key(op->user_data, NULL, NULL, NULL, &rc);
	}
	return rc;
	}

	gboolean
	did_rsc_op_fail(lrmd_event_data_t * op, int target_rc)
	{
	switch (op->op_status) {
	case PCMK_EXEC_CANCELLED:
	case PCMK_EXEC_PENDING:
	return FALSE;

	case PCMK_EXEC_NOT_SUPPORTED:
	case PCMK_EXEC_TIMEOUT:
	case PCMK_EXEC_ERROR:
	case PCMK_EXEC_NOT_CONNECTED:
	case PCMK_EXEC_NO_FENCE_DEVICE:
	case PCMK_EXEC_NO_SECRETS:
	case PCMK_EXEC_INVALID:
	return TRUE;

	default:
	if (target_rc != op->rc) {
	return TRUE;
	}
	}

	return FALSE;
	}

	/*!
	* \brief Create a CIB XML element for an operation
	*
	* \param[in,out] parent If not NULL, make new XML node a child of this
	* \param[in] prefix Generate an ID using this prefix
	* \param[in] task Operation task to set
	* \param[in] interval_spec Operation interval to set
	* \param[in] timeout If not NULL, operation timeout to set
	*
	* \return New XML object on success, NULL otherwise
	*/
	xmlNode *
	crm_create_op_xml(xmlNode parent, const char prefix, const char *task,
	const char interval_spec, const char timeout)
	{
	xmlNode *xml_op;

	CRM_CHECK(prefix && task && interval_spec, return NULL);

	xml_op = pcmk__xe_create(parent, PCMK_XE_OP);
	- crm_xml_set_id(xml_op, "%s-%s-%s", prefix, task, interval_spec);
	+ pcmk__xe_set_id(xml_op, "%s-%s-%s", prefix, task, interval_spec);
	crm_xml_add(xml_op, PCMK_META_INTERVAL, interval_spec);
	crm_xml_add(xml_op, PCMK_XA_NAME, task);
	if (timeout) {
	crm_xml_add(xml_op, PCMK_META_TIMEOUT, timeout);
	}
	return xml_op;
	}

	/*!
	* \brief Check whether an operation requires resource agent meta-data
	*
	* \param[in] rsc_class Resource agent class (or NULL to skip class check)
	* \param[in] op Operation action (or NULL to skip op check)
	*
	* \return true if operation needs meta-data, false otherwise
	* \note At least one of rsc_class and op must be specified.
	*/
	bool
	crm_op_needs_metadata(const char rsc_class, const char op)
	{
	/* Agent metadata is used to determine whether an agent reload is possible,
	* so if this op is not relevant to that feature, we don't need metadata.
	*/

	CRM_CHECK((rsc_class != NULL) \|\| (op != NULL), return false);

	if ((rsc_class != NULL)
	&& !pcmk_is_set(pcmk_get_ra_caps(rsc_class), pcmk_ra_cap_params)) {
	// Metadata is needed only for resource classes that use parameters
	return false;
	}
	if (op == NULL) {
	return true;
	}

	// Metadata is needed only for these actions
	return pcmk__str_any_of(op, PCMK_ACTION_START, PCMK_ACTION_MONITOR,
	PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
	PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT,
	PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
	PCMK_ACTION_NOTIFY, NULL);
	}

	/*!
	* \internal
	* \brief Check whether an action name is for a fencing action
	*
	* \param[in] action Action name to check
	*
	* \return \c true if \p action is \c PCMK_ACTION_OFF, \c PCMK_ACTION_REBOOT,
	* or \c PCMK__ACTION_POWEROFF, otherwise \c false
	*/
	bool
	pcmk__is_fencing_action(const char *action)
	{
	return pcmk__str_any_of(action, PCMK_ACTION_OFF, PCMK_ACTION_REBOOT,
	PCMK__ACTION_POWEROFF, NULL);
	}
	diff --git a/lib/common/crmcommon_private.h b/lib/common/crmcommon_private.h
	index d90a64ef3d..8a5cb13b2b 100644
	--- a/lib/common/crmcommon_private.h
	+++ b/lib/common/crmcommon_private.h
	@@ -1,422 +1,428 @@
	/*
	* Copyright 2018-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef CRMCOMMON_PRIVATE__H
	# define CRMCOMMON_PRIVATE__H

	/* This header is for the sole use of libcrmcommon, so that functions can be
	* declared with G_GNUC_INTERNAL for efficiency.
	*/

	#include <stdint.h> // uint8_t, uint32_t
	#include <stdbool.h> // bool
	#include <sys/types.h> // size_t
	#include <glib.h> // gchar, GList
	#include <libxml/tree.h> // xmlNode, xmlAttr
	#include <qb/qbipcc.h> // struct qb_ipc_response_header

	// Decent chunk size for processing large amounts of data
	#define PCMK__BUFFER_SIZE 4096

	#if defined(PCMK__UNIT_TESTING)
	#undef G_GNUC_INTERNAL
	#define G_GNUC_INTERNAL
	#endif

	/* When deleting portions of an XML tree, we keep a record so we can know later
	* (e.g. when checking differences) that something was deleted.
	*/
	typedef struct pcmk__deleted_xml_s {
	gchar *path;
	int position;
	} pcmk__deleted_xml_t;

	typedef struct xml_node_private_s {
	uint32_t check;
	uint32_t flags;
	} xml_node_private_t;

	typedef struct xml_doc_private_s {
	uint32_t check;
	uint32_t flags;
	char *user;
	GList *acls;
	GList *deleted_objs; // List of pcmk__deleted_xml_t
	} xml_doc_private_t;

	// XML entity references

	#define PCMK__XML_ENTITY_AMP "&"
	#define PCMK__XML_ENTITY_GT ">"
	#define PCMK__XML_ENTITY_LT "<"
	#define PCMK__XML_ENTITY_QUOT """

	//! libxml2 supports only XML version 1.0, at least as of libxml2-2.12.5
	#define PCMK__XML_VERSION ((pcmkXmlStr) "1.0")

	#define pcmk__set_xml_flags(xml_priv, flags_to_set) do { \
	(xml_priv)->flags = pcmk__set_flags_as(__func__, __LINE__, \
	LOG_NEVER, "XML", "XML node", (xml_priv)->flags, \
	(flags_to_set), #flags_to_set); \
	} while (0)

	#define pcmk__clear_xml_flags(xml_priv, flags_to_clear) do { \
	(xml_priv)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
	LOG_NEVER, "XML", "XML node", (xml_priv)->flags, \
	(flags_to_clear), #flags_to_clear); \
	} while (0)

	G_GNUC_INTERNAL
	bool pcmk__tracking_xml_changes(xmlNode *xml, bool lazy);

	G_GNUC_INTERNAL
	void pcmk__xml_mark_created(xmlNode *xml);

	G_GNUC_INTERNAL
	int pcmk__xml_position(const xmlNode *xml,
	enum xml_private_flags ignore_if_set);

	G_GNUC_INTERNAL
	xmlNode pcmk__xml_match(const xmlNode haystack, const xmlNode *needle,
	bool exact);

	G_GNUC_INTERNAL
	void pcmk__xml_update(xmlNode parent, xmlNode target, xmlNode *update,
	uint32_t flags, bool as_diff);

	G_GNUC_INTERNAL
	xmlNode pcmk__xc_match(const xmlNode root, const xmlNode *search_comment,
	bool exact);

	G_GNUC_INTERNAL
	void pcmk__xc_update(xmlNode parent, xmlNode target, xmlNode *update);

	G_GNUC_INTERNAL
	void pcmk__free_acls(GList *acls);

	G_GNUC_INTERNAL
	void pcmk__unpack_acl(xmlNode source, xmlNode target, const char *user);

	G_GNUC_INTERNAL
	bool pcmk__is_user_in_group(const char user, const char group);

	G_GNUC_INTERNAL
	void pcmk__apply_acl(xmlNode *xml);

	G_GNUC_INTERNAL
	void pcmk__apply_creation_acl(xmlNode *xml, bool check_top);

	G_GNUC_INTERNAL
	void pcmk__mark_xml_attr_dirty(xmlAttr *a);

	G_GNUC_INTERNAL
	bool pcmk__xa_filterable(const char *name);

	G_GNUC_INTERNAL
	void pcmk__log_xmllib_err(void ctx, const char fmt, ...)
	G_GNUC_PRINTF(2, 3);

	G_GNUC_INTERNAL
	void pcmk__mark_xml_node_dirty(xmlNode *xml);

	G_GNUC_INTERNAL
	bool pcmk__marked_as_deleted(xmlAttrPtr a, void *user_data);

	G_GNUC_INTERNAL
	void pcmk__dump_xml_attr(const xmlAttr attr, GString buffer);

	G_GNUC_INTERNAL
	int pcmk__xe_set_score(xmlNode target, const char name, const char *value);

	+G_GNUC_INTERNAL
	+bool pcmk__xml_is_name_start_char(const char utf8, int len);
	+
	+G_GNUC_INTERNAL
	+bool pcmk__xml_is_name_char(const char utf8, int len);
	+
	/*
	* Date/times
	*/

	// For use with pcmk__add_time_from_xml()
	enum pcmk__time_component {
	pcmk__time_unknown,
	pcmk__time_years,
	pcmk__time_months,
	pcmk__time_weeks,
	pcmk__time_days,
	pcmk__time_hours,
	pcmk__time_minutes,
	pcmk__time_seconds,
	};

	G_GNUC_INTERNAL
	const char *pcmk__time_component_attr(enum pcmk__time_component component);

	G_GNUC_INTERNAL
	int pcmk__add_time_from_xml(crm_time_t *t, enum pcmk__time_component component,
	const xmlNode *xml);

	G_GNUC_INTERNAL
	void pcmk__set_time_if_earlier(crm_time_t target, const crm_time_t source);


	/*
	* IPC
	*/

	#define PCMK__IPC_VERSION 1

	#define PCMK__CONTROLD_API_MAJOR "1"
	#define PCMK__CONTROLD_API_MINOR "0"

	// IPC behavior that varies by daemon
	typedef struct pcmk__ipc_methods_s {
	/*!
	* \internal
	* \brief Allocate any private data needed by daemon IPC
	*
	* \param[in,out] api IPC API connection
	*
	* \return Standard Pacemaker return code
	*/
	int (new_data)(pcmk_ipc_api_t api);

	/*!
	* \internal
	* \brief Free any private data used by daemon IPC
	*
	* \param[in,out] api_data Data allocated by new_data() method
	*/
	void (free_data)(void api_data);

	/*!
	* \internal
	* \brief Perform daemon-specific handling after successful connection
	*
	* Some daemons require clients to register before sending any other
	* commands. The controller requires a CRM_OP_HELLO (with no reply), and
	* the CIB manager, executor, and fencer require a CRM_OP_REGISTER (with a
	* reply). Ideally this would be consistent across all daemons, but for now
	* this allows each to do its own authorization.
	*
	* \param[in,out] api IPC API connection
	*
	* \return Standard Pacemaker return code
	*/
	int (post_connect)(pcmk_ipc_api_t api);

	/*!
	* \internal
	* \brief Check whether an IPC request results in a reply
	*
	* \param[in,out] api IPC API connection
	* \param[in] request IPC request XML
	*
	* \return true if request would result in an IPC reply, false otherwise
	*/
	bool (reply_expected)(pcmk_ipc_api_t api, const xmlNode *request);

	/*!
	* \internal
	* \brief Perform daemon-specific handling of an IPC message
	*
	* \param[in,out] api IPC API connection
	* \param[in,out] msg Message read from IPC connection
	*
	* \return true if more IPC reply messages should be expected
	*/
	bool (dispatch)(pcmk_ipc_api_t api, xmlNode *msg);

	/*!
	* \internal
	* \brief Perform daemon-specific handling of an IPC disconnect
	*
	* \param[in,out] api IPC API connection
	*/
	void (post_disconnect)(pcmk_ipc_api_t api);
	} pcmk__ipc_methods_t;

	// Implementation of pcmk_ipc_api_t
	struct pcmk_ipc_api_s {
	enum pcmk_ipc_server server; // Daemon this IPC API instance is for
	enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched
	size_t ipc_size_max; // maximum IPC buffer size
	crm_ipc_t *ipc; // IPC connection
	mainloop_io_t *mainloop_io; // If using mainloop, I/O source for IPC
	bool free_on_disconnect; // Whether disconnect should free object
	pcmk_ipc_callback_t cb; // Caller-registered callback (if any)
	void *user_data; // Caller-registered data (if any)
	void *api_data; // For daemon-specific use
	pcmk__ipc_methods_t *cmds; // Behavior that varies by daemon
	};

	typedef struct pcmk__ipc_header_s {
	struct qb_ipc_response_header qb;
	uint32_t size_uncompressed;
	uint32_t size_compressed;
	uint32_t flags;
	uint8_t version;
	} pcmk__ipc_header_t;

	G_GNUC_INTERNAL
	int pcmk__send_ipc_request(pcmk_ipc_api_t api, const xmlNode request);

	G_GNUC_INTERNAL
	void pcmk__call_ipc_callback(pcmk_ipc_api_t *api,
	enum pcmk_ipc_event event_type,
	crm_exit_t status, void *event_data);

	G_GNUC_INTERNAL
	unsigned int pcmk__ipc_buffer_size(unsigned int max);

	G_GNUC_INTERNAL
	bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header);

	G_GNUC_INTERNAL
	pcmk__ipc_methods_t *pcmk__attrd_api_methods(void);

	G_GNUC_INTERNAL
	pcmk__ipc_methods_t *pcmk__controld_api_methods(void);

	G_GNUC_INTERNAL
	pcmk__ipc_methods_t *pcmk__pacemakerd_api_methods(void);

	G_GNUC_INTERNAL
	pcmk__ipc_methods_t *pcmk__schedulerd_api_methods(void);


	/*
	* Logging
	*/

	//! XML is newly created
	#define PCMK__XML_PREFIX_CREATED "++"

	//! XML has been deleted
	#define PCMK__XML_PREFIX_DELETED "--"

	//! XML has been modified
	#define PCMK__XML_PREFIX_MODIFIED "+ "

	//! XML has been moved
	#define PCMK__XML_PREFIX_MOVED "+~"

	/*
	* Output
	*/
	G_GNUC_INTERNAL
	int pcmk__bare_output_new(pcmk__output_t *out, const char fmt_name,
	const char filename, char *argv);

	G_GNUC_INTERNAL
	void pcmk__register_option_messages(pcmk__output_t *out);

	G_GNUC_INTERNAL
	void pcmk__register_patchset_messages(pcmk__output_t *out);

	G_GNUC_INTERNAL
	bool pcmk__output_text_get_fancy(pcmk__output_t *out);

	/*
	* Rules
	*/

	// How node attribute values may be compared in rules
	enum pcmk__comparison {
	pcmk__comparison_unknown,
	pcmk__comparison_defined,
	pcmk__comparison_undefined,
	pcmk__comparison_eq,
	pcmk__comparison_ne,
	pcmk__comparison_lt,
	pcmk__comparison_lte,
	pcmk__comparison_gt,
	pcmk__comparison_gte,
	};

	// How node attribute values may be parsed in rules
	enum pcmk__type {
	pcmk__type_unknown,
	pcmk__type_string,
	pcmk__type_integer,
	pcmk__type_number,
	pcmk__type_version,
	};

	// Where to obtain reference value for a node attribute comparison
	enum pcmk__reference_source {
	pcmk__source_unknown,
	pcmk__source_literal,
	pcmk__source_instance_attrs,
	pcmk__source_meta_attrs,
	};

	G_GNUC_INTERNAL
	enum pcmk__comparison pcmk__parse_comparison(const char *op);

	G_GNUC_INTERNAL
	enum pcmk__type pcmk__parse_type(const char *type, enum pcmk__comparison op,
	const char value1, const char value2);

	G_GNUC_INTERNAL
	enum pcmk__reference_source pcmk__parse_source(const char *source);

	G_GNUC_INTERNAL
	int pcmk__cmp_by_type(const char value1, const char value2,
	enum pcmk__type type);

	G_GNUC_INTERNAL
	int pcmk__unpack_duration(const xmlNode duration, const crm_time_t start,
	crm_time_t **end);

	G_GNUC_INTERNAL
	int pcmk__evaluate_date_spec(const xmlNode date_spec, const crm_time_t now);

	G_GNUC_INTERNAL
	int pcmk__evaluate_attr_expression(const xmlNode *expression,
	const pcmk_rule_input_t *rule_input);

	G_GNUC_INTERNAL
	int pcmk__evaluate_rsc_expression(const xmlNode *expr,
	const pcmk_rule_input_t *rule_input);

	G_GNUC_INTERNAL
	int pcmk__evaluate_op_expression(const xmlNode *expr,
	const pcmk_rule_input_t *rule_input);


	/*
	* Utils
	*/
	#define PCMK__PW_BUFFER_LEN 500


	/*
	* Schemas
	*/
	typedef struct {
	unsigned char v[2];
	} pcmk__schema_version_t;

	enum pcmk__schema_validator {
	pcmk__schema_validator_none,
	pcmk__schema_validator_rng
	};

	typedef struct {
	int schema_index;
	char *name;
	char *transform;
	void *cache;
	enum pcmk__schema_validator validator;
	pcmk__schema_version_t version;
	char *transform_enter;
	bool transform_onleave;
	} pcmk__schema_t;

	G_GNUC_INTERNAL
	GList *pcmk__find_x_0_schema(void);


	#endif // CRMCOMMON_PRIVATE__H
	diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
	index 044850e8ce..f1ee56c6c5 100644
	--- a/lib/common/ipc_client.c
	+++ b/lib/common/ipc_client.c
	@@ -1,1694 +1,1694 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#if defined(HAVE_UCRED) \|\| defined(HAVE_SOCKPEERCRED)
	# ifdef HAVE_UCRED
	# ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	# endif
	# endif
	# include <sys/socket.h>
	#elif defined(HAVE_GETPEERUCRED)
	# include <ucred.h>
	#endif

	#include <stdio.h>
	#include <sys/types.h>
	#include <errno.h>
	#include <bzlib.h>

	#include <crm/crm.h> /* indirectly: pcmk_err_generic */
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include "crmcommon_private.h"

	static int is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
	uid_t refuid, gid_t refgid, pid_t *gotpid,
	uid_t gotuid, gid_t gotgid);

	/*!
	* \brief Create a new object for using Pacemaker daemon IPC
	*
	* \param[out] api Where to store new IPC object
	* \param[in] server Which Pacemaker daemon the object is for
	*
	* \return Standard Pacemaker result code
	*
	* \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
	* \note This is intended to supersede crm_ipc_new() but currently only
	* supports the controller, pacemakerd, and schedulerd IPC API.
	*/
	int
	pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
	{
	if (api == NULL) {
	return EINVAL;
	}

	*api = calloc(1, sizeof(pcmk_ipc_api_t));
	if (*api == NULL) {
	return errno;
	}

	(*api)->server = server;
	if (pcmk_ipc_name(*api, false) == NULL) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return EOPNOTSUPP;
	}

	(*api)->ipc_size_max = 0;

	// Set server methods and max_size (if not default)
	switch (server) {
	case pcmk_ipc_attrd:
	(*api)->cmds = pcmk__attrd_api_methods();
	break;

	case pcmk_ipc_based:
	(api)->ipc_size_max = 512 1024; // 512KB
	break;

	case pcmk_ipc_controld:
	(*api)->cmds = pcmk__controld_api_methods();
	break;

	case pcmk_ipc_execd:
	break;

	case pcmk_ipc_fenced:
	break;

	case pcmk_ipc_pacemakerd:
	(*api)->cmds = pcmk__pacemakerd_api_methods();
	break;

	case pcmk_ipc_schedulerd:
	(*api)->cmds = pcmk__schedulerd_api_methods();
	// @TODO max_size could vary by client, maybe take as argument?
	(api)->ipc_size_max = 5 1024 * 1024; // 5MB
	break;
	}
	if ((*api)->cmds == NULL) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return ENOMEM;
	}

	(api)->ipc = crm_ipc_new(pcmk_ipc_name(api, false),
	(*api)->ipc_size_max);
	if ((*api)->ipc == NULL) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return ENOMEM;
	}

	// If daemon API has its own data to track, allocate it
	if ((*api)->cmds->new_data != NULL) {
	if ((api)->cmds->new_data(api) != pcmk_rc_ok) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return ENOMEM;
	}
	}
	crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true));
	return pcmk_rc_ok;
	}

	static void
	free_daemon_specific_data(pcmk_ipc_api_t *api)
	{
	if ((api != NULL) && (api->cmds != NULL)) {
	if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) {
	api->cmds->free_data(api->api_data);
	api->api_data = NULL;
	}
	free(api->cmds);
	api->cmds = NULL;
	}
	}

	/*!
	* \internal
	* \brief Call an IPC API event callback, if one is registed
	*
	* \param[in,out] api IPC API connection
	* \param[in] event_type The type of event that occurred
	* \param[in] status Event status
	* \param[in,out] event_data Event-specific data
	*/
	void
	pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
	crm_exit_t status, void *event_data)
	{
	if ((api != NULL) && (api->cb != NULL)) {
	api->cb(api, event_type, status, event_data, api->user_data);
	}
	}

	/*!
	* \internal
	* \brief Clean up after an IPC disconnect
	*
	* \param[in,out] user_data IPC API connection that disconnected
	*
	* \note This function can be used as a main loop IPC destroy callback.
	*/
	static void
	ipc_post_disconnect(gpointer user_data)
	{
	pcmk_ipc_api_t *api = user_data;

	crm_info("Disconnected from %s", pcmk_ipc_name(api, true));

	// Perform any daemon-specific handling needed
	if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) {
	api->cmds->post_disconnect(api);
	}

	// Call client's registered event callback
	pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT,
	NULL);

	/* If this is being called from a running main loop, mainloop_gio_destroy()
	* will free ipc and mainloop_io immediately after calling this function.
	* If this is called from a stopped main loop, these will leak, so the best
	* practice is to close the connection before stopping the main loop.
	*/
	api->ipc = NULL;
	api->mainloop_io = NULL;

	if (api->free_on_disconnect) {
	/* pcmk_free_ipc_api() has already been called, but did not free api
	* or api->cmds because this function needed them. Do that now.
	*/
	free_daemon_specific_data(api);
	crm_trace("Freeing IPC API object after disconnect");
	free(api);
	}
	}

	/*!
	* \brief Free the contents of an IPC API object
	*
	* \param[in,out] api IPC API object to free
	*/
	void
	pcmk_free_ipc_api(pcmk_ipc_api_t *api)
	{
	bool free_on_disconnect = false;

	if (api == NULL) {
	return;
	}
	crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true));

	if (api->ipc != NULL) {
	if (api->mainloop_io != NULL) {
	/* We need to keep the api pointer itself around, because it is the
	* user data for the IPC client destroy callback. That will be
	* triggered by the pcmk_disconnect_ipc() call below, but it might
	* happen later in the main loop (if still running).
	*
	* This flag tells the destroy callback to free the object. It can't
	* do that unconditionally, because the application might call this
	* function after a disconnect that happened by other means.
	*/
	free_on_disconnect = api->free_on_disconnect = true;
	}
	pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true
	}
	if (!free_on_disconnect) {
	free_daemon_specific_data(api);
	crm_trace("Freeing IPC API object");
	free(api);
	}
	}

	/*!
	* \brief Get the IPC name used with an IPC API connection
	*
	* \param[in] api IPC API connection
	* \param[in] for_log If true, return human-friendly name instead of IPC name
	*
	* \return IPC API's human-friendly or connection name, or if none is available,
	* "Pacemaker" if for_log is true and NULL if for_log is false
	*/
	const char *
	pcmk_ipc_name(const pcmk_ipc_api_t *api, bool for_log)
	{
	if (api == NULL) {
	return for_log? "Pacemaker" : NULL;
	}
	switch (api->server) {
	case pcmk_ipc_attrd:
	return for_log? "attribute manager" : PCMK__VALUE_ATTRD;

	case pcmk_ipc_based:
	return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */;

	case pcmk_ipc_controld:
	return for_log? "controller" : CRM_SYSTEM_CRMD;

	case pcmk_ipc_execd:
	return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */;

	case pcmk_ipc_fenced:
	return for_log? "fencer" : NULL /* "stonith-ng" */;

	case pcmk_ipc_pacemakerd:
	return for_log? "launcher" : CRM_SYSTEM_MCP;

	case pcmk_ipc_schedulerd:
	return for_log? "scheduler" : CRM_SYSTEM_PENGINE;

	default:
	return for_log? "Pacemaker" : NULL;
	}
	}

	/*!
	* \brief Check whether an IPC API connection is active
	*
	* \param[in,out] api IPC API connection
	*
	* \return true if IPC is connected, false otherwise
	*/
	bool
	pcmk_ipc_is_connected(pcmk_ipc_api_t *api)
	{
	return (api != NULL) && crm_ipc_connected(api->ipc);
	}

	/*!
	* \internal
	* \brief Call the daemon-specific API's dispatch function
	*
	* Perform daemon-specific handling of IPC reply dispatch. It is the daemon
	* method's responsibility to call the client's registered event callback, as
	* well as allocate and free any event data.
	*
	* \param[in,out] api IPC API connection
	* \param[in,out] message IPC reply XML to dispatch
	*/
	static bool
	call_api_dispatch(pcmk_ipc_api_t api, xmlNode message)
	{
	crm_log_xml_trace(message, "ipc-received");
	if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) {
	return api->cmds->dispatch(api, message);
	}

	return false;
	}

	/*!
	* \internal
	* \brief Dispatch previously read IPC data
	*
	* \param[in] buffer Data read from IPC
	* \param[in,out] api IPC object
	*
	* \return Standard Pacemaker return code. In particular:
	*
	* pcmk_rc_ok: There are no more messages expected from the server. Quit
	* reading.
	* EINPROGRESS: There are more messages expected from the server. Keep reading.
	*
	* All other values indicate an error.
	*/
	static int
	dispatch_ipc_data(const char buffer, pcmk_ipc_api_t api)
	{
	bool more = false;
	xmlNode *msg;

	if (buffer == NULL) {
	crm_warn("Empty message received from %s IPC",
	pcmk_ipc_name(api, true));
	return ENOMSG;
	}

	msg = pcmk__xml_parse(buffer);
	if (msg == NULL) {
	crm_warn("Malformed message received from %s IPC",
	pcmk_ipc_name(api, true));
	return EPROTO;
	}

	more = call_api_dispatch(api, msg);
	pcmk__xml_free(msg);

	if (more) {
	return EINPROGRESS;
	} else {
	return pcmk_rc_ok;
	}
	}

	/*!
	* \internal
	* \brief Dispatch data read from IPC source
	*
	* \param[in] buffer Data read from IPC
	* \param[in] length Number of bytes of data in buffer (ignored)
	* \param[in,out] user_data IPC object
	*
	* \return Always 0 (meaning connection is still required)
	*
	* \note This function can be used as a main loop IPC dispatch callback.
	*/
	static int
	dispatch_ipc_source_data(const char *buffer, ssize_t length, gpointer user_data)
	{
	pcmk_ipc_api_t *api = user_data;

	CRM_CHECK(api != NULL, return 0);
	dispatch_ipc_data(buffer, api);
	return 0;
	}

	/*!
	* \brief Check whether an IPC connection has data available (without main loop)
	*
	* \param[in] api IPC API connection
	* \param[in] timeout_ms If less than 0, poll indefinitely; if 0, poll once
	* and return immediately; otherwise, poll for up to
	* this many milliseconds
	*
	* \return Standard Pacemaker return code
	*
	* \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
	* this function to check whether IPC data is available. Return values of
	* interest include pcmk_rc_ok meaning data is available, and EAGAIN
	* meaning no data is available; all other values indicate errors.
	* \todo This does not allow the caller to poll multiple file descriptors at
	* once. If there is demand for that, we could add a wrapper for
	* pcmk__ipc_fd(api->ipc), so the caller can call poll() themselves.
	*/
	int
	pcmk_poll_ipc(const pcmk_ipc_api_t *api, int timeout_ms)
	{
	int rc;
	struct pollfd pollfd = { 0, };

	if ((api == NULL) \|\| (api->dispatch_type != pcmk_ipc_dispatch_poll)) {
	return EINVAL;
	}

	rc = pcmk__ipc_fd(api->ipc, &(pollfd.fd));
	if (rc != pcmk_rc_ok) {
	crm_debug("Could not obtain file descriptor for %s IPC: %s",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc));
	return rc;
	}

	pollfd.events = POLLIN;
	rc = poll(&pollfd, 1, timeout_ms);
	if (rc < 0) {
	/* Some UNIX systems return negative and set EAGAIN for failure to
	* allocate memory; standardize the return code in that case
	*/
	return (errno == EAGAIN)? ENOMEM : errno;
	} else if (rc == 0) {
	return EAGAIN;
	}
	return pcmk_rc_ok;
	}

	/*!
	* \brief Dispatch available messages on an IPC connection (without main loop)
	*
	* \param[in,out] api IPC API connection
	*
	* \return Standard Pacemaker return code
	*
	* \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
	* this function when IPC data is available.
	*/
	void
	pcmk_dispatch_ipc(pcmk_ipc_api_t *api)
	{
	if (api == NULL) {
	return;
	}
	while (crm_ipc_ready(api->ipc) > 0) {
	if (crm_ipc_read(api->ipc) > 0) {
	dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);
	}
	}
	}

	// \return Standard Pacemaker return code
	static int
	connect_with_main_loop(pcmk_ipc_api_t *api)
	{
	int rc;

	struct ipc_client_callbacks callbacks = {
	.dispatch = dispatch_ipc_source_data,
	.destroy = ipc_post_disconnect,
	};

	rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api,
	&callbacks, &(api->mainloop_io));
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	crm_debug("Connected to %s IPC (attached to main loop)",
	pcmk_ipc_name(api, true));
	/* After this point, api->mainloop_io owns api->ipc, so api->ipc
	* should not be explicitly freed.
	*/
	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	connect_without_main_loop(pcmk_ipc_api_t *api)
	{
	int rc = pcmk__connect_generic_ipc(api->ipc);

	if (rc != pcmk_rc_ok) {
	crm_ipc_close(api->ipc);
	} else {
	crm_debug("Connected to %s IPC (without main loop)",
	pcmk_ipc_name(api, true));
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Connect to a Pacemaker daemon via IPC (retrying after soft errors)
	*
	* \param[in,out] api IPC API instance
	* \param[in] dispatch_type How IPC replies should be dispatched
	* \param[in] attempts How many times to try (in case of soft error)
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type,
	int attempts)
	{
	int rc = pcmk_rc_ok;

	if ((api == NULL) \|\| (attempts < 1)) {
	return EINVAL;
	}

	if (api->ipc == NULL) {
	api->ipc = crm_ipc_new(pcmk_ipc_name(api, false), api->ipc_size_max);
	if (api->ipc == NULL) {
	return ENOMEM;
	}
	}

	if (crm_ipc_connected(api->ipc)) {
	crm_trace("Already connected to %s", pcmk_ipc_name(api, true));
	return pcmk_rc_ok;
	}

	api->dispatch_type = dispatch_type;

	crm_debug("Attempting connection to %s (up to %d time%s)",
	pcmk_ipc_name(api, true), attempts, pcmk__plural_s(attempts));
	for (int remaining = attempts - 1; remaining >= 0; --remaining) {
	switch (dispatch_type) {
	case pcmk_ipc_dispatch_main:
	rc = connect_with_main_loop(api);
	break;

	case pcmk_ipc_dispatch_sync:
	case pcmk_ipc_dispatch_poll:
	rc = connect_without_main_loop(api);
	break;
	}

	if ((remaining == 0) \|\| ((rc != EAGAIN) && (rc != EALREADY))) {
	break; // Result is final
	}

	// Retry after soft error (interrupted by signal, etc.)
	pcmk__sleep_ms((attempts - remaining) * 500);
	crm_debug("Re-attempting connection to %s (%d attempt%s remaining)",
	pcmk_ipc_name(api, true), remaining,
	pcmk__plural_s(remaining));
	}

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) {
	rc = api->cmds->post_connect(api);
	if (rc != pcmk_rc_ok) {
	crm_ipc_close(api->ipc);
	}
	}
	return rc;
	}

	/*!
	* \brief Connect to a Pacemaker daemon via IPC
	*
	* \param[in,out] api IPC API instance
	* \param[in] dispatch_type How IPC replies should be dispatched
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type)
	{
	int rc = pcmk__connect_ipc(api, dispatch_type, 2);

	if (rc != pcmk_rc_ok) {
	crm_err("Connection to %s failed: %s",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc));
	}
	return rc;
	}

	/*!
	* \brief Disconnect an IPC API instance
	*
	* \param[in,out] api IPC API connection
	*
	* \return Standard Pacemaker return code
	*
	* \note If the connection is attached to a main loop, this function should be
	* called before quitting the main loop, to ensure that all memory is
	* freed.
	*/
	void
	pcmk_disconnect_ipc(pcmk_ipc_api_t *api)
	{
	if ((api == NULL) \|\| (api->ipc == NULL)) {
	return;
	}
	switch (api->dispatch_type) {
	case pcmk_ipc_dispatch_main:
	{
	mainloop_io_t *mainloop_io = api->mainloop_io;

	// Make sure no code with access to api can use these again
	api->mainloop_io = NULL;
	api->ipc = NULL;

	mainloop_del_ipc_client(mainloop_io);
	// After this point api might have already been freed
	}
	break;

	case pcmk_ipc_dispatch_poll:
	case pcmk_ipc_dispatch_sync:
	{
	crm_ipc_t *ipc = api->ipc;

	// Make sure no code with access to api can use ipc again
	api->ipc = NULL;

	// This should always be the case already, but to be safe
	api->free_on_disconnect = false;

	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	ipc_post_disconnect(api);
	}
	break;
	}
	}

	/*!
	* \brief Register a callback for IPC API events
	*
	* \param[in,out] api IPC API connection
	* \param[in] callback Callback to register
	* \param[in] userdata Caller data to pass to callback
	*
	* \note This function may be called multiple times to update the callback
	* and/or user data. The caller remains responsible for freeing
	* userdata in any case (after the IPC is disconnected, if the
	* user data is still registered with the IPC).
	*/
	void
	pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
	void *user_data)
	{
	if (api == NULL) {
	return;
	}
	api->cb = cb;
	api->user_data = user_data;
	}

	/*!
	* \internal
	* \brief Send an XML request across an IPC API connection
	*
	* \param[in,out] api IPC API connection
	* \param[in] request XML request to send
	*
	* \return Standard Pacemaker return code
	*
	* \note Daemon-specific IPC API functions should call this function to send
	* requests, because it handles different dispatch types appropriately.
	*/
	int
	pcmk__send_ipc_request(pcmk_ipc_api_t api, const xmlNode request)
	{
	int rc;
	xmlNode *reply = NULL;
	enum crm_ipc_flags flags = crm_ipc_flags_none;

	if ((api == NULL) \|\| (api->ipc == NULL) \|\| (request == NULL)) {
	return EINVAL;
	}
	crm_log_xml_trace(request, "ipc-sent");

	// Synchronous dispatch requires waiting for a reply
	if ((api->dispatch_type == pcmk_ipc_dispatch_sync)
	&& (api->cmds != NULL)
	&& (api->cmds->reply_expected != NULL)
	&& (api->cmds->reply_expected(api, request))) {
	flags = crm_ipc_client_response;
	}

	// The 0 here means a default timeout of 5 seconds
	rc = crm_ipc_send(api->ipc, request, flags, 0, &reply);

	if (rc < 0) {
	return pcmk_legacy2rc(rc);
	} else if (rc == 0) {
	return ENODATA;
	}

	// With synchronous dispatch, we dispatch any reply now
	if (reply != NULL) {
	bool more = call_api_dispatch(api, reply);

	pcmk__xml_free(reply);

	while (more) {
	rc = crm_ipc_read(api->ipc);

	if (rc == -EAGAIN) {
	continue;
	} else if (rc == -ENOMSG \|\| rc == pcmk_ok) {
	return pcmk_rc_ok;
	} else if (rc < 0) {
	return -rc;
	}

	rc = dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);

	if (rc == pcmk_rc_ok) {
	more = false;
	} else if (rc == EINPROGRESS) {
	more = true;
	} else {
	continue;
	}
	}
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Create the XML for an IPC request to purge a node from the peer cache
	*
	* \param[in] api IPC API connection
	* \param[in] node_name If not NULL, name of node to purge
	* \param[in] nodeid If not 0, node ID of node to purge
	*
	* \return Newly allocated IPC request XML
	*
	* \note The controller, fencer, and pacemakerd use the same request syntax, but
	* the attribute manager uses a different one. The CIB manager doesn't
	* have any syntax for it. The executor and scheduler don't connect to the
	* cluster layer and thus don't have or need any syntax for it.
	*
	* \todo Modify the attribute manager to accept the common syntax (as well
	* as its current one, for compatibility with older clients). Modify
	* the CIB manager to accept and honor the common syntax. Modify the
	* executor and scheduler to accept the syntax (immediately returning
	* success), just for consistency. Modify this function to use the
	* common syntax with all daemons if their version supports it.
	*/
	static xmlNode *
	create_purge_node_request(const pcmk_ipc_api_t api, const char node_name,
	uint32_t nodeid)
	{
	xmlNode *request = NULL;
	const char *client = crm_system_name? crm_system_name : "client";

	switch (api->server) {
	case pcmk_ipc_attrd:
	request = pcmk__xe_create(NULL, __func__);
	crm_xml_add(request, PCMK__XA_T, PCMK__VALUE_ATTRD);
	crm_xml_add(request, PCMK__XA_SRC, crm_system_name);
	crm_xml_add(request, PCMK_XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE);
	pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, true);
	pcmk__xe_add_node(request, node_name, nodeid);
	break;

	case pcmk_ipc_controld:
	case pcmk_ipc_fenced:
	case pcmk_ipc_pacemakerd:
	request = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL,
	pcmk_ipc_name(api, false), client, NULL);
	if (nodeid > 0) {
	- crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
	+ pcmk__xe_set_id(request, "%lu", (unsigned long) nodeid);
	}
	crm_xml_add(request, PCMK_XA_UNAME, node_name);
	break;

	case pcmk_ipc_based:
	case pcmk_ipc_execd:
	case pcmk_ipc_schedulerd:
	break;
	}
	return request;
	}

	/*!
	* \brief Ask a Pacemaker daemon to purge a node from its peer cache
	*
	* \param[in,out] api IPC API connection
	* \param[in] node_name If not NULL, name of node to purge
	* \param[in] nodeid If not 0, node ID of node to purge
	*
	* \return Standard Pacemaker return code
	*
	* \note At least one of node_name or nodeid must be specified.
	*/
	int
	pcmk_ipc_purge_node(pcmk_ipc_api_t api, const char node_name, uint32_t nodeid)
	{
	int rc = 0;
	xmlNode *request = NULL;

	if (api == NULL) {
	return EINVAL;
	}
	if ((node_name == NULL) && (nodeid == 0)) {
	return EINVAL;
	}

	request = create_purge_node_request(api, node_name, nodeid);
	if (request == NULL) {
	return EOPNOTSUPP;
	}
	rc = pcmk__send_ipc_request(api, request);
	pcmk__xml_free(request);

	crm_debug("%s peer cache purge of node %s[%lu]: rc=%d",
	pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc);
	return rc;
	}

	/*
	* Generic IPC API (to eventually be deprecated as public API and made internal)
	*/

	struct crm_ipc_s {
	struct pollfd pfd;
	unsigned int max_buf_size; // maximum bytes we can send or receive over IPC
	unsigned int buf_size; // size of allocated buffer
	int msg_size;
	int need_reply;
	char *buffer;
	char *server_name; // server IPC name being connected to
	qb_ipcc_connection_t *ipc;
	};

	/*!
	* \brief Create a new (legacy) object for using Pacemaker daemon IPC
	*
	* \param[in] name IPC system name to connect to
	* \param[in] max_size Use a maximum IPC buffer size of at least this size
	*
	* \return Newly allocated IPC object on success, NULL otherwise
	*
	* \note The caller is responsible for freeing the result using
	* crm_ipc_destroy().
	* \note This should be considered deprecated for use with daemons supported by
	* pcmk_new_ipc_api().
	*/
	crm_ipc_t *
	crm_ipc_new(const char *name, size_t max_size)
	{
	crm_ipc_t *client = NULL;

	client = calloc(1, sizeof(crm_ipc_t));
	if (client == NULL) {
	crm_err("Could not create IPC connection: %s", strerror(errno));
	return NULL;
	}

	client->server_name = strdup(name);
	if (client->server_name == NULL) {
	crm_err("Could not create %s IPC connection: %s",
	name, strerror(errno));
	free(client);
	return NULL;
	}
	client->buf_size = pcmk__ipc_buffer_size(max_size);
	client->buffer = malloc(client->buf_size);
	if (client->buffer == NULL) {
	crm_err("Could not create %s IPC connection: %s",
	name, strerror(errno));
	free(client->server_name);
	free(client);
	return NULL;
	}

	/* Clients initiating connection pick the max buf size */
	client->max_buf_size = client->buf_size;

	client->pfd.fd = -1;
	client->pfd.events = POLLIN;
	client->pfd.revents = 0;

	return client;
	}

	/*!
	* \internal
	* \brief Connect a generic (not daemon-specific) IPC object
	*
	* \param[in,out] ipc Generic IPC object to connect
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__connect_generic_ipc(crm_ipc_t *ipc)
	{
	uid_t cl_uid = 0;
	gid_t cl_gid = 0;
	pid_t found_pid = 0;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	int rc = pcmk_rc_ok;

	if (ipc == NULL) {
	return EINVAL;
	}

	ipc->need_reply = FALSE;
	ipc->ipc = qb_ipcc_connect(ipc->server_name, ipc->buf_size);
	if (ipc->ipc == NULL) {
	return errno;
	}

	rc = qb_ipcc_fd_get(ipc->ipc, &ipc->pfd.fd);
	if (rc < 0) { // -errno
	crm_ipc_close(ipc);
	return -rc;
	}

	rc = pcmk_daemon_user(&cl_uid, &cl_gid);
	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	crm_ipc_close(ipc);
	return rc;
	}

	rc = is_ipc_provider_expected(ipc->ipc, ipc->pfd.fd, cl_uid, cl_gid,
	&found_pid, &found_uid, &found_gid);
	if (rc != pcmk_rc_ok) {
	if (rc == pcmk_rc_ipc_unauthorized) {
	crm_info("%s IPC provider authentication failed: process %lld has "
	"uid %lld (expected %lld) and gid %lld (expected %lld)",
	ipc->server_name,
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) cl_uid,
	(long long) found_gid, (long long) cl_gid);
	}
	crm_ipc_close(ipc);
	return rc;
	}

	ipc->max_buf_size = qb_ipcc_get_buffer_size(ipc->ipc);
	if (ipc->max_buf_size > ipc->buf_size) {
	free(ipc->buffer);
	ipc->buffer = calloc(ipc->max_buf_size, sizeof(char));
	if (ipc->buffer == NULL) {
	rc = errno;
	crm_ipc_close(ipc);
	return rc;
	}
	ipc->buf_size = ipc->max_buf_size;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \brief Establish an IPC connection to a Pacemaker component
	*
	* \param[in,out] client Connection instance obtained from crm_ipc_new()
	*
	* \return true on success, false otherwise (in which case errno will be set;
	* specifically, in case of discovering the remote side is not
	* authentic, its value is set to ECONNABORTED).
	*/
	bool
	crm_ipc_connect(crm_ipc_t *client)
	{
	int rc = pcmk__connect_generic_ipc(client);

	if (rc == pcmk_rc_ok) {
	return true;
	}
	if ((client != NULL) && (client->ipc == NULL)) {
	errno = (rc > 0)? rc : ENOTCONN;
	crm_debug("Could not establish %s IPC connection: %s (%d)",
	client->server_name, pcmk_rc_str(errno), errno);
	} else if (rc == pcmk_rc_ipc_unauthorized) {
	crm_err("%s IPC provider authentication failed",
	(client == NULL)? "Pacemaker" : client->server_name);
	errno = ECONNABORTED;
	} else {
	crm_perror(LOG_ERR,
	"Could not verify authenticity of %s IPC provider",
	(client == NULL)? "Pacemaker" : client->server_name);
	errno = ENOTCONN;
	}
	return false;
	}

	void
	crm_ipc_close(crm_ipc_t * client)
	{
	if (client) {
	if (client->ipc) {
	qb_ipcc_connection_t *ipc = client->ipc;

	client->ipc = NULL;
	qb_ipcc_disconnect(ipc);
	}
	}
	}

	void
	crm_ipc_destroy(crm_ipc_t * client)
	{
	if (client) {
	if (client->ipc && qb_ipcc_is_connected(client->ipc)) {
	crm_notice("Destroying active %s IPC connection",
	client->server_name);
	/* The next line is basically unsafe
	*
	* If this connection was attached to mainloop and mainloop is active,
	* the 'disconnected' callback will end up back here and we'll end
	* up free'ing the memory twice - something that can still happen
	* even without this if we destroy a connection and it closes before
	* we call exit
	*/
	/* crm_ipc_close(client); */
	} else {
	crm_trace("Destroying inactive %s IPC connection",
	client->server_name);
	}
	free(client->buffer);
	free(client->server_name);
	free(client);
	}
	}

	/*!
	* \internal
	* \brief Get the file descriptor for a generic IPC object
	*
	* \param[in,out] ipc Generic IPC object to get file descriptor for
	* \param[out] fd Where to store file descriptor
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__ipc_fd(crm_ipc_t ipc, int fd)
	{
	if ((ipc == NULL) \|\| (fd == NULL)) {
	return EINVAL;
	}
	if ((ipc->ipc == NULL) \|\| (ipc->pfd.fd < 0)) {
	return ENOTCONN;
	}
	*fd = ipc->pfd.fd;
	return pcmk_rc_ok;
	}

	int
	crm_ipc_get_fd(crm_ipc_t * client)
	{
	int fd = -1;

	if (pcmk__ipc_fd(client, &fd) != pcmk_rc_ok) {
	crm_err("Could not obtain file descriptor for %s IPC",
	((client == NULL)? "unspecified" : client->server_name));
	errno = EINVAL;
	return -EINVAL;
	}
	return fd;
	}

	bool
	crm_ipc_connected(crm_ipc_t * client)
	{
	bool rc = FALSE;

	if (client == NULL) {
	crm_trace("No client");
	return FALSE;

	} else if (client->ipc == NULL) {
	crm_trace("No connection");
	return FALSE;

	} else if (client->pfd.fd < 0) {
	crm_trace("Bad descriptor");
	return FALSE;
	}

	rc = qb_ipcc_is_connected(client->ipc);
	if (rc == FALSE) {
	client->pfd.fd = -EINVAL;
	}
	return rc;
	}

	/*!
	* \brief Check whether an IPC connection is ready to be read
	*
	* \param[in,out] client Connection to check
	*
	* \return Positive value if ready to be read, 0 if not ready, -errno on error
	*/
	int
	crm_ipc_ready(crm_ipc_t *client)
	{
	int rc;

	CRM_ASSERT(client != NULL);

	if (!crm_ipc_connected(client)) {
	return -ENOTCONN;
	}

	client->pfd.revents = 0;
	rc = poll(&(client->pfd), 1, 0);
	return (rc < 0)? -errno : rc;
	}

	// \return Standard Pacemaker return code
	static int
	crm_ipc_decompress(crm_ipc_t * client)
	{
	pcmk__ipc_header_t header = (pcmk__ipc_header_t )(void*)client->buffer;

	if (header->size_compressed) {
	int rc = 0;
	unsigned int size_u = 1 + header->size_uncompressed;
	/* never let buf size fall below our max size required for ipc reads. */
	unsigned int new_buf_size = QB_MAX((sizeof(pcmk__ipc_header_t) + size_u), client->max_buf_size);
	char *uncompressed = pcmk__assert_alloc(1, new_buf_size);

	crm_trace("Decompressing message data %u bytes into %u bytes",
	header->size_compressed, size_u);

	rc = BZ2_bzBuffToBuffDecompress(uncompressed + sizeof(pcmk__ipc_header_t), &size_u,
	client->buffer + sizeof(pcmk__ipc_header_t), header->size_compressed, 1, 0);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_err("Decompression failed: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	free(uncompressed);
	return rc;
	}

	/*
	* This assert no longer holds true. For an identical msg, some clients may
	* require compression, and others may not. If that same msg (event) is sent
	* to multiple clients, it could result in some clients receiving a compressed
	* msg even though compression was not explicitly required for them.
	*
	* CRM_ASSERT((header->size_uncompressed + sizeof(pcmk__ipc_header_t)) >= ipc_buffer_max);
	*/
	CRM_ASSERT(size_u == header->size_uncompressed);

	memcpy(uncompressed, client->buffer, sizeof(pcmk__ipc_header_t)); /* Preserve the header */
	header = (pcmk__ipc_header_t )(void)uncompressed;

	free(client->buffer);
	client->buf_size = new_buf_size;
	client->buffer = uncompressed;
	}

	CRM_ASSERT(client->buffer[sizeof(pcmk__ipc_header_t) + header->size_uncompressed - 1] == 0);
	return pcmk_rc_ok;
	}

	long
	crm_ipc_read(crm_ipc_t * client)
	{
	pcmk__ipc_header_t *header = NULL;

	CRM_ASSERT(client != NULL);
	CRM_ASSERT(client->ipc != NULL);
	CRM_ASSERT(client->buffer != NULL);

	client->buffer[0] = 0;
	client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer,
	client->buf_size, 0);
	if (client->msg_size >= 0) {
	int rc = crm_ipc_decompress(client);

	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}

	header = (pcmk__ipc_header_t )(void)client->buffer;
	if (!pcmk__valid_ipc_header(header)) {
	return -EBADMSG;
	}

	crm_trace("Received %s IPC event %d size=%u rc=%d text='%.100s'",
	client->server_name, header->qb.id, header->qb.size,
	client->msg_size,
	client->buffer + sizeof(pcmk__ipc_header_t));

	} else {
	crm_trace("No message received from %s IPC: %s",
	client->server_name, pcmk_strerror(client->msg_size));

	if (client->msg_size == -EAGAIN) {
	return -EAGAIN;
	}
	}

	if (!crm_ipc_connected(client) \|\| client->msg_size == -ENOTCONN) {
	crm_err("Connection to %s IPC failed", client->server_name);
	}

	if (header) {
	/* Data excluding the header */
	return header->size_uncompressed;
	}
	return -ENOMSG;
	}

	const char *
	crm_ipc_buffer(crm_ipc_t * client)
	{
	CRM_ASSERT(client != NULL);
	return client->buffer + sizeof(pcmk__ipc_header_t);
	}

	uint32_t
	crm_ipc_buffer_flags(crm_ipc_t * client)
	{
	pcmk__ipc_header_t *header = NULL;

	CRM_ASSERT(client != NULL);
	if (client->buffer == NULL) {
	return 0;
	}

	header = (pcmk__ipc_header_t )(void)client->buffer;
	return header->flags;
	}

	const char *
	crm_ipc_name(crm_ipc_t * client)
	{
	CRM_ASSERT(client != NULL);
	return client->server_name;
	}

	// \return Standard Pacemaker return code
	static int
	internal_ipc_get_reply(crm_ipc_t *client, int request_id, int ms_timeout,
	ssize_t *bytes)
	{
	time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
	int rc = pcmk_rc_ok;

	/* get the reply */
	crm_trace("Waiting on reply to %s IPC message %d",
	client->server_name, request_id);
	do {

	*bytes = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000);
	if (*bytes > 0) {
	pcmk__ipc_header_t *hdr = NULL;

	rc = crm_ipc_decompress(client);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	hdr = (pcmk__ipc_header_t )(void)client->buffer;
	if (hdr->qb.id == request_id) {
	/* Got it */
	break;
	} else if (hdr->qb.id < request_id) {
	xmlNode *bad = pcmk__xml_parse(crm_ipc_buffer(client));

	crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id);
	crm_log_xml_notice(bad, "OldIpcReply");

	} else {
	xmlNode *bad = pcmk__xml_parse(crm_ipc_buffer(client));

	crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id);
	crm_log_xml_notice(bad, "ImpossibleReply");
	CRM_ASSERT(hdr->qb.id <= request_id);
	}
	} else if (!crm_ipc_connected(client)) {
	crm_err("%s IPC provider disconnected while waiting for message %d",
	client->server_name, request_id);
	break;
	}

	} while (time(NULL) < timeout);

	if (*bytes < 0) {
	rc = (int) -*bytes; // System errno
	}
	return rc;
	}

	/*!
	* \brief Send an IPC XML message
	*
	* \param[in,out] client Connection to IPC server
	* \param[in] message XML message to send
	* \param[in] flags Bitmask of crm_ipc_flags
	* \param[in] ms_timeout Give up if not sent within this much time
	* (5 seconds if 0, or no timeout if negative)
	* \param[out] reply Reply from server (or NULL if none)
	*
	* \return Negative errno on error, otherwise size of reply received in bytes
	* if reply was needed, otherwise number of bytes sent
	*/
	int
	crm_ipc_send(crm_ipc_t client, const xmlNode message,
	enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply)
	{
	int rc = 0;
	ssize_t qb_rc = 0;
	ssize_t bytes = 0;
	struct iovec *iov;
	static uint32_t id = 0;
	static int factor = 8;
	pcmk__ipc_header_t *header;

	if (client == NULL) {
	crm_notice("Can't send IPC request without connection (bug?): %.100s",
	message);
	return -ENOTCONN;

	} else if (!crm_ipc_connected(client)) {
	/* Don't even bother */
	crm_notice("Can't send %s IPC requests: Connection closed",
	client->server_name);
	return -ENOTCONN;
	}

	if (ms_timeout == 0) {
	ms_timeout = 5000;
	}

	if (client->need_reply) {
	qb_rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout);
	if (qb_rc < 0) {
	crm_warn("Sending %s IPC disabled until pending reply received",
	client->server_name);
	return -EALREADY;

	} else {
	crm_notice("Sending %s IPC re-enabled after pending reply received",
	client->server_name);
	client->need_reply = FALSE;
	}
	}

	id++;
	CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */
	rc = pcmk__ipc_prepare_iov(id, message, client->max_buf_size, &iov, &bytes);
	if (rc != pcmk_rc_ok) {
	crm_warn("Couldn't prepare %s IPC request: %s " CRM_XS " rc=%d",
	client->server_name, pcmk_rc_str(rc), rc);
	return pcmk_rc2legacy(rc);
	}

	header = iov[0].iov_base;
	pcmk__set_ipc_flags(header->flags, client->server_name, flags);

	if (pcmk_is_set(flags, crm_ipc_proxied)) {
	/* Don't look for a synchronous response */
	pcmk__clear_ipc_flags(flags, "client", crm_ipc_client_response);
	}

	if(header->size_compressed) {
	if(factor < 10 && (client->max_buf_size / 10) < (bytes / factor)) {
	crm_notice("Compressed message exceeds %d0%% of configured IPC "
	"limit (%u bytes); consider setting PCMK_ipc_buffer to "
	"%u or higher",
	factor, client->max_buf_size, 2 * client->max_buf_size);
	factor++;
	}
	}

	crm_trace("Sending %s IPC request %d of %u bytes using %dms timeout",
	client->server_name, header->qb.id, header->qb.size, ms_timeout);

	if ((ms_timeout > 0) \|\| !pcmk_is_set(flags, crm_ipc_client_response)) {

	time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);

	do {
	/* @TODO Is this check really needed? Won't qb_ipcc_sendv() return
	* an error if it's not connected?
	*/
	if (!crm_ipc_connected(client)) {
	goto send_cleanup;
	}

	qb_rc = qb_ipcc_sendv(client->ipc, iov, 2);
	} while ((qb_rc == -EAGAIN) && (time(NULL) < timeout));

	rc = (int) qb_rc; // Negative of system errno, or bytes sent
	if (qb_rc <= 0) {
	goto send_cleanup;

	} else if (!pcmk_is_set(flags, crm_ipc_client_response)) {
	crm_trace("Not waiting for reply to %s IPC request %d",
	client->server_name, header->qb.id);
	goto send_cleanup;
	}

	rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout, &bytes);
	if (rc != pcmk_rc_ok) {
	/* We didn't get the reply in time, so disable future sends for now.
	* The only alternative would be to close the connection since we
	* don't know how to detect and discard out-of-sequence replies.
	*
	* @TODO Implement out-of-sequence detection
	*/
	client->need_reply = TRUE;
	}
	rc = (int) bytes; // Negative system errno, or size of reply received

	} else {
	// No timeout, and client response needed
	do {
	qb_rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer,
	client->buf_size, -1);
	} while ((qb_rc == -EAGAIN) && crm_ipc_connected(client));
	rc = (int) qb_rc; // Negative system errno, or size of reply received
	}

	if (rc > 0) {
	pcmk__ipc_header_t hdr = (pcmk__ipc_header_t )(void*)client->buffer;

	crm_trace("Received %d-byte reply %d to %s IPC %d: %.100s",
	rc, hdr->qb.id, client->server_name, header->qb.id,
	crm_ipc_buffer(client));

	if (reply) {
	*reply = pcmk__xml_parse(crm_ipc_buffer(client));
	}

	} else {
	crm_trace("No reply to %s IPC %d: rc=%d",
	client->server_name, header->qb.id, rc);
	}

	send_cleanup:
	if (!crm_ipc_connected(client)) {
	crm_notice("Couldn't send %s IPC request %d: Connection closed "
	CRM_XS " rc=%d", client->server_name, header->qb.id, rc);

	} else if (rc == -ETIMEDOUT) {
	crm_warn("%s IPC request %d failed: %s after %dms " CRM_XS " rc=%d",
	client->server_name, header->qb.id, pcmk_strerror(rc),
	ms_timeout, rc);
	crm_write_blackbox(0, NULL);

	} else if (rc <= 0) {
	crm_warn("%s IPC request %d failed: %s " CRM_XS " rc=%d",
	client->server_name, header->qb.id,
	((rc == 0)? "No bytes sent" : pcmk_strerror(rc)), rc);
	}

	pcmk_free_ipc_event(iov);
	return rc;
	}

	/*!
	* \brief Ensure an IPC provider has expected user or group
	*
	* \param[in] qb_ipc libqb client connection if available
	* \param[in] sock Connected Unix socket for IPC
	* \param[in] refuid Expected user ID
	* \param[in] refgid Expected group ID
	* \param[out] gotpid If not NULL, where to store provider's actual process ID
	* (or 1 on platforms where ID is not available)
	* \param[out] gotuid If not NULL, where to store provider's actual user ID
	* \param[out] gotgid If not NULL, where to store provider's actual group ID
	*
	* \return Standard Pacemaker return code
	* \note An actual user ID of 0 (root) will always be considered authorized,
	* regardless of the expected values provided. The caller can use the
	* output arguments to be stricter than this function.
	*/
	static int
	is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
	uid_t refuid, gid_t refgid,
	pid_t gotpid, uid_t gotuid, gid_t *gotgid)
	{
	int rc = EOPNOTSUPP;
	pid_t found_pid = 0;
	uid_t found_uid = 0;
	gid_t found_gid = 0;

	#ifdef HAVE_QB_IPCC_AUTH_GET
	if (qb_ipc != NULL) {
	rc = qb_ipcc_auth_get(qb_ipc, &found_pid, &found_uid, &found_gid);
	rc = -rc; // libqb returns 0 or -errno
	if (rc == pcmk_rc_ok) {
	goto found;
	}
	}
	#endif

	#ifdef HAVE_UCRED
	{
	struct ucred ucred;
	socklen_t ucred_len = sizeof(ucred);

	if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &ucred_len) < 0) {
	rc = errno;
	} else if (ucred_len != sizeof(ucred)) {
	rc = EOPNOTSUPP;
	} else {
	found_pid = ucred.pid;
	found_uid = ucred.uid;
	found_gid = ucred.gid;
	goto found;
	}
	}
	#endif

	#ifdef HAVE_SOCKPEERCRED
	{
	struct sockpeercred sockpeercred;
	socklen_t sockpeercred_len = sizeof(sockpeercred);

	if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED,
	&sockpeercred, &sockpeercred_len) < 0) {
	rc = errno;
	} else if (sockpeercred_len != sizeof(sockpeercred)) {
	rc = EOPNOTSUPP;
	} else {
	found_pid = sockpeercred.pid;
	found_uid = sockpeercred.uid;
	found_gid = sockpeercred.gid;
	goto found;
	}
	}
	#endif

	#ifdef HAVE_GETPEEREID // For example, FreeBSD
	if (getpeereid(sock, &found_uid, &found_gid) < 0) {
	rc = errno;
	} else {
	found_pid = PCMK__SPECIAL_PID;
	goto found;
	}
	#endif

	#ifdef HAVE_GETPEERUCRED
	{
	ucred_t *ucred = NULL;

	if (getpeerucred(sock, &ucred) < 0) {
	rc = errno;
	} else {
	found_pid = ucred_getpid(ucred);
	found_uid = ucred_geteuid(ucred);
	found_gid = ucred_getegid(ucred);
	ucred_free(ucred);
	goto found;
	}
	}
	#endif

	return rc; // If we get here, nothing succeeded

	found:
	if (gotpid != NULL) {
	*gotpid = found_pid;
	}
	if (gotuid != NULL) {
	*gotuid = found_uid;
	}
	if (gotgid != NULL) {
	*gotgid = found_gid;
	}
	if ((found_uid != 0) && (found_uid != refuid) && (found_gid != refgid)) {
	return pcmk_rc_ipc_unauthorized;
	}
	return pcmk_rc_ok;
	}

	int
	crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid,
	pid_t gotpid, uid_t gotuid, gid_t *gotgid)
	{
	int ret = is_ipc_provider_expected(NULL, sock, refuid, refgid,
	gotpid, gotuid, gotgid);

	/* The old function had some very odd return codes*/
	if (ret == 0) {
	return 1;
	} else if (ret == pcmk_rc_ipc_unauthorized) {
	return 0;
	} else {
	return pcmk_rc2legacy(ret);
	}
	}

	int
	pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid,
	gid_t refgid, pid_t *gotpid)
	{
	static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */
	int fd;
	int rc = pcmk_rc_ipc_unresponsive;
	int auth_rc = 0;
	int32_t qb_rc;
	pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0;
	qb_ipcc_connection_t *c;
	#ifdef HAVE_QB_IPCC_CONNECT_ASYNC
	struct pollfd pollfd = { 0, };
	int poll_rc;

	c = qb_ipcc_connect_async(name, 0,
	&(pollfd.fd));
	#else
	c = qb_ipcc_connect(name, 0);
	#endif
	if (c == NULL) {
	crm_info("Could not connect to %s IPC: %s", name, strerror(errno));
	rc = pcmk_rc_ipc_unresponsive;
	goto bail;
	}
	#ifdef HAVE_QB_IPCC_CONNECT_ASYNC
	pollfd.events = POLLIN;
	do {
	poll_rc = poll(&pollfd, 1, 2000);
	} while ((poll_rc == -1) && (errno == EINTR));

	/* If poll() failed, given that disconnect function is not registered yet,
	* qb_ipcc_disconnect() won't clean up the socket. In any case, call
	* qb_ipcc_connect_continue() here so that it may fail and do the cleanup
	* for us.
	*/
	if (qb_ipcc_connect_continue(c) != 0) {
	crm_info("Could not connect to %s IPC: %s", name,
	(poll_rc == 0)?"timeout":strerror(errno));
	rc = pcmk_rc_ipc_unresponsive;
	c = NULL; // qb_ipcc_connect_continue cleaned up for us
	goto bail;
	}
	#endif

	qb_rc = qb_ipcc_fd_get(c, &fd);
	if (qb_rc != 0) {
	rc = (int) -qb_rc; // System errno
	crm_err("Could not get fd from %s IPC: %s " CRM_XS " rc=%d",
	name, pcmk_rc_str(rc), rc);
	goto bail;
	}

	auth_rc = is_ipc_provider_expected(c, fd, refuid, refgid,
	&found_pid, &found_uid, &found_gid);
	if (auth_rc == pcmk_rc_ipc_unauthorized) {
	crm_err("Daemon (IPC %s) effectively blocked with unauthorized"
	" process %lld (uid: %lld, gid: %lld)",
	name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	rc = pcmk_rc_ipc_unauthorized;
	goto bail;
	}

	if (auth_rc != pcmk_rc_ok) {
	rc = auth_rc;
	crm_err("Could not get peer credentials from %s IPC: %s "
	CRM_XS " rc=%d", name, pcmk_rc_str(rc), rc);
	goto bail;
	}

	if (gotpid != NULL) {
	*gotpid = found_pid;
	}

	rc = pcmk_rc_ok;
	if ((found_uid != refuid \|\| found_gid != refgid)
	&& strncmp(last_asked_name, name, sizeof(last_asked_name))) {
	if ((found_uid == 0) && (refuid != 0)) {
	crm_warn("Daemon (IPC %s) runs as root, whereas the expected"
	" credentials are %lld:%lld, hazard of violating"
	" the least privilege principle",
	name, (long long) refuid, (long long) refgid);
	} else {
	crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the"
	" expected credentials are %lld:%lld, which may"
	" mean a different set of privileges than expected",
	name, (long long) found_uid, (long long) found_gid,
	(long long) refuid, (long long) refgid);
	}
	memccpy(last_asked_name, name, '\0', sizeof(last_asked_name));
	}

	bail:
	if (c != NULL) {
	qb_ipcc_disconnect(c);
	}
	return rc;
	}
	diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
	index 5cac86030f..dc173377fc 100644
	--- a/lib/common/ipc_controld.c
	+++ b/lib/common/ipc_controld.c
	@@ -1,657 +1,657 @@
	/*
	* Copyright 2020-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <stdbool.h>
	#include <errno.h>
	#include <libxml/tree.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/ipc_controld.h>
	#include "crmcommon_private.h"

	struct controld_api_private_s {
	char *client_uuid;
	unsigned int replies_expected;
	};

	/*!
	* \internal
	* \brief Get a string representation of a controller API reply type
	*
	* \param[in] reply Controller API reply type
	*
	* \return String representation of a controller API reply type
	*/
	const char *
	pcmk__controld_api_reply2str(enum pcmk_controld_api_reply reply)
	{
	switch (reply) {
	case pcmk_controld_reply_reprobe:
	return "reprobe";
	case pcmk_controld_reply_info:
	return "info";
	case pcmk_controld_reply_resource:
	return "resource";
	case pcmk_controld_reply_ping:
	return "ping";
	case pcmk_controld_reply_nodes:
	return "nodes";
	default:
	return "unknown";
	}
	}

	// \return Standard Pacemaker return code
	static int
	new_data(pcmk_ipc_api_t *api)
	{
	struct controld_api_private_s *private = NULL;

	api->api_data = calloc(1, sizeof(struct controld_api_private_s));

	if (api->api_data == NULL) {
	return errno;
	}

	private = api->api_data;

	/* This is set to the PID because that's how it was always done, but PIDs
	* are not unique because clients can be remote. The value appears to be
	* unused other than as part of PCMK__XA_CRM_SYS_FROM in IPC requests, which
	* is only compared against the internal system names (CRM_SYSTEM_TENGINE,
	* etc.), so it shouldn't be a problem.
	*/
	private->client_uuid = pcmk__getpid_s();

	/* @TODO Implement a call ID model similar to the CIB, executor, and fencer
	* IPC APIs, so that requests and replies can be matched, and
	* duplicate replies can be discarded.
	*/
	return pcmk_rc_ok;
	}

	static void
	free_data(void *data)
	{
	free(((struct controld_api_private_s *) data)->client_uuid);
	free(data);
	}

	// \return Standard Pacemaker return code
	static int
	post_connect(pcmk_ipc_api_t *api)
	{
	/* The controller currently requires clients to register via a hello
	* request, but does not reply back.
	*/
	struct controld_api_private_s *private = api->api_data;
	const char *client_name = crm_system_name? crm_system_name : "client";
	xmlNode *hello;
	int rc;

	hello = create_hello_message(private->client_uuid, client_name,
	PCMK__CONTROLD_API_MAJOR,
	PCMK__CONTROLD_API_MINOR);
	rc = pcmk__send_ipc_request(api, hello);
	pcmk__xml_free(hello);
	if (rc != pcmk_rc_ok) {
	crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
	} else {
	crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
	}
	return rc;
	}

	static void
	set_node_info_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	data->reply_type = pcmk_controld_reply_info;
	if (msg_data == NULL) {
	return;
	}
	data->data.node_info.have_quorum =
	pcmk__xe_attr_is_true(msg_data, PCMK_XA_HAVE_QUORUM);
	data->data.node_info.is_remote =
	pcmk__xe_attr_is_true(msg_data, PCMK_XA_REMOTE_NODE);

	/* Integer node_info.id is currently valid only for Corosync nodes.
	*
	* @TODO: Improve handling after crm_node_t is refactored to handle layer-
	* specific data better.
	*/
	crm_element_value_int(msg_data, PCMK_XA_ID, &(data->data.node_info.id));

	data->data.node_info.uuid = crm_element_value(msg_data, PCMK_XA_ID);
	data->data.node_info.uname = crm_element_value(msg_data, PCMK_XA_UNAME);
	data->data.node_info.state = crm_element_value(msg_data, PCMK_XA_CRMD);
	}

	static void
	set_ping_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	data->reply_type = pcmk_controld_reply_ping;
	if (msg_data == NULL) {
	return;
	}
	data->data.ping.sys_from = crm_element_value(msg_data,
	PCMK__XA_CRM_SUBSYSTEM);
	data->data.ping.fsa_state = crm_element_value(msg_data,
	PCMK__XA_CRMD_STATE);
	data->data.ping.result = crm_element_value(msg_data, PCMK_XA_RESULT);
	}

	static void
	set_nodes_data(pcmk_controld_api_reply_t data, xmlNode msg_data)
	{
	pcmk_controld_api_node_t *node_info;

	data->reply_type = pcmk_controld_reply_nodes;
	for (xmlNode *node = pcmk__xe_first_child(msg_data, PCMK_XE_NODE, NULL,
	NULL);
	node != NULL; node = pcmk__xe_next_same(node)) {

	long long id_ll = 0;

	node_info = pcmk__assert_alloc(1, sizeof(pcmk_controld_api_node_t));
	crm_element_value_ll(node, PCMK_XA_ID, &id_ll);
	if (id_ll > 0) {
	node_info->id = id_ll;
	}
	node_info->uname = crm_element_value(node, PCMK_XA_UNAME);
	node_info->state = crm_element_value(node, PCMK__XA_IN_CCM);
	data->data.nodes = g_list_prepend(data->data.nodes, node_info);
	}
	}

	static bool
	reply_expected(pcmk_ipc_api_t api, const xmlNode request)
	{
	// We only need to handle commands that API functions can send
	return pcmk__str_any_of(crm_element_value(request, PCMK__XA_CRM_TASK),
	PCMK__CONTROLD_CMD_NODES,
	CRM_OP_LRM_DELETE,
	CRM_OP_LRM_FAIL,
	CRM_OP_NODE_INFO,
	CRM_OP_PING,
	CRM_OP_REPROBE,
	CRM_OP_RM_NODE_CACHE,
	NULL);
	}

	static bool
	dispatch(pcmk_ipc_api_t api, xmlNode reply)
	{
	struct controld_api_private_s *private = api->api_data;
	crm_exit_t status = CRM_EX_OK;
	xmlNode *wrapper = NULL;
	xmlNode *msg_data = NULL;
	const char *value = NULL;
	pcmk_controld_api_reply_t reply_data = {
	pcmk_controld_reply_unknown, NULL, NULL,
	};

	if (pcmk__xe_is(reply, PCMK__XE_ACK)) {
	/* ACKs are trivial responses that do not count toward expected replies,
	* and do not have all the fields that validation requires, so skip that
	* processing.
	*/
	return private->replies_expected > 0;
	}

	if (private->replies_expected > 0) {
	private->replies_expected--;
	}

	// Do some basic validation of the reply

	/* @TODO We should be able to verify that value is always a response, but
	* currently the controller doesn't always properly set the type. Even
	* if we fix the controller, we'll still need to handle replies from
	* old versions (feature set could be used to differentiate).
	*/
	value = crm_element_value(reply, PCMK__XA_SUBT);
	if (!pcmk__str_any_of(value, PCMK__VALUE_REQUEST, PCMK__VALUE_RESPONSE,
	NULL)) {
	crm_info("Unrecognizable message from controller: "
	"invalid message type '%s'", pcmk__s(value, ""));
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	if (pcmk__str_empty(crm_element_value(reply, PCMK_XA_REFERENCE))) {
	crm_info("Unrecognizable message from controller: no reference");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	value = crm_element_value(reply, PCMK__XA_CRM_TASK);
	if (pcmk__str_empty(value)) {
	crm_info("Unrecognizable message from controller: no command name");
	status = CRM_EX_PROTOCOL;
	goto done;
	}

	// Parse useful info from reply

	reply_data.feature_set = crm_element_value(reply, PCMK_XA_VERSION);
	reply_data.host_from = crm_element_value(reply, PCMK__XA_SRC);

	wrapper = pcmk__xe_first_child(reply, PCMK__XE_CRM_XML, NULL, NULL);
	msg_data = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);

	if (!strcmp(value, CRM_OP_REPROBE)) {
	reply_data.reply_type = pcmk_controld_reply_reprobe;

	} else if (!strcmp(value, CRM_OP_NODE_INFO)) {
	set_node_info_data(&reply_data, msg_data);

	} else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
	reply_data.reply_type = pcmk_controld_reply_resource;
	reply_data.data.resource.node_state = msg_data;

	} else if (!strcmp(value, CRM_OP_PING)) {
	set_ping_data(&reply_data, msg_data);

	} else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
	set_nodes_data(&reply_data, msg_data);

	} else {
	crm_info("Unrecognizable message from controller: unknown command '%s'",
	value);
	status = CRM_EX_PROTOCOL;
	}

	done:
	pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);

	// Free any reply data that was allocated
	if (pcmk__str_eq(value, PCMK__CONTROLD_CMD_NODES, pcmk__str_casei)) {
	g_list_free_full(reply_data.data.nodes, free);
	}

	return false; // No further replies needed
	}

	pcmk__ipc_methods_t *
	pcmk__controld_api_methods(void)
	{
	pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));

	if (cmds != NULL) {
	cmds->new_data = new_data;
	cmds->free_data = free_data;
	cmds->post_connect = post_connect;
	cmds->reply_expected = reply_expected;
	cmds->dispatch = dispatch;
	}
	return cmds;
	}

	/*!
	* \internal
	* \brief Create XML for a controller IPC request
	*
	* \param[in] api Controller connection
	* \param[in] op Controller IPC command name
	* \param[in] node Node name to set as destination host
	* \param[in] msg_data XML to attach to request as message data
	*
	* \return Newly allocated XML for request
	*/
	static xmlNode *
	create_controller_request(const pcmk_ipc_api_t api, const char op,
	const char node, xmlNode msg_data)
	{
	struct controld_api_private_s *private = NULL;
	const char *sys_to = NULL;

	if (api == NULL) {
	return NULL;
	}
	private = api->api_data;
	if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
	sys_to = CRM_SYSTEM_DC;
	} else {
	sys_to = CRM_SYSTEM_CRMD;
	}
	return create_request(op, msg_data, node, sys_to,
	(crm_system_name? crm_system_name : "client"),
	private->client_uuid);
	}

	// \return Standard Pacemaker return code
	static int
	send_controller_request(pcmk_ipc_api_t api, const xmlNode request,
	bool reply_is_expected)
	{
	if (crm_element_value(request, PCMK_XA_REFERENCE) == NULL) {
	return EINVAL;
	}
	if (reply_is_expected) {
	struct controld_api_private_s *private = api->api_data;

	private->replies_expected++;
	}
	return pcmk__send_ipc_request(api, request);
	}

	static xmlNode *
	create_reprobe_message_data(const char target_node, const char router_node)
	{
	xmlNode *msg_data;

	msg_data = pcmk__xe_create(NULL, "data_for_" CRM_OP_REPROBE);
	crm_xml_add(msg_data, PCMK__META_ON_NODE, target_node);
	if ((router_node != NULL) && !pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
	crm_xml_add(msg_data, PCMK__XA_ROUTER_NODE, router_node);
	}
	return msg_data;
	}

	/*!
	* \brief Send a reprobe controller operation
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node to reprobe
	* \param[in] router_node Router node for host
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
	*/
	int
	pcmk_controld_api_reprobe(pcmk_ipc_api_t api, const char target_node,
	const char *router_node)
	{
	xmlNode *request;
	xmlNode *msg_data;
	int rc = pcmk_rc_ok;

	if (api == NULL) {
	return EINVAL;
	}
	if (router_node == NULL) {
	router_node = target_node;
	}
	crm_debug("Sending %s IPC request to reprobe %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(target_node, "local node"),
	pcmk__s(router_node, "local node"));
	msg_data = create_reprobe_message_data(target_node, router_node);
	request = create_controller_request(api, CRM_OP_REPROBE, router_node,
	msg_data);
	rc = send_controller_request(api, request, true);
	pcmk__xml_free(msg_data);
	pcmk__xml_free(request);
	return rc;
	}

	/*!
	* \brief Send a "node info" controller operation
	*
	* \param[in,out] api Controller connection
	* \param[in] nodeid ID of node to get info for (or 0 for local node)
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_info.
	*/
	int
	pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
	{
	xmlNode *request;
	int rc = pcmk_rc_ok;

	request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
	if (request == NULL) {
	return EINVAL;
	}
	if (nodeid > 0) {
	- crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
	+ pcmk__xe_set_id(request, "%lu", (unsigned long) nodeid);
	}

	rc = send_controller_request(api, request, true);
	pcmk__xml_free(request);
	return rc;
	}

	/*!
	* \brief Ask the controller for status
	*
	* \param[in,out] api Controller connection
	* \param[in] node_name Name of node whose status is desired (NULL for DC)
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_ping.
	*/
	int
	pcmk_controld_api_ping(pcmk_ipc_api_t api, const char node_name)
	{
	xmlNode *request;
	int rc = pcmk_rc_ok;

	request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
	if (request == NULL) {
	return EINVAL;
	}
	rc = send_controller_request(api, request, true);
	pcmk__xml_free(request);
	return rc;
	}

	/*!
	* \brief Ask the controller for cluster information
	*
	* \param[in,out] api Controller connection
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_nodes.
	*/
	int
	pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api)
	{
	xmlNode *request;
	int rc = EINVAL;

	request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL,
	NULL);
	if (request != NULL) {
	rc = send_controller_request(api, request, true);
	pcmk__xml_free(request);
	}
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	controller_resource_op(pcmk_ipc_api_t api, const char op,
	const char target_node, const char router_node,
	bool cib_only, const char *rsc_id,
	const char rsc_long_id, const char standard,
	const char provider, const char type)
	{
	int rc = pcmk_rc_ok;
	char *key;
	xmlNode request, msg_data, xml_rsc, params;

	if (api == NULL) {
	return EINVAL;
	}
	if (router_node == NULL) {
	router_node = target_node;
	}

	msg_data = pcmk__xe_create(NULL, PCMK__XE_RSC_OP);

	/* The controller logs the transition key from resource op requests, so we
	* need to have something for it.
	* @TODO don't use "crm-resource"
	*/
	key = pcmk__transition_key(0, getpid(), 0,
	"xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
	crm_xml_add(msg_data, PCMK__XA_TRANSITION_KEY, key);
	free(key);

	crm_xml_add(msg_data, PCMK__META_ON_NODE, target_node);
	if (!pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
	crm_xml_add(msg_data, PCMK__XA_ROUTER_NODE, router_node);
	}

	if (cib_only) {
	// Indicate that only the CIB needs to be cleaned
	crm_xml_add(msg_data, PCMK__XA_MODE, PCMK__VALUE_CIB);
	}

	xml_rsc = pcmk__xe_create(msg_data, PCMK_XE_PRIMITIVE);
	crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
	crm_xml_add(xml_rsc, PCMK__XA_LONG_ID, rsc_long_id);
	crm_xml_add(xml_rsc, PCMK_XA_CLASS, standard);
	crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(xml_rsc, PCMK_XA_TYPE, type);

	params = pcmk__xe_create(msg_data, PCMK__XE_ATTRIBUTES);
	crm_xml_add(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);

	// The controller parses the timeout from the request
	key = crm_meta_name(PCMK_META_TIMEOUT);
	crm_xml_add(params, key, "60000"); /* 1 minute */ //@TODO pass as arg
	free(key);

	request = create_controller_request(api, op, router_node, msg_data);
	rc = send_controller_request(api, request, true);
	pcmk__xml_free(msg_data);
	pcmk__xml_free(request);
	return rc;
	}

	/*!
	* \brief Ask the controller to fail a resource
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node resource is on
	* \param[in] router_node Router node for target
	* \param[in] rsc_id ID of resource to fail
	* \param[in] rsc_long_id Long ID of resource (if any)
	* \param[in] standard Standard of resource
	* \param[in] provider Provider of resource (if any)
	* \param[in] type Type of resource to fail
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_resource.
	*/
	int
	pcmk_controld_api_fail(pcmk_ipc_api_t *api,
	const char target_node, const char router_node,
	const char rsc_id, const char rsc_long_id,
	const char standard, const char provider,
	const char *type)
	{
	crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
	pcmk__s(rsc_long_id, "no other names"),
	pcmk__s(target_node, "unspecified node"),
	pcmk__s(router_node, "unspecified node"));
	return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
	router_node, false, rsc_id, rsc_long_id,
	standard, provider, type);
	}

	/*!
	* \brief Ask the controller to refresh a resource
	*
	* \param[in,out] api Controller connection
	* \param[in] target_node Name of node resource is on
	* \param[in] router_node Router node for target
	* \param[in] rsc_id ID of resource to refresh
	* \param[in] rsc_long_id Long ID of resource (if any)
	* \param[in] standard Standard of resource
	* \param[in] provider Provider of resource (if any)
	* \param[in] type Type of resource
	* \param[in] cib_only If true, clean resource from CIB only
	*
	* \return Standard Pacemaker return code
	* \note Event callback will get a reply of type pcmk_controld_reply_resource.
	*/
	int
	pcmk_controld_api_refresh(pcmk_ipc_api_t api, const char target_node,
	const char *router_node,
	const char rsc_id, const char rsc_long_id,
	const char standard, const char provider,
	const char *type, bool cib_only)
	{
	crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
	pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
	pcmk__s(rsc_long_id, "no other names"),
	pcmk__s(target_node, "unspecified node"),
	pcmk__s(router_node, "unspecified node"));
	return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
	router_node, cib_only, rsc_id, rsc_long_id,
	standard, provider, type);
	}

	/*!
	* \brief Get the number of IPC replies currently expected from the controller
	*
	* \param[in] api Controller IPC API connection
	*
	* \return Number of replies expected
	*/
	unsigned int
	pcmk_controld_api_replies_expected(const pcmk_ipc_api_t *api)
	{
	struct controld_api_private_s *private = api->api_data;

	return private->replies_expected;
	}

	/*!
	* \brief Create XML for a controller IPC "hello" message
	*
	* \deprecated This function is deprecated as part of the public C API.
	*/
	// \todo make this static to this file when breaking API backward compatibility
	xmlNode *
	create_hello_message(const char uuid, const char client_name,
	const char major_version, const char minor_version)
	{
	xmlNode *hello_node = NULL;
	xmlNode *hello = NULL;

	if (pcmk__str_empty(uuid) \|\| pcmk__str_empty(client_name)
	\|\| pcmk__str_empty(major_version) \|\| pcmk__str_empty(minor_version)) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"missing information",
	client_name? client_name : "unknown client",
	uuid? uuid : "unknown");
	return NULL;
	}

	hello_node = pcmk__xe_create(NULL, PCMK__XE_OPTIONS);
	crm_xml_add(hello_node, PCMK__XA_MAJOR_VERSION, major_version);
	crm_xml_add(hello_node, PCMK__XA_MINOR_VERSION, minor_version);
	crm_xml_add(hello_node, PCMK__XA_CLIENT_NAME, client_name);

	// @TODO Nothing uses this. Drop, or keep for debugging?
	crm_xml_add(hello_node, PCMK__XA_CLIENT_UUID, uuid);

	hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
	if (hello == NULL) {
	crm_err("Could not create IPC hello message from %s (UUID %s): "
	"Request creation failed", client_name, uuid);
	return NULL;
	}
	pcmk__xml_free(hello_node);

	crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
	return hello;
	}
	diff --git a/lib/common/nvpair.c b/lib/common/nvpair.c
	index 8d020c1aaf..cfe5b9aa55 100644
	--- a/lib/common/nvpair.c
	+++ b/lib/common/nvpair.c
	@@ -1,997 +1,997 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <string.h>
	#include <ctype.h>
	#include <glib.h>
	#include <libxml/tree.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>
	#include "crmcommon_private.h"

	/*
	* This file isolates handling of various kinds of name/value pairs:
	*
	* - pcmk_nvpair_t data type
	* - XML attributes (<TAG ... NAME=VALUE ...>)
	* - XML nvpair elements (<nvpair id=ID name=NAME value=VALUE>)
	* - Meta-attributes (for resources and actions)
	*/

	// pcmk_nvpair_t handling

	/*!
	* \internal
	* \brief Allocate a new name/value pair
	*
	* \param[in] name New name (required)
	* \param[in] value New value
	*
	* \return Newly allocated name/value pair
	* \note The caller is responsible for freeing the result with
	* \c pcmk__free_nvpair().
	*/
	static pcmk_nvpair_t *
	pcmk__new_nvpair(const char name, const char value)
	{
	pcmk_nvpair_t *nvpair = NULL;

	CRM_ASSERT(name);

	nvpair = pcmk__assert_alloc(1, sizeof(pcmk_nvpair_t));

	nvpair->name = pcmk__str_copy(name);
	nvpair->value = pcmk__str_copy(value);
	return nvpair;
	}

	/*!
	* \internal
	* \brief Free a name/value pair
	*
	* \param[in,out] nvpair Name/value pair to free
	*/
	static void
	pcmk__free_nvpair(gpointer data)
	{
	if (data) {
	pcmk_nvpair_t *nvpair = data;

	free(nvpair->name);
	free(nvpair->value);
	free(nvpair);
	}
	}

	/*!
	* \brief Prepend a name/value pair to a list
	*
	* \param[in,out] nvpairs List to modify
	* \param[in] name New entry's name
	* \param[in] value New entry's value
	*
	* \return New head of list
	* \note The caller is responsible for freeing the list with
	* \c pcmk_free_nvpairs().
	*/
	GSList *
	pcmk_prepend_nvpair(GSList nvpairs, const char name, const char *value)
	{
	return g_slist_prepend(nvpairs, pcmk__new_nvpair(name, value));
	}

	/*!
	* \brief Free a list of name/value pairs
	*
	* \param[in,out] list List to free
	*/
	void
	pcmk_free_nvpairs(GSList *nvpairs)
	{
	g_slist_free_full(nvpairs, pcmk__free_nvpair);
	}

	/*!
	* \internal
	* \brief Compare two name/value pairs
	*
	* \param[in] a First name/value pair to compare
	* \param[in] b Second name/value pair to compare
	*
	* \return 0 if a == b, 1 if a > b, -1 if a < b
	*/
	static gint
	pcmk__compare_nvpair(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const pcmk_nvpair_t *pair_a = a;
	const pcmk_nvpair_t *pair_b = b;

	CRM_ASSERT(a != NULL);
	CRM_ASSERT(pair_a->name != NULL);

	CRM_ASSERT(b != NULL);
	CRM_ASSERT(pair_b->name != NULL);

	rc = strcmp(pair_a->name, pair_b->name);
	if (rc < 0) {
	return -1;
	} else if (rc > 0) {
	return 1;
	}
	return 0;
	}

	/*!
	* \brief Sort a list of name/value pairs
	*
	* \param[in,out] list List to sort
	*
	* \return New head of list
	*/
	GSList *
	pcmk_sort_nvpairs(GSList *list)
	{
	return g_slist_sort(list, pcmk__compare_nvpair);
	}

	/*!
	* \brief Create a list of name/value pairs from an XML node's attributes
	*
	* \param[in] XML to parse
	*
	* \return New list of name/value pairs
	* \note It is the caller's responsibility to free the list with
	* \c pcmk_free_nvpairs().
	*/
	GSList *
	pcmk_xml_attrs2nvpairs(const xmlNode *xml)
	{
	GSList *result = NULL;

	for (xmlAttrPtr iter = pcmk__xe_first_attr(xml); iter != NULL;
	iter = iter->next) {

	result = pcmk_prepend_nvpair(result,
	(const char *) iter->name,
	(const char *) pcmk__xml_attr_value(iter));
	}
	return result;
	}

	/*!
	* \internal
	* \brief Add an XML attribute corresponding to a name/value pair
	*
	* Suitable for glib list iterators, this function adds a NAME=VALUE
	* XML attribute based on a given name/value pair.
	*
	* \param[in] data Name/value pair
	* \param[out] user_data XML node to add attributes to
	*/
	static void
	pcmk__nvpair_add_xml_attr(gpointer data, gpointer user_data)
	{
	pcmk_nvpair_t *pair = data;
	xmlNode *parent = user_data;

	crm_xml_add(parent, pair->name, pair->value);
	}

	/*!
	* \brief Add XML attributes based on a list of name/value pairs
	*
	* \param[in,out] list List of name/value pairs
	* \param[in,out] xml XML node to add attributes to
	*/
	void
	pcmk_nvpairs2xml_attrs(GSList list, xmlNode xml)
	{
	g_slist_foreach(list, pcmk__nvpair_add_xml_attr, xml);
	}

	// convenience function for name=value strings

	/*!
	* \internal
	* \brief Extract the name and value from an input string formatted as "name=value".
	* If unable to extract them, they are returned as NULL.
	*
	* \param[in] input The input string, likely from the command line
	* \param[out] name Everything before the first '=' in the input string
	* \param[out] value Everything after the first '=' in the input string
	*
	* \return 2 if both name and value could be extracted, 1 if only one could, and
	* and error code otherwise
	*/
	int
	pcmk__scan_nvpair(const char input, char name, char *value)
	{
	#ifdef HAVE_SSCANF_M
	*name = NULL;
	*value = NULL;
	if (sscanf(input, "%m[^=]=%m[^\n]", name, value) <= 0) {
	return -pcmk_err_bad_nvpair;
	}
	#else
	char *sep = NULL;
	*name = NULL;
	*value = NULL;

	sep = strstr(optarg, "=");
	if (sep == NULL) {
	return -pcmk_err_bad_nvpair;
	}

	*name = strndup(input, sep-input);

	if (*name == NULL) {
	return -ENOMEM;
	}

	/* If the last char in optarg is =, the user gave no
	* value for the option. Leave it as NULL.
	*/
	if (*(sep+1) != '\0') {
	*value = strdup(sep+1);

	if (*value == NULL) {
	return -ENOMEM;
	}
	}
	#endif

	if (name != NULL && value != NULL) {
	return 2;
	} else if (name != NULL \|\| value != NULL) {
	return 1;
	} else {
	return -pcmk_err_bad_nvpair;
	}
	}

	/*!
	* \internal
	* \brief Format a name/value pair.
	*
	* Units can optionally be provided for the value. Note that unlike most
	* formatting functions, this one returns the formatted string. It is
	* assumed that the most common use of this function will be to build up
	* a string to be output as part of other functions.
	*
	* \note The caller is responsible for freeing the return value after use.
	*
	* \param[in] name The name of the nvpair.
	* \param[in] value The value of the nvpair.
	* \param[in] units Optional units for the value, or NULL.
	*
	* \return Newly allocated string with name/value pair
	*/
	char *
	pcmk__format_nvpair(const char name, const char value, const char *units)
	{
	return crm_strdup_printf("%s=\"%s%s\"", name, value, units ? units : "");
	}

	// XML attribute handling

	/*!
	* \brief Create an XML attribute with specified name and value
	*
	* \param[in,out] node XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] value Attribute value to set
	*
	* \return New value on success, \c NULL otherwise
	* \note This does nothing if node, name, or value are \c NULL or empty.
	*/
	const char *
	crm_xml_add(xmlNode node, const char name, const char *value)
	{
	bool dirty = FALSE;
	xmlAttr *attr = NULL;

	CRM_CHECK(node != NULL, return NULL);
	CRM_CHECK(name != NULL, return NULL);

	if (value == NULL) {
	return NULL;
	}

	if (pcmk__tracking_xml_changes(node, FALSE)) {
	const char *old = crm_element_value(node, name);

	if (old == NULL \|\| value == NULL \|\| strcmp(old, value) != 0) {
	dirty = TRUE;
	}
	}

	if (dirty && (pcmk__check_acl(node, name, pcmk__xf_acl_create) == FALSE)) {
	crm_trace("Cannot add %s=%s to %s", name, value, node->name);
	return NULL;
	}

	attr = xmlSetProp(node, (pcmkXmlStr) name, (pcmkXmlStr) value);
	if (dirty) {
	pcmk__mark_xml_attr_dirty(attr);
	}

	CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
	return (char *)attr->children->content;
	}

	/*!
	* \brief Create an XML attribute with specified name and integer value
	*
	* This is like \c crm_xml_add() but taking an integer value.
	*
	* \param[in,out] node XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] value Attribute value to set
	*
	* \return New value as string on success, \c NULL otherwise
	* \note This does nothing if node or name are \c NULL or empty.
	*/
	const char *
	crm_xml_add_int(xmlNode node, const char name, int value)
	{
	char *number = pcmk__itoa(value);
	const char *added = crm_xml_add(node, name, number);

	free(number);
	return added;
	}

	/*!
	* \brief Create an XML attribute with specified name and unsigned value
	*
	* This is like \c crm_xml_add() but taking a guint value.
	*
	* \param[in,out] node XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] ms Attribute value to set
	*
	* \return New value as string on success, \c NULL otherwise
	* \note This does nothing if node or name are \c NULL or empty.
	*/
	const char *
	crm_xml_add_ms(xmlNode node, const char name, guint ms)
	{
	char *number = crm_strdup_printf("%u", ms);
	const char *added = crm_xml_add(node, name, number);

	free(number);
	return added;
	}

	// Maximum size of null-terminated string representation of 64-bit integer
	// -9223372036854775808
	#define LLSTRSIZE 21

	/*!
	* \brief Create an XML attribute with specified name and long long int value
	*
	* This is like \c crm_xml_add() but taking a long long int value. It is a
	* useful equivalent for defined types like time_t, etc.
	*
	* \param[in,out] xml XML node to modify
	* \param[in] name Attribute name to set
	* \param[in] value Attribute value to set
	*
	* \return New value as string on success, \c NULL otherwise
	* \note This does nothing if xml or name are \c NULL or empty.
	* This does not support greater than 64-bit values.
	*/
	const char *
	crm_xml_add_ll(xmlNode xml, const char name, long long value)
	{
	char s[LLSTRSIZE] = { '\0', };

	if (snprintf(s, LLSTRSIZE, "%lld", (long long) value) == LLSTRSIZE) {
	return NULL;
	}
	return crm_xml_add(xml, name, s);
	}

	/*!
	* \brief Create XML attributes for seconds and microseconds
	*
	* This is like \c crm_xml_add() but taking a struct timeval.
	*
	* \param[in,out] xml XML node to modify
	* \param[in] name_sec Name of XML attribute for seconds
	* \param[in] name_usec Name of XML attribute for microseconds (or NULL)
	* \param[in] value Time value to set
	*
	* \return New seconds value as string on success, \c NULL otherwise
	* \note This does nothing if xml, name_sec, or value is \c NULL.
	*/
	const char *
	crm_xml_add_timeval(xmlNode xml, const char name_sec, const char *name_usec,
	const struct timeval *value)
	{
	const char *added = NULL;

	if (xml && name_sec && value) {
	added = crm_xml_add_ll(xml, name_sec, (long long) value->tv_sec);
	if (added && name_usec) {
	// Any error is ignored (we successfully added seconds)
	crm_xml_add_ll(xml, name_usec, (long long) value->tv_usec);
	}
	}
	return added;
	}

	/*!
	* \brief Retrieve the value of an XML attribute
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	*
	* \return Value of specified attribute (may be \c NULL)
	*/
	const char *
	crm_element_value(const xmlNode data, const char name)
	{
	xmlAttr *attr = NULL;

	if (data == NULL) {
	crm_err("Couldn't find %s in NULL", name ? name : "<null>");
	CRM_LOG_ASSERT(data != NULL);
	return NULL;

	} else if (name == NULL) {
	crm_err("Couldn't find NULL in %s", data->name);
	return NULL;
	}

	/* The first argument to xmlHasProp() has always been const,
	* but libxml2 <2.9.2 didn't declare that, so cast it
	*/
	attr = xmlHasProp((xmlNode *) data, (pcmkXmlStr) name);
	if (!attr \|\| !attr->children) {
	return NULL;
	}
	return (const char *) attr->children->content;
	}

	/*!
	* \brief Retrieve the integer value of an XML attribute
	*
	* This is like \c crm_element_value() but getting the value as an integer.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store element value
	*
	* \return 0 on success, -1 otherwise
	*/
	int
	crm_element_value_int(const xmlNode data, const char name, int *dest)
	{
	const char *value = NULL;

	CRM_CHECK(dest != NULL, return -1);
	value = crm_element_value(data, name);
	if (value) {
	long long value_ll;

	if ((pcmk__scan_ll(value, &value_ll, 0LL) != pcmk_rc_ok)
	\|\| (value_ll < INT_MIN) \|\| (value_ll > INT_MAX)) {
	*dest = PCMK__PARSE_INT_DEFAULT;
	} else {
	*dest = (int) value_ll;
	return 0;
	}
	}
	return -1;
	}

	/*!
	* \brief Retrieve the long long integer value of an XML attribute
	*
	* This is like \c crm_element_value() but getting the value as a long long int.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store element value
	*
	* \return 0 on success, -1 otherwise
	*/
	int
	crm_element_value_ll(const xmlNode data, const char name, long long *dest)
	{
	const char *value = NULL;

	CRM_CHECK(dest != NULL, return -1);
	value = crm_element_value(data, name);
	if ((value != NULL)
	&& (pcmk__scan_ll(value, dest, PCMK__PARSE_INT_DEFAULT) == pcmk_rc_ok)) {
	return 0;
	}
	return -1;
	}

	/*!
	* \brief Retrieve the millisecond value of an XML attribute
	*
	* This is like \c crm_element_value() but returning the value as a guint.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store attribute value
	*
	* \return \c pcmk_ok on success, -1 otherwise
	*/
	int
	crm_element_value_ms(const xmlNode data, const char name, guint *dest)
	{
	const char *value = NULL;
	long long value_ll;

	CRM_CHECK(dest != NULL, return -1);
	*dest = 0;
	value = crm_element_value(data, name);
	if ((pcmk__scan_ll(value, &value_ll, 0LL) != pcmk_rc_ok)
	\|\| (value_ll < 0) \|\| (value_ll > G_MAXUINT)) {
	return -1;
	}
	*dest = (guint) value_ll;
	return pcmk_ok;
	}

	/*!
	* \brief Retrieve the seconds-since-epoch value of an XML attribute
	*
	* This is like \c crm_element_value() but returning the value as a time_t.
	*
	* \param[in] xml XML node to check
	* \param[in] name Attribute name to check
	* \param[out] dest Where to store attribute value
	*
	* \return \c pcmk_ok on success, -1 otherwise
	*/
	int
	crm_element_value_epoch(const xmlNode xml, const char name, time_t *dest)
	{
	long long value_ll = 0;

	if (crm_element_value_ll(xml, name, &value_ll) < 0) {
	return -1;
	}

	/* Unfortunately, we can't do any bounds checking, since time_t has neither
	* standardized bounds nor constants defined for them.
	*/
	*dest = (time_t) value_ll;
	return pcmk_ok;
	}

	/*!
	* \brief Retrieve the value of XML second/microsecond attributes as time
	*
	* This is like \c crm_element_value() but returning value as a struct timeval.
	*
	* \param[in] xml XML to parse
	* \param[in] name_sec Name of XML attribute for seconds
	* \param[in] name_usec Name of XML attribute for microseconds
	* \param[out] dest Where to store result
	*
	* \return \c pcmk_ok on success, -errno on error
	* \note Values default to 0 if XML or XML attribute does not exist
	*/
	int
	crm_element_value_timeval(const xmlNode xml, const char name_sec,
	const char name_usec, struct timeval dest)
	{
	long long value_i = 0;

	CRM_CHECK(dest != NULL, return -EINVAL);
	dest->tv_sec = 0;
	dest->tv_usec = 0;

	if (xml == NULL) {
	return pcmk_ok;
	}

	/* Unfortunately, we can't do any bounds checking, since there are no
	* constants provided for the bounds of time_t and suseconds_t, and
	* calculating them isn't worth the effort. If there are XML values
	* beyond the native sizes, there will probably be worse problems anyway.
	*/

	// Parse seconds
	errno = 0;
	if (crm_element_value_ll(xml, name_sec, &value_i) < 0) {
	return -errno;
	}
	dest->tv_sec = (time_t) value_i;

	// Parse microseconds
	if (crm_element_value_ll(xml, name_usec, &value_i) < 0) {
	return -errno;
	}
	dest->tv_usec = (suseconds_t) value_i;

	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Get a date/time object from an XML attribute value
	*
	* \param[in] xml XML with attribute to parse (from CIB)
	* \param[in] attr Name of attribute to parse
	* \param[out] t Where to create date/time object
	* (\p *t must be NULL initially)
	*
	* \return Standard Pacemaker return code
	* \note The caller is responsible for freeing \p *t using crm_time_free().
	*/
	int
	pcmk__xe_get_datetime(const xmlNode xml, const char attr, crm_time_t **t)
	{
	const char *value = NULL;

	if ((t == NULL) \|\| (*t != NULL) \|\| (xml == NULL) \|\| (attr == NULL)) {
	return EINVAL;
	}

	value = crm_element_value(xml, attr);
	if (value != NULL) {
	*t = crm_time_new(value);
	if (*t == NULL) {
	return pcmk_rc_unpack_error;
	}
	}
	return pcmk_rc_ok;
	}

	/*!
	* \brief Retrieve a copy of the value of an XML attribute
	*
	* This is like \c crm_element_value() but allocating new memory for the result.
	*
	* \param[in] data XML node to check
	* \param[in] name Attribute name to check
	*
	* \return Value of specified attribute (may be \c NULL)
	* \note The caller is responsible for freeing the result.
	*/
	char *
	crm_element_value_copy(const xmlNode data, const char name)
	{
	return pcmk__str_copy(crm_element_value(data, name));
	}

	/*!
	* \brief Safely add hash table entry to XML as attribute or name-value pair
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as user data, and adds an XML attribute
	* with the specified name and value if it does not already exist. If the key
	* name starts with a digit, then it's not a valid XML attribute name. In that
	* case, this will instead add a <tt><param name=NAME value=VALUE/></tt> child
	* to the XML.
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2smartfield(gpointer key, gpointer value, gpointer user_data)
	{
	/* @TODO Generate PCMK__XE_PARAM nodes for all keys that aren't valid XML
	* attribute names (not just those that start with digits), or possibly for
	* all keys to simplify parsing.
	*
	* Consider either deprecating as public API or exposing PCMK__XE_PARAM.
	* PCMK__XE_PARAM is currently private because it doesn't appear in any
	* output that Pacemaker generates.
	*/
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if (isdigit(name[0])) {
	xmlNode *tmp = pcmk__xe_create(xml_node, PCMK__XE_PARAM);

	crm_xml_add(tmp, PCMK_XA_NAME, name);
	crm_xml_add(tmp, PCMK_XA_VALUE, s_value);

	} else if (crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);
	crm_trace("dumped: %s=%s", name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	/*!
	* \brief Set XML attribute based on hash table entry
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as user data, and adds an XML attribute
	* with the specified name and value if it does not already exist.
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2field(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if (crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	/*!
	* \brief Set XML attribute based on hash table entry, as meta-attribute name
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as user data, and adds an XML attribute
	* with the meta-attribute version of the specified name and value if it does
	* not already exist and if the name does not appear to be cluster-internal.
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2metafield(gpointer key, gpointer value, gpointer user_data)
	{
	char *crm_name = NULL;

	if (key == NULL \|\| value == NULL) {
	return;
	}

	/* Filter out cluster-generated attributes that contain a '#' or ':'
	* (like fail-count and last-failure).
	*/
	for (crm_name = key; *crm_name; ++crm_name) {
	if ((crm_name == '#') \|\| (crm_name == ':')) {
	return;
	}
	}

	crm_name = crm_meta_name(key);
	hash2field(crm_name, value, user_data);
	free(crm_name);
	}

	// nvpair handling

	/*!
	* \brief Create an XML name/value pair
	*
	* \param[in,out] parent If not \c NULL, make new XML node a child of this one
	* \param[in] id Set this as XML ID (or NULL to auto-generate)
	* \param[in] name Name to use
	* \param[in] value Value to use
	*
	* \return New XML object on success, \c NULL otherwise
	*/
	xmlNode *
	crm_create_nvpair_xml(xmlNode parent, const char id, const char *name,
	const char *value)
	{
	xmlNode *nvp;

	/* id can be NULL so we auto-generate one, and name can be NULL if this
	* will be used to delete a name/value pair by ID, but both can't be NULL
	*/
	CRM_CHECK(id \|\| name, return NULL);

	nvp = pcmk__xe_create(parent, PCMK_XE_NVPAIR);

	if (id) {
	crm_xml_add(nvp, PCMK_XA_ID, id);
	} else {
	- crm_xml_set_id(nvp, "%s-%s",
	- pcmk__s(pcmk__xe_id(parent), PCMK_XE_NVPAIR), name);
	+ pcmk__xe_set_id(nvp, "%s-%s",
	+ pcmk__s(pcmk__xe_id(parent), PCMK_XE_NVPAIR), name);
	}
	crm_xml_add(nvp, PCMK_XA_NAME, name);
	crm_xml_add(nvp, PCMK_XA_VALUE, value);
	return nvp;
	}

	/*!
	* \brief Add XML nvpair element based on hash table entry
	*
	* Suitable for \c g_hash_table_foreach(), this function takes a hash table key
	* and value, with an XML node passed as the user data, and adds an \c nvpair
	* XML element with the specified name and value.
	*
	* \param[in] key Key of hash table entry
	* \param[in] value Value of hash table entry
	* \param[in,out] user_data XML node
	*/
	void
	hash2nvpair(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;
	xmlNode *xml_node = user_data;

	crm_create_nvpair_xml(xml_node, name, name, s_value);
	crm_trace("dumped: name=%s value=%s", name, s_value);
	}

	/*!
	* \brief Retrieve XML attributes as a hash table
	*
	* Given an XML element, this will look for any \<attributes> element child,
	* creating a hash table of (newly allocated string) name/value pairs taken
	* first from the attributes element's NAME=VALUE XML attributes, and then
	* from any \<param name=NAME value=VALUE> children of attributes.
	*
	* \param[in] XML node to parse
	*
	* \return Hash table with name/value pairs
	* \note It is the caller's responsibility to free the result using
	* \c g_hash_table_destroy().
	*/
	GHashTable *
	xml2list(const xmlNode *parent)
	{
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	xmlNode *nvpair_list = NULL;
	GHashTable *nvpair_hash = pcmk__strkey_table(free, free);

	CRM_CHECK(parent != NULL, return nvpair_hash);

	nvpair_list = pcmk__xe_first_child(parent, PCMK__XE_ATTRIBUTES, NULL, NULL);
	if (nvpair_list == NULL) {
	crm_trace("No attributes in %s", parent->name);
	crm_log_xml_trace(parent, "No attributes for resource op");
	}

	crm_log_xml_trace(nvpair_list, "Unpacking");

	for (pIter = pcmk__xe_first_attr(nvpair_list); pIter != NULL;
	pIter = pIter->next) {

	const char p_name = (const char )pIter->name;
	const char *p_value = pcmk__xml_attr_value(pIter);

	crm_trace("Added %s=%s", p_name, p_value);

	pcmk__insert_dup(nvpair_hash, p_name, p_value);
	}

	for (child = pcmk__xe_first_child(nvpair_list, PCMK__XE_PARAM, NULL, NULL);
	child != NULL; child = pcmk__xe_next_same(child)) {

	const char *key = crm_element_value(child, PCMK_XA_NAME);
	const char *value = crm_element_value(child, PCMK_XA_VALUE);

	crm_trace("Added %s=%s", key, value);
	if (key != NULL && value != NULL) {
	pcmk__insert_dup(nvpair_hash, key, value);
	}
	}

	return nvpair_hash;
	}

	void
	pcmk__xe_set_bool_attr(xmlNodePtr node, const char *name, bool value)
	{
	crm_xml_add(node, name, pcmk__btoa(value));
	}

	int
	pcmk__xe_get_bool_attr(const xmlNode node, const char name, bool *value)
	{
	const char *xml_value = NULL;
	int ret, rc;

	if (node == NULL) {
	return ENODATA;
	} else if (name == NULL \|\| value == NULL) {
	return EINVAL;
	}

	xml_value = crm_element_value(node, name);

	if (xml_value == NULL) {
	return ENODATA;
	}

	rc = crm_str_to_boolean(xml_value, &ret);
	if (rc == 1) {
	*value = ret;
	return pcmk_rc_ok;
	} else {
	return pcmk_rc_bad_input;
	}
	}

	bool
	pcmk__xe_attr_is_true(const xmlNode node, const char name)
	{
	bool value = false;
	int rc;

	rc = pcmk__xe_get_bool_attr(node, name, &value);
	return rc == pcmk_rc_ok && value == true;
	}

	// Meta-attribute handling

	/*!
	* \brief Get the environment variable equivalent of a meta-attribute name
	*
	* \param[in] attr_name Name of meta-attribute
	*
	* \return Newly allocated string for \p attr_name with "CRM_meta_" prefix and
	* underbars instead of dashes
	* \note This asserts on an invalid argument or memory allocation error, so
	* callers can assume the result is non-NULL. The caller is responsible
	* for freeing the result using free().
	*/
	char *
	crm_meta_name(const char *attr_name)
	{
	char *env_name = NULL;

	CRM_ASSERT(!pcmk__str_empty(attr_name));

	env_name = crm_strdup_printf(CRM_META "_%s", attr_name);
	for (char c = env_name; c != '\0'; ++c) {
	if (*c == '-') {
	*c = '_';
	}
	}
	return env_name;
	}

	/*!
	* \brief Get the value of a meta-attribute
	*
	* Get the value of a meta-attribute from a hash table whose keys are
	* meta-attribute environment variable names (as crm_meta_name() would
	* create, like pcmk__graph_action_t:params, not pcmk_resource_t:meta).
	*
	* \param[in] meta Hash table of meta-attributes
	* \param[in] attr_name Name of meta-attribute to get
	*
	* \return Value of given meta-attribute
	*/
	const char *
	crm_meta_value(GHashTable meta, const char attr_name)
	{
	if ((meta != NULL) && (attr_name != NULL)) {
	char *key = crm_meta_name(attr_name);
	const char *value = g_hash_table_lookup(meta, key);

	free(key);
	return value;
	}
	return NULL;
	}
	diff --git a/lib/common/tests/xml/Makefile.am b/lib/common/tests/xml/Makefile.am
	index 9afd05d6e4..db0cbaa120 100644
	--- a/lib/common/tests/xml/Makefile.am
	+++ b/lib/common/tests/xml/Makefile.am
	@@ -1,22 +1,26 @@
	#
	# Copyright 2022-2024 the Pacemaker project contributors
	#
	# The version control history for this file may have further details.
	#
	# This source code is licensed under the GNU General Public License version 2
	# or later (GPLv2+) WITHOUT ANY WARRANTY.
	#

	include $(top_srcdir)/mk/tap.mk
	include $(top_srcdir)/mk/unittest.mk

	# Add "_test" to the end of all test program names to simplify .gitignore.
	check_PROGRAMS = pcmk__xe_copy_attrs_test \
	pcmk__xe_first_child_test \
	pcmk__xe_foreach_child_test \
	+ pcmk__xe_set_id_test \
	pcmk__xe_set_score_test \
	pcmk__xml_escape_test \
	pcmk__xml_init_test \
	- pcmk__xml_needs_escape_test
	+ pcmk__xml_is_name_char_test \
	+ pcmk__xml_is_name_start_char_test \
	+ pcmk__xml_needs_escape_test \
	+ pcmk__xml_sanitize_id_test

	TESTS = $(check_PROGRAMS)
	diff --git a/lib/common/tests/xml/pcmk__xe_set_id_test.c b/lib/common/tests/xml/pcmk__xe_set_id_test.c
	new file mode 100644
	index 0000000000..8c7f6c19e8
	--- /dev/null
	+++ b/lib/common/tests/xml/pcmk__xe_set_id_test.c
	@@ -0,0 +1,102 @@
	+/*
	+ * Copyright 2024 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU General Public License version 2
	+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#include <crm_internal.h>
	+
	+#include <crm/common/unittest_internal.h>
	+#include <crm/common/xml_internal.h>
	+
	+static void
	+null_node(void **state)
	+{
	+ pcmk__xe_set_id(NULL, "test_id");
	+}
	+
	+static void
	+null_format(void **state)
	+{
	+ xmlNode *test_xml = pcmk__xe_create(NULL, "test");
	+
	+ pcmk__assert_asserts(pcmk__xe_set_id(NULL, NULL));
	+ pcmk__assert_asserts(pcmk__xe_set_id(test_xml, NULL));
	+
	+#ifdef HAVE_FORMAT_NONLITERAL
	+#pragma GCC diagnostic push
	+#pragma GCC diagnostic ignored "-Wformat-zero-length"
	+#endif // HAVE_FORMAT_NONLITERAL
	+ pcmk__assert_asserts(pcmk__xe_set_id(test_xml, ""));
	+#ifdef HAVE_FORMAT_NONLITERAL
	+#pragma GCC diagnostic pop
	+#endif // HAVE_FORMAT_NONLITERAL
	+
	+ pcmk__xml_free(test_xml);
	+}
	+
	+static void
	+valid_id(void **state)
	+{
	+ // IDs that are already valid XML names
	+ xmlNode *test_xml = pcmk__xe_create(NULL, "test");
	+ const char *id = NULL;
	+
	+ // No ID set initially
	+ pcmk__xe_set_id(test_xml, "test_id");
	+ assert_string_equal(pcmk__xe_id(test_xml), "test_id");
	+
	+ // ID already set: test overwrite
	+ // #xEFFFF (NameStartChar), #xB7 (NameChar)
	+ id = "\xF3\xAF\xBF\xBF" "\xC2\xB7";
	+ pcmk__xe_set_id(test_xml, "%s", id);
	+ assert_string_equal(pcmk__xe_id(test_xml), id);
	+
	+ pcmk__xml_free(test_xml);
	+}
	+
	+static void
	+invalid_id(void **state)
	+{
	+ // IDs that need sanitization
	+ xmlNode *test_xml = pcmk__xe_create(NULL, "test");
	+
	+ // No ID set initially
	+ pcmk__xe_set_id(test_xml, "-ab");
	+ assert_string_equal(pcmk__xe_id(test_xml), "_ab");
	+
	+ // ID already set: test overwrite
	+ pcmk__xe_set_id(test_xml, "a$b");
	+ assert_string_equal(pcmk__xe_id(test_xml), "a.b");
	+
	+ pcmk__xe_set_id(test_xml, "ab$");
	+ assert_string_equal(pcmk__xe_id(test_xml), "ab.");
	+
	+ pcmk__xe_set_id(test_xml, "$$$");
	+ assert_string_equal(pcmk__xe_id(test_xml), "_..");
	+
	+ pcmk__xml_free(test_xml);
	+}
	+
	+static void
	+format_args(void **state)
	+{
	+ xmlNode *test_xml = pcmk__xe_create(NULL, "test");
	+ const char *str = "test";
	+ int num = 42;
	+
	+ pcmk__xe_set_id(test_xml, "rsc-%s-%d", str, num);
	+ assert_string_equal(pcmk__xe_id(test_xml), "rsc-test-42");
	+
	+ pcmk__xml_free(test_xml);
	+}
	+
	+PCMK__UNIT_TEST(pcmk__xml_test_setup_group, pcmk__xml_test_teardown_group,
	+ cmocka_unit_test(null_node),
	+ cmocka_unit_test(null_format),
	+ cmocka_unit_test(valid_id),
	+ cmocka_unit_test(invalid_id),
	+ cmocka_unit_test(format_args))
	diff --git a/lib/common/tests/xml/pcmk__xml_is_name_char_test.c b/lib/common/tests/xml/pcmk__xml_is_name_char_test.c
	new file mode 100644
	index 0000000000..49715a3c2e
	--- /dev/null
	+++ b/lib/common/tests/xml/pcmk__xml_is_name_char_test.c
	@@ -0,0 +1,334 @@
	+/*
	+ * Copyright 2024 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU General Public License version 2
	+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#include <crm_internal.h>
	+
	+#include <glib.h> // gchar, g_ascii_isalnum(), etc.
	+
	+#include <crm/common/unittest_internal.h>
	+#include <crm/common/xml_internal.h>
	+
	+#include "crmcommon_private.h" // pcmk__xml_is_name_char()
	+
	+/*!
	+ * \internal
	+ * \brief Assert that a Unicode character is (not) a valid XML \c NameChar
	+ *
	+ * \param[in] c Unicode code point of character to check
	+ * \param[in] reference If \c true, \p c should be considered a valid
	+ * \c NameChar; if \c false, it should not
	+ */
	+static void
	+assert_name_char(int c, bool reference)
	+{
	+ gchar utf8_buf[6] = { 0, };
	+ int len = 4;
	+ int ref_len = g_unichar_to_utf8(c, utf8_buf);
	+ bool result = pcmk__xml_is_name_char(utf8_buf, &len);
	+
	+ if (reference) {
	+ assert_true(result);
	+ } else {
	+ assert_false(result);
	+ }
	+
	+ if ((c < 0xD800) \|\| (c > 0xDFFF)) {
	+ /* Unicode code points in the range D800 to DFFF are UTF-16 surrogate
	+ * pair halves. They can be represented in UTF-8, but they shouldn't
	+ * appear in valid UTF-8-encoded text. RFC 3629 (Nov 2003) says they
	+ * should be treated as invalid:
	+ * https://en.wikipedia.org/wiki/UTF-8#Invalid_sequences_and_error_handling.
	+ *
	+ * GLib treats these characters as valid and returns a length of 3
	+ * bytes. So did libxml until v2.12 (commit 845bd99). Since that commit,
	+ * libxml treats these characters as invalid and returns a length of 0.
	+ * To avoid version-dependent testing behavior, skip the length check
	+ * for code points in that range.
	+ */
	+ assert_int_equal(len, ref_len);
	+ }
	+}
	+
	+static void
	+null_len(void **state)
	+{
	+ assert_true(pcmk__xml_is_name_char("a", NULL));
	+ assert_false(pcmk__xml_is_name_char("@", NULL));
	+}
	+
	+static void
	+ascii(void **state)
	+{
	+ for (int c = 0x00; c <= 0x7F; c++) {
	+ if (g_ascii_isalnum(c)
	+ \|\| c == ':' \|\| c == '_' \|\| c == '-' \|\| c == '.') {
	+
	+ assert_name_char(c, true);
	+ } else {
	+ assert_name_char(c, false);
	+ }
	+ }
	+}
	+
	+static void
	+unicode_0x80_to_0xB6(void **state)
	+{
	+ for (int c = 0x80; c <= 0xB6; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0xB7(void **state)
	+{
	+ assert_name_char(0xB7, true);
	+}
	+
	+static void
	+unicode_0xB8_to_0xBF(void **state)
	+{
	+ for (int c = 0xB8; c <= 0xBF; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0xC0_to_0xD6(void **state)
	+{
	+ for (int c = 0xC0; c <= 0xD6; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xD7(void **state)
	+{
	+ assert_name_char(0xD7, false);
	+}
	+
	+static void
	+unicode_0xD8_to_0xF6(void **state)
	+{
	+ for (int c = 0xD8; c <= 0xF6; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xF7(void **state)
	+{
	+ assert_name_char(0xF7, false);
	+}
	+
	+static void
	+unicode_0xF8_to_0x2FF(void **state)
	+{
	+ for (int c = 0xF8; c <= 0x2FF; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x300_to_0x36F(void **state)
	+{
	+ for (int c = 0x300; c <= 0x36F; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x370_to_0x37D(void **state)
	+{
	+ for (int c = 0x370; c <= 0x37D; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x37E(void **state)
	+{
	+ assert_name_char(0x37E, false);
	+}
	+
	+static void
	+unicode_0x37F_to_0x1FFF(void **state)
	+{
	+ for (int c = 0x37F; c <= 0x1FFF; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x2000_to_0x200B(void **state)
	+{
	+ for (int c = 0x2000; c <= 0x200B; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x200C_to_0x200D(void **state)
	+{
	+ for (int c = 0x200C; c <= 0x200D; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x200E_to_0x203E(void **state)
	+{
	+ for (int c = 0x200E; c <= 0x203E; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x203F_to_0x2040(void **state)
	+{
	+ for (int c = 0x203F; c <= 0x2040; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x2041_to_0x206F(void **state)
	+{
	+ for (int c = 0x2041; c <= 0x206F; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x2070_to_0x218F(void **state)
	+{
	+ for (int c = 0x2070; c <= 0x218F; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x2190_to_0x2BFF(void **state)
	+{
	+ for (int c = 0x2190; c <= 0x2BFF; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x2C00_to_0x2FEF(void **state)
	+{
	+ for (int c = 0x2C00; c <= 0x2FEF; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x2FF0_to_0x3000(void **state)
	+{
	+ for (int c = 0x2FF0; c <= 0x3000; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x3001_to_0xD7FF(void **state)
	+{
	+ for (int c = 0x3001; c <= 0xD7FF; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xD800_to_0xF8FF(void **state)
	+{
	+ for (int c = 0xD800; c <= 0xF8FF; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0xF900_to_0xFDCF(void **state)
	+{
	+ for (int c = 0xF900; c <= 0xFDCF; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xFDD0_to_0xFDEF(void **state)
	+{
	+ for (int c = 0xFDD0; c <= 0xFDEF; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0xFDF0_to_0xFFFD(void **state)
	+{
	+ for (int c = 0xFDF0; c <= 0xFFFD; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xFFFE_to_0xFFFF(void **state)
	+{
	+ for (int c = 0xFFFE; c <= 0xFFFF; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x10000_to_0xEFFFF(void **state)
	+{
	+ for (int c = 0x10000; c <= 0xEFFFF; c++) {
	+ assert_name_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xF0000_to_0x10FFFF(void **state)
	+{
	+ for (int c = 0xF0000; c <= 0x10FFFF; c++) {
	+ assert_name_char(c, false);
	+ }
	+}
	+
	+PCMK__UNIT_TEST(NULL, NULL,
	+ cmocka_unit_test(null_len),
	+ cmocka_unit_test(ascii),
	+ cmocka_unit_test(unicode_0x80_to_0xB6),
	+ cmocka_unit_test(unicode_0xB7),
	+ cmocka_unit_test(unicode_0xB8_to_0xBF),
	+ cmocka_unit_test(unicode_0xC0_to_0xD6),
	+ cmocka_unit_test(unicode_0xD7),
	+ cmocka_unit_test(unicode_0xD8_to_0xF6),
	+ cmocka_unit_test(unicode_0xF7),
	+ cmocka_unit_test(unicode_0xF8_to_0x2FF),
	+ cmocka_unit_test(unicode_0x300_to_0x36F),
	+ cmocka_unit_test(unicode_0x370_to_0x37D),
	+ cmocka_unit_test(unicode_0x37E),
	+ cmocka_unit_test(unicode_0x37F_to_0x1FFF),
	+ cmocka_unit_test(unicode_0x2000_to_0x200B),
	+ cmocka_unit_test(unicode_0x200C_to_0x200D),
	+ cmocka_unit_test(unicode_0x200E_to_0x203E),
	+ cmocka_unit_test(unicode_0x203F_to_0x2040),
	+ cmocka_unit_test(unicode_0x2041_to_0x206F),
	+ cmocka_unit_test(unicode_0x2070_to_0x218F),
	+ cmocka_unit_test(unicode_0x2190_to_0x2BFF),
	+ cmocka_unit_test(unicode_0x2C00_to_0x2FEF),
	+ cmocka_unit_test(unicode_0x2FF0_to_0x3000),
	+ cmocka_unit_test(unicode_0x3001_to_0xD7FF),
	+ cmocka_unit_test(unicode_0xD800_to_0xF8FF),
	+ cmocka_unit_test(unicode_0xF900_to_0xFDCF),
	+ cmocka_unit_test(unicode_0xFDD0_to_0xFDEF),
	+ cmocka_unit_test(unicode_0xFDF0_to_0xFFFD),
	+ cmocka_unit_test(unicode_0xFFFE_to_0xFFFF),
	+ cmocka_unit_test(unicode_0x10000_to_0xEFFFF),
	+ cmocka_unit_test(unicode_0xF0000_to_0x10FFFF))
	diff --git a/lib/common/tests/xml/pcmk__xml_is_name_start_char_test.c b/lib/common/tests/xml/pcmk__xml_is_name_start_char_test.c
	new file mode 100644
	index 0000000000..ca62adb9cc
	--- /dev/null
	+++ b/lib/common/tests/xml/pcmk__xml_is_name_start_char_test.c
	@@ -0,0 +1,299 @@
	+/*
	+ * Copyright 2024 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU General Public License version 2
	+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#include <crm_internal.h>
	+
	+#include <glib.h> // gchar, g_ascii_isalpha(), etc.
	+#include <libxml/xmlstring.h> // xmlGetUTF8Char()
	+
	+#include <crm/common/unittest_internal.h>
	+#include <crm/common/xml_internal.h>
	+
	+#include "crmcommon_private.h" // pcmk__xml_is_name_start_char()
	+
	+/*!
	+ * \internal
	+ * \brief Assert that a Unicode character is (not) a valid XML \c NameStartChar
	+ *
	+ * \param[in] c Unicode code point of character to check
	+ * \param[in] reference If \c true, \p c should be considered a valid
	+ * \c NameStartChar; if \c false, it should not
	+ */
	+static void
	+assert_name_start_char(int c, bool reference)
	+{
	+ gchar utf8_buf[6] = { 0, };
	+ int len = 4;
	+ int ref_len = g_unichar_to_utf8(c, utf8_buf);
	+ bool result = pcmk__xml_is_name_start_char(utf8_buf, &len);
	+
	+ if (reference) {
	+ assert_true(result);
	+ } else {
	+ assert_false(result);
	+ }
	+
	+ if ((c < 0xD800) \|\| (c > 0xDFFF)) {
	+ /* Unicode code points in the range D800 to DFFF are UTF-16 surrogate
	+ * pair halves. They can be represented in UTF-8, but they shouldn't
	+ * appear in valid UTF-8-encoded text. RFC 3629 (Nov 2003) says they
	+ * should be treated as invalid:
	+ * https://en.wikipedia.org/wiki/UTF-8#Invalid_sequences_and_error_handling.
	+ *
	+ * GLib treats these characters as valid and returns a length of 3
	+ * bytes. So did libxml until v2.12 (commit 845bd99). Since that commit,
	+ * libxml treats these characters as invalid and returns a length of 0.
	+ * To avoid version-dependent testing behavior, skip the length check
	+ * for code points in that range.
	+ */
	+ assert_int_equal(len, ref_len);
	+ }
	+}
	+
	+static void
	+null_len(void **state)
	+{
	+ assert_true(pcmk__xml_is_name_start_char("a", NULL));
	+ assert_false(pcmk__xml_is_name_start_char("0", NULL));
	+}
	+
	+static void
	+ascii(void **state)
	+{
	+ for (int c = 0x00; c <= 0x7F; c++) {
	+ if (g_ascii_isalpha(c) \|\| c == ':' \|\| c == '_') {
	+ assert_name_start_char(c, true);
	+ } else {
	+ assert_name_start_char(c, false);
	+ }
	+ }
	+}
	+
	+static void
	+unicode_0x80_to_0xBF(void **state)
	+{
	+ for (int c = 0x80; c <= 0xBF; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0xC0_to_0xD6(void **state)
	+{
	+ for (int c = 0xC0; c <= 0xD6; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xD7(void **state)
	+{
	+ assert_name_start_char(0xD7, false);
	+}
	+
	+static void
	+unicode_0xD8_to_0xF6(void **state)
	+{
	+ for (int c = 0xD8; c <= 0xF6; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xF7(void **state)
	+{
	+ assert_name_start_char(0xF7, false);
	+}
	+
	+static void
	+unicode_0xF8_to_0x2FF(void **state)
	+{
	+ for (int c = 0xF8; c <= 0x2FF; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x300_to_0x36F(void **state)
	+{
	+ for (int c = 0x300; c <= 0x36F; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x370_to_0x37D(void **state)
	+{
	+ for (int c = 0x370; c <= 0x37D; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x37E(void **state)
	+{
	+ assert_name_start_char(0x37E, false);
	+}
	+
	+static void
	+unicode_0x37F_to_0x1FFF(void **state)
	+{
	+ for (int c = 0x37F; c <= 0x1FFF; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x2000_to_0x200B(void **state)
	+{
	+ for (int c = 0x2000; c <= 0x200B; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x200C_to_0x200D(void **state)
	+{
	+ for (int c = 0x200C; c <= 0x200D; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x200E_to_0x206F(void **state)
	+{
	+ for (int c = 0x200E; c <= 0x206F; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x2070_to_0x218F(void **state)
	+{
	+ for (int c = 0x2070; c <= 0x218F; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x2190_to_0x2BFF(void **state)
	+{
	+ for (int c = 0x2190; c <= 0x2BFF; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x2C00_to_0x2FEF(void **state)
	+{
	+ for (int c = 0x2C00; c <= 0x2FEF; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0x2FF0_to_0x3000(void **state)
	+{
	+ for (int c = 0x2FF0; c <= 0x3000; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x3001_to_0xD7FF(void **state)
	+{
	+ for (int c = 0x3001; c <= 0xD7FF; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xD800_to_0xF8FF(void **state)
	+{
	+ for (int c = 0xD800; c <= 0xF8FF; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0xF900_to_0xFDCF(void **state)
	+{
	+ for (int c = 0xF900; c <= 0xFDCF; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xFDD0_to_0xFDEF(void **state)
	+{
	+ for (int c = 0xFDD0; c <= 0xFDEF; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0xFDF0_to_0xFFFD(void **state)
	+{
	+ for (int c = 0xFDF0; c <= 0xFFFD; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xFFFE_to_0xFFFF(void **state)
	+{
	+ for (int c = 0xFFFE; c <= 0xFFFF; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+static void
	+unicode_0x10000_to_0xEFFFF(void **state)
	+{
	+ for (int c = 0x10000; c <= 0xEFFFF; c++) {
	+ assert_name_start_char(c, true);
	+ }
	+}
	+
	+static void
	+unicode_0xF0000_to_0x10FFFF(void **state)
	+{
	+ for (int c = 0xF0000; c <= 0x10FFFF; c++) {
	+ assert_name_start_char(c, false);
	+ }
	+}
	+
	+PCMK__UNIT_TEST(NULL, NULL,
	+ cmocka_unit_test(null_len),
	+ cmocka_unit_test(ascii),
	+ cmocka_unit_test(unicode_0x80_to_0xBF),
	+ cmocka_unit_test(unicode_0xC0_to_0xD6),
	+ cmocka_unit_test(unicode_0xD7),
	+ cmocka_unit_test(unicode_0xD8_to_0xF6),
	+ cmocka_unit_test(unicode_0xF7),
	+ cmocka_unit_test(unicode_0xF8_to_0x2FF),
	+ cmocka_unit_test(unicode_0x300_to_0x36F),
	+ cmocka_unit_test(unicode_0x370_to_0x37D),
	+ cmocka_unit_test(unicode_0x37E),
	+ cmocka_unit_test(unicode_0x37F_to_0x1FFF),
	+ cmocka_unit_test(unicode_0x2000_to_0x200B),
	+ cmocka_unit_test(unicode_0x200C_to_0x200D),
	+ cmocka_unit_test(unicode_0x200E_to_0x206F),
	+ cmocka_unit_test(unicode_0x2070_to_0x218F),
	+ cmocka_unit_test(unicode_0x2190_to_0x2BFF),
	+ cmocka_unit_test(unicode_0x2C00_to_0x2FEF),
	+ cmocka_unit_test(unicode_0x2FF0_to_0x3000),
	+ cmocka_unit_test(unicode_0x3001_to_0xD7FF),
	+ cmocka_unit_test(unicode_0xD800_to_0xF8FF),
	+ cmocka_unit_test(unicode_0xF900_to_0xFDCF),
	+ cmocka_unit_test(unicode_0xFDD0_to_0xFDEF),
	+ cmocka_unit_test(unicode_0xFDF0_to_0xFFFD),
	+ cmocka_unit_test(unicode_0xFFFE_to_0xFFFF),
	+ cmocka_unit_test(unicode_0x10000_to_0xEFFFF),
	+ cmocka_unit_test(unicode_0xF0000_to_0x10FFFF))
	diff --git a/lib/common/tests/xml/pcmk__xml_sanitize_id_test.c b/lib/common/tests/xml/pcmk__xml_sanitize_id_test.c
	new file mode 100644
	index 0000000000..0fa10c54f3
	--- /dev/null
	+++ b/lib/common/tests/xml/pcmk__xml_sanitize_id_test.c
	@@ -0,0 +1,120 @@
	+/*
	+ * Copyright 2024 the Pacemaker project contributors
	+ *
	+ * The version control history for this file may have further details.
	+ *
	+ * This source code is licensed under the GNU General Public License version 2
	+ * or later (GPLv2+) WITHOUT ANY WARRANTY.
	+ */
	+
	+#include <crm_internal.h>
	+
	+#include <crm/common/unittest_internal.h>
	+#include <crm/common/xml_internal.h>
	+
	+static void
	+assert_sanitized(const char str, const char reference)
	+{
	+ char *buf = pcmk__str_copy(str);
	+
	+ pcmk__xml_sanitize_id(buf);
	+ assert_string_equal(buf, reference);
	+ free(buf);
	+}
	+
	+static void
	+null_empty(void **state)
	+{
	+ char *buf = pcmk__str_copy("");
	+
	+ pcmk__assert_asserts(pcmk__xml_sanitize_id(NULL));
	+ pcmk__assert_asserts(pcmk__xml_sanitize_id(buf));
	+
	+ free(buf);
	+}
	+
	+static void
	+all_valid(void **state)
	+{
	+ const char *str = NULL;
	+
	+ // All NameStartChars
	+ str = "abc";
	+ assert_sanitized(str, str);
	+
	+ // '-' NameChar but not NameStartChar
	+ str = "b-c";
	+ assert_sanitized(str, str);
	+ str = "bc-";
	+ assert_sanitized(str, str);
	+
	+ // #xC2 (NameStartChar), #xB7 (NameChar)
	+ str = "a" "\xC2\xB7" "b";
	+ assert_sanitized(str, str);
	+
	+ // #xEFFFF (NameStartChar)
	+ str = "\xF3\xAF\xBF\xBF" "a";
	+ assert_sanitized(str, str);
	+
	+ // #xEFFFF (NameStartChar), #xB7 (NameChar)
	+ str = "\xF3\xAF\xBF\xBF" "\xC2\xB7";
	+ assert_sanitized(str, str);
	+}
	+
	+static void
	+start_invalid(void **state)
	+{
	+ // '-' NameChar but not NameStartChar
	+ assert_sanitized("-ab", "_ab");
	+
	+ // '$' neither NameChar nor NameStartChar
	+ assert_sanitized("$ab", "_ab");
	+
	+ // #xB7 NameChar but not NameStartChar (two-byte character)
	+ assert_sanitized("\xC2\xB7" "ab", "_.ab");
	+
	+ // #xB8 neither NameChar nor NameStartChar (two-byte character)
	+ assert_sanitized("\xC2\xB8" "ab", "_.ab");
	+}
	+
	+static void
	+middle_invalid(void **state)
	+{
	+ // '$' not a NameChar
	+ assert_sanitized("a$b", "a.b");
	+
	+ // #xB8 not a NameChar (two-byte character)
	+ assert_sanitized("a" "\xC2\xB8" "b", "a..b");
	+}
	+
	+static void
	+end_invalid(void **state)
	+{
	+ // '$' not a NameChar
	+ assert_sanitized("ab$", "ab.");
	+
	+ // #xB8 not a NameChar (two-byte character)
	+ assert_sanitized("ab" "\xC2\xB8", "ab..");
	+}
	+
	+static void
	+all_invalid(void **state)
	+{
	+ // None are NameChars (all ASCII)
	+ assert_sanitized("$!%", "_..");
	+
	+ // None are NameChars (#xB8 two-byte character)
	+ assert_sanitized("$!" "\xC2\xB8", "_...");
	+
	+ // None are NameChars (all multi-byte characters)
	+ assert_sanitized("\xC2\xB7" "\xCD\xBE" "\xF3\xB0\x80\x80",
	+ "_." ".." "....");
	+}
	+
	+PCMK__UNIT_TEST(pcmk__xml_test_setup_group, pcmk__xml_test_teardown_group,
	+ cmocka_unit_test(null_empty),
	+ cmocka_unit_test(all_valid),
	+ cmocka_unit_test(start_invalid),
	+ cmocka_unit_test(middle_invalid),
	+ cmocka_unit_test(end_invalid),
	+ cmocka_unit_test(all_invalid))
	diff --git a/lib/common/xml.c b/lib/common/xml.c
	index b0ca4d4230..ae05e2b7a0 100644
	--- a/lib/common/xml.c
	+++ b/lib/common/xml.c
	@@ -1,2400 +1,2610 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdarg.h>
	#include <stdint.h> // uint32_t
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <sys/stat.h> // stat(), S_ISREG, etc.
	#include <sys/types.h>

	+#include <glib.h> // gboolean, GString
	#include <libxml/parser.h>
	#include <libxml/tree.h>
	+#include <libxml/xmlstring.h> // xmlGetUTF8Char()

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h> // PCMK__XML_LOG_BASE, etc.
	#include "crmcommon_private.h"

	/*!
	* \internal
	* \brief Apply a function to each XML node in a tree (pre-order, depth-first)
	*
	* \param[in,out] xml XML tree to traverse
	* \param[in,out] fn Function to call for each node (returns \c true to
	* continue traversing the tree or \c false to stop)
	* \param[in,out] user_data Argument to \p fn
	*
	* \return \c false if any \p fn call returned \c false, or \c true otherwise
	*
	* \note This function is recursive.
	*/
	bool
	pcmk__xml_tree_foreach(xmlNode xml, bool (fn)(xmlNode , void ),
	void *user_data)
	{
	if (!fn(xml, user_data)) {
	return false;
	}

	for (xml = pcmk__xml_first_child(xml); xml != NULL;
	xml = pcmk__xml_next(xml)) {

	if (!pcmk__xml_tree_foreach(xml, fn, user_data)) {
	return false;
	}
	}
	return true;
	}

	bool
	pcmk__tracking_xml_changes(xmlNode *xml, bool lazy)
	{
	if(xml == NULL \|\| xml->doc == NULL \|\| xml->doc->_private == NULL) {
	return FALSE;
	} else if (!pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
	pcmk__xf_tracking)) {
	return FALSE;
	} else if (lazy && !pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
	pcmk__xf_lazy)) {
	return FALSE;
	}
	return TRUE;
	}

	static inline void
	set_parent_flag(xmlNode *xml, long flag)
	{
	for(; xml; xml = xml->parent) {
	xml_node_private_t *nodepriv = xml->_private;

	if (nodepriv == NULL) {
	/* During calls to xmlDocCopyNode(), _private will be unset for parent nodes */
	} else {
	pcmk__set_xml_flags(nodepriv, flag);
	}
	}
	}

	void
	pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag)
	{
	if(xml && xml->doc && xml->doc->_private){
	/* During calls to xmlDocCopyNode(), xml->doc may be unset */
	xml_doc_private_t *docpriv = xml->doc->_private;

	pcmk__set_xml_flags(docpriv, flag);
	}
	}

	// Mark document, element, and all element's parents as changed
	void
	pcmk__mark_xml_node_dirty(xmlNode *xml)
	{
	pcmk__set_xml_doc_flag(xml, pcmk__xf_dirty);
	set_parent_flag(xml, pcmk__xf_dirty);
	}

	/*!
	* \internal
	* \brief Clear flags on an XML node
	*
	* \param[in,out] xml XML node whose flags to reset
	* \param[in,out] user_data Ignored
	*
	* \return \c true (to continue traversing the tree)
	*
	* \note This is compatible with \c pcmk__xml_tree_foreach().
	*/
	static bool
	reset_xml_node_flags(xmlNode xml, void user_data)
	{
	xml_node_private_t *nodepriv = xml->_private;

	if (nodepriv != NULL) {
	nodepriv->flags = pcmk__xf_none;
	}
	return true;
	}

	/*!
	* \internal
	* \brief Set the \c pcmk__xf_dirty and \c pcmk__xf_created flags on an XML node
	*
	* \param[in,out] xml Node whose flags to set
	* \param[in] user_data Ignored
	*
	* \return \c true (to continue traversing the tree)
	*
	* \note This is compatible with \c pcmk__xml_tree_foreach().
	*/
	static bool
	mark_xml_dirty_created(xmlNode xml, void user_data)
	{
	xml_node_private_t *nodepriv = xml->_private;

	if (nodepriv != NULL) {
	pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty\|pcmk__xf_created);
	}
	return true;
	}

	/*!
	* \internal
	* \brief Mark an XML tree as dirty and created, and mark its parents dirty
	*
	* Also mark the document dirty.
	*
	* \param[in,out] xml Tree to mark as dirty and created
	*/
	void
	pcmk__xml_mark_created(xmlNode *xml)
	{
	CRM_ASSERT(xml != NULL);

	if (!pcmk__tracking_xml_changes(xml, false)) {
	// Tracking is disabled for entire document
	return;
	}

	// Mark all parents and document dirty
	pcmk__mark_xml_node_dirty(xml);

	pcmk__xml_tree_foreach(xml, mark_xml_dirty_created, NULL);
	}

	#define XML_DOC_PRIVATE_MAGIC 0x81726354UL
	#define XML_NODE_PRIVATE_MAGIC 0x54637281UL

	// Free an XML object previously marked as deleted
	static void
	free_deleted_object(void *data)
	{
	if(data) {
	pcmk__deleted_xml_t *deleted_obj = data;

	g_free(deleted_obj->path);
	free(deleted_obj);
	}
	}

	// Free and NULL user, ACLs, and deleted objects in an XML node's private data
	static void
	reset_xml_private_data(xml_doc_private_t *docpriv)
	{
	if (docpriv != NULL) {
	CRM_ASSERT(docpriv->check == XML_DOC_PRIVATE_MAGIC);

	free(docpriv->user);
	docpriv->user = NULL;

	if (docpriv->acls != NULL) {
	pcmk__free_acls(docpriv->acls);
	docpriv->acls = NULL;
	}

	if(docpriv->deleted_objs) {
	g_list_free_full(docpriv->deleted_objs, free_deleted_object);
	docpriv->deleted_objs = NULL;
	}
	}
	}

	// Free all private data associated with an XML node
	static void
	free_private_data(xmlNode *node)
	{
	/* Note:

	This function frees private data assosciated with an XML node,
	unless the function is being called as a result of internal
	XSLT cleanup.

	That could happen through, for example, the following chain of
	function calls:

	xsltApplyStylesheetInternal
	-> xsltFreeTransformContext
	-> xsltFreeRVTs
	-> xmlFreeDoc

	And in that case, the node would fulfill three conditions:

	1. It would be a standalone document (i.e. it wouldn't be
	part of a document)
	2. It would have a space-prefixed name (for reference, please
	see xsltInternals.h: XSLT_MARK_RES_TREE_FRAG)
	3. It would carry its own payload in the _private field.

	We do not free data in this circumstance to avoid a failed
	assertion on the XML_*_PRIVATE_MAGIC later.

	*/
	if (node->name == NULL \|\| node->name[0] != ' ') {
	if (node->_private) {
	if (node->type == XML_DOCUMENT_NODE) {
	reset_xml_private_data(node->_private);
	} else {
	CRM_ASSERT(((xml_node_private_t *) node->_private)->check
	== XML_NODE_PRIVATE_MAGIC);
	/* nothing dynamically allocated nested */
	}
	free(node->_private);
	node->_private = NULL;
	}
	}
	}

	// Allocate and initialize private data for an XML node
	static void
	new_private_data(xmlNode *node)
	{
	switch (node->type) {
	case XML_DOCUMENT_NODE: {
	xml_doc_private_t *docpriv =
	pcmk__assert_alloc(1, sizeof(xml_doc_private_t));

	docpriv->check = XML_DOC_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	pcmk__set_xml_flags(docpriv, pcmk__xf_dirty\|pcmk__xf_created);
	node->_private = docpriv;
	break;
	}
	case XML_ELEMENT_NODE:
	case XML_ATTRIBUTE_NODE:
	case XML_COMMENT_NODE: {
	xml_node_private_t *nodepriv =
	pcmk__assert_alloc(1, sizeof(xml_node_private_t));

	nodepriv->check = XML_NODE_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty\|pcmk__xf_created);
	node->_private = nodepriv;
	if (pcmk__tracking_xml_changes(node, FALSE)) {
	/* XML_ELEMENT_NODE doesn't get picked up here, node->doc is
	* not hooked up at the point we are called
	*/
	pcmk__mark_xml_node_dirty(node);
	}
	break;
	}
	case XML_TEXT_NODE:
	case XML_DTD_NODE:
	case XML_CDATA_SECTION_NODE:
	break;
	default:
	/* Ignore */
	crm_trace("Ignoring %p %d", node, node->type);
	CRM_LOG_ASSERT(node->type == XML_ELEMENT_NODE);
	break;
	}
	}

	void
	xml_track_changes(xmlNode * xml, const char user, xmlNode acl_source, bool enforce_acls)
	{
	xml_accept_changes(xml);
	crm_trace("Tracking changes%s to %p", enforce_acls?" with ACLs":"", xml);
	pcmk__set_xml_doc_flag(xml, pcmk__xf_tracking);
	if(enforce_acls) {
	if(acl_source == NULL) {
	acl_source = xml;
	}
	pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_enabled);
	pcmk__unpack_acl(acl_source, xml, user);
	pcmk__apply_acl(xml);
	}
	}

	bool xml_tracking_changes(xmlNode * xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_tracking);
	}

	bool xml_document_dirty(xmlNode *xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_dirty);
	}

	/*!
	* \internal
	* \brief Return ordinal position of an XML node among its siblings
	*
	* \param[in] xml XML node to check
	* \param[in] ignore_if_set Don't count siblings with this flag set
	*
	* \return Ordinal position of \p xml (starting with 0)
	*/
	int
	pcmk__xml_position(const xmlNode *xml, enum xml_private_flags ignore_if_set)
	{
	int position = 0;

	for (const xmlNode *cIter = xml; cIter->prev; cIter = cIter->prev) {
	xml_node_private_t nodepriv = ((xmlNode)cIter->prev)->_private;

	if (!pcmk_is_set(nodepriv->flags, ignore_if_set)) {
	position++;
	}
	}

	return position;
	}

	/*!
	* \internal
	* \brief Remove all attributes marked as deleted from an XML node
	*
	* \param[in,out] xml XML node whose deleted attributes to remove
	* \param[in,out] user_data Ignored
	*
	* \return \c true (to continue traversing the tree)
	*
	* \note This is compatible with \c pcmk__xml_tree_foreach().
	*/
	static bool
	accept_attr_deletions(xmlNode xml, void user_data)
	{
	reset_xml_node_flags(xml, NULL);
	pcmk__xe_remove_matching_attrs(xml, pcmk__marked_as_deleted, NULL);
	return true;
	}

	/*!
	* \internal
	* \brief Find first child XML node matching another given XML node
	*
	* \param[in] haystack XML whose children should be checked
	* \param[in] needle XML to match (comment content or element name and ID)
	* \param[in] exact If true and needle is a comment, position must match
	*/
	xmlNode *
	pcmk__xml_match(const xmlNode haystack, const xmlNode needle, bool exact)
	{
	CRM_CHECK(needle != NULL, return NULL);

	if (needle->type == XML_COMMENT_NODE) {
	return pcmk__xc_match(haystack, needle, exact);

	} else {
	const char *id = pcmk__xe_id(needle);
	const char *attr = (id == NULL)? NULL : PCMK_XA_ID;

	return pcmk__xe_first_child(haystack, (const char *) needle->name, attr,
	id);
	}
	}

	void
	xml_accept_changes(xmlNode * xml)
	{
	xmlNode *top = NULL;
	xml_doc_private_t *docpriv = NULL;

	if(xml == NULL) {
	return;
	}

	crm_trace("Accepting changes to %p", xml);
	docpriv = xml->doc->_private;
	top = xmlDocGetRootElement(xml->doc);

	reset_xml_private_data(xml->doc->_private);

	if (!pcmk_is_set(docpriv->flags, pcmk__xf_dirty)) {
	docpriv->flags = pcmk__xf_none;
	return;
	}

	docpriv->flags = pcmk__xf_none;
	pcmk__xml_tree_foreach(top, accept_attr_deletions, NULL);
	}

	/*!
	* \internal
	* \brief Find first XML child element matching given criteria
	*
	* \param[in] parent XML element to search (can be \c NULL)
	* \param[in] node_name If not \c NULL, only match children of this type
	* \param[in] attr_n If not \c NULL, only match children with an attribute
	* of this name.
	* \param[in] attr_v If \p attr_n and this are not NULL, only match children
	* with an attribute named \p attr_n and this value
	*
	* \return Matching XML child element, or \c NULL if none found
	*/
	xmlNode *
	pcmk__xe_first_child(const xmlNode parent, const char node_name,
	const char attr_n, const char attr_v)
	{
	xmlNode *child = NULL;
	const char *parent_name = "<null>";

	CRM_CHECK((attr_v == NULL) \|\| (attr_n != NULL), return NULL);

	if (parent != NULL) {
	child = parent->children;
	while ((child != NULL) && (child->type != XML_ELEMENT_NODE)) {
	child = child->next;
	}

	parent_name = (const char *) parent->name;
	}

	for (; child != NULL; child = pcmk__xe_next(child)) {
	const char *value = NULL;

	if ((node_name != NULL) && !pcmk__xe_is(child, node_name)) {
	// Node name mismatch
	continue;
	}
	if (attr_n == NULL) {
	// No attribute match needed
	return child;
	}

	value = crm_element_value(child, attr_n);

	if ((attr_v == NULL) && (value != NULL)) {
	// attr_v == NULL: Attribute attr_n must be set (to any value)
	return child;
	}
	if ((attr_v != NULL) && (pcmk__str_eq(value, attr_v, pcmk__str_none))) {
	// attr_v != NULL: Attribute attr_n must be set to value attr_v
	return child;
	}
	}

	if (node_name == NULL) {
	node_name = "(any)"; // For logging
	}
	if (attr_n != NULL) {
	crm_trace("XML child node <%s %s=%s> not found in %s",
	node_name, attr_n, attr_v, parent_name);
	} else {
	crm_trace("XML child node <%s> not found in %s",
	node_name, parent_name);
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Set an XML attribute, expanding \c ++ and \c += where appropriate
	*
	* If \p target already has an attribute named \p name set to an integer value
	* and \p value is an addition assignment expression on \p name, then expand
	* \p value to an integer and set attribute \p name to the expanded value in
	* \p target.
	*
	* Otherwise, set attribute \p name on \p target using the literal \p value.
	*
	* The original attribute value in \p target and the number in an assignment
	* expression in \p value are parsed and added as scores (that is, their values
	* are capped at \c INFINITY and \c -INFINITY). For more details, refer to
	* \c char2score().
	*
	* For example, suppose \p target has an attribute named \c "X" with value
	* \c "5", and that \p name is \c "X".
	* * If \p value is \c "X++", the new value of \c "X" in \p target is \c "6".
	* * If \p value is \c "X+=3", the new value of \c "X" in \p target is \c "8".
	* * If \p value is \c "val", the new value of \c "X" in \p target is \c "val".
	* * If \p value is \c "Y++", the new value of \c "X" in \p target is \c "Y++".
	*
	* \param[in,out] target XML node whose attribute to set
	* \param[in] name Name of the attribute to set
	* \param[in] value New value of attribute to set
	*
	* \return Standard Pacemaker return code (specifically, \c EINVAL on invalid
	* argument, or \c pcmk_rc_ok otherwise)
	*/
	int
	pcmk__xe_set_score(xmlNode target, const char name, const char *value)
	{
	const char *old_value = NULL;

	CRM_CHECK((target != NULL) && (name != NULL), return EINVAL);

	if (value == NULL) {
	return pcmk_rc_ok;
	}

	old_value = crm_element_value(target, name);

	// If no previous value, skip to default case and set the value unexpanded.
	if (old_value != NULL) {
	const char *n = name;
	const char *v = value;

	// Stop at first character that differs between name and value
	for (; (n == v) && (*n != '\0'); n++, v++);

	// If value begins with name followed by a "++" or "+="
	if ((*n == '\0')
	&& (*v++ == '+')
	&& ((v == '+') \|\| (v == '='))) {

	// If we're expanding ourselves, no previous value was set; use 0
	int old_value_i = (old_value != value)? char2score(old_value) : 0;

	/* value="X++": new value of X is old_value + 1
	* value="X+=Y": new value of X is old_value + Y (for some number Y)
	*/
	int add = (*v == '+')? 1 : char2score(++v);

	crm_xml_add_int(target, name, pcmk__add_scores(old_value_i, add));
	return pcmk_rc_ok;
	}
	}

	// Default case: set the attribute unexpanded (with value treated literally)
	if (old_value != value) {
	crm_xml_add(target, name, value);
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Copy XML attributes from a source element to a target element
	*
	* This is similar to \c xmlCopyPropList() except that attributes are marked
	* as dirty for change tracking purposes.
	*
	* \param[in,out] target XML element to receive copied attributes from \p src
	* \param[in] src XML element whose attributes to copy to \p target
	* \param[in] flags Group of <tt>enum pcmk__xa_flags</tt>
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__xe_copy_attrs(xmlNode target, const xmlNode src, uint32_t flags)
	{
	CRM_CHECK((src != NULL) && (target != NULL), return EINVAL);

	for (xmlAttr *attr = pcmk__xe_first_attr(src); attr != NULL;
	attr = attr->next) {

	const char name = (const char ) attr->name;
	const char *value = pcmk__xml_attr_value(attr);

	if (pcmk_is_set(flags, pcmk__xaf_no_overwrite)
	&& (crm_element_value(target, name) != NULL)) {
	continue;
	}

	if (pcmk_is_set(flags, pcmk__xaf_score_update)) {
	pcmk__xe_set_score(target, name, value);
	} else {
	crm_xml_add(target, name, value);
	}
	}

	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Remove an XML attribute from an element
	*
	* \param[in,out] element XML element that owns \p attr
	* \param[in,out] attr XML attribute to remove from \p element
	*
	* \return Standard Pacemaker return code (\c EPERM if ACLs prevent removal of
	* attributes from \p element, or \c pcmk_rc_ok otherwise)
	*/
	static int
	remove_xe_attr(xmlNode element, xmlAttr attr)
	{
	if (attr == NULL) {
	return pcmk_rc_ok;
	}

	if (!pcmk__check_acl(element, NULL, pcmk__xf_acl_write)) {
	// ACLs apply to element, not to particular attributes
	crm_trace("ACLs prevent removal of attributes from %s element",
	(const char *) element->name);
	return EPERM;
	}

	if (pcmk__tracking_xml_changes(element, false)) {
	// Leave in place (marked for removal) until after diff is calculated
	set_parent_flag(element, pcmk__xf_dirty);
	pcmk__set_xml_flags((xml_node_private_t *) attr->_private,
	pcmk__xf_deleted);
	} else {
	xmlRemoveProp(attr);
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Remove a named attribute from an XML element
	*
	* \param[in,out] element XML element to remove an attribute from
	* \param[in] name Name of attribute to remove
	*/
	void
	pcmk__xe_remove_attr(xmlNode element, const char name)
	{
	if (name != NULL) {
	remove_xe_attr(element, xmlHasProp(element, (pcmkXmlStr) name));
	}
	}

	/*!
	* \internal
	* \brief Remove a named attribute from an XML element
	*
	* This is a wrapper for \c pcmk__xe_remove_attr() for use with
	* \c pcmk__xml_tree_foreach().
	*
	* \param[in,out] xml XML element to remove an attribute from
	* \param[in] user_data Name of attribute to remove
	*
	* \return \c true (to continue traversing the tree)
	*
	* \note This is compatible with \c pcmk__xml_tree_foreach().
	*/
	bool
	pcmk__xe_remove_attr_cb(xmlNode xml, void user_data)
	{
	const char *name = user_data;

	pcmk__xe_remove_attr(xml, name);
	return true;
	}

	/*!
	* \internal
	* \brief Remove an XML element's attributes that match some criteria
	*
	* \param[in,out] element XML element to modify
	* \param[in] match If not NULL, only remove attributes for which
	* this function returns true
	* \param[in,out] user_data Data to pass to \p match
	*/
	void
	pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data)
	{
	xmlAttrPtr next = NULL;

	for (xmlAttrPtr a = pcmk__xe_first_attr(element); a != NULL; a = next) {
	next = a->next; // Grab now because attribute might get removed
	if ((match == NULL) \|\| match(a, user_data)) {
	if (remove_xe_attr(element, a) != pcmk_rc_ok) {
	return;
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Create a new XML element under a given parent
	*
	* \param[in,out] parent XML element that will be the new element's parent
	* (\c NULL to create a new XML document with the new
	* node as root)
	* \param[in] name Name of new element
	*
	* \return Newly created XML element (guaranteed not to be \c NULL)
	*/
	xmlNode *
	pcmk__xe_create(xmlNode parent, const char name)
	{
	xmlNode *node = NULL;

	CRM_ASSERT(!pcmk__str_empty(name));

	if (parent == NULL) {
	xmlDoc *doc = xmlNewDoc(PCMK__XML_VERSION);

	pcmk__mem_assert(doc);

	node = xmlNewDocRawNode(doc, NULL, (pcmkXmlStr) name, NULL);
	pcmk__mem_assert(node);

	xmlDocSetRootElement(doc, node);

	} else {
	node = xmlNewChild(parent, NULL, (pcmkXmlStr) name, NULL);
	pcmk__mem_assert(node);
	}

	pcmk__xml_mark_created(node);
	return node;
	}

	/*!
	* \internal
	* \brief Set a formatted string as an XML node's content
	*
	* \param[in,out] node Node whose content to set
	* \param[in] format <tt>printf(3)</tt>-style format string
	* \param[in] ... Arguments for \p format
	*
	* \note This function escapes special characters. \c xmlNodeSetContent() does
	* not.
	*/
	G_GNUC_PRINTF(2, 3)
	void
	pcmk__xe_set_content(xmlNode node, const char format, ...)
	{
	if (node != NULL) {
	const char *content = NULL;
	char *buf = NULL;

	if (strchr(format, '%') == NULL) {
	// Nothing to format
	content = format;

	} else {
	va_list ap;

	va_start(ap, format);

	if (pcmk__str_eq(format, "%s", pcmk__str_none)) {
	// No need to make a copy
	content = va_arg(ap, const char *);

	} else {
	CRM_ASSERT(vasprintf(&buf, format, ap) >= 0);
	content = buf;
	}
	va_end(ap);
	}

	xmlNodeSetContent(node, (pcmkXmlStr) content);
	free(buf);
	}
	}

	+/*!
	+ * \internal
	+ * \brief Check whether the first character of a string is an XML NameStartChar
	+ *
	+ * See https://www.w3.org/TR/xml/#NT-NameStartChar.
	+ *
	+ * This is almost identical to libxml2's \c xmlIsDocNameStartChar(), but they
	+ * don't expose it as part of the public API.
	+ *
	+ * \param[in] utf8 UTF-8 encoded string
	+ * \param[out] len If not \c NULL, where to store size in bytes of first
	+ * character in \p utf8
	+ *
	+ * \return \c true if \p utf8 begins with a valid XML NameStartChar, or \c false
	+ * otherwise
	+ */
	+bool
	+pcmk__xml_is_name_start_char(const char utf8, int len)
	+{
	+ int c = 0;
	+ int local_len = 0;
	+
	+ if (len == NULL) {
	+ len = &local_len;
	+ }
	+
	+ /* xmlGetUTF8Char() abuses the len argument. At call time, it must be set to
	+ * "the minimum number of bytes present in the sequence... to assure the
	+ * next character is completely contained within the sequence." It's similar
	+ * to the "n" in the strn*() functions. However, this doesn't make any sense
	+ * for null-terminated strings, and there's no value that indicates "keep
	+ * going until '\0'." So we set it to 4, the max number of bytes in a UTF-8
	+ * character.
	+ *
	+ * At return, it's set to the actual number of bytes in the char, or 0 on
	+ * error.
	+ */
	+ *len = 4;
	+
	+ // Note: xmlGetUTF8Char() assumes a 32-bit int
	+ c = xmlGetUTF8Char((pcmkXmlStr) utf8, len);
	+ if (c < 0) {
	+ GString *buf = g_string_sized_new(32);
	+
	+ for (int i = 0; (i < 4) && (utf8[i] != '\0'); i++) {
	+ g_string_append_printf(buf, " 0x%.2X", utf8[i]);
	+ }
	+ crm_info("Invalid UTF-8 character (bytes:%s)",
	+ (pcmk__str_empty(buf->str)? " <none>" : buf->str));
	+ g_string_free(buf, TRUE);
	+ return false;
	+ }
	+
	+ return (c == '_')
	+ \|\| (c == ':')
	+ \|\| ((c >= 'a') && (c <= 'z'))
	+ \|\| ((c >= 'A') && (c <= 'Z'))
	+ \|\| ((c >= 0xC0) && (c <= 0xD6))
	+ \|\| ((c >= 0xD8) && (c <= 0xF6))
	+ \|\| ((c >= 0xF8) && (c <= 0x2FF))
	+ \|\| ((c >= 0x370) && (c <= 0x37D))
	+ \|\| ((c >= 0x37F) && (c <= 0x1FFF))
	+ \|\| ((c >= 0x200C) && (c <= 0x200D))
	+ \|\| ((c >= 0x2070) && (c <= 0x218F))
	+ \|\| ((c >= 0x2C00) && (c <= 0x2FEF))
	+ \|\| ((c >= 0x3001) && (c <= 0xD7FF))
	+ \|\| ((c >= 0xF900) && (c <= 0xFDCF))
	+ \|\| ((c >= 0xFDF0) && (c <= 0xFFFD))
	+ \|\| ((c >= 0x10000) && (c <= 0xEFFFF));
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Check whether the first character of a string is an XML NameChar
	+ *
	+ * See https://www.w3.org/TR/xml/#NT-NameChar.
	+ *
	+ * This is almost identical to libxml2's \c xmlIsDocNameChar(), but they don't
	+ * expose it as part of the public API.
	+ *
	+ * \param[in] utf8 UTF-8 encoded string
	+ * \param[out] len If not \c NULL, where to store size in bytes of first
	+ * character in \p utf8
	+ *
	+ * \return \c true if \p utf8 begins with a valid XML NameChar, or \c false
	+ * otherwise
	+ */
	+bool
	+pcmk__xml_is_name_char(const char utf8, int len)
	+{
	+ int c = 0;
	+ int local_len = 0;
	+
	+ if (len == NULL) {
	+ len = &local_len;
	+ }
	+
	+ // See comment regarding len in pcmk__xml_is_name_start_char()
	+ *len = 4;
	+
	+ // Note: xmlGetUTF8Char() assumes a 32-bit int
	+ c = xmlGetUTF8Char((pcmkXmlStr) utf8, len);
	+ if (c < 0) {
	+ GString *buf = g_string_sized_new(32);
	+
	+ for (int i = 0; (i < 4) && (utf8[i] != '\0'); i++) {
	+ g_string_append_printf(buf, " 0x%.2X", utf8[i]);
	+ }
	+ crm_info("Invalid UTF-8 character (bytes:%s)",
	+ (pcmk__str_empty(buf->str)? " <none>" : buf->str));
	+ g_string_free(buf, TRUE);
	+ return false;
	+ }
	+
	+ return ((c >= 'a') && (c <= 'z'))
	+ \|\| ((c >= 'A') && (c <= 'Z'))
	+ \|\| ((c >= '0') && (c <= '9'))
	+ \|\| (c == '_')
	+ \|\| (c == ':')
	+ \|\| (c == '-')
	+ \|\| (c == '.')
	+ \|\| (c == 0xB7)
	+ \|\| ((c >= 0xC0) && (c <= 0xD6))
	+ \|\| ((c >= 0xD8) && (c <= 0xF6))
	+ \|\| ((c >= 0xF8) && (c <= 0x2FF))
	+ \|\| ((c >= 0x300) && (c <= 0x36F))
	+ \|\| ((c >= 0x370) && (c <= 0x37D))
	+ \|\| ((c >= 0x37F) && (c <= 0x1FFF))
	+ \|\| ((c >= 0x200C) && (c <= 0x200D))
	+ \|\| ((c >= 0x203F) && (c <= 0x2040))
	+ \|\| ((c >= 0x2070) && (c <= 0x218F))
	+ \|\| ((c >= 0x2C00) && (c <= 0x2FEF))
	+ \|\| ((c >= 0x3001) && (c <= 0xD7FF))
	+ \|\| ((c >= 0xF900) && (c <= 0xFDCF))
	+ \|\| ((c >= 0xFDF0) && (c <= 0xFFFD))
	+ \|\| ((c >= 0x10000) && (c <= 0xEFFFF));
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Sanitize a string so it is usable as an XML ID
	+ *
	+ * An ID must match the Name production as defined here:
	+ * https://www.w3.org/TR/xml/#NT-Name.
	+ *
	+ * Convert an invalid start character to \c '_'. Convert an invalid character
	+ * after the start character to \c '.'.
	+ *
	+ * \param[in,out] id String to sanitize
	+ */
	+void
	+pcmk__xml_sanitize_id(char *id)
	+{
	+ bool valid = true;
	+ int len = 0;
	+
	+ // If id is empty or NULL, there's no way to make it a valid XML ID
	+ CRM_ASSERT(!pcmk__str_empty(id));
	+
	+ /* @TODO Suppose there are two strings and each has an invalid ID character
	+ * in the same position. The strings are otherwise identical. Both strings
	+ * will be sanitized to the same valid ID, which is incorrect.
	+ *
	+ * The caller is responsible for ensuring the sanitized ID does not already
	+ * exist in a given XML document before using it, if uniqueness is desired.
	+ */
	+ valid = pcmk__xml_is_name_start_char(id, &len);
	+ CRM_CHECK(len > 0, return); // UTF-8 encoding error
	+ if (!valid) {
	+ *id = '_';
	+ for (int i = 1; i < len; i++) {
	+ id[i] = '.';
	+ }
	+ }
	+
	+ for (id += len; *id != '\0'; id += len) {
	+ valid = pcmk__xml_is_name_char(id, &len);
	+ CRM_CHECK(len > 0, return); // UTF-8 encoding error
	+ if (!valid) {
	+ for (int i = 0; i < len; i++) {
	+ id[i] = '.';
	+ }
	+ }
	+ }
	+}
	+
	+/*!
	+ * \internal
	+ * \brief Set a formatted string as an XML element's ID
	+ *
	+ * If the formatted string would not be a valid ID, it's first sanitized by
	+ * \c pcmk__xml_sanitize_id().
	+ *
	+ * \param[in,out] node Node whose ID to set
	+ * \param[in] format <tt>printf(3)</tt>-style format string
	+ * \param[in] ... Arguments for \p format
	+ */
	+G_GNUC_PRINTF(2, 3)
	+void
	+pcmk__xe_set_id(xmlNode node, const char format, ...)
	+{
	+ char *id = NULL;
	+ va_list ap;
	+
	+ CRM_ASSERT(!pcmk__str_empty(format));
	+
	+ if (node == NULL) {
	+ return;
	+ }
	+
	+ va_start(ap, format);
	+ CRM_ASSERT(vasprintf(&id, format, ap) >= 0);
	+ va_end(ap);
	+
	+ if (!xmlValidateNameValue((pcmkXmlStr) id)) {
	+ pcmk__xml_sanitize_id(id);
	+ }
	+ crm_xml_add(node, PCMK_XA_ID, id);
	+ free(id);
	+}
	+
	/*!
	* \internal
	* \brief Free an XML tree if ACLs allow; track deletion if tracking is enabled
	*
	* If \p node is the root of its document, free the entire document.
	*
	* \param[in,out] node XML node to free
	* \param[in] position Position of \p node among its siblings for change
	* tracking (negative to calculate automatically if
	* needed)
	*/
	static void
	free_xml_with_position(xmlNode *node, int position)
	{
	xmlDoc *doc = NULL;
	xml_node_private_t *nodepriv = NULL;

	if (node == NULL) {
	return;
	}
	doc = node->doc;
	nodepriv = node->_private;

	if ((doc != NULL) && (xmlDocGetRootElement(doc) == node)) {
	/* @TODO Should we check ACLs first? Otherwise it seems like we could
	* free the root element without write permission.
	*/
	xmlFreeDoc(doc);
	return;
	}

	if (!pcmk__check_acl(node, NULL, pcmk__xf_acl_write)) {
	GString *xpath = NULL;

	pcmk__if_tracing({}, return);
	xpath = pcmk__element_xpath(node);
	qb_log_from_external_source(__func__, __FILE__,
	"Cannot remove %s %x", LOG_TRACE,
	__LINE__, 0, xpath->str, nodepriv->flags);
	g_string_free(xpath, TRUE);
	return;
	}

	if ((doc != NULL) && pcmk__tracking_xml_changes(node, false)
	&& !pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {

	xml_doc_private_t *docpriv = doc->_private;
	GString *xpath = pcmk__element_xpath(node);

	if (xpath != NULL) {
	pcmk__deleted_xml_t *deleted_obj = NULL;

	crm_trace("Deleting %s %p from %p", xpath->str, node, doc);

	deleted_obj = pcmk__assert_alloc(1, sizeof(pcmk__deleted_xml_t));
	deleted_obj->path = g_string_free(xpath, FALSE);
	deleted_obj->position = -1;

	// Record the position only for XML comments for now
	if (node->type == XML_COMMENT_NODE) {
	if (position >= 0) {
	deleted_obj->position = position;

	} else {
	deleted_obj->position = pcmk__xml_position(node,
	pcmk__xf_skip);
	}
	}

	docpriv->deleted_objs = g_list_append(docpriv->deleted_objs,
	deleted_obj);
	pcmk__set_xml_doc_flag(node, pcmk__xf_dirty);
	}
	}
	xmlUnlinkNode(node);
	xmlFreeNode(node);
	}

	/*!
	* \internal
	* \brief Free an XML tree if ACLs allow; track deletion if tracking is enabled
	*
	* If \p xml is the root of its document, free the entire document.
	*
	* \param[in,out] xml XML node to free
	*/
	void
	pcmk__xml_free(xmlNode *xml)
	{
	free_xml_with_position(xml, -1);
	}

	/*!
	* \internal
	* \brief Make a deep copy of an XML node under a given parent
	*
	* \param[in,out] parent XML element that will be the copy's parent (\c NULL
	* to create a new XML document with the copy as root)
	* \param[in] src XML node to copy
	*
	* \return Deep copy of \p src, or \c NULL if \p src is \c NULL
	*/
	xmlNode *
	pcmk__xml_copy(xmlNode parent, xmlNode src)
	{
	xmlNode *copy = NULL;

	if (src == NULL) {
	return NULL;
	}

	if (parent == NULL) {
	xmlDoc *doc = NULL;

	// The copy will be the root element of a new document
	CRM_ASSERT(src->type == XML_ELEMENT_NODE);

	doc = xmlNewDoc(PCMK__XML_VERSION);
	pcmk__mem_assert(doc);

	copy = xmlDocCopyNode(src, doc, 1);
	pcmk__mem_assert(copy);

	xmlDocSetRootElement(doc, copy);

	} else {
	copy = xmlDocCopyNode(src, parent->doc, 1);
	pcmk__mem_assert(copy);

	xmlAddChild(parent, copy);
	}

	pcmk__xml_mark_created(copy);
	return copy;
	}

	/*!
	* \internal
	* \brief Remove XML text nodes from specified XML and all its children
	*
	* \param[in,out] xml XML to strip text from
	*/
	void
	pcmk__strip_xml_text(xmlNode *xml)
	{
	xmlNode *iter = xml->children;

	while (iter) {
	xmlNode *next = iter->next;

	switch (iter->type) {
	case XML_TEXT_NODE:
	xmlUnlinkNode(iter);
	xmlFreeNode(iter);
	break;

	case XML_ELEMENT_NODE:
	/* Search it */
	pcmk__strip_xml_text(iter);
	break;

	default:
	/* Leave it */
	break;
	}

	iter = next;
	}
	}

	/*!
	* \internal
	* \brief Add a "last written" attribute to an XML element, set to current time
	*
	* \param[in,out] xe XML element to add attribute to
	*
	* \return Value that was set, or NULL on error
	*/
	const char *
	pcmk__xe_add_last_written(xmlNode *xe)
	{
	char *now_s = pcmk__epoch2str(NULL, 0);
	const char *result = NULL;

	result = crm_xml_add(xe, PCMK_XA_CIB_LAST_WRITTEN,
	pcmk__s(now_s, "Could not determine current time"));
	free(now_s);
	return result;
	}

	-/*!
	- * \brief Sanitize a string so it is usable as an XML ID
	- *
	- * \param[in,out] id String to sanitize
	- */
	-void
	-crm_xml_sanitize_id(char *id)
	-{
	- char *c;
	-
	- for (c = id; *c; ++c) {
	- /* @TODO Sanitize more comprehensively */
	- switch (*c) {
	- case ':':
	- case '#':
	- *c = '.';
	- }
	- }
	-}
	-
	-/*!
	- * \brief Set the ID of an XML element using a format
	- *
	- * \param[in,out] xml XML element
	- * \param[in] fmt printf-style format
	- * \param[in] ... any arguments required by format
	- */
	-void
	-crm_xml_set_id(xmlNode xml, const char format, ...)
	-{
	- va_list ap;
	- int len = 0;
	- char *id = NULL;
	-
	- /* equivalent to crm_strdup_printf() */
	- va_start(ap, format);
	- len = vasprintf(&id, format, ap);
	- va_end(ap);
	- CRM_ASSERT(len > 0);
	-
	- crm_xml_sanitize_id(id);
	- crm_xml_add(xml, PCMK_XA_ID, id);
	- free(id);
	-}
	-
	/*!
	* \internal
	* \brief Check whether a string has XML special characters that must be escaped
	*
	* See \c pcmk__xml_escape() and \c pcmk__xml_escape_type for more details.
	*
	* \param[in] text String to check
	* \param[in] type Type of escaping
	*
	* \return \c true if \p text has special characters that need to be escaped, or
	* \c false otherwise
	*/
	bool
	pcmk__xml_needs_escape(const char *text, enum pcmk__xml_escape_type type)
	{
	if (text == NULL) {
	return false;
	}

	while (*text != '\0') {
	switch (type) {
	case pcmk__xml_escape_text:
	switch (*text) {
	case '<':
	case '>':
	case '&':
	return true;
	case '\n':
	case '\t':
	break;
	default:
	if (g_ascii_iscntrl(*text)) {
	return true;
	}
	break;
	}
	break;

	case pcmk__xml_escape_attr:
	switch (*text) {
	case '<':
	case '>':
	case '&':
	case '"':
	return true;
	default:
	if (g_ascii_iscntrl(*text)) {
	return true;
	}
	break;
	}
	break;

	case pcmk__xml_escape_attr_pretty:
	switch (*text) {
	case '\n':
	case '\r':
	case '\t':
	case '"':
	return true;
	default:
	break;
	}
	break;

	default: // Invalid enum value
	CRM_ASSERT(false);
	break;
	}

	text = g_utf8_next_char(text);
	}
	return false;
	}

	/*!
	* \internal
	* \brief Replace special characters with their XML escape sequences
	*
	* \param[in] text Text to escape
	* \param[in] type Type of escaping
	*
	* \return Newly allocated string equivalent to \p text but with special
	* characters replaced with XML escape sequences (or \c NULL if \p text
	* is \c NULL). If \p text is not \c NULL, the return value is
	* guaranteed not to be \c NULL.
	*
	* \note There are libxml functions that purport to do this:
	* \c xmlEncodeEntitiesReentrant() and \c xmlEncodeSpecialChars().
	* However, their escaping is incomplete. See:
	* https://discourse.gnome.org/t/intended-use-of-xmlencodeentitiesreentrant-vs-xmlencodespecialchars/19252
	* \note The caller is responsible for freeing the return value using
	* \c g_free().
	*/
	gchar *
	pcmk__xml_escape(const char *text, enum pcmk__xml_escape_type type)
	{
	GString *copy = NULL;

	if (text == NULL) {
	return NULL;
	}
	copy = g_string_sized_new(strlen(text));

	while (*text != '\0') {
	// Don't escape any non-ASCII characters
	if ((*text & 0x80) != 0) {
	size_t bytes = g_utf8_next_char(text) - text;

	g_string_append_len(copy, text, bytes);
	text += bytes;
	continue;
	}

	switch (type) {
	case pcmk__xml_escape_text:
	switch (*text) {
	case '<':
	g_string_append(copy, PCMK__XML_ENTITY_LT);
	break;
	case '>':
	g_string_append(copy, PCMK__XML_ENTITY_GT);
	break;
	case '&':
	g_string_append(copy, PCMK__XML_ENTITY_AMP);
	break;
	case '\n':
	case '\t':
	g_string_append_c(copy, *text);
	break;
	default:
	if (g_ascii_iscntrl(*text)) {
	g_string_append_printf(copy, "&#x%.2X;", *text);
	} else {
	g_string_append_c(copy, *text);
	}
	break;
	}
	break;

	case pcmk__xml_escape_attr:
	switch (*text) {
	case '<':
	g_string_append(copy, PCMK__XML_ENTITY_LT);
	break;
	case '>':
	g_string_append(copy, PCMK__XML_ENTITY_GT);
	break;
	case '&':
	g_string_append(copy, PCMK__XML_ENTITY_AMP);
	break;
	case '"':
	g_string_append(copy, PCMK__XML_ENTITY_QUOT);
	break;
	default:
	if (g_ascii_iscntrl(*text)) {
	g_string_append_printf(copy, "&#x%.2X;", *text);
	} else {
	g_string_append_c(copy, *text);
	}
	break;
	}
	break;

	case pcmk__xml_escape_attr_pretty:
	switch (*text) {
	case '"':
	g_string_append(copy, "\\\"");
	break;
	case '\n':
	g_string_append(copy, "\\n");
	break;
	case '\r':
	g_string_append(copy, "\\r");
	break;
	case '\t':
	g_string_append(copy, "\\t");
	break;
	default:
	g_string_append_c(copy, *text);
	break;
	}
	break;

	default: // Invalid enum value
	CRM_ASSERT(false);
	break;
	}

	text = g_utf8_next_char(text);
	}
	return g_string_free(copy, FALSE);
	}

	/*!
	* \internal
	* \brief Set a flag on all attributes of an XML element
	*
	* \param[in,out] xml XML node to set flags on
	* \param[in] flag XML private flag to set
	*/
	static void
	set_attrs_flag(xmlNode *xml, enum xml_private_flags flag)
	{
	for (xmlAttr *attr = pcmk__xe_first_attr(xml); attr; attr = attr->next) {
	pcmk__set_xml_flags((xml_node_private_t *) (attr->_private), flag);
	}
	}

	/*!
	* \internal
	* \brief Add an XML attribute to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when an attribute has been
	* deleted. Add the attribute back to the new XML, so that we can check the
	* removal against ACLs, and mark it as deleted for later removal after
	* differences have been calculated.
	*
	* \param[in,out] new_xml XML to modify
	* \param[in] element Name of XML element that changed (for logging)
	* \param[in] attr_name Name of attribute that was deleted
	* \param[in] old_value Value of attribute that was deleted
	*/
	static void
	mark_attr_deleted(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	xml_doc_private_t *docpriv = new_xml->doc->_private;
	xmlAttr *attr = NULL;
	xml_node_private_t *nodepriv;

	// Prevent the dirty flag being set recursively upwards
	pcmk__clear_xml_flags(docpriv, pcmk__xf_tracking);

	// Restore the old value (and the tracking flag)
	attr = xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);
	pcmk__set_xml_flags(docpriv, pcmk__xf_tracking);

	// Reset flags (so the attribute doesn't appear as newly created)
	nodepriv = attr->_private;
	nodepriv->flags = 0;

	// Check ACLs and mark restored value for later removal
	remove_xe_attr(new_xml, attr);

	crm_trace("XML attribute %s=%s was removed from %s",
	attr_name, old_value, element);
	}

	/*
	* \internal
	* \brief Check ACLs for a changed XML attribute
	*/
	static void
	mark_attr_changed(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	char *vcopy = crm_element_value_copy(new_xml, attr_name);

	crm_trace("XML attribute %s was changed from '%s' to '%s' in %s",
	attr_name, old_value, vcopy, element);

	// Restore the original value
	xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);

	// Change it back to the new value, to check ACLs
	crm_xml_add(new_xml, attr_name, vcopy);
	free(vcopy);
	}

	/*!
	* \internal
	* \brief Mark an XML attribute as having changed position
	*
	* \param[in,out] new_xml XML to modify
	* \param[in] element Name of XML element that changed (for logging)
	* \param[in,out] old_attr Attribute that moved, in original XML
	* \param[in,out] new_attr Attribute that moved, in \p new_xml
	* \param[in] p_old Ordinal position of \p old_attr in original XML
	* \param[in] p_new Ordinal position of \p new_attr in \p new_xml
	*/
	static void
	mark_attr_moved(xmlNode new_xml, const char element, xmlAttr *old_attr,
	xmlAttr *new_attr, int p_old, int p_new)
	{
	xml_node_private_t *nodepriv = new_attr->_private;

	crm_trace("XML attribute %s moved from position %d to %d in %s",
	old_attr->name, p_old, p_new, element);

	// Mark document, element, and all element's parents as changed
	pcmk__mark_xml_node_dirty(new_xml);

	// Mark attribute as changed
	pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty\|pcmk__xf_moved);

	nodepriv = (p_old > p_new)? old_attr->_private : new_attr->_private;
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	}

	/*!
	* \internal
	* \brief Calculate differences in all previously existing XML attributes
	*
	* \param[in,out] old_xml Original XML to compare
	* \param[in,out] new_xml New XML to compare
	*/
	static void
	xml_diff_old_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(old_xml);

	while (attr_iter != NULL) {
	const char name = (const char ) attr_iter->name;
	xmlAttr *old_attr = attr_iter;
	xmlAttr *new_attr = xmlHasProp(new_xml, attr_iter->name);
	const char *old_value = pcmk__xml_attr_value(attr_iter);

	attr_iter = attr_iter->next;
	if (new_attr == NULL) {
	mark_attr_deleted(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else {
	xml_node_private_t *nodepriv = new_attr->_private;
	int new_pos = pcmk__xml_position((xmlNode*) new_attr,
	pcmk__xf_skip);
	int old_pos = pcmk__xml_position((xmlNode*) old_attr,
	pcmk__xf_skip);
	const char *new_value = crm_element_value(new_xml, name);

	// This attribute isn't new
	pcmk__clear_xml_flags(nodepriv, pcmk__xf_created);

	if (strcmp(new_value, old_value) != 0) {
	mark_attr_changed(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else if ((old_pos != new_pos)
	&& !pcmk__tracking_xml_changes(new_xml, TRUE)) {
	mark_attr_moved(new_xml, (const char *) old_xml->name,
	old_attr, new_attr, old_pos, new_pos);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Check all attributes in new XML for creation
	*
	* For each of a given XML element's attributes marked as newly created, accept
	* (and mark as dirty) or reject the creation according to ACLs.
	*
	* \param[in,out] new_xml XML to check
	*/
	static void
	mark_created_attrs(xmlNode *new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(new_xml);

	while (attr_iter != NULL) {
	xmlAttr *new_attr = attr_iter;
	xml_node_private_t *nodepriv = attr_iter->_private;

	attr_iter = attr_iter->next;
	if (pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
	const char attr_name = (const char ) new_attr->name;

	crm_trace("Created new attribute %s=%s in %s",
	attr_name, pcmk__xml_attr_value(new_attr),
	new_xml->name);

	/* Check ACLs (we can't use the remove-then-create trick because it
	* would modify the attribute position).
	*/
	if (pcmk__check_acl(new_xml, attr_name, pcmk__xf_acl_write)) {
	pcmk__mark_xml_attr_dirty(new_attr);
	} else {
	// Creation was not allowed, so remove the attribute
	xmlUnsetProp(new_xml, new_attr->name);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Calculate differences in attributes between two XML nodes
	*
	* \param[in,out] old_xml Original XML to compare
	* \param[in,out] new_xml New XML to compare
	*/
	static void
	xml_diff_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	set_attrs_flag(new_xml, pcmk__xf_created); // cleared later if not really new
	xml_diff_old_attrs(old_xml, new_xml);
	mark_created_attrs(new_xml);
	}

	/*!
	* \internal
	* \brief Add an XML child element to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when a child element has been
	* deleted. Add the child back to the new XML, so that we can check the removal
	* against ACLs, and mark it as deleted for later removal after differences have
	* been calculated.
	*
	* \param[in,out] old_child Child element from original XML
	* \param[in,out] new_parent New XML to add marked copy to
	*/
	static void
	mark_child_deleted(xmlNode old_child, xmlNode new_parent)
	{
	// Re-create the child element so we can check ACLs
	xmlNode *candidate = pcmk__xml_copy(new_parent, old_child);

	// Clear flags on new child and its children
	pcmk__xml_tree_foreach(candidate, reset_xml_node_flags, NULL);

	// Check whether ACLs allow the deletion
	pcmk__apply_acl(xmlDocGetRootElement(candidate->doc));

	// Remove the child again (which will track it in document's deleted_objs)
	free_xml_with_position(candidate,
	pcmk__xml_position(old_child, pcmk__xf_skip));

	if (pcmk__xml_match(new_parent, old_child, true) == NULL) {
	pcmk__set_xml_flags((xml_node_private_t *) (old_child->_private),
	pcmk__xf_skip);
	}
	}

	static void
	mark_child_moved(xmlNode old_child, xmlNode new_parent, xmlNode *new_child,
	int p_old, int p_new)
	{
	xml_node_private_t *nodepriv = new_child->_private;

	crm_trace("Child element %s with "
	PCMK_XA_ID "='%s' moved from position %d to %d under %s",
	new_child->name, pcmk__s(pcmk__xe_id(new_child), "<no id>"),
	p_old, p_new, new_parent->name);
	pcmk__mark_xml_node_dirty(new_parent);
	pcmk__set_xml_flags(nodepriv, pcmk__xf_moved);

	if (p_old > p_new) {
	nodepriv = old_child->_private;
	} else {
	nodepriv = new_child->_private;
	}
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	}

	// Given original and new XML, mark new XML portions that have changed
	static void
	mark_xml_changes(xmlNode old_xml, xmlNode new_xml, bool check_top)
	{
	xmlNode *old_child = NULL;
	xmlNode *new_child = NULL;
	xml_node_private_t *nodepriv = NULL;

	CRM_CHECK(new_xml != NULL, return);
	if (old_xml == NULL) {
	pcmk__xml_mark_created(new_xml);
	pcmk__apply_creation_acl(new_xml, check_top);
	return;
	}

	nodepriv = new_xml->_private;
	CRM_CHECK(nodepriv != NULL, return);

	if(nodepriv->flags & pcmk__xf_processed) {
	/* Avoid re-comparing nodes */
	return;
	}
	pcmk__set_xml_flags(nodepriv, pcmk__xf_processed);

	xml_diff_attrs(old_xml, new_xml);

	// Check for differences in the original children
	for (old_child = pcmk__xml_first_child(old_xml); old_child != NULL;
	old_child = pcmk__xml_next(old_child)) {

	new_child = pcmk__xml_match(new_xml, old_child, true);

	if (new_child != NULL) {
	mark_xml_changes(old_child, new_child, true);

	} else {
	mark_child_deleted(old_child, new_xml);
	}
	}

	// Check for moved or created children
	new_child = pcmk__xml_first_child(new_xml);
	while (new_child != NULL) {
	xmlNode *next = pcmk__xml_next(new_child);

	old_child = pcmk__xml_match(old_xml, new_child, true);

	if (old_child == NULL) {
	// This is a newly created child
	nodepriv = new_child->_private;
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);

	// May free new_child
	mark_xml_changes(old_child, new_child, true);

	} else {
	/* Check for movement, we already checked for differences */
	int p_new = pcmk__xml_position(new_child, pcmk__xf_skip);
	int p_old = pcmk__xml_position(old_child, pcmk__xf_skip);

	if(p_old != p_new) {
	mark_child_moved(old_child, new_xml, new_child, p_old, p_new);
	}
	}

	new_child = next;
	}
	}

	void
	xml_calculate_significant_changes(xmlNode old_xml, xmlNode new_xml)
	{
	pcmk__set_xml_doc_flag(new_xml, pcmk__xf_lazy);
	xml_calculate_changes(old_xml, new_xml);
	}

	// Called functions may set the \p pcmk__xf_skip flag on parts of \p old_xml
	void
	xml_calculate_changes(xmlNode old_xml, xmlNode new_xml)
	{
	CRM_CHECK((old_xml != NULL) && (new_xml != NULL)
	&& pcmk__xe_is(old_xml, (const char *) new_xml->name)
	&& pcmk__str_eq(pcmk__xe_id(old_xml), pcmk__xe_id(new_xml),
	pcmk__str_none),
	return);

	if(xml_tracking_changes(new_xml) == FALSE) {
	xml_track_changes(new_xml, NULL, NULL, FALSE);
	}

	mark_xml_changes(old_xml, new_xml, FALSE);
	}

	/*!
	* \internal
	* \brief Find a comment with matching content in specified XML
	*
	* \param[in] root XML to search
	* \param[in] search_comment Comment whose content should be searched for
	* \param[in] exact If true, comment must also be at same position
	*/
	xmlNode *
	pcmk__xc_match(const xmlNode root, const xmlNode search_comment, bool exact)
	{
	xmlNode *a_child = NULL;
	int search_offset = pcmk__xml_position(search_comment, pcmk__xf_skip);

	CRM_CHECK(search_comment->type == XML_COMMENT_NODE, return NULL);

	for (a_child = pcmk__xml_first_child(root); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	if (exact) {
	int offset = pcmk__xml_position(a_child, pcmk__xf_skip);
	xml_node_private_t *nodepriv = a_child->_private;

	if (offset < search_offset) {
	continue;

	} else if (offset > search_offset) {
	return NULL;
	}

	if (pcmk_is_set(nodepriv->flags, pcmk__xf_skip)) {
	continue;
	}
	}

	if (a_child->type == XML_COMMENT_NODE
	&& pcmk__str_eq((const char )a_child->content, (const char )search_comment->content, pcmk__str_casei)) {
	return a_child;

	} else if (exact) {
	return NULL;
	}
	}

	return NULL;
	}

	/*!
	* \internal
	* \brief Make one XML comment match another (in content)
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* comment child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML comment node
	* \param[in] update Make comment content match this (must not be NULL)
	*
	* \note At least one of \parent and \target must be non-NULL
	*/
	void
	pcmk__xc_update(xmlNode parent, xmlNode target, xmlNode *update)
	{
	CRM_CHECK(update != NULL, return);
	CRM_CHECK(update->type == XML_COMMENT_NODE, return);

	if (target == NULL) {
	target = pcmk__xc_match(parent, update, false);
	}

	if (target == NULL) {
	pcmk__xml_copy(parent, update);

	} else if (!pcmk__str_eq((const char )target->content, (const char )update->content, pcmk__str_casei)) {
	xmlFree(target->content);
	target->content = xmlStrdup(update->content);
	}
	}

	/*!
	* \internal
	* \brief Merge one XML tree into another
	*
	* Here, "merge" means:
	* 1. Copy attribute values from \p update to the target, overwriting in case of
	* conflict.
	* 2. Descend through \p update and the target in parallel. At each level, for
	* each child of \p update, look for a matching child of the target.
	* a. For each child, if a match is found, go to step 1, recursively merging
	* the child of \p update into the child of the target.
	* b. Otherwise, copy the child of \p update as a child of the target.
	*
	* A match is defined as the first child of the same type within the target,
	* with:
	* * the \c PCMK_XA_ID attribute matching, if set in \p update; otherwise,
	* * the \c PCMK_XA_ID_REF attribute matching, if set in \p update
	*
	* This function does not delete any elements or attributes from the target. It
	* may add elements or overwrite attributes, as described above.
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML
	* \param[in] update Make the desired XML match this (must not be \c NULL)
	* \param[in] flags Group of <tt>enum pcmk__xa_flags</tt>
	* \param[in] as_diff If \c true, preserve order of attributes (deprecated
	* since 2.0.5)
	*
	* \note At least one of \p parent and \p target must be non-<tt>NULL</tt>.
	* \note This function is recursive. For the top-level call, \p parent is
	* \c NULL and \p target is not \c NULL. For recursive calls, \p target is
	* \c NULL and \p parent is not \c NULL.
	*/
	void
	pcmk__xml_update(xmlNode parent, xmlNode target, xmlNode *update,
	uint32_t flags, bool as_diff)
	{
	/* @COMPAT Refactor further and staticize after v1 patchset deprecation.
	*
	* @COMPAT Drop as_diff argument when apply_xml_diff() is dropped.
	*/
	const char *update_name = NULL;
	const char *update_id_attr = NULL;
	const char *update_id_val = NULL;
	char *trace_s = NULL;

	crm_log_xml_trace(update, "update");
	crm_log_xml_trace(target, "target");

	CRM_CHECK(update != NULL, goto done);

	if (update->type == XML_COMMENT_NODE) {
	pcmk__xc_update(parent, target, update);
	goto done;
	}

	update_name = (const char *) update->name;

	CRM_CHECK(update_name != NULL, goto done);
	CRM_CHECK((target != NULL) \|\| (parent != NULL), goto done);

	update_id_val = pcmk__xe_id(update);
	if (update_id_val != NULL) {
	update_id_attr = PCMK_XA_ID;

	} else {
	update_id_val = crm_element_value(update, PCMK_XA_ID_REF);
	if (update_id_val != NULL) {
	update_id_attr = PCMK_XA_ID_REF;
	}
	}

	pcmk__if_tracing(
	{
	if (update_id_attr != NULL) {
	trace_s = crm_strdup_printf("<%s %s=%s/>",
	update_name, update_id_attr,
	update_id_val);
	} else {
	trace_s = crm_strdup_printf("<%s/>", update_name);
	}
	},
	{}
	);

	if (target == NULL) {
	// Recursive call
	target = pcmk__xe_first_child(parent, update_name, update_id_attr,
	update_id_val);
	}

	if (target == NULL) {
	// Recursive call with no existing matching child
	target = pcmk__xe_create(parent, update_name);
	crm_trace("Added %s", pcmk__s(trace_s, update_name));

	} else {
	// Either recursive call with match, or top-level call
	crm_trace("Found node %s to update", pcmk__s(trace_s, update_name));
	}

	CRM_CHECK(pcmk__xe_is(target, (const char *) update->name), return);

	if (!as_diff) {
	pcmk__xe_copy_attrs(target, update, flags);

	} else {
	// Preserve order of attributes. Don't use pcmk__xe_copy_attrs().
	for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
	a = a->next) {
	const char *p_value = pcmk__xml_attr_value(a);

	/* Remove it first so the ordering of the update is preserved */
	xmlUnsetProp(target, a->name);
	xmlSetProp(target, a->name, (pcmkXmlStr) p_value);
	}
	}

	for (xmlNode *child = pcmk__xml_first_child(update); child != NULL;
	child = pcmk__xml_next(child)) {

	crm_trace("Updating child of %s", pcmk__s(trace_s, update_name));
	pcmk__xml_update(target, NULL, child, flags, as_diff);
	}

	crm_trace("Finished with %s", pcmk__s(trace_s, update_name));

	done:
	free(trace_s);
	}

	/*!
	* \internal
	* \brief Delete an XML subtree if it matches a search element
	*
	* A match is defined as follows:
	* * \p xml and \p user_data are both element nodes of the same type.
	* * If \p user_data has attributes set, \p xml has those attributes set to the
	* same values. (\p xml may have additional attributes set to arbitrary
	* values.)
	*
	* \param[in,out] xml XML subtree to delete upon match
	* \param[in] user_data Search element
	*
	* \return \c true to continue traversing the tree, or \c false to stop (because
	* \p xml was deleted)
	*
	* \note This is compatible with \c pcmk__xml_tree_foreach().
	*/
	static bool
	delete_xe_if_matching(xmlNode xml, void user_data)
	{
	xmlNode *search = user_data;

	if (!pcmk__xe_is(search, (const char *) xml->name)) {
	// No match: either not both elements, or different element types
	return true;
	}

	for (const xmlAttr *attr = pcmk__xe_first_attr(search); attr != NULL;
	attr = attr->next) {

	const char *search_val = pcmk__xml_attr_value(attr);
	const char xml_val = crm_element_value(xml, (const char ) attr->name);

	if (!pcmk__str_eq(search_val, xml_val, pcmk__str_casei)) {
	// No match: an attr in xml doesn't match the attr in search
	return true;
	}
	}

	crm_log_xml_trace(xml, "delete-match");
	crm_log_xml_trace(search, "delete-search");
	pcmk__xml_free(xml);

	// Found a match and deleted it; stop traversing tree
	return false;
	}

	/*!
	* \internal
	* \brief Search an XML tree depth-first and delete the first matching element
	*
	* This function does not attempt to match the tree root (\p xml).
	*
	* A match with a node \c node is defined as follows:
	* * \c node and \p search are both element nodes of the same type.
	* * If \p search has attributes set, \c node has those attributes set to the
	* same values. (\c node may have additional attributes set to arbitrary
	* values.)
	*
	* \param[in,out] xml XML subtree to search
	* \param[in] search Element to match against
	*
	* \return Standard Pacemaker return code (specifically, \c pcmk_rc_ok on
	* successful deletion and an error code otherwise)
	*/
	int
	pcmk__xe_delete_match(xmlNode xml, xmlNode search)
	{
	// See @COMPAT comment in pcmk__xe_replace_match()
	CRM_CHECK((xml != NULL) && (search != NULL), return EINVAL);

	for (xml = pcmk__xe_first_child(xml, NULL, NULL, NULL); xml != NULL;
	xml = pcmk__xe_next(xml)) {

	if (!pcmk__xml_tree_foreach(xml, delete_xe_if_matching, search)) {
	// Found and deleted an element
	return pcmk_rc_ok;
	}
	}

	// No match found in this subtree
	return ENXIO;
	}

	/*!
	* \internal
	* \brief Replace one XML node with a copy of another XML node
	*
	* This function handles change tracking and applies ACLs.
	*
	* \param[in,out] old XML node to replace
	* \param[in] new XML node to copy as replacement for \p old
	*
	* \note This frees \p old.
	*/
	static void
	replace_node(xmlNode old, xmlNode new)
	{
	new = xmlCopyNode(new, 1);
	pcmk__mem_assert(new);

	// May be unnecessary but avoids slight changes to some test outputs
	pcmk__xml_tree_foreach(new, reset_xml_node_flags, NULL);

	old = xmlReplaceNode(old, new);

	if (xml_tracking_changes(new)) {
	// Replaced sections may have included relevant ACLs
	pcmk__apply_acl(new);
	}
	xml_calculate_changes(old, new);
	xmlFreeNode(old);
	}

	/*!
	* \internal
	* \brief Replace one XML subtree with a copy of another if the two match
	*
	* A match is defined as follows:
	* * \p xml and \p user_data are both element nodes of the same type.
	* * If \p user_data has the \c PCMK_XA_ID attribute set, then \p xml has
	* \c PCMK_XA_ID set to the same value.
	*
	* \param[in,out] xml XML subtree to replace with \p user_data upon match
	* \param[in] user_data XML to replace \p xml with a copy of upon match
	*
	* \return \c true to continue traversing the tree, or \c false to stop (because
	* \p xml was replaced by \p user_data)
	*
	* \note This is compatible with \c pcmk__xml_tree_foreach().
	*/
	static bool
	replace_xe_if_matching(xmlNode xml, void user_data)
	{
	xmlNode *replace = user_data;
	const char *xml_id = NULL;
	const char *replace_id = NULL;

	xml_id = pcmk__xe_id(xml);
	replace_id = pcmk__xe_id(replace);

	if (!pcmk__xe_is(replace, (const char *) xml->name)) {
	// No match: either not both elements, or different element types
	return true;
	}

	if ((replace_id != NULL)
	&& !pcmk__str_eq(replace_id, xml_id, pcmk__str_none)) {

	// No match: ID was provided in replace and doesn't match xml's ID
	return true;
	}

	crm_log_xml_trace(xml, "replace-match");
	crm_log_xml_trace(replace, "replace-with");
	replace_node(xml, replace);

	// Found a match and replaced it; stop traversing tree
	return false;
	}

	/*!
	* \internal
	* \brief Search an XML tree depth-first and replace the first matching element
	*
	* This function does not attempt to match the tree root (\p xml).
	*
	* A match with a node \c node is defined as follows:
	* * \c node and \p replace are both element nodes of the same type.
	* * If \p replace has the \c PCMK_XA_ID attribute set, then \c node has
	* \c PCMK_XA_ID set to the same value.
	*
	* \param[in,out] xml XML tree to search
	* \param[in] replace XML to replace a matching element with a copy of
	*
	* \return Standard Pacemaker return code (specifically, \c pcmk_rc_ok on
	* successful replacement and an error code otherwise)
	*/
	int
	pcmk__xe_replace_match(xmlNode xml, xmlNode replace)
	{
	/* @COMPAT Some of this behavior (like not matching the tree root, which is
	* allowed by pcmk__xe_update_match()) is questionable for general use but
	* required for backward compatibility by cib_process_replace() and
	* cib_process_delete(). Behavior can change at a major version release if
	* desired.
	*/
	CRM_CHECK((xml != NULL) && (replace != NULL), return EINVAL);

	for (xml = pcmk__xe_first_child(xml, NULL, NULL, NULL); xml != NULL;
	xml = pcmk__xe_next(xml)) {

	if (!pcmk__xml_tree_foreach(xml, replace_xe_if_matching, replace)) {
	// Found and replaced an element
	return pcmk_rc_ok;
	}
	}

	// No match found in this subtree
	return ENXIO;
	}

	//! User data for \c update_xe_if_matching()
	struct update_data {
	xmlNode *update; //!< Update source
	uint32_t flags; //!< Group of <tt>enum pcmk__xa_flags</tt>
	};

	/*!
	* \internal
	* \brief Update one XML subtree with another if the two match
	*
	* "Update" means to merge a source subtree into a target subtree (see
	* \c pcmk__xml_update()).
	*
	* A match is defined as follows:
	* * \p xml and \p user_data->update are both element nodes of the same type.
	* * \p xml and \p user_data->update have the same \c PCMK_XA_ID attribute
	* value, or \c PCMK_XA_ID is unset in both
	*
	* \param[in,out] xml XML subtree to update with \p user_data->update
	* upon match
	* \param[in] user_data <tt>struct update_data</tt> object
	*
	* \return \c true to continue traversing the tree, or \c false to stop (because
	* \p xml was updated by \p user_data->update)
	*
	* \note This is compatible with \c pcmk__xml_tree_foreach().
	*/
	static bool
	update_xe_if_matching(xmlNode xml, void user_data)
	{
	struct update_data *data = user_data;
	xmlNode *update = data->update;

	if (!pcmk__xe_is(update, (const char *) xml->name)) {
	// No match: either not both elements, or different element types
	return true;
	}

	if (!pcmk__str_eq(pcmk__xe_id(xml), pcmk__xe_id(update), pcmk__str_none)) {
	// No match: ID mismatch
	return true;
	}

	crm_log_xml_trace(xml, "update-match");
	crm_log_xml_trace(update, "update-with");
	pcmk__xml_update(NULL, xml, update, data->flags, false);

	// Found a match and replaced it; stop traversing tree
	return false;
	}

	/*!
	* \internal
	* \brief Search an XML tree depth-first and update the first matching element
	*
	* "Update" means to merge a source subtree into a target subtree (see
	* \c pcmk__xml_update()).
	*
	* A match with a node \c node is defined as follows:
	* * \c node and \p update are both element nodes of the same type.
	* * \c node and \p update have the same \c PCMK_XA_ID attribute value, or
	* \c PCMK_XA_ID is unset in both
	*
	* \param[in,out] xml XML tree to search
	* \param[in] update XML to update a matching element with
	* \param[in] flags Group of <tt>enum pcmk__xa_flags</tt>
	*
	* \return Standard Pacemaker return code (specifically, \c pcmk_rc_ok on
	* successful update and an error code otherwise)
	*/
	int
	pcmk__xe_update_match(xmlNode xml, xmlNode update, uint32_t flags)
	{
	/* @COMPAT In pcmk__xe_delete_match() and pcmk__xe_replace_match(), we
	* compare IDs only if the equivalent of the update argument has an ID.
	* Here, we're stricter: we consider it a mismatch if only one element has
	* an ID attribute, or if both elements have IDs but they don't match.
	*
	* Perhaps we should align the behavior at a major version release.
	*/
	struct update_data data = {
	.update = update,
	.flags = flags,
	};

	CRM_CHECK((xml != NULL) && (update != NULL), return EINVAL);

	if (!pcmk__xml_tree_foreach(xml, update_xe_if_matching, &data)) {
	// Found and updated an element
	return pcmk_rc_ok;
	}

	// No match found in this subtree
	return ENXIO;
	}

	xmlNode *
	sorted_xml(xmlNode input, xmlNode parent, gboolean recursive)
	{
	xmlNode *child = NULL;
	GSList *nvpairs = NULL;
	xmlNode *result = NULL;

	CRM_CHECK(input != NULL, return NULL);

	result = pcmk__xe_create(parent, (const char *) input->name);
	nvpairs = pcmk_xml_attrs2nvpairs(input);
	nvpairs = pcmk_sort_nvpairs(nvpairs);
	pcmk_nvpairs2xml_attrs(nvpairs, result);
	pcmk_free_nvpairs(nvpairs);

	for (child = pcmk__xe_first_child(input, NULL, NULL, NULL); child != NULL;
	child = pcmk__xe_next(child)) {

	if (recursive) {
	sorted_xml(child, result, recursive);
	} else {
	pcmk__xml_copy(result, child);
	}
	}

	return result;
	}

	/*!
	* \internal
	* \brief Get next sibling XML element with the same name as a given element
	*
	* \param[in] node XML element to start from
	*
	* \return Next sibling XML element with same name
	*/
	xmlNode *
	pcmk__xe_next_same(const xmlNode *node)
	{
	for (xmlNode *match = pcmk__xe_next(node); match != NULL;
	match = pcmk__xe_next(match)) {

	if (pcmk__xe_is(match, (const char *) node->name)) {
	return match;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Initialize the Pacemaker XML environment
	*
	* Set an XML buffer allocation scheme, set XML node create and destroy
	* callbacks, and load schemas into the cache.
	*/
	void
	pcmk__xml_init(void)
	{
	// @TODO Try to find a better caller than crm_log_preinit()
	static bool initialized = false;

	if (!initialized) {
	initialized = true;

	/* Double the buffer size when the buffer needs to grow. The default
	* allocator XML_BUFFER_ALLOC_EXACT was found to cause poor performance
	* due to the number of reallocs.
	*/
	xmlSetBufferAllocationScheme(XML_BUFFER_ALLOC_DOUBLEIT);

	// Initialize private data at node creation
	xmlRegisterNodeDefault(new_private_data);

	// Free private data at node destruction
	xmlDeregisterNodeDefault(free_private_data);

	// Load schemas into the cache
	pcmk__schema_init();
	}
	}

	/*!
	* \internal
	* \brief Tear down the Pacemaker XML environment
	*
	* Destroy schema cache and clean up memory allocated by libxml2.
	*/
	void
	pcmk__xml_cleanup(void)
	{
	pcmk__schema_cleanup();
	xmlCleanupParser();
	}

	/*!
	* \internal
	* \brief Get the XML element whose \c PCMK_XA_ID matches an \c PCMK_XA_ID_REF
	*
	* \param[in] xml Element whose \c PCMK_XA_ID_REF attribute to check
	* \param[in] search Node whose document to search for node with matching
	* \c PCMK_XA_ID (\c NULL to use \p xml)
	*
	* \return If \p xml has a \c PCMK_XA_ID_REF attribute, node in
	* <tt>search</tt>'s document whose \c PCMK_XA_ID attribute matches;
	* otherwise, \p xml
	*/
	xmlNode *
	pcmk__xe_resolve_idref(xmlNode xml, xmlNode search)
	{
	char *xpath = NULL;
	const char *ref = NULL;
	xmlNode *result = NULL;

	if (xml == NULL) {
	return NULL;
	}

	ref = crm_element_value(xml, PCMK_XA_ID_REF);
	if (ref == NULL) {
	return xml;
	}

	if (search == NULL) {
	search = xml;
	}

	xpath = crm_strdup_printf("//%s[@" PCMK_XA_ID "='%s']", xml->name, ref);
	result = get_xpath_object(xpath, search, LOG_DEBUG);
	if (result == NULL) {
	// Not possible with schema validation enabled
	pcmk__config_err("Ignoring invalid %s configuration: "
	PCMK_XA_ID_REF " '%s' does not reference "
	"a valid object " CRM_XS " xpath=%s",
	xml->name, ref, xpath);
	}
	free(xpath);
	return result;
	}

	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns)
	{
	static const char *base = NULL;
	char *ret = NULL;

	if (base == NULL) {
	base = pcmk__env_option(PCMK__ENV_SCHEMA_DIRECTORY);
	}
	if (pcmk__str_empty(base)) {
	base = CRM_SCHEMA_DIRECTORY;
	}

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_legacy_xslt:
	ret = strdup(base);
	break;
	case pcmk__xml_artefact_ns_base_rng:
	case pcmk__xml_artefact_ns_base_xslt:
	ret = crm_strdup_printf("%s/base", base);
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}
	return ret;
	}

	static char *
	find_artefact(enum pcmk__xml_artefact_ns ns, const char path, const char filespec)
	{
	char *ret = NULL;

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_base_rng:
	if (pcmk__ends_with(filespec, ".rng")) {
	ret = crm_strdup_printf("%s/%s", path, filespec);
	} else {
	ret = crm_strdup_printf("%s/%s.rng", path, filespec);
	}
	break;
	case pcmk__xml_artefact_ns_legacy_xslt:
	case pcmk__xml_artefact_ns_base_xslt:
	if (pcmk__ends_with(filespec, ".xsl")) {
	ret = crm_strdup_printf("%s/%s", path, filespec);
	} else {
	ret = crm_strdup_printf("%s/%s.xsl", path, filespec);
	}
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}

	return ret;
	}

	char *
	pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec)
	{
	struct stat sb;
	char *base = pcmk__xml_artefact_root(ns);
	char *ret = NULL;

	ret = find_artefact(ns, base, filespec);
	free(base);

	if (stat(ret, &sb) != 0 \|\| !S_ISREG(sb.st_mode)) {
	const char *remote_schema_dir = pcmk__remote_schema_dir();

	free(ret);
	ret = find_artefact(ns, remote_schema_dir, filespec);
	}

	return ret;
	}

	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs)
	{
	while (true) {
	const char name, value;

	name = va_arg(pairs, const char *);
	if (name == NULL) {
	return;
	}

	value = va_arg(pairs, const char *);
	if (value != NULL) {
	crm_xml_add(node, name, value);
	}
	}
	}

	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	{
	va_list pairs;
	va_start(pairs, node);
	pcmk__xe_set_propv(node, pairs);
	va_end(pairs);
	}

	int
	pcmk__xe_foreach_child(xmlNode xml, const char child_element_name,
	int (handler)(xmlNode xml, void *userdata),
	void *userdata)
	{
	xmlNode *children = (xml? xml->children : NULL);

	CRM_ASSERT(handler != NULL);

	for (xmlNode *node = children; node != NULL; node = node->next) {
	if ((node->type == XML_ELEMENT_NODE)
	&& ((child_element_name == NULL)
	\|\| pcmk__xe_is(node, child_element_name))) {
	int rc = handler(node, userdata);

	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	}

	return pcmk_rc_ok;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/common/xml_compat.h>

	xmlNode *
	copy_xml(xmlNode *src)
	{
	xmlDoc *doc = xmlNewDoc(PCMK__XML_VERSION);
	xmlNode *copy = NULL;

	pcmk__mem_assert(doc);

	copy = xmlDocCopyNode(src, doc, 1);
	pcmk__mem_assert(copy);

	xmlDocSetRootElement(doc, copy);
	return copy;
	}

	void
	crm_xml_init(void)
	{
	pcmk__xml_init();
	}

	void
	crm_xml_cleanup(void)
	{
	pcmk__xml_cleanup();
	}

	void
	pcmk_free_xml_subtree(xmlNode *xml)
	{
	xmlUnlinkNode(xml); // Detaches from parent and siblings
	xmlFreeNode(xml); // Frees
	}

	void
	free_xml(xmlNode *child)
	{
	pcmk__xml_free(child);
	}

	xmlNode *
	expand_idref(xmlNode input, xmlNode top)
	{
	return pcmk__xe_resolve_idref(input, top);
	}

	+void
	+crm_xml_sanitize_id(char *id)
	+{
	+ char *c;
	+
	+ for (c = id; *c; ++c) {
	+ switch (*c) {
	+ case ':':
	+ case '#':
	+ *c = '.';
	+ }
	+ }
	+}
	+
	+void
	+crm_xml_set_id(xmlNode xml, const char format, ...)
	+{
	+ va_list ap;
	+ int len = 0;
	+ char *id = NULL;
	+
	+ /* equivalent to crm_strdup_printf() */
	+ va_start(ap, format);
	+ len = vasprintf(&id, format, ap);
	+ va_end(ap);
	+ CRM_ASSERT(len > 0);
	+
	+ crm_xml_sanitize_id(id);
	+ crm_xml_add(xml, PCMK_XA_ID, id);
	+ free(id);
	+}
	+
	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
	index 05dc838dcf..67340ca1c5 100644
	--- a/lib/pengine/bundle.c
	+++ b/lib/pengine/bundle.c
	@@ -1,2133 +1,2138 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <ctype.h>
	#include <stdint.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/output.h>
	#include <crm/common/xml_internal.h>
	#include <pe_status_private.h>

	enum pe__bundle_mount_flags {
	pe__bundle_mount_none = 0x00,

	// mount instance-specific subdirectory rather than source directly
	pe__bundle_mount_subdir = 0x01
	};

	typedef struct {
	char *source;
	char *target;
	char *options;
	uint32_t flags; // bitmask of pe__bundle_mount_flags
	} pe__bundle_mount_t;

	typedef struct {
	char *source;
	char *target;
	} pe__bundle_port_t;

	enum pe__container_agent {
	PE__CONTAINER_AGENT_UNKNOWN,
	PE__CONTAINER_AGENT_DOCKER,
	PE__CONTAINER_AGENT_RKT,
	PE__CONTAINER_AGENT_PODMAN,
	};

	#define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
	#define PE__CONTAINER_AGENT_DOCKER_S "docker"
	#define PE__CONTAINER_AGENT_RKT_S "rkt"
	#define PE__CONTAINER_AGENT_PODMAN_S "podman"

	typedef struct pe__bundle_variant_data_s {
	int promoted_max;
	int nreplicas;
	int nreplicas_per_host;
	char *prefix;
	char *image;
	const char *ip_last;
	char *host_network;
	char *host_netmask;
	char *control_port;
	char *container_network;
	char *ip_range_start;
	gboolean add_host;
	gchar *container_host_options;
	char *container_command;
	char *launcher_options;
	const char *attribute_target;

	pcmk_resource_t *child;

	GList replicas; // pcmk__bundle_replica_t
	GList ports; // pe__bundle_port_t
	GList mounts; // pe__bundle_mount_t

	enum pe__container_agent agent_type;
	} pe__bundle_variant_data_t;

	#define get_bundle_variant_data(data, rsc) do { \
	CRM_ASSERT(pcmk__is_bundle(rsc)); \
	data = rsc->private->variant_opaque; \
	} while (0)

	/*!
	* \internal
	* \brief Get maximum number of bundle replicas allowed to run
	*
	* \param[in] rsc Bundle or bundled resource to check
	*
	* \return Maximum replicas for bundle corresponding to \p rsc
	*/
	int
	pe__bundle_max(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->nreplicas;
	}

	/*!
	* \internal
	* \brief Get the resource inside a bundle
	*
	* \param[in] bundle Bundle to check
	*
	* \return Resource inside \p bundle if any, otherwise NULL
	*/
	pcmk_resource_t *
	pe__bundled_resource(const pcmk_resource_t *rsc)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
	return bundle_data->child;
	}

	/*!
	* \internal
	* \brief Get containerized resource corresponding to a given bundle container
	*
	* \param[in] instance Collective instance that might be a bundle container
	*
	* \return Bundled resource instance inside \p instance if it is a bundle
	* container instance, otherwise NULL
	*/
	const pcmk_resource_t *
	pe__get_rsc_in_container(const pcmk_resource_t *instance)
	{
	const pe__bundle_variant_data_t *data = NULL;
	const pcmk_resource_t *top = pe__const_top_resource(instance, true);

	if (!pcmk__is_bundle(top)) {
	return NULL;
	}
	get_bundle_variant_data(data, top);

	for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	const pcmk__bundle_replica_t *replica = iter->data;

	if (instance == replica->container) {
	return replica->child;
	}
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Check whether a given node is created by a bundle
	*
	* \param[in] bundle Bundle resource to check
	* \param[in] node Node to check
	*
	* \return true if \p node is an instance of \p bundle, otherwise false
	*/
	bool
	pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
	const pcmk_node_t *node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (pcmk__same_node(node, replica->node)) {
	return true;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Get the container of a bundle's first replica
	*
	* \param[in] bundle Bundle resource to get container for
	*
	* \return Container resource from first replica of \p bundle if any,
	* otherwise NULL
	*/
	pcmk_resource_t *
	pe__first_container(const pcmk_resource_t *bundle)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;
	const pcmk__bundle_replica_t *replica = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	if (bundle_data->replicas == NULL) {
	return NULL;
	}
	replica = bundle_data->replicas->data;
	return replica->container;
	}

	/*!
	* \internal
	* \brief Iterate over bundle replicas
	*
	* \param[in,out] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_bundle_replica(pcmk_resource_t *bundle,
	bool (fn)(pcmk__bundle_replica_t , void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	/*!
	* \internal
	* \brief Iterate over const bundle replicas
	*
	* \param[in] bundle Bundle to iterate over
	* \param[in] fn Function to call for each replica (its return value
	* indicates whether to continue iterating)
	* \param[in,out] user_data Pointer to pass to \p fn
	*/
	void
	pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
	bool (fn)(const pcmk__bundle_replica_t ,
	void *),
	void *user_data)
	{
	const pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, bundle);
	for (const GList *iter = bundle_data->replicas; iter != NULL;
	iter = iter->next) {

	if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
	break;
	}
	}
	}

	static char *
	next_ip(const char *last_ip)
	{
	unsigned int oct1 = 0;
	unsigned int oct2 = 0;
	unsigned int oct3 = 0;
	unsigned int oct4 = 0;
	int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);

	if (rc != 4) {
	/@ TODO check for IPv6 /
	return NULL;

	} else if (oct3 > 253) {
	return NULL;

	} else if (oct4 > 253) {
	++oct3;
	oct4 = 1;

	} else {
	++oct4;
	}

	return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
	}

	static void
	allocate_ip(pe__bundle_variant_data_t data, pcmk__bundle_replica_t replica,
	GString *buffer)
	{
	if(data->ip_range_start == NULL) {
	return;

	} else if(data->ip_last) {
	replica->ipaddr = next_ip(data->ip_last);

	} else {
	replica->ipaddr = strdup(data->ip_range_start);
	}

	data->ip_last = replica->ipaddr;
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (data->add_host) {
	g_string_append_printf(buffer, " --add-host=%s-%d:%s",
	data->prefix, replica->offset,
	replica->ipaddr);
	} else {
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	}
	break;

	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
	replica->ipaddr, data->prefix,
	replica->offset);
	break;

	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	}

	static xmlNode *
	create_resource(const char name, const char provider, const char *kind)
	{
	xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);

	crm_xml_add(rsc, PCMK_XA_ID, name);
	crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
	crm_xml_add(rsc, PCMK_XA_TYPE, kind);

	return rsc;
	}

	/*!
	* \internal
	* \brief Check whether cluster can manage resource inside container
	*
	* \param[in,out] data Container variant data
	*
	* \return TRUE if networking configuration is acceptable, FALSE otherwise
	*
	* \note The resource is manageable if an IP range or control port has been
	* specified. If a control port is used without an IP range, replicas per
	* host must be 1.
	*/
	static bool
	valid_network(pe__bundle_variant_data_t *data)
	{
	if(data->ip_range_start) {
	return TRUE;
	}
	if(data->control_port) {
	if(data->nreplicas_per_host > 1) {
	pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
	"requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
	data->prefix);
	data->nreplicas_per_host = 1;
	// @TODO to be sure:
	// pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}
	return TRUE;
	}
	return FALSE;
	}

	static int
	create_ip_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if(data->ip_range_start) {
	char *id = NULL;
	xmlNode *xml_ip = NULL;
	xmlNode *xml_obj = NULL;

	id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
	- crm_xml_sanitize_id(id);
	+ pcmk__xml_sanitize_id(id);
	xml_ip = create_resource(id, "heartbeat", "IPaddr2");
	free(id);

	xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
	- crm_xml_set_id(xml_obj, "%s-attributes-%d",
	- data->prefix, replica->offset);
	+ pcmk__xe_set_id(xml_obj, "%s-attributes-%d",
	+ data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
	if(data->host_network) {
	crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
	}

	if(data->host_netmask) {
	crm_create_nvpair_xml(xml_obj, NULL,
	"cidr_netmask", data->host_netmask);

	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
	}

	xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
	crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
	"60s", NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?

	if (pe__unpack_resource(xml_ip, &replica->ip, parent,
	parent->private->scheduler) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	parent->children = g_list_append(parent->children, replica->ip);
	}
	return pcmk_rc_ok;
	}

	static const char*
	container_agent_str(enum pe__container_agent t)
	{
	switch (t) {
	case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
	case PE__CONTAINER_AGENT_RKT: return PE__CONTAINER_AGENT_RKT_S;
	case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	break;
	}
	return PE__CONTAINER_AGENT_UNKNOWN_S;
	}

	static int
	create_container_resource(pcmk_resource_t *parent,
	const pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	char *id = NULL;
	xmlNode *xml_container = NULL;
	xmlNode *xml_obj = NULL;

	// Agent-specific
	const char *hostname_opt = NULL;
	const char *env_opt = NULL;
	const char *agent_str = NULL;
	int volid = 0; // rkt-only

	GString *buffer = NULL;
	GString *dbuffer = NULL;

	// Where syntax differences are drop-in replacements, set them now
	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	hostname_opt = "-h ";
	env_opt = "-e ";
	break;
	case PE__CONTAINER_AGENT_RKT:
	hostname_opt = "--hostname=";
	env_opt = "--environment=";
	break;
	default: // PE__CONTAINER_AGENT_UNKNOWN
	return pcmk_rc_unpack_error;
	}
	agent_str = container_agent_str(data->agent_type);

	buffer = g_string_sized_new(4096);

	id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
	replica->offset);
	- crm_xml_sanitize_id(id);
	+ pcmk__xml_sanitize_id(id);
	xml_container = create_resource(id, "heartbeat", agent_str);
	free(id);

	xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
	- crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
	+ pcmk__xe_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);

	crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
	crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
	crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
	crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);

	if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
	g_string_append(buffer, " --restart=no");
	}

	/* Set a container hostname only if we have an IP to map it to. The user can
	* set -h or --uts=host themselves if they want a nicer name for logs, but
	* this makes applications happy who need their hostname to match the IP
	* they bind to.
	*/
	if (data->ip_range_start != NULL) {
	g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
	replica->offset);
	}
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);

	if (data->container_network != NULL) {
	pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
	}

	if (data->control_port != NULL) {
	pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
	data->control_port, NULL);
	} else {
	g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
	env_opt, DEFAULT_REMOTE_PORT);
	}

	for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
	pe__bundle_mount_t mount = (pe__bundle_mount_t ) iter->data;
	char *source = NULL;

	if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
	source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
	replica->offset);
	pcmk__add_separated_word(&dbuffer, 1024, source, ",");
	}

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	pcmk__g_strcat(buffer,
	" -v ", pcmk__s(source, mount->source),
	":", mount->target, NULL);

	if (mount->options != NULL) {
	pcmk__g_strcat(buffer, ":", mount->options, NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	g_string_append_printf(buffer,
	" --volume vol%d,kind=host,"
	"source=%s%s%s "
	"--mount volume=vol%d,target=%s",
	volid, pcmk__s(source, mount->source),
	(mount->options != NULL)? "," : "",
	pcmk__s(mount->options, ""),
	volid, mount->target);
	volid++;
	break;
	default:
	break;
	}
	free(source);
	}

	for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
	pe__bundle_port_t port = (pe__bundle_port_t ) iter->data;

	switch (data->agent_type) {
	case PE__CONTAINER_AGENT_DOCKER:
	case PE__CONTAINER_AGENT_PODMAN:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" -p ", replica->ipaddr, ":", port->source,
	":", port->target, NULL);

	} else if (!pcmk__str_eq(data->container_network,
	PCMK_VALUE_HOST, pcmk__str_none)) {
	// No need to do port mapping if net == host
	pcmk__g_strcat(buffer,
	" -p ", port->source, ":", port->target,
	NULL);
	}
	break;
	case PE__CONTAINER_AGENT_RKT:
	if (replica->ipaddr != NULL) {
	pcmk__g_strcat(buffer,
	" --port=", port->target,
	":", replica->ipaddr, ":", port->source,
	NULL);
	} else {
	pcmk__g_strcat(buffer,
	" --port=", port->target, ":", port->source,
	NULL);
	}
	break;
	default:
	break;
	}
	}

	/* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
	* it would cause restarts during rolling upgrades.
	*
	* In a previous version of the container resource creation logic, if
	* data->launcher_options is not NULL, we append
	* (" %s", data->launcher_options) even if data->launcher_options is an
	* empty string. Likewise for data->container_host_options. Using
	*
	* pcmk__add_word(buffer, 0, data->launcher_options)
	*
	* removes that extra trailing space, causing a resource definition change.
	*/
	if (data->launcher_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
	}

	if (data->container_host_options != NULL) {
	pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
	}

	crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
	(const char *) buffer->str);
	g_string_free(buffer, TRUE);

	crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
	(dbuffer != NULL)? (const char *) dbuffer->str : "");
	if (dbuffer != NULL) {
	g_string_free(dbuffer, TRUE);
	}

	if (replica->child != NULL) {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	} else {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	SBIN_DIR "/pacemaker-remoted");
	}

	/* TODO: Allow users to specify their own?
	*
	* We just want to know if the container is alive; we'll monitor the
	* child independently.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	#if 0
	/* @TODO Consider supporting the use case where we can start and stop
	* resources, but not proxy local commands (such as setting node
	* attributes), by running the local executor in stand-alone mode.
	* However, this would probably be better done via ACLs as with other
	* Pacemaker Remote nodes.
	*/
	} else if ((child != NULL) && data->untrusted) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	CRM_DAEMON_DIR "/pacemaker-execd");
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
	CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
	#endif
	} else {
	if (data->container_command != NULL) {
	crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
	data->container_command);
	}

	/* TODO: Allow users to specify their own?
	*
	* We don't know what's in the container, so we just want to know if it
	* is alive.
	*/
	crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
	}

	xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
	crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
	"60s", NULL);

	// TODO: Other ops? Timeouts and intervals from underlying resource?
	if (pe__unpack_resource(xml_container, &replica->container, parent,
	parent->private->scheduler) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}
	pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
	parent->children = g_list_append(parent->children, replica->container);

	return pcmk_rc_ok;
	}

	/*!
	* \brief Ban a node from a resource's (and its children's) allowed nodes list
	*
	* \param[in,out] rsc Resource to modify
	* \param[in] uname Name of node to ban
	*/
	static void
	disallow_node(pcmk_resource_t rsc, const char uname)
	{
	gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);

	if (match) {
	((pcmk_node_t *) match)->weight = -PCMK_SCORE_INFINITY;
	((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
	}
	if (rsc->children) {
	g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
	}
	}

	static int
	create_remote_resource(pcmk_resource_t parent, pe__bundle_variant_data_t data,
	pcmk__bundle_replica_t *replica)
	{
	if (replica->child && valid_network(data)) {
	GHashTableIter gIter;
	pcmk_node_t *node = NULL;
	xmlNode *xml_remote = NULL;
	char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
	char *port_s = NULL;
	const char *uname = NULL;
	const char *connect_name = NULL;

	if (pe_find_resource(parent->private->scheduler->resources,
	id) != NULL) {
	free(id);
	// The biggest hammer we have
	id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
	replica->child->id, replica->offset);
	//@TODO return error instead of asserting?
	CRM_ASSERT(pe_find_resource(parent->private->scheduler->resources,
	id) == NULL);
	}

	/* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
	* connection does not have its own IP is a magic string that we use to
	* support nested remotes (i.e. a bundle running on a remote node).
	*/
	connect_name = (replica->ipaddr? replica->ipaddr : "#uname");

	if (data->control_port == NULL) {
	port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}

	/* This sets replica->container as replica->remote's container, which is
	* similar to what happens with guest nodes. This is how the scheduler
	* knows that the bundle node is fenced by recovering the container, and
	* that remote should be ordered relative to the container.
	*/
	xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
	NULL, NULL, NULL,
	connect_name, (data->control_port?
	data->control_port : port_s));
	free(port_s);

	/* Abandon our created ID, and pull the copy from the XML, because we
	* need something that will get freed during scheduler data cleanup to
	* use as the node ID and uname.
	*/
	free(id);
	id = NULL;
	uname = pcmk__xe_id(xml_remote);

	/* Ensure a node has been created for the guest (it may have already
	* been, if it has a permanent node attribute), and ensure its weight is
	* -INFINITY so no other resources can run on it.
	*/
	node = pcmk_find_node(parent->private->scheduler, uname);
	if (node == NULL) {
	node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
	PCMK_VALUE_MINUS_INFINITY,
	parent->private->scheduler);
	} else {
	node->weight = -PCMK_SCORE_INFINITY;
	}
	node->rsc_discover_mode = pcmk_probe_never;

	/* unpack_remote_nodes() ensures that each remote node and guest node
	* has a pcmk_node_t entry. Ideally, it would do the same for bundle
	* nodes. Unfortunately, a bundle has to be mostly unpacked before it's
	* obvious what nodes will be needed, so we do it just above.
	*
	* Worse, that means that the node may have been utilized while
	* unpacking other resources, without our weight correction. The most
	* likely place for this to happen is when pe__unpack_resource() calls
	* resource_location() to set a default score in symmetric clusters.
	* This adds a node copy to each resource's allowed nodes, and these
	* copies will have the wrong weight.
	*
	* As a hacky workaround, fix those copies here.
	*
	* @TODO Possible alternative: ensure bundles are unpacked before other
	* resources, so the weight is correct before any copies are made.
	*/
	g_list_foreach(parent->private->scheduler->resources,
	(GFunc) disallow_node, (gpointer) uname);

	replica->node = pe__copy_node(node);
	replica->node->weight = 500;
	replica->node->rsc_discover_mode = pcmk_probe_exclusive;

	/* Ensure the node shows up as allowed and with the correct discovery set */
	if (replica->child->allowed_nodes != NULL) {
	g_hash_table_destroy(replica->child->allowed_nodes);
	}
	replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
	g_hash_table_insert(replica->child->allowed_nodes,
	(gpointer) replica->node->details->id,
	pe__copy_node(replica->node));

	{
	pcmk_node_t *copy = pe__copy_node(replica->node);
	copy->weight = -PCMK_SCORE_INFINITY;
	g_hash_table_insert(replica->child->private->parent->allowed_nodes,
	(gpointer) replica->node->details->id, copy);
	}
	if (pe__unpack_resource(xml_remote, &replica->remote, parent,
	parent->private->scheduler) != pcmk_rc_ok) {
	return pcmk_rc_unpack_error;
	}

	g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
	if (pcmk__is_pacemaker_remote_node(node)) {
	/* Remote resources can only run on 'normal' cluster node */
	node->weight = -PCMK_SCORE_INFINITY;
	}
	}

	replica->node->details->remote_rsc = replica->remote;

	// Ensure pcmk__is_guest_or_bundle_node() functions correctly
	replica->remote->container = replica->container;

	/* A bundle's #kind is closer to "container" (guest node) than the
	* "remote" set by pe_create_node().
	*/
	pcmk__insert_dup(replica->node->details->attrs,
	CRM_ATTR_KIND, "container");

	/* One effect of this is that setup_container() will add
	* replica->remote to replica->container's fillers, which will make
	* pe__resource_contains_guest_node() true for replica->container.
	*
	* replica->child does NOT get added to replica->container's fillers.
	* The only noticeable effect if it did would be for its fail count to
	* be taken into account when checking replica->container's migration
	* threshold.
	*/
	parent->children = g_list_append(parent->children, replica->remote);
	}
	return pcmk_rc_ok;
	}

	static int
	create_replica_resources(pcmk_resource_t *parent,
	pe__bundle_variant_data_t *data,
	pcmk__bundle_replica_t *replica)
	{
	int rc = pcmk_rc_ok;

	rc = create_container_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_ip_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	rc = create_remote_resource(parent, data, replica);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
	pcmk__insert_meta(replica->child, "external-ip", replica->ipaddr);
	}

	if (replica->remote != NULL) {
	/*
	* Allow the remote connection resource to be allocated to a
	* different node than the one on which the container is active.
	*
	* This makes it possible to have Pacemaker Remote nodes running
	* containers with pacemaker-remoted inside in order to start
	* services inside those containers.
	*/
	pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
	}
	return rc;
	}

	static void
	mount_add(pe__bundle_variant_data_t bundle_data, const char source,
	const char target, const char options, uint32_t flags)
	{
	pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
	sizeof(pe__bundle_mount_t));

	mount->source = pcmk__str_copy(source);
	mount->target = pcmk__str_copy(target);
	mount->options = pcmk__str_copy(options);
	mount->flags = flags;
	bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
	}

	static void
	mount_free(pe__bundle_mount_t *mount)
	{
	free(mount->source);
	free(mount->target);
	free(mount->options);
	free(mount);
	}

	static void
	port_free(pe__bundle_port_t *port)
	{
	free(port->source);
	free(port->target);
	free(port);
	}

	static pcmk__bundle_replica_t *
	replica_for_remote(pcmk_resource_t *remote)
	{
	pcmk_resource_t *top = remote;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (top == NULL) {
	return NULL;
	}
	while (top->private->parent != NULL) {
	top = top->private->parent;
	}

	get_bundle_variant_data(bundle_data, top);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (replica->remote == remote) {
	return replica;
	}
	}
	CRM_LOG_ASSERT(FALSE);
	return NULL;
	}

	bool
	pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
	{
	const char *value;
	GHashTable *params = NULL;

	if (rsc == NULL) {
	return false;
	}

	// Use NULL node since pcmk__bundle_expand() uses that to set value
	params = pe_rsc_params(rsc, NULL, rsc->private->scheduler);
	value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);

	return pcmk__str_eq(value, "#uname", pcmk__str_casei)
	&& xml_contains_remote_node(rsc->private->xml);
	}

	const char *
	pe__add_bundle_remote_name(pcmk_resource_t rsc, xmlNode xml,
	const char *field)
	{
	// REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside

	pcmk_node_t *node = NULL;
	pcmk__bundle_replica_t *replica = NULL;

	if (!pe__bundle_needs_remote_name(rsc)) {
	return NULL;
	}

	replica = replica_for_remote(rsc);
	if (replica == NULL) {
	return NULL;
	}

	node = replica->container->allocated_to;
	if (node == NULL) {
	/* If it won't be running anywhere after the
	* transition, go with where it's running now.
	*/
	node = pcmk__current_node(replica->container);
	}

	if(node == NULL) {
	crm_trace("Cannot determine address for bundle connection %s", rsc->id);
	return NULL;
	}

	crm_trace("Setting address for bundle connection %s to bundle host %s",
	rsc->id, pcmk__node_name(node));
	if(xml != NULL && field != NULL) {
	crm_xml_add(xml, field, node->details->uname);
	}

	return node->details->uname;
	}

	#define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do { \
	flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE, \
	"Bundle mount", pcmk__xe_id(mount_xml), \
	flags, (flags_to_set), #flags_to_set); \
	} while (0)

	gboolean
	pe__unpack_bundle(pcmk_resource_t rsc, pcmk_scheduler_t scheduler)
	{
	const char *value = NULL;
	xmlNode *xml_obj = NULL;
	const xmlNode *xml_child = NULL;
	xmlNode *xml_resource = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	bool need_log_mount = TRUE;

	CRM_ASSERT(rsc != NULL);
	pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);

	bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
	rsc->private->variant_opaque = bundle_data;
	bundle_data->prefix = strdup(rsc->id);

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_DOCKER, NULL,
	NULL);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
	} else {
	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK__XE_RKT, NULL,
	NULL);
	if (xml_obj != NULL) {
	pcmk__warn_once(pcmk__wo_rkt,
	"Support for " PCMK__XE_RKT " in bundles "
	"(such as %s) is deprecated and will be "
	"removed in a future release", rsc->id);
	bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
	} else {
	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_PODMAN,
	NULL, NULL);
	if (xml_obj != NULL) {
	bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
	} else {
	return FALSE;
	}
	}
	}

	// Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
	value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
	if (value == NULL) {
	// @COMPAT deprecated since 2.0.0
	value = crm_element_value(xml_obj, PCMK__XA_PROMOTED_MAX_LEGACY);

	if (value != NULL) {
	pcmk__warn_once(pcmk__wo_bundle_master,
	"Support for the " PCMK__XA_PROMOTED_MAX_LEGACY
	" attribute (such as in %s) is deprecated and "
	"will be removed in a future release. Use "
	PCMK_XA_PROMOTED_MAX " instead.",
	rsc->id);
	}
	}
	pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);

	/* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
	* otherwise
	*/
	value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
	if ((value == NULL) && (bundle_data->promoted_max > 0)) {
	bundle_data->nreplicas = bundle_data->promoted_max;
	} else {
	pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
	}

	/*
	* Communication between containers on the same host via the
	* floating IPs only works if the container is started with:
	* --userland-proxy=false --ip-masq=false
	*/
	value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
	pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
	if (bundle_data->nreplicas_per_host == 1) {
	pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
	}

	bundle_data->container_command =
	crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
	bundle_data->launcher_options = crm_element_value_copy(xml_obj,
	PCMK_XA_OPTIONS);
	bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
	bundle_data->container_network = crm_element_value_copy(xml_obj,
	PCMK_XA_NETWORK);

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_NETWORK, NULL,
	NULL);
	if(xml_obj) {
	bundle_data->ip_range_start =
	crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
	bundle_data->host_netmask =
	crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
	bundle_data->host_network =
	crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
	bundle_data->control_port =
	crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
	value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
	if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
	bundle_data->add_host = TRUE;
	}

	for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
	NULL, NULL);
	xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {

	pe__bundle_port_t *port =
	pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));

	port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);

	if(port->source == NULL) {
	port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
	} else {
	port->target = crm_element_value_copy(xml_child,
	PCMK_XA_INTERNAL_PORT);
	}

	if(port->source != NULL && strlen(port->source) > 0) {
	if(port->target == NULL) {
	port->target = strdup(port->source);
	}
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	} else {
	pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
	pcmk__xe_id(xml_child));
	port_free(port);
	}
	}
	}

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_STORAGE, NULL,
	NULL);
	for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
	NULL, NULL);
	xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {

	const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
	const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
	const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
	int flags = pe__bundle_mount_none;

	if (source == NULL) {
	source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
	pe__set_bundle_mount_flags(xml_child, flags,
	pe__bundle_mount_subdir);
	}

	if (source && target) {
	mount_add(bundle_data, source, target, options, flags);
	if (strcmp(target, "/var/log") == 0) {
	need_log_mount = FALSE;
	}
	} else {
	pcmk__config_err("Invalid mount directive %s",
	pcmk__xe_id(xml_child));
	}
	}

	xml_obj = pcmk__xe_first_child(rsc->private->xml, PCMK_XE_PRIMITIVE, NULL,
	NULL);
	if (xml_obj && valid_network(bundle_data)) {
	+ const char *suffix = NULL;
	char *value = NULL;
	xmlNode *xml_set = NULL;

	xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);

	/* @COMPAT We no longer use the <master> tag, but we need to keep it as
	* part of the resource name, so that bundles don't restart in a rolling
	* upgrade. (It also avoids needing to change regression tests.)
	*/
	- crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
	- (bundle_data->promoted_max? "master"
	- : (const char *)xml_resource->name));
	+ suffix = (const char *) xml_resource->name;
	+ if (bundle_data->promoted_max > 0) {
	+ suffix = "master";
	+ }
	+
	+ pcmk__xe_set_id(xml_resource, "%s-%s", bundle_data->prefix, suffix);

	xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
	- crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
	+ pcmk__xe_set_id(xml_set, "%s-%s-meta",
	+ bundle_data->prefix, xml_resource->name);

	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_ORDERED, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->nreplicas);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
	free(value);

	value = pcmk__itoa(bundle_data->nreplicas_per_host);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
	free(value);

	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
	pcmk__btoa(bundle_data->nreplicas_per_host > 1));

	if (bundle_data->promoted_max) {
	crm_create_nvpair_xml(xml_set, NULL,
	PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);

	value = pcmk__itoa(bundle_data->promoted_max);
	crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
	free(value);
	}

	//crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
	pcmk__xml_copy(xml_resource, xml_obj);

	} else if(xml_obj) {
	pcmk__config_err("Cannot control %s inside %s without either "
	PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
	rsc->id, pcmk__xe_id(xml_obj));
	return FALSE;
	}

	if(xml_resource) {
	int lpc = 0;
	GList *childIter = NULL;
	pe__bundle_port_t *port = NULL;
	GString *buffer = NULL;

	if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
	scheduler) != pcmk_rc_ok) {
	return FALSE;
	}

	/* Currently, we always map the default authentication key location
	* into the same location inside the container.
	*
	* Ideally, we would respect the host's PCMK_authkey_location, but:
	* - it may be different on different nodes;
	* - the actual connection will do extra checking to make sure the key
	* file exists and is readable, that we can't do here on the DC
	* - tools such as crm_resource and crm_simulate may not have the same
	* environment variables as the cluster, causing operation digests to
	* differ
	*
	* Always using the default location inside the container is fine,
	* because we control the pacemaker_remote environment, and it avoids
	* having to pass another environment variable to the container.
	*
	* @TODO A better solution may be to have only pacemaker_remote use the
	* environment variable, and have the cluster nodes use a new
	* cluster option for key location. This would introduce the limitation
	* of the location being the same on all cluster nodes, but that's
	* reasonable.
	*/
	mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
	DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);

	if (need_log_mount) {
	mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
	pe__bundle_mount_subdir);
	}

	port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
	if(bundle_data->control_port) {
	port->source = strdup(bundle_data->control_port);
	} else {
	/* If we wanted to respect PCMK_remote_port, we could use
	* crm_default_remote_port() here and elsewhere in this file instead
	* of DEFAULT_REMOTE_PORT.
	*
	* However, it gains nothing, since we control both the container
	* environment and the connection resource parameters, and the user
	* can use a different port if desired by setting
	* PCMK_XA_CONTROL_PORT.
	*/
	port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
	}
	port->target = strdup(port->source);
	bundle_data->ports = g_list_append(bundle_data->ports, port);

	buffer = g_string_sized_new(1024);
	for (childIter = bundle_data->child->children; childIter != NULL;
	childIter = childIter->next) {

	pcmk__bundle_replica_t *replica = NULL;

	replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
	replica->child = childIter->data;
	replica->child->exclusive_discover = TRUE;
	replica->offset = lpc++;

	// Ensure the child's notify gets set based on the underlying primitive's value
	if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
	pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
	}

	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	bundle_data->attribute_target =
	g_hash_table_lookup(replica->child->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);

	if (bundle_data->attribute_target) {
	pcmk__insert_dup(rsc->meta, PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
	bundle_data->attribute_target);
	pcmk__insert_dup(bundle_data->child->meta,
	PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
	bundle_data->attribute_target);
	}

	} else {
	// Just a naked container, no pacemaker-remote
	GString *buffer = g_string_sized_new(1024);

	for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
	pcmk__bundle_replica_t *replica = NULL;

	replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
	replica->offset = lpc;
	allocate_ip(bundle_data, replica, buffer);
	bundle_data->replicas = g_list_append(bundle_data->replicas,
	replica);
	}
	bundle_data->container_host_options = g_string_free(buffer, FALSE);
	}

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
	pcmk__config_err("Failed unpacking resource %s", rsc->id);
	rsc->private->fns->free(rsc);
	return FALSE;
	}

	/* Utilization needs special handling for bundles. It makes no sense for
	* the inner primitive to have utilization, because it is tied
	* one-to-one to the guest node created by the container resource -- and
	* there's no way to set capacities for that guest node anyway.
	*
	* What the user really wants is to configure utilization for the
	* container. However, the schema only allows utilization for
	* primitives, and the container resource is implicit anyway, so the
	* user can only configure utilization for the inner primitive. If
	* they do, move the primitive's utilization values to the container.
	*
	* @TODO This means that bundles without an inner primitive can't have
	* utilization. An alternative might be to allow utilization values in
	* the top-level bundle XML in the schema, and copy those to each
	* container.
	*/
	if (replica->child != NULL) {
	GHashTable *empty = replica->container->utilization;

	replica->container->utilization = replica->child->utilization;
	replica->child->utilization = empty;
	}
	}

	if (bundle_data->child) {
	rsc->children = g_list_append(rsc->children, bundle_data->child);
	}
	return TRUE;
	}

	static int
	replica_resource_active(pcmk_resource_t *rsc, gboolean all)
	{
	if (rsc) {
	gboolean child_active = rsc->private->fns->active(rsc, all);

	if (child_active && !all) {
	return TRUE;
	} else if (!child_active && all) {
	return FALSE;
	}
	}
	return -1;
	}

	gboolean
	pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	GList *iter = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;
	int rsc_active;

	rsc_active = replica_resource_active(replica->ip, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->child, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->container, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}

	rsc_active = replica_resource_active(replica->remote, all);
	if (rsc_active >= 0) {
	return (gboolean) rsc_active;
	}
	}

	/* If "all" is TRUE, we've already checked that no resources were inactive,
	* so return TRUE; if "all" is FALSE, we didn't find any active resources,
	* so return FALSE.
	*/
	return all;
	}

	/*!
	* \internal
	* \brief Find the bundle replica corresponding to a given node
	*
	* \param[in] bundle Top-level bundle resource
	* \param[in] node Node to search for
	*
	* \return Bundle replica if found, NULL otherwise
	*/
	pcmk_resource_t *
	pe__find_bundle_replica(const pcmk_resource_t bundle, const pcmk_node_t node)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_ASSERT(bundle && node);

	get_bundle_variant_data(bundle_data, bundle);
	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;

	CRM_ASSERT(replica && replica->node);
	if (pcmk__same_node(replica->node, node)) {
	return replica->child;
	}
	}
	return NULL;
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_xml(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean printed_header = FALSE;
	gboolean print_everything = TRUE;

	const char *desc = NULL;

	CRM_ASSERT(rsc != NULL);
	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;
	char *id = NULL;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(container, only_node)) {
	continue;
	}

	print_ip = (ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc,
	print_everything);
	print_child = (child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc,
	print_everything);
	print_ctnr = !container->private->fns->is_filtered(container, only_rsc,
	print_everything);
	print_remote = (remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	print_everything);

	if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
	continue;
	}

	if (!printed_header) {
	const char *type = container_agent_str(bundle_data->agent_type);
	const char *unique = pcmk__flag_text(rsc->flags, pcmk_rsc_unique);
	const char *maintenance = pcmk__flag_text(rsc->flags,
	pcmk_rsc_maintenance);
	const char *managed = pcmk__flag_text(rsc->flags, pcmk_rsc_managed);
	const char *failed = pcmk__flag_text(rsc->flags, pcmk_rsc_failed);

	printed_header = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_TYPE, type,
	PCMK_XA_IMAGE, bundle_data->image,
	PCMK_XA_UNIQUE, unique,
	PCMK_XA_MAINTENANCE, maintenance,
	PCMK_XA_MANAGED, managed,
	PCMK_XA_FAILED, failed,
	PCMK_XA_DESCRIPTION, desc,
	NULL);
	CRM_ASSERT(rc == pcmk_rc_ok);
	}

	id = pcmk__itoa(replica->offset);
	rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA,
	PCMK_XA_ID, id,
	NULL);
	free(id);
	CRM_ASSERT(rc == pcmk_rc_ok);

	if (print_ip) {
	out->message(out, (const char *) ip->private->xml->name, show_opts,
	ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, (const char *) child->private->xml->name,
	show_opts, child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, (const char *) container->private->xml->name,
	show_opts, container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, (const char *) remote->private->xml->name,
	show_opts, remote, only_node, only_rsc);
	}

	pcmk__output_xml_pop_parent(out); // replica
	}

	if (printed_header) {
	pcmk__output_xml_pop_parent(out); // bundle
	}

	return rc;
	}

	static void
	pe__bundle_replica_output_html(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_html(out, rsc, buffer, node, show_opts);
	}

	/*!
	* \internal
	* \brief Get a string describing a resource's unmanaged state or lack thereof
	*
	* \param[in] rsc Resource to describe
	*
	* \return A string indicating that a resource is in maintenance mode or
	* otherwise unmanaged, or an empty string otherwise
	*/
	static const char *
	get_unmanaged_str(const pcmk_resource_t *rsc)
	{
	if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
	return " (maintenance)";
	}
	if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
	return " (unmanaged)";
	}
	return "";
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_html(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	CRM_ASSERT(rsc != NULL);
	get_bundle_variant_data(bundle_data, rsc);

	desc = pe__resource_description(rsc, show_opts);

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(container, only_node)) {
	continue;
	}

	print_ip = (ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc,
	print_everything);
	print_child = (child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc,
	print_everything);
	print_ctnr = !container->private->fns->is_filtered(container, only_rsc,
	print_everything);
	print_remote = (remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
	}

	if (print_ip) {
	out->message(out, (const char *) ip->private->xml->name,
	new_show_opts, ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, (const char *) child->private->xml->name,
	new_show_opts, child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, (const char *) container->private->xml->name,
	new_show_opts, container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, (const char *) remote->private->xml->name,
	new_show_opts, remote, only_node, only_rsc);
	}

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->end_list(out);
	}
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_html(out, replica,
	pcmk__current_node(container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	pe__bundle_replica_output_text(pcmk__output_t *out,
	pcmk__bundle_replica_t *replica,
	pcmk_node_t *node, uint32_t show_opts)
	{
	const pcmk_resource_t *rsc = replica->child;

	int offset = 0;
	char buffer[LINE_MAX];

	if(rsc == NULL) {
	rsc = replica->container;
	}

	if (replica->remote) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->remote));
	} else {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
	rsc_printable_id(replica->container));
	}
	if (replica->ipaddr) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
	replica->ipaddr);
	}

	pe__common_output_text(out, rsc, buffer, node, show_opts);
	}

	PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t ", "GList ",
	"GList *")
	int
	pe__bundle_text(pcmk__output_t *out, va_list args)
	{
	uint32_t show_opts = va_arg(args, uint32_t);
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	GList only_node = va_arg(args, GList );
	GList only_rsc = va_arg(args, GList );

	const char *desc = NULL;
	pe__bundle_variant_data_t *bundle_data = NULL;
	int rc = pcmk_rc_no_output;
	gboolean print_everything = TRUE;

	desc = pe__resource_description(rsc, show_opts);

	CRM_ASSERT(rsc != NULL);
	get_bundle_variant_data(bundle_data, rsc);

	if (rsc->private->fns->is_filtered(rsc, only_rsc, TRUE)) {
	return rc;
	}

	print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);

	for (GList *gIter = bundle_data->replicas; gIter != NULL;
	gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;
	gboolean print_ip, print_child, print_ctnr, print_remote;

	CRM_ASSERT(replica);

	if (pcmk__rsc_filtered_by_node(container, only_node)) {
	continue;
	}

	print_ip = (ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc,
	print_everything);
	print_child = (child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc,
	print_everything);
	print_ctnr = !container->private->fns->is_filtered(container, only_rsc,
	print_everything);
	print_remote = (remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	print_everything);

	if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) \|\|
	(print_everything == FALSE && (print_ip \|\| print_child \|\| print_ctnr \|\| print_remote))) {
	/* The text output messages used below require pe_print_implicit to
	* be set to do anything.
	*/
	uint32_t new_show_opts = show_opts \| pcmk_show_implicit_rscs;

	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	if (pcmk__list_of_multiple(bundle_data->replicas)) {
	out->list_item(out, NULL, "Replica[%d]", replica->offset);
	}

	out->begin_list(out, NULL, NULL, NULL);

	if (print_ip) {
	out->message(out, (const char *) ip->private->xml->name,
	new_show_opts, ip, only_node, only_rsc);
	}

	if (print_child) {
	out->message(out, (const char *) child->private->xml->name,
	new_show_opts, child, only_node, only_rsc);
	}

	if (print_ctnr) {
	out->message(out, (const char *) container->private->xml->name,
	new_show_opts, container, only_node, only_rsc);
	}

	if (print_remote) {
	out->message(out, (const char *) remote->private->xml->name,
	new_show_opts, remote, only_node, only_rsc);
	}

	out->end_list(out);
	} else if (print_everything == FALSE && !(print_ip \|\| print_child \|\| print_ctnr \|\| print_remote)) {
	continue;
	} else {
	PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
	(bundle_data->nreplicas > 1)? " set" : "",
	rsc->id, bundle_data->image,
	pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
	desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
	get_unmanaged_str(rsc));

	pe__bundle_replica_output_text(out, replica,
	pcmk__current_node(container),
	show_opts);
	}
	}

	PCMK__OUTPUT_LIST_FOOTER(out, rc);
	return rc;
	}

	static void
	free_bundle_replica(pcmk__bundle_replica_t *replica)
	{
	if (replica == NULL) {
	return;
	}

	if (replica->node) {
	free(replica->node);
	replica->node = NULL;
	}

	if (replica->ip) {
	pcmk__xml_free(replica->ip->private->xml);
	replica->ip->private->xml = NULL;
	replica->ip->private->fns->free(replica->ip);
	}
	if (replica->container) {
	pcmk__xml_free(replica->container->private->xml);
	replica->container->private->xml = NULL;
	replica->container->private->fns->free(replica->container);
	}
	if (replica->remote) {
	pcmk__xml_free(replica->remote->private->xml);
	replica->remote->private->xml = NULL;
	replica->remote->private->fns->free(replica->remote);
	}
	free(replica->ipaddr);
	free(replica);
	}

	void
	pe__free_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;
	CRM_CHECK(rsc != NULL, return);

	get_bundle_variant_data(bundle_data, rsc);
	pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);

	free(bundle_data->prefix);
	free(bundle_data->image);
	free(bundle_data->control_port);
	free(bundle_data->host_network);
	free(bundle_data->host_netmask);
	free(bundle_data->ip_range_start);
	free(bundle_data->container_network);
	free(bundle_data->launcher_options);
	free(bundle_data->container_command);
	g_free(bundle_data->container_host_options);

	g_list_free_full(bundle_data->replicas,
	(GDestroyNotify) free_bundle_replica);
	g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
	g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
	g_list_free(rsc->children);

	if(bundle_data->child) {
	pcmk__xml_free(bundle_data->child->private->xml);
	bundle_data->child->private->xml = NULL;
	bundle_data->child->private->fns->free(bundle_data->child);
	}
	common_free(rsc);
	}

	enum rsc_role_e
	pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
	{
	enum rsc_role_e container_role = pcmk_role_unknown;
	return container_role;
	}

	/*!
	* \brief Get the number of configured replicas in a bundle
	*
	* \param[in] rsc Bundle resource
	*
	* \return Number of configured replicas, or 0 on error
	*/
	int
	pe_bundle_replicas(const pcmk_resource_t *rsc)
	{
	if (pcmk__is_bundle(rsc)) {
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	return bundle_data->nreplicas;
	}
	return 0;
	}

	void
	pe__count_bundle(pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
	pcmk__bundle_replica_t *replica = item->data;

	if (replica->ip) {
	replica->ip->private->fns->count(replica->ip);
	}
	if (replica->child) {
	replica->child->private->fns->count(replica->child);
	}
	if (replica->container) {
	replica->container->private->fns->count(replica->container);
	}
	if (replica->remote) {
	replica->remote->private->fns->count(replica->remote);
	}
	}
	}

	gboolean
	pe__bundle_is_filtered(const pcmk_resource_t rsc, GList only_rsc,
	gboolean check_parent)
	{
	gboolean passes = FALSE;
	pe__bundle_variant_data_t *bundle_data = NULL;

	if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
	passes = TRUE;
	} else {
	get_bundle_variant_data(bundle_data, rsc);

	for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
	pcmk__bundle_replica_t *replica = gIter->data;
	pcmk_resource_t *ip = replica->ip;
	pcmk_resource_t *child = replica->child;
	pcmk_resource_t *container = replica->container;
	pcmk_resource_t *remote = replica->remote;

	if ((ip != NULL)
	&& !ip->private->fns->is_filtered(ip, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	if ((child != NULL)
	&& !child->private->fns->is_filtered(child, only_rsc, FALSE)) {
	passes = TRUE;
	break;
	}
	if (!container->private->fns->is_filtered(container, only_rsc,
	FALSE)) {
	passes = TRUE;
	break;
	}
	if ((remote != NULL)
	&& !remote->private->fns->is_filtered(remote, only_rsc,
	FALSE)) {
	passes = TRUE;
	break;
	}
	}
	}

	return !passes;
	}

	/*!
	* \internal
	* \brief Get a list of a bundle's containers
	*
	* \param[in] bundle Bundle resource
	*
	* \return Newly created list of \p bundle's containers
	* \note It is the caller's responsibility to free the result with
	* g_list_free().
	*/
	GList *
	pe__bundle_containers(const pcmk_resource_t *bundle)
	{
	GList *containers = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	get_bundle_variant_data(data, bundle);
	for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	containers = g_list_append(containers, replica->container);
	}
	return containers;
	}

	// Bundle implementation of pcmk__rsc_methods_t:active_node()
	pcmk_node_t *
	pe__bundle_active_node(const pcmk_resource_t rsc, unsigned int count_all,
	unsigned int *count_clean)
	{
	pcmk_node_t *active = NULL;
	pcmk_node_t *node = NULL;
	pcmk_resource_t *container = NULL;
	GList *containers = NULL;
	GList *iter = NULL;
	GHashTable *nodes = NULL;
	const pe__bundle_variant_data_t *data = NULL;

	if (count_all != NULL) {
	*count_all = 0;
	}
	if (count_clean != NULL) {
	*count_clean = 0;
	}
	if (rsc == NULL) {
	return NULL;
	}

	/* For the purposes of this method, we only care about where the bundle's
	* containers are active, so build a list of active containers.
	*/
	get_bundle_variant_data(data, rsc);
	for (iter = data->replicas; iter != NULL; iter = iter->next) {
	pcmk__bundle_replica_t *replica = iter->data;

	if (replica->container->running_on != NULL) {
	containers = g_list_append(containers, replica->container);
	}
	}
	if (containers == NULL) {
	return NULL;
	}

	/* If the bundle has only a single active container, just use that
	* container's method. If live migration is ever supported for bundle
	* containers, this will allow us to prefer the migration source when there
	* is only one container and it is migrating. For now, this just lets us
	* avoid creating the nodes table.
	*/
	if (pcmk__list_of_1(containers)) {
	container = containers->data;
	node = container->private->fns->active_node(container, count_all,
	count_clean);
	g_list_free(containers);
	return node;
	}

	// Add all containers' active nodes to a hash table (for uniqueness)
	nodes = g_hash_table_new(NULL, NULL);
	for (iter = containers; iter != NULL; iter = iter->next) {
	container = iter->data;

	for (GList *node_iter = container->running_on; node_iter != NULL;
	node_iter = node_iter->next) {
	node = node_iter->data;

	// If insert returns true, we haven't counted this node yet
	if (g_hash_table_insert(nodes, (gpointer) node->details,
	(gpointer) node)
	&& !pe__count_active_node(rsc, node, &active, count_all,
	count_clean)) {
	goto done;
	}
	}
	}

	done:
	g_list_free(containers);
	g_hash_table_destroy(nodes);
	return active;
	}

	/*!
	* \internal
	* \brief Get maximum bundle resource instances per node
	*
	* \param[in] rsc Bundle resource to check
	*
	* \return Maximum number of \p rsc instances that can be active on one node
	*/
	unsigned int
	pe__bundle_max_per_node(const pcmk_resource_t *rsc)
	{
	pe__bundle_variant_data_t *bundle_data = NULL;

	get_bundle_variant_data(bundle_data, rsc);
	CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
	return (unsigned int) bundle_data->nreplicas_per_host;
	}
	diff --git a/lib/pengine/remote.c b/lib/pengine/remote.c
	index 983ce804db..8120b8ee73 100644
	--- a/lib/pengine/remote.c
	+++ b/lib/pengine/remote.c
	@@ -1,238 +1,238 @@
	/*
	* Copyright 2013-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/scheduler_internal.h>
	#include <crm/pengine/internal.h>
	#include <glib.h>

	/*!
	* \internal
	* \brief Check whether a resource creates a guest node
	*
	* If a given resource contains a filler resource that is a remote connection,
	* return that filler resource (or NULL if none is found).
	*
	* \param[in] scheduler Scheduler data
	* \param[in] rsc Resource to check
	*
	* \return Filler resource with remote connection, or NULL if none found
	*/
	pcmk_resource_t *
	pe__resource_contains_guest_node(const pcmk_scheduler_t *scheduler,
	const pcmk_resource_t *rsc)
	{
	if ((rsc != NULL) && (scheduler != NULL)
	&& pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {

	for (GList *gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
	pcmk_resource_t *filler = gIter->data;

	if (filler->is_remote_node) {
	return filler;
	}
	}
	}
	return NULL;
	}

	bool
	xml_contains_remote_node(xmlNode *xml)
	{
	const char *value = NULL;

	if (xml == NULL) {
	return false;
	}

	value = crm_element_value(xml, PCMK_XA_TYPE);
	if (!pcmk__str_eq(value, "remote", pcmk__str_casei)) {
	return false;
	}

	value = crm_element_value(xml, PCMK_XA_CLASS);
	if (!pcmk__str_eq(value, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
	return false;
	}

	value = crm_element_value(xml, PCMK_XA_PROVIDER);
	if (!pcmk__str_eq(value, "pacemaker", pcmk__str_casei)) {
	return false;
	}

	return true;
	}

	/*!
	* \internal
	* \brief Execute a supplied function for each guest node running on a host
	*
	* \param[in] scheduler Scheduler data
	* \param[in] host Host node to check
	* \param[in] helper Function to call for each guest node
	* \param[in,out] user_data Pointer to pass to helper function
	*/
	void
	pe_foreach_guest_node(const pcmk_scheduler_t *scheduler,
	const pcmk_node_t *host,
	void (helper)(const pcmk_node_t, void*),
	void *user_data)
	{
	GList *iter;

	CRM_CHECK(scheduler && host && host->details && helper, return);
	if (!pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
	return;
	}
	for (iter = host->details->running_rsc; iter != NULL; iter = iter->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) iter->data;

	if (rsc->is_remote_node && (rsc->container != NULL)) {
	pcmk_node_t *guest_node = pcmk_find_node(scheduler, rsc->id);

	if (guest_node) {
	(*helper)(guest_node, user_data);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Create CIB XML for an implicit remote connection
	*
	* \param[in,out] parent If not \c NULL, use as parent XML element
	* \param[in] uname Name of Pacemaker Remote node
	* \param[in] container_id If not \c NULL, use this as connection container
	* \param[in] migrateable If not \c NULL, use as remote
	* \c PCMK_META_ALLOW_MIGRATE value
	* \param[in] is_managed If not \c NULL, use as remote
	* \c PCMK_META_IS_MANAGED value
	* \param[in] start_timeout If not \c NULL, use as remote connect timeout
	* \param[in] server If not \c NULL, use as \c PCMK_REMOTE_RA_ADDR
	* \param[in] port If not \c NULL, use as \c PCMK_REMOTE_RA_PORT
	*
	* \return Newly created XML
	*/
	xmlNode *
	pe_create_remote_xml(xmlNode parent, const char uname,
	const char container_id, const char migrateable,
	const char is_managed, const char start_timeout,
	const char server, const char port)
	{
	xmlNode *remote;
	xmlNode *xml_sub;

	remote = pcmk__xe_create(parent, PCMK_XE_PRIMITIVE);

	// Add identity
	crm_xml_add(remote, PCMK_XA_ID, uname);
	crm_xml_add(remote, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
	crm_xml_add(remote, PCMK_XA_PROVIDER, "pacemaker");
	crm_xml_add(remote, PCMK_XA_TYPE, "remote");

	// Add meta-attributes
	xml_sub = pcmk__xe_create(remote, PCMK_XE_META_ATTRIBUTES);
	- crm_xml_set_id(xml_sub, "%s-%s", uname, PCMK_XE_META_ATTRIBUTES);
	+ pcmk__xe_set_id(xml_sub, "%s-%s", uname, PCMK_XE_META_ATTRIBUTES);
	crm_create_nvpair_xml(xml_sub, NULL,
	PCMK__META_INTERNAL_RSC, PCMK_VALUE_TRUE);
	if (container_id) {
	crm_create_nvpair_xml(xml_sub, NULL,
	PCMK__META_CONTAINER, container_id);
	}
	if (migrateable) {
	crm_create_nvpair_xml(xml_sub, NULL,
	PCMK_META_ALLOW_MIGRATE, migrateable);
	}
	if (is_managed) {
	crm_create_nvpair_xml(xml_sub, NULL, PCMK_META_IS_MANAGED, is_managed);
	}

	// Add instance attributes
	if (port \|\| server) {
	xml_sub = pcmk__xe_create(remote, PCMK_XE_INSTANCE_ATTRIBUTES);
	- crm_xml_set_id(xml_sub, "%s-%s", uname, PCMK_XE_INSTANCE_ATTRIBUTES);
	+ pcmk__xe_set_id(xml_sub, "%s-%s", uname, PCMK_XE_INSTANCE_ATTRIBUTES);
	if (server) {
	crm_create_nvpair_xml(xml_sub, NULL, PCMK_REMOTE_RA_ADDR, server);
	}
	if (port) {
	crm_create_nvpair_xml(xml_sub, NULL, PCMK_REMOTE_RA_PORT, port);
	}
	}

	// Add operations
	xml_sub = pcmk__xe_create(remote, PCMK_XE_OPERATIONS);
	crm_create_op_xml(xml_sub, uname, PCMK_ACTION_MONITOR, "30s", "30s");
	if (start_timeout) {
	crm_create_op_xml(xml_sub, uname, PCMK_ACTION_START, "0",
	start_timeout);
	}
	return remote;
	}

	// History entry to be checked for fail count clearing
	struct check_op {
	const xmlNode *rsc_op; // History entry XML
	pcmk_resource_t *rsc; // Known resource corresponding to history entry
	pcmk_node_t *node; // Known node corresponding to history entry
	enum pcmk__check_parameters check_type; // What needs checking
	};

	void
	pe__add_param_check(const xmlNode rsc_op, pcmk_resource_t rsc,
	pcmk_node_t *node, enum pcmk__check_parameters flag,
	pcmk_scheduler_t *scheduler)
	{
	struct check_op *check_op = NULL;

	CRM_CHECK(scheduler && rsc_op && rsc && node, return);

	check_op = pcmk__assert_alloc(1, sizeof(struct check_op));

	crm_trace("Deferring checks of %s until after allocation",
	pcmk__xe_id(rsc_op));
	check_op->rsc_op = rsc_op;
	check_op->rsc = rsc;
	check_op->node = node;
	check_op->check_type = flag;
	scheduler->param_check = g_list_prepend(scheduler->param_check, check_op);
	}

	/*!
	* \internal
	* \brief Call a function for each action to be checked for addr substitution
	*
	* \param[in,out] scheduler Scheduler data
	* \param[in] cb Function to be called
	*/
	void
	pe__foreach_param_check(pcmk_scheduler_t *scheduler,
	void (cb)(pcmk_resource_t, pcmk_node_t*,
	const xmlNode*, enum pcmk__check_parameters))
	{
	CRM_CHECK(scheduler && cb, return);

	for (GList *item = scheduler->param_check;
	item != NULL; item = item->next) {
	struct check_op *check_op = item->data;

	cb(check_op->rsc, check_op->node, check_op->rsc_op,
	check_op->check_type);
	}
	}

	void
	pe__free_param_checks(pcmk_scheduler_t *scheduler)
	{
	if (scheduler && scheduler->param_check) {
	g_list_free_full(scheduler->param_check, free);
	scheduler->param_check = NULL;
	}
	}
	diff --git a/tools/crm_node.c b/tools/crm_node.c
	index a3d79b7180..cef2ad197d 100644
	--- a/tools/crm_node.c
	+++ b/tools/crm_node.c
	@@ -1,875 +1,875 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <inttypes.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/xml.h>
	#include <crm/cib.h>
	#include <crm/cib/internal.h>
	#include <crm/common/ipc_controld.h>
	#include <crm/common/attrs_internal.h>

	#include <pacemaker-internal.h>

	#define SUMMARY "crm_node - Tool for displaying low-level node information"

	struct {
	gboolean corosync;
	gboolean dangerous_cmd;
	gboolean force_flag;
	char command;
	int nodeid;
	char *target_uname;
	} options = {
	.command = '\0',
	.force_flag = FALSE
	};

	gboolean command_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean name_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean remove_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	static GError *error = NULL;
	static GMainLoop *mainloop = NULL;
	static crm_exit_t exit_code = CRM_EX_OK;
	static pcmk__output_t *out = NULL;

	#define INDENT " "

	static GOptionEntry command_entries[] = {
	{ "cluster-id", 'i', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display this node's cluster id",
	NULL },
	{ "list", 'l', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display all known members (past and present) of this cluster",
	NULL },
	{ "name", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the name used by the cluster for this node",
	NULL },
	{ "partition", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display the members of this partition",
	NULL },
	{ "quorum", 'q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Display a 1 if our partition has quorum, 0 if not",
	NULL },
	{ "name-for-id", 'N', 0, G_OPTION_ARG_CALLBACK, name_cb,
	"Display the name used by the cluster for the node with the specified ID",
	"ID" },
	{ "remove", 'R', 0, G_OPTION_ARG_CALLBACK, remove_cb,
	"(Advanced) Remove the (stopped) node with the specified name from Pacemaker's\n"
	INDENT "configuration and caches (the node must already have been removed from\n"
	INDENT "the underlying cluster stack configuration",
	"NAME" },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "force", 'f', 0, G_OPTION_ARG_NONE, &options.force_flag,
	NULL,
	NULL },
	#if SUPPORT_COROSYNC
	/* Unused and deprecated */
	{ "corosync", 'C', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.corosync,
	NULL,
	NULL },
	#endif

	// @TODO add timeout option for when IPC replies are needed

	{ NULL }
	};

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	gboolean
	command_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (pcmk__str_eq("-i", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--cluster-id", option_name, pcmk__str_casei)) {
	options.command = 'i';
	} else if (pcmk__str_eq("-l", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--list", option_name, pcmk__str_casei)) {
	options.command = 'l';
	} else if (pcmk__str_eq("-n", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--name", option_name, pcmk__str_casei)) {
	options.command = 'n';
	} else if (pcmk__str_eq("-p", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--partition", option_name, pcmk__str_casei)) {
	options.command = 'p';
	} else if (pcmk__str_eq("-q", option_name, pcmk__str_casei) \|\| pcmk__str_eq("--quorum", option_name, pcmk__str_casei)) {
	options.command = 'q';
	} else {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Unknown param passed to command_cb: %s", option_name);
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	name_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.command = 'N';
	pcmk__scan_min_int(optarg, &(options.nodeid), 0);
	return TRUE;
	}

	gboolean
	remove_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (optarg == NULL) {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "-R option requires an argument");
	return FALSE;
	}

	options.command = 'R';
	options.dangerous_cmd = TRUE;
	pcmk__str_update(&options.target_uname, optarg);
	return TRUE;
	}

	PCMK__OUTPUT_ARGS("node-id", "uint32_t")
	static int
	node_id_default(pcmk__output_t *out, va_list args) {
	uint32_t node_id = va_arg(args, uint32_t);

	out->info(out, "%" PRIu32, node_id);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-id", "uint32_t")
	static int
	node_id_xml(pcmk__output_t *out, va_list args) {
	uint32_t node_id = va_arg(args, uint32_t);

	char *id_s = crm_strdup_printf("%" PRIu32, node_id);

	pcmk__output_create_xml_node(out, PCMK_XE_NODE_INFO,
	PCMK_XA_NODEID, id_s,
	NULL);

	free(id_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("simple-node-list", "GList *")
	static int
	simple_node_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;
	out->info(out, "%" PRIu32 " %s %s", node->id, pcmk__s(node->uname, ""),
	pcmk__s(node->state, ""));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("simple-node-list", "GList *")
	static int
	simple_node_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	out->begin_list(out, NULL, NULL, PCMK_XE_NODES);

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;
	char *id_s = crm_strdup_printf("%" PRIu32, node->id);

	pcmk__output_create_xml_node(out, PCMK_XE_NODE,
	PCMK_XA_ID, id_s,
	PCMK_XA_NAME, node->uname,
	PCMK_XA_STATE, node->state,
	NULL);

	free(id_s);
	}

	out->end_list(out);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-name", "uint32_t", "const char *")
	static int
	node_name_default(pcmk__output_t *out, va_list args) {
	uint32_t node_id G_GNUC_UNUSED = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );

	out->info(out, "%s", node_name);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("node-name", "uint32_t", "const char *")
	static int
	node_name_xml(pcmk__output_t *out, va_list args) {
	uint32_t node_id = va_arg(args, uint32_t);
	const char node_name = va_arg(args, const char );

	char *id_s = crm_strdup_printf("%" PRIu32, node_id);

	pcmk__output_create_xml_node(out, PCMK_XE_NODE_INFO,
	PCMK_XA_NODEID, id_s,
	PCMK_XA_UNAME, node_name,
	NULL);

	free(id_s);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("partition-list", "GList *")
	static int
	partition_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	GString *buffer = NULL;

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;
	if (pcmk__str_eq(node->state, "member", pcmk__str_none)) {
	pcmk__add_separated_word(&buffer, 128, pcmk__s(node->uname, ""), " ");
	}
	}

	if (buffer != NULL) {
	out->info(out, "%s", buffer->str);
	g_string_free(buffer, TRUE);
	return pcmk_rc_ok;
	}

	return pcmk_rc_no_output;
	}

	PCMK__OUTPUT_ARGS("partition-list", "GList *")
	static int
	partition_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );

	out->begin_list(out, NULL, NULL, PCMK_XE_NODES);

	for (GList *node_iter = nodes; node_iter != NULL; node_iter = node_iter->next) {
	pcmk_controld_api_node_t *node = node_iter->data;

	if (pcmk__str_eq(node->state, "member", pcmk__str_none)) {
	char *id_s = crm_strdup_printf("%" PRIu32, node->id);

	pcmk__output_create_xml_node(out, PCMK_XE_NODE,
	PCMK_XA_ID, id_s,
	PCMK_XA_NAME, node->uname,
	PCMK_XA_STATE, node->state,
	NULL);
	free(id_s);
	}
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("quorum", "bool")
	static int
	quorum_default(pcmk__output_t *out, va_list args) {
	bool have_quorum = va_arg(args, int);

	out->info(out, "%d", have_quorum);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("quorum", "bool")
	static int
	quorum_xml(pcmk__output_t *out, va_list args) {
	bool have_quorum = va_arg(args, int);

	pcmk__output_create_xml_node(out, PCMK_XE_CLUSTER_INFO,
	PCMK_XA_QUORUM, pcmk__btoa(have_quorum),
	NULL);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "node-id", "default", node_id_default },
	{ "node-id", "xml", node_id_xml },
	{ "node-name", "default", node_name_default },
	{ "node-name", "xml", node_name_xml },
	{ "partition-list", "default", partition_list_default },
	{ "partition-list", "xml", partition_list_xml },
	{ "quorum", "default", quorum_default },
	{ "quorum", "xml", quorum_xml },
	{ "simple-node-list", "default", simple_node_list_default },
	{ "simple-node-list", "xml", simple_node_list_xml },

	{ NULL, NULL, NULL }
	};

	static gint
	sort_node(gconstpointer a, gconstpointer b)
	{
	const pcmk_controld_api_node_t *node_a = a;
	const pcmk_controld_api_node_t *node_b = b;

	return pcmk__numeric_strcasecmp((node_a->uname? node_a->uname : ""),
	(node_b->uname? node_b->uname : ""));
	}

	static void
	controller_event_cb(pcmk_ipc_api_t *controld_api,
	enum pcmk_ipc_event event_type, crm_exit_t status,
	void event_data, void user_data)
	{
	pcmk_controld_api_reply_t *reply = event_data;

	switch (event_type) {
	case pcmk_ipc_event_disconnect:
	if (exit_code == CRM_EX_DISCONNECT) { // Unexpected
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Lost connection to controller");
	}
	goto done;
	break;

	case pcmk_ipc_event_reply:
	break;

	default:
	return;
	}

	if (status != CRM_EX_OK) {
	exit_code = status;
	g_set_error(&error, PCMK__EXITC_ERROR, status,
	"Bad reply from controller: %s",
	crm_exit_str(status));
	goto done;
	}

	if (reply->reply_type != pcmk_controld_reply_nodes) {
	g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_INDETERMINATE,
	"Unknown reply type %d from controller",
	reply->reply_type);
	goto done;
	}

	reply->data.nodes = g_list_sort(reply->data.nodes, sort_node);

	if (options.command == 'p') {
	out->message(out, "partition-list", reply->data.nodes);
	} else if (options.command == 'l') {
	out->message(out, "simple-node-list", reply->data.nodes);
	}

	// Success
	exit_code = CRM_EX_OK;
	done:
	pcmk_disconnect_ipc(controld_api);
	pcmk_quit_main_loop(mainloop, 10);
	}

	static void
	run_controller_mainloop(void)
	{
	pcmk_ipc_api_t *controld_api = NULL;
	int rc;

	// Set disconnect exit code to handle unexpected disconnects
	exit_code = CRM_EX_DISCONNECT;

	// Create controller IPC object
	rc = pcmk_new_ipc_api(&controld_api, pcmk_ipc_controld);
	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not connect to controller: %s",
	pcmk_rc_str(rc));
	return;
	}
	pcmk_register_ipc_callback(controld_api, controller_event_cb, NULL);

	// Connect to controller
	rc = pcmk__connect_ipc(controld_api, pcmk_ipc_dispatch_main, 5);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not connect to %s: %s",
	pcmk_ipc_name(controld_api, true), pcmk_rc_str(rc));
	return;
	}

	rc = pcmk_controld_api_list_nodes(controld_api);

	if (rc != pcmk_rc_ok) {
	pcmk_disconnect_ipc(controld_api);
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not ping controller: %s", pcmk_rc_str(rc));
	return;
	}

	// Run main loop to get controller reply via controller_event_cb()
	mainloop = g_main_loop_new(NULL, FALSE);
	g_main_loop_run(mainloop);
	g_main_loop_unref(mainloop);
	mainloop = NULL;
	pcmk_free_ipc_api(controld_api);
	}

	static void
	print_node_id(void)
	{
	uint32_t nodeid;
	int rc = pcmk__query_node_info(out, &nodeid, NULL, NULL, NULL, NULL, NULL,
	false, 0);

	if (rc != pcmk_rc_ok) {
	/* pcmk__query_node_info already sets an error message on the output object,
	* so there's no need to call g_set_error here. That would just create a
	* duplicate error message in the output.
	*/
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	rc = out->message(out, "node-id", nodeid);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print node ID: %s",
	pcmk_rc_str(rc));
	}

	exit_code = pcmk_rc2exitc(rc);
	}

	static void
	print_node_name(uint32_t nodeid)
	{
	int rc = pcmk_rc_ok;
	char *node_name = NULL;

	if (nodeid == 0) {
	// Check environment first (i.e. when called by resource agent)
	const char *name = getenv("OCF_RESKEY_" CRM_META "_"
	PCMK__META_ON_NODE);

	if (name != NULL) {
	rc = out->message(out, "node-name", 0UL, name);
	goto done;
	}
	}

	// Otherwise ask the controller

	/* pcmk__query_node_name already sets an error message on the output object,
	* so there's no need to call g_set_error here. That would just create a
	* duplicate error message in the output.
	*/
	rc = pcmk__query_node_name(out, nodeid, &node_name, 0);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	rc = out->message(out, "node-name", 0UL, node_name);

	done:
	if (node_name != NULL) {
	free(node_name);
	}

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print node name: %s",
	pcmk_rc_str(rc));
	}

	exit_code = pcmk_rc2exitc(rc);
	}

	static void
	print_quorum(void)
	{
	bool quorum;
	int rc = pcmk__query_node_info(out, NULL, NULL, NULL, NULL, &quorum, NULL,
	false, 0);

	if (rc != pcmk_rc_ok) {
	/* pcmk__query_node_info already sets an error message on the output object,
	* so there's no need to call g_set_error here. That would just create a
	* duplicate error message in the output.
	*/
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	rc = out->message(out, "quorum", quorum);

	if (rc != pcmk_rc_ok) {
	g_set_error(&error, PCMK__RC_ERROR, rc, "Could not print quorum status: %s",
	pcmk_rc_str(rc));
	}

	exit_code = pcmk_rc2exitc(rc);
	}

	/*!
	* \internal
	* \brief Extend a transaction by removing a node from a CIB section
	*
	* \param[in,out] cib Active CIB connection
	* \param[in] element CIB element containing node name and/or ID
	* \param[in] section CIB section that \p element is in
	* \param[in] node_name Name of node to purge (NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	remove_from_section(cib_t cib, const char element, const char *section,
	const char *node_name, long node_id)
	{
	int rc = pcmk_rc_ok;
	xmlNode *xml = pcmk__xe_create(NULL, element);

	crm_xml_add(xml, PCMK_XA_UNAME, node_name);
	if (node_id > 0) {
	- crm_xml_set_id(xml, "%ld", node_id);
	+ pcmk__xe_set_id(xml, "%ld", node_id);
	}

	rc = cib->cmds->remove(cib, section, xml, cib_transaction);
	pcmk__xml_free(xml);
	return (rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(rc);
	}

	/*!
	* \internal
	* \brief Purge a node from CIB
	*
	* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	purge_node_from_cib(const char *node_name, long node_id)
	{
	int rc = pcmk_rc_ok;
	int commit_rc = pcmk_rc_ok;
	cib_t *cib = NULL;

	// Connect to CIB and start a transaction
	cib = cib_new();
	if (cib == NULL) {
	return ENOTCONN;
	}
	rc = cib->cmds->signon(cib, crm_system_name, cib_command);
	if (rc == pcmk_ok) {
	rc = cib->cmds->init_transaction(cib);
	}
	if (rc != pcmk_ok) {
	rc = pcmk_legacy2rc(rc);
	cib__clean_up_connection(&cib);
	return rc;
	}

	// Remove from configuration and status
	rc = remove_from_section(cib, PCMK_XE_NODE, PCMK_XE_NODES, node_name,
	node_id);
	if (rc == pcmk_rc_ok) {
	rc = remove_from_section(cib, PCMK__XE_NODE_STATE, PCMK_XE_STATUS,
	node_name, node_id);
	}

	// Commit the transaction
	commit_rc = cib->cmds->end_transaction(cib, (rc == pcmk_rc_ok),
	cib_sync_call);
	cib__clean_up_connection(&cib);

	if ((rc == pcmk_rc_ok) && (commit_rc == pcmk_ok)) {
	crm_debug("Purged node %s (%ld) from CIB",
	pcmk__s(node_name, "by ID"), node_id);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Purge a node from a single server's peer cache
	*
	* \param[in] server IPC server to send request to
	* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	purge_node_from(enum pcmk_ipc_server server, const char *node_name,
	long node_id)
	{
	pcmk_ipc_api_t *api = NULL;
	int rc;

	rc = pcmk_new_ipc_api(&api, server);
	if (rc != pcmk_rc_ok) {
	goto done;
	}

	rc = pcmk__connect_ipc(api, pcmk_ipc_dispatch_sync, 5);
	if (rc != pcmk_rc_ok) {
	goto done;
	}

	rc = pcmk_ipc_purge_node(api, node_name, node_id);
	done:
	if (rc != pcmk_rc_ok) { // Debug message already logged on success
	g_set_error(&error, PCMK__RC_ERROR, rc,
	"Could not purge node %s from %s: %s",
	pcmk__s(node_name, "by ID"), pcmk_ipc_name(api, true),
	pcmk_rc_str(rc));
	}
	pcmk_free_ipc_api(api);
	return rc;
	}

	/*!
	* \internal
	* \brief Purge a node from the fencer's peer cache
	*
	* \param[in] node_name Name of node to purge (or NULL to leave unspecified)
	* \param[in] node_id Node ID of node to purge (or 0 to leave unspecified)
	*
	* \note At least one of node_name and node_id must be specified.
	* \return Standard Pacemaker return code
	*/
	static int
	purge_node_from_fencer(const char *node_name, long node_id)
	{
	int rc = pcmk_rc_ok;
	crm_ipc_t *conn = NULL;
	xmlNode *cmd = NULL;

	conn = crm_ipc_new("stonith-ng", 0);
	if (conn == NULL) {
	rc = ENOTCONN;
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not connect to fencer to purge node %s",
	pcmk__s(node_name, "by ID"));
	return rc;
	}

	rc = pcmk__connect_generic_ipc(conn);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not connect to fencer to purge node %s: %s",
	pcmk__s(node_name, "by ID"), pcmk_rc_str(rc));
	crm_ipc_destroy(conn);
	return rc;
	}

	cmd = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, "stonith-ng",
	crm_system_name, NULL);
	if (node_id > 0) {
	- crm_xml_set_id(cmd, "%ld", node_id);
	+ pcmk__xe_set_id(cmd, "%ld", node_id);
	}
	crm_xml_add(cmd, PCMK_XA_UNAME, node_name);

	rc = crm_ipc_send(conn, cmd, 0, 0, NULL);
	if (rc >= 0) {
	rc = pcmk_rc_ok;
	crm_debug("Purged node %s (%ld) from fencer",
	pcmk__s(node_name, "by ID"), node_id);
	} else {
	rc = pcmk_legacy2rc(rc);
	fprintf(stderr, "Could not purge node %s from fencer: %s\n",
	pcmk__s(node_name, "by ID"), pcmk_rc_str(rc));
	}
	pcmk__xml_free(cmd);
	crm_ipc_close(conn);
	crm_ipc_destroy(conn);
	return rc;
	}

	static void
	remove_node(const char *target_uname)
	{
	int rc = pcmk_rc_ok;
	long nodeid = 0;
	const char *node_name = NULL;
	char *endptr = NULL;
	const enum pcmk_ipc_server servers[] = {
	pcmk_ipc_controld,
	pcmk_ipc_attrd,
	};

	// Check whether node was specified by name or numeric ID
	errno = 0;
	nodeid = strtol(target_uname, &endptr, 10);
	if ((errno != 0) \|\| (endptr == target_uname) \|\| (*endptr != '\0')
	\|\| (nodeid <= 0)) {
	// It's not a positive integer, so assume it's a node name
	nodeid = 0;
	node_name = target_uname;
	}

	for (int i = 0; i < PCMK__NELEM(servers); ++i) {
	rc = purge_node_from(servers[i], node_name, nodeid);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	return;
	}
	}

	// The fencer hasn't been converted to pcmk_ipc_api_t yet
	rc = purge_node_from_fencer(node_name, nodeid);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	return;
	}

	// Lastly, purge the node from the CIB itself
	rc = purge_node_from_cib(node_name, nodeid);
	exit_code = pcmk_rc2exitc(rc);
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	{ "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
	"Be less descriptive in output.",
	NULL },

	{ NULL }
	};

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);

	/* Add the -q option, which cannot be part of the globally supported options
	* because some tools use that flag for something else.
	*/
	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "commands", "Commands:",
	"Show command help", command_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "NR");
	GOptionContext *context = build_arg_context(args, &output_group);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_node", args->verbosity);

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = pcmk_rc2exitc(rc);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error creating output format %s: %s", args->output_ty,
	pcmk_rc_str(rc));
	goto done;
	}

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if (options.command == 0) {
	char *help = g_option_context_get_help(context, TRUE, NULL);

	out->err(out, "%s", help);
	g_free(help);
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	if (options.dangerous_cmd && options.force_flag == FALSE) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command is considered dangerous."
	" To prevent accidental destruction of the cluster,"
	" the --force flag is required in order to proceed.");
	goto done;
	}

	pcmk__register_lib_messages(out);
	pcmk__register_messages(out, fmt_functions);

	switch (options.command) {
	case 'i':
	print_node_id();
	break;

	case 'n':
	print_node_name(0);
	break;

	case 'q':
	print_quorum();
	break;

	case 'N':
	print_node_name(options.nodeid);
	break;

	case 'R':
	remove_node(options.target_uname);
	break;

	case 'l':
	case 'p':
	run_controller_mainloop();
	break;

	default:
	break;
	}

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}
	pcmk__unregister_formats();
	return crm_exit(exit_code);
	}
	diff --git a/tools/crm_resource_ban.c b/tools/crm_resource_ban.c
	index a3177e99f3..bc2334a489 100644
	--- a/tools/crm_resource_ban.c
	+++ b/tools/crm_resource_ban.c
	@@ -1,514 +1,514 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm_resource.h>

	static char *
	parse_cli_lifetime(pcmk__output_t out, const char move_lifetime)
	{
	char *later_s = NULL;
	crm_time_t *now = NULL;
	crm_time_t *later = NULL;
	crm_time_t *duration = NULL;

	if (move_lifetime == NULL) {
	return NULL;
	}

	duration = crm_time_parse_duration(move_lifetime);
	if (duration == NULL) {
	out->err(out, "Invalid duration specified: %s\n"
	"Please refer to https://en.wikipedia.org/wiki/ISO_8601#Durations "
	"for examples of valid durations", move_lifetime);
	return NULL;
	}

	now = crm_time_new(NULL);
	later = crm_time_add(now, duration);
	if (later == NULL) {
	out->err(out, "Unable to add %s to current time\n"
	"Please report to " PACKAGE_BUGREPORT " as possible bug",
	move_lifetime);
	crm_time_free(now);
	crm_time_free(duration);
	return NULL;
	}

	crm_time_log(LOG_INFO, "now ", now,
	crm_time_log_date \| crm_time_log_timeofday \| crm_time_log_with_timezone);
	crm_time_log(LOG_INFO, "later ", later,
	crm_time_log_date \| crm_time_log_timeofday \| crm_time_log_with_timezone);
	crm_time_log(LOG_INFO, "duration", duration, crm_time_log_date \| crm_time_log_timeofday);
	later_s = crm_time_as_string(later, crm_time_log_date \| crm_time_log_timeofday \| crm_time_log_with_timezone);
	out->info(out, "Migration will take effect until: %s", later_s);

	crm_time_free(duration);
	crm_time_free(later);
	crm_time_free(now);
	return later_s;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_ban(pcmk__output_t out, const char rsc_id, const char *host,
	const char move_lifetime, cib_t cib_conn, int cib_options,
	gboolean promoted_role_only, const char *promoted_role)
	{
	char *later_s = NULL;
	int rc = pcmk_rc_ok;
	xmlNode *fragment = NULL;
	xmlNode *location = NULL;

	later_s = parse_cli_lifetime(out, move_lifetime);
	if(move_lifetime && later_s == NULL) {
	return EINVAL;
	}

	fragment = pcmk__xe_create(NULL, PCMK_XE_CONSTRAINTS);

	location = pcmk__xe_create(fragment, PCMK_XE_RSC_LOCATION);
	- crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);
	+ pcmk__xe_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);

	out->info(out,
	"WARNING: Creating " PCMK_XE_RSC_LOCATION " constraint '%s' with "
	"a score of " PCMK_VALUE_MINUS_INFINITY " for resource %s on %s."
	"\n\tThis will prevent %s from %s on %s until the constraint is "
	"removed using the clear option or by editing the CIB with an "
	"appropriate tool.\n"
	"\tThis will be the case even if %s is the last node in the "
	"cluster",
	pcmk__xe_id(location), rsc_id, host, rsc_id,
	(promoted_role_only? "being promoted" : "running"), host, host);

	crm_xml_add(location, PCMK_XA_RSC, rsc_id);
	if(promoted_role_only) {
	crm_xml_add(location, PCMK_XA_ROLE, promoted_role);
	} else {
	crm_xml_add(location, PCMK_XA_ROLE, PCMK_ROLE_STARTED);
	}

	if (later_s == NULL) {
	/* Short form */
	crm_xml_add(location, PCMK_XE_NODE, host);
	crm_xml_add(location, PCMK_XA_SCORE, PCMK_VALUE_MINUS_INFINITY);

	} else {
	xmlNode *rule = pcmk__xe_create(location, PCMK_XE_RULE);
	xmlNode *expr = pcmk__xe_create(rule, PCMK_XE_EXPRESSION);

	- crm_xml_set_id(rule, "cli-ban-%s-on-%s-rule", rsc_id, host);
	+ pcmk__xe_set_id(rule, "cli-ban-%s-on-%s-rule", rsc_id, host);
	crm_xml_add(rule, PCMK_XA_SCORE, PCMK_VALUE_MINUS_INFINITY);
	crm_xml_add(rule, PCMK_XA_BOOLEAN_OP, PCMK_VALUE_AND);

	- crm_xml_set_id(expr, "cli-ban-%s-on-%s-expr", rsc_id, host);
	+ pcmk__xe_set_id(expr, "cli-ban-%s-on-%s-expr", rsc_id, host);
	crm_xml_add(expr, PCMK_XA_ATTRIBUTE, CRM_ATTR_UNAME);
	crm_xml_add(expr, PCMK_XA_OPERATION, PCMK_VALUE_EQ);
	crm_xml_add(expr, PCMK_XA_VALUE, host);
	crm_xml_add(expr, PCMK_XA_TYPE, PCMK_VALUE_STRING);

	expr = pcmk__xe_create(rule, PCMK_XE_DATE_EXPRESSION);
	- crm_xml_set_id(expr, "cli-ban-%s-on-%s-lifetime", rsc_id, host);
	+ pcmk__xe_set_id(expr, "cli-ban-%s-on-%s-lifetime", rsc_id, host);
	crm_xml_add(expr, PCMK_XA_OPERATION, PCMK_VALUE_LT);
	crm_xml_add(expr, PCMK_XA_END, later_s);
	}

	crm_log_xml_notice(fragment, "Modify");
	rc = cib_conn->cmds->modify(cib_conn, PCMK_XE_CONSTRAINTS, fragment,
	cib_options);
	rc = pcmk_legacy2rc(rc);

	pcmk__xml_free(fragment);
	free(later_s);

	if ((rc != pcmk_rc_ok)
	&& promoted_role_only
	&& (strcmp(promoted_role, PCMK_ROLE_PROMOTED) == 0)) {

	int banrc = cli_resource_ban(out, rsc_id, host, move_lifetime,
	cib_conn, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED_LEGACY);
	if (banrc == pcmk_rc_ok) {
	rc = banrc;
	}
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_prefer(pcmk__output_t out,const char rsc_id, const char *host,
	const char move_lifetime, cib_t cib_conn, int cib_options,
	gboolean promoted_role_only, const char *promoted_role)
	{
	char *later_s = parse_cli_lifetime(out, move_lifetime);
	int rc = pcmk_rc_ok;
	xmlNode *location = NULL;
	xmlNode *fragment = NULL;

	if(move_lifetime && later_s == NULL) {
	return EINVAL;
	}

	if(cib_conn == NULL) {
	free(later_s);
	return ENOTCONN;
	}

	fragment = pcmk__xe_create(NULL, PCMK_XE_CONSTRAINTS);

	location = pcmk__xe_create(fragment, PCMK_XE_RSC_LOCATION);
	- crm_xml_set_id(location, "cli-prefer-%s", rsc_id);
	+ pcmk__xe_set_id(location, "cli-prefer-%s", rsc_id);

	crm_xml_add(location, PCMK_XA_RSC, rsc_id);
	if(promoted_role_only) {
	crm_xml_add(location, PCMK_XA_ROLE, promoted_role);
	} else {
	crm_xml_add(location, PCMK_XA_ROLE, PCMK_ROLE_STARTED);
	}

	if (later_s == NULL) {
	/* Short form */
	crm_xml_add(location, PCMK_XE_NODE, host);
	crm_xml_add(location, PCMK_XA_SCORE, PCMK_VALUE_INFINITY);

	} else {
	xmlNode *rule = pcmk__xe_create(location, PCMK_XE_RULE);
	xmlNode *expr = pcmk__xe_create(rule, PCMK_XE_EXPRESSION);

	- crm_xml_set_id(rule, "cli-prefer-rule-%s", rsc_id);
	+ pcmk__xe_set_id(rule, "cli-prefer-rule-%s", rsc_id);
	crm_xml_add(rule, PCMK_XA_SCORE, PCMK_VALUE_INFINITY);
	crm_xml_add(rule, PCMK_XA_BOOLEAN_OP, PCMK_VALUE_AND);

	- crm_xml_set_id(expr, "cli-prefer-expr-%s", rsc_id);
	+ pcmk__xe_set_id(expr, "cli-prefer-expr-%s", rsc_id);
	crm_xml_add(expr, PCMK_XA_ATTRIBUTE, CRM_ATTR_UNAME);
	crm_xml_add(expr, PCMK_XA_OPERATION, PCMK_VALUE_EQ);
	crm_xml_add(expr, PCMK_XA_VALUE, host);
	crm_xml_add(expr, PCMK_XA_TYPE, PCMK_VALUE_STRING);

	expr = pcmk__xe_create(rule, PCMK_XE_DATE_EXPRESSION);
	- crm_xml_set_id(expr, "cli-prefer-lifetime-end-%s", rsc_id);
	+ pcmk__xe_set_id(expr, "cli-prefer-lifetime-end-%s", rsc_id);
	crm_xml_add(expr, PCMK_XA_OPERATION, PCMK_VALUE_LT);
	crm_xml_add(expr, PCMK_XA_END, later_s);
	}

	crm_log_xml_info(fragment, "Modify");
	rc = cib_conn->cmds->modify(cib_conn, PCMK_XE_CONSTRAINTS, fragment,
	cib_options);
	rc = pcmk_legacy2rc(rc);

	pcmk__xml_free(fragment);
	free(later_s);

	if ((rc != pcmk_rc_ok)
	&& promoted_role_only
	&& (strcmp(promoted_role, PCMK_ROLE_PROMOTED) == 0)) {

	int preferrc = cli_resource_prefer(out, rsc_id, host, move_lifetime,
	cib_conn, cib_options, promoted_role_only,
	PCMK__ROLE_PROMOTED_LEGACY);
	if (preferrc == pcmk_rc_ok) {
	rc = preferrc;
	}
	}

	return rc;
	}

	/* Nodes can be specified two different ways in the CIB, so we have two different
	* functions to try clearing out any constraints on them:
	*
	* (1) The node could be given by attribute=/value= in an expression XML node.
	* That's what resource_clear_node_in_expr handles. That XML looks like this:
	*
	* <rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started">
	* <rule id="cli-prefer-rule-dummy" score="INFINITY" boolean-op="and">
	* <expression id="cli-prefer-expr-dummy" attribute="#uname" operation="eq" value="test02" type="string"/>
	* <date_expression id="cli-prefer-lifetime-end-dummy" operation="lt" end="2018-12-12 14:05:37 -05:00"/>
	* </rule>
	* </rsc_location>
	*
	* (2) The node could be given by node= in a PCMK_XE_RSC_LOCATION XML node.
	* That's what resource_clear_node_in_location handles. That XML looks like
	* this:
	*
	* <rsc_location id="cli-prefer-dummy" rsc="dummy" role="Started" node="node1" score="INFINITY"/>
	*
	* \return Standard Pacemaker return code
	*/
	static int
	resource_clear_node_in_expr(const char rsc_id, const char host, cib_t * cib_conn,
	int cib_options)
	{
	int rc = pcmk_rc_ok;
	char *xpath_string = NULL;

	#define XPATH_FMT \
	"//" PCMK_XE_RSC_LOCATION "[@" PCMK_XA_ID "='cli-prefer-%s']" \
	"[" PCMK_XE_RULE \
	"[@" PCMK_XA_ID "='cli-prefer-rule-%s']" \
	"/" PCMK_XE_EXPRESSION \
	"[@" PCMK_XA_ATTRIBUTE "='" CRM_ATTR_UNAME "' " \
	"and @" PCMK_XA_VALUE "='%s']" \
	"]"

	xpath_string = crm_strdup_printf(XPATH_FMT, rsc_id, rsc_id, host);

	rc = cib_conn->cmds->remove(cib_conn, xpath_string, NULL, cib_xpath \| cib_options);
	if (rc == -ENXIO) {
	rc = pcmk_rc_ok;
	} else {
	rc = pcmk_legacy2rc(rc);
	}

	free(xpath_string);
	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	resource_clear_node_in_location(const char rsc_id, const char host, cib_t * cib_conn,
	int cib_options, bool clear_ban_constraints, gboolean force)
	{
	int rc = pcmk_rc_ok;
	xmlNode *fragment = NULL;
	xmlNode *location = NULL;

	fragment = pcmk__xe_create(NULL, PCMK_XE_CONSTRAINTS);

	if (clear_ban_constraints == TRUE) {
	location = pcmk__xe_create(fragment, PCMK_XE_RSC_LOCATION);
	- crm_xml_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);
	+ pcmk__xe_set_id(location, "cli-ban-%s-on-%s", rsc_id, host);
	}

	location = pcmk__xe_create(fragment, PCMK_XE_RSC_LOCATION);
	- crm_xml_set_id(location, "cli-prefer-%s", rsc_id);
	+ pcmk__xe_set_id(location, "cli-prefer-%s", rsc_id);
	if (force == FALSE) {
	crm_xml_add(location, PCMK_XE_NODE, host);
	}

	crm_log_xml_info(fragment, "Delete");
	rc = cib_conn->cmds->remove(cib_conn, PCMK_XE_CONSTRAINTS, fragment,
	cib_options);
	if (rc == -ENXIO) {
	rc = pcmk_rc_ok;
	} else {
	rc = pcmk_legacy2rc(rc);
	}

	pcmk__xml_free(fragment);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_clear(const char rsc_id, const char host, GList allnodes, cib_t cib_conn,
	int cib_options, bool clear_ban_constraints, gboolean force)
	{
	int rc = pcmk_rc_ok;

	if(cib_conn == NULL) {
	return ENOTCONN;
	}

	if (host) {
	rc = resource_clear_node_in_expr(rsc_id, host, cib_conn, cib_options);

	/* rc does not tell us whether the previous operation did anything, only
	* whether it failed or not. Thus, as long as it did not fail, we need
	* to try the second clear method.
	*/
	if (rc == pcmk_rc_ok) {
	rc = resource_clear_node_in_location(rsc_id, host, cib_conn,
	cib_options, clear_ban_constraints,
	force);
	}

	} else {
	GList *n = allnodes;

	/* Iterate over all nodes, attempting to clear the constraint from each.
	* On the first error, abort.
	*/
	for(; n; n = n->next) {
	pcmk_node_t *target = n->data;

	rc = cli_resource_clear(rsc_id, target->details->uname, NULL,
	cib_conn, cib_options, clear_ban_constraints,
	force);
	if (rc != pcmk_rc_ok) {
	break;
	}
	}
	}

	return rc;
	}

	static void
	build_clear_xpath_string(GString buf, const xmlNode constraint_node,
	const char rsc, const char node,
	bool promoted_role_only)
	{
	const char *cons_id = pcmk__xe_id(constraint_node);
	const char *cons_rsc = crm_element_value(constraint_node, PCMK_XA_RSC);
	GString *rsc_role_substr = NULL;
	const char *promoted_role_rule = "@" PCMK_XA_ROLE "='" PCMK_ROLE_PROMOTED
	"' or @" PCMK_XA_ROLE "='"
	PCMK__ROLE_PROMOTED_LEGACY "'";

	CRM_ASSERT(buf != NULL);
	g_string_truncate(buf, 0);

	if (!pcmk__starts_with(cons_id, "cli-ban-")
	&& !pcmk__starts_with(cons_id, "cli-prefer-")) {
	return;
	}

	g_string_append(buf, "//" PCMK_XE_RSC_LOCATION);

	if ((node != NULL) \|\| (rsc != NULL) \|\| promoted_role_only) {
	g_string_append_c(buf, '[');

	if (node != NULL) {
	pcmk__g_strcat(buf, "@" PCMK_XE_NODE "='", node, "'", NULL);

	if (promoted_role_only \|\| (rsc != NULL)) {
	g_string_append(buf, " and ");
	}
	}

	if ((rsc != NULL) && promoted_role_only) {
	rsc_role_substr = g_string_sized_new(64);
	pcmk__g_strcat(rsc_role_substr,
	"@" PCMK_XA_RSC "='", rsc, "' "
	"and (" , promoted_role_rule, ")", NULL);

	} else if (rsc != NULL) {
	rsc_role_substr = g_string_sized_new(64);
	pcmk__g_strcat(rsc_role_substr,
	"@" PCMK_XA_RSC "='", rsc, "'", NULL);

	} else if (promoted_role_only) {
	rsc_role_substr = g_string_sized_new(64);
	g_string_append(rsc_role_substr, promoted_role_rule);
	}

	if (rsc_role_substr != NULL) {
	g_string_append(buf, rsc_role_substr->str);
	}
	g_string_append_c(buf, ']');
	}

	if (node != NULL) {
	g_string_append(buf, "\|//" PCMK_XE_RSC_LOCATION);

	if (rsc_role_substr != NULL) {
	pcmk__g_strcat(buf, "[", rsc_role_substr, "]", NULL);
	}
	pcmk__g_strcat(buf,
	"/" PCMK_XE_RULE "[" PCMK_XE_EXPRESSION
	"[@" PCMK_XA_ATTRIBUTE "='" CRM_ATTR_UNAME "' "
	"and @" PCMK_XA_VALUE "='", node, "']]", NULL);
	}

	g_string_append(buf, "//" PCMK_XE_DATE_EXPRESSION "[@" PCMK_XA_ID "='");
	if (pcmk__starts_with(cons_id, "cli-ban-")) {
	pcmk__g_strcat(buf, cons_id, "-lifetime']", NULL);

	} else { // starts with "cli-prefer-"
	pcmk__g_strcat(buf,
	"cli-prefer-lifetime-end-", cons_rsc, "']", NULL);
	}

	if (rsc_role_substr != NULL) {
	g_string_free(rsc_role_substr, TRUE);
	}
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_clear_all_expired(xmlNode root, cib_t cib_conn, int cib_options,
	const char rsc, const char node, gboolean promoted_role_only)
	{
	GString *buf = NULL;
	xmlXPathObject *xpathObj = NULL;
	xmlNode *cib_constraints = NULL;
	crm_time_t *now = crm_time_new(NULL);
	int i;
	int rc = pcmk_rc_ok;

	cib_constraints = pcmk_find_cib_element(root, PCMK_XE_CONSTRAINTS);
	xpathObj = xpath_search(cib_constraints, "//" PCMK_XE_RSC_LOCATION);

	for (i = 0; i < numXpathResults(xpathObj); i++) {
	xmlNode *constraint_node = getXpathResult(xpathObj, i);
	xmlNode *date_expr_node = NULL;
	crm_time_t *end = NULL;
	int rc = pcmk_rc_ok;

	if (buf == NULL) {
	buf = g_string_sized_new(1024);
	}

	build_clear_xpath_string(buf, constraint_node, rsc, node,
	promoted_role_only);
	if (buf->len == 0) {
	continue;
	}

	date_expr_node = get_xpath_object((const char *) buf->str,
	constraint_node, LOG_DEBUG);
	if (date_expr_node == NULL) {
	continue;
	}

	/* And then finally, see if the date expression is expired. If so,
	* clear the constraint.
	*
	* @COMPAT Check for error once we are rejecting rules with invalid end
	*/
	rc = pcmk__xe_get_datetime(date_expr_node, PCMK_XA_END, &end);
	if (rc != pcmk_rc_ok) {
	crm_trace("Invalid " PCMK_XA_END ": %s", pcmk_rc_str(rc));
	}

	if (crm_time_compare(now, end) == 1) {
	xmlNode *fragment = NULL;
	xmlNode *location = NULL;

	fragment = pcmk__xe_create(NULL, PCMK_XE_CONSTRAINTS);
	location = pcmk__xe_create(fragment, PCMK_XE_RSC_LOCATION);
	- crm_xml_set_id(location, "%s", pcmk__xe_id(constraint_node));
	+ pcmk__xe_set_id(location, "%s", pcmk__xe_id(constraint_node));
	crm_log_xml_info(fragment, "Delete");

	rc = cib_conn->cmds->remove(cib_conn, PCMK_XE_CONSTRAINTS, fragment,
	cib_options);
	rc = pcmk_legacy2rc(rc);

	if (rc != pcmk_rc_ok) {
	goto done;
	}

	pcmk__xml_free(fragment);
	}

	crm_time_free(end);
	}

	done:
	if (buf != NULL) {
	g_string_free(buf, TRUE);
	}
	freeXpathObject(xpathObj);
	crm_time_free(now);
	return rc;
	}

File Metadata

Mime Type: text/x-diff
Expires: Wed, Oct 15, 11:53 PM (2 h, 53 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2530590
Default Alt Text: (521 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions