No OneTemporary
Actions

Size

829 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/daemons/attrd/attrd_corosync.c b/daemons/attrd/attrd_corosync.c
	index 6b8fbfe490..6a08f2e4ab 100644
	--- a/daemons/attrd/attrd_corosync.c
	+++ b/daemons/attrd/attrd_corosync.c
	@@ -1,603 +1,603 @@
	/*
	* Copyright 2013-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <errno.h>
	#include <stdbool.h>
	#include <stdint.h>
	#include <stdlib.h>

	#include <crm/cluster.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/logging.h>
	#include <crm/common/results.h>
	#include <crm/common/strings_internal.h>
	#include <crm/common/xml.h>

	#include "pacemaker-attrd.h"

	static xmlNode *
	attrd_confirmation(int callid)
	{
	xmlNode *node = create_xml_node(NULL, __func__);

	crm_xml_add(node, PCMK__XA_T, PCMK__VALUE_ATTRD);
	crm_xml_add(node, PCMK__XA_SRC, get_local_node_name());
	crm_xml_add(node, PCMK_XA_TASK, PCMK__ATTRD_CMD_CONFIRM);
	crm_xml_add_int(node, PCMK__XA_CALL_ID, callid);

	return node;
	}

	static void
	attrd_peer_message(crm_node_t peer, xmlNode xml)
	{
	const char *election_op = crm_element_value(xml, PCMK__XA_CRM_TASK);

	if (election_op) {
	attrd_handle_election_op(peer, xml);
	return;
	}

	if (attrd_shutting_down(false)) {
	/* If we're shutting down, we want to continue responding to election
	* ops as long as we're a cluster member (because our vote may be
	* needed). Ignore all other messages.
	*/
	return;

	} else {
	pcmk__request_t request = {
	.ipc_client = NULL,
	.ipc_id = 0,
	.ipc_flags = 0,
	.peer = peer->uname,
	.xml = xml,
	.call_options = 0,
	.result = PCMK__UNKNOWN_RESULT,
	};

	request.op = crm_element_value_copy(request.xml, PCMK_XA_TASK);
	CRM_CHECK(request.op != NULL, return);

	attrd_handle_request(&request);

	/* Having finished handling the request, check to see if the originating
	* peer requested confirmation. If so, send that confirmation back now.
	*/
	if (pcmk__xe_attr_is_true(xml, PCMK__XA_CONFIRM) &&
	!pcmk__str_eq(request.op, PCMK__ATTRD_CMD_CONFIRM, pcmk__str_none)) {
	int callid = 0;
	xmlNode *reply = NULL;

	/* Add the confirmation ID for the message we are confirming to the
	* response so the originating peer knows what they're a confirmation
	* for.
	*/
	crm_element_value_int(xml, PCMK__XA_CALL_ID, &callid);
	reply = attrd_confirmation(callid);

	/* And then send the confirmation back to the originating peer. This
	* ends up right back in this same function (attrd_peer_message) on the
	* peer where it will have to do something with a PCMK__XA_CONFIRM type
	* message.
	*/
	crm_debug("Sending %s a confirmation", peer->uname);
	attrd_send_message(peer, reply, false);
	free_xml(reply);
	}

	pcmk__reset_request(&request);
	}
	}

	static void
	attrd_cpg_dispatch(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	xmlNode *xml = NULL;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}

	if (kind == crm_class_cluster) {
	- xml = string2xml(data);
	+ xml = pcmk__xml_parse_string(data);
	}

	if (xml == NULL) {
	crm_err("Bad message of class %d received from %s[%u]: '%.120s'", kind, from, nodeid, data);
	} else {
	attrd_peer_message(pcmk__get_node(nodeid, from, NULL,
	pcmk__node_search_cluster),
	xml);
	}

	free_xml(xml);
	free(data);
	}

	static void
	attrd_cpg_destroy(gpointer unused)
	{
	if (attrd_shutting_down(false)) {
	crm_info("Disconnected from Corosync process group");

	} else {
	crm_crit("Lost connection to Corosync process group, shutting down");
	attrd_exit_status = CRM_EX_DISCONNECT;
	attrd_shutdown(0);
	}
	}

	/*!
	* \internal
	* \brief Broadcast an update for a single attribute value
	*
	* \param[in] a Attribute to broadcast
	* \param[in] v Attribute value to broadcast
	*/
	void
	attrd_broadcast_value(const attribute_t a, const attribute_value_t v)
	{
	xmlNode *op = create_xml_node(NULL, PCMK_XE_OP);

	crm_xml_add(op, PCMK_XA_TASK, PCMK__ATTRD_CMD_UPDATE);
	attrd_add_value_xml(op, a, v, false);
	attrd_send_message(NULL, op, false);
	free_xml(op);
	}

	#define state_text(state) pcmk__s((state), "in unknown state")

	static void
	attrd_peer_change_cb(enum crm_status_type kind, crm_node_t peer, const void data)
	{
	bool gone = false;
	bool is_remote = pcmk_is_set(peer->flags, crm_remote_node);

	switch (kind) {
	case crm_status_uname:
	crm_debug("%s node %s is now %s",
	(is_remote? "Remote" : "Cluster"),
	peer->uname, state_text(peer->state));
	break;

	case crm_status_processes:
	if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
	gone = true;
	}
	crm_debug("Node %s is %s a peer",
	peer->uname, (gone? "no longer" : "now"));
	break;

	case crm_status_nstate:
	crm_debug("%s node %s is now %s (was %s)",
	(is_remote? "Remote" : "Cluster"),
	peer->uname, state_text(peer->state), state_text(data));
	if (pcmk__str_eq(peer->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
	/* If we're the writer, send new peers a list of all attributes
	* (unless it's a remote node, which doesn't run its own attrd)
	*/
	if (attrd_election_won()
	&& !pcmk_is_set(peer->flags, crm_remote_node)) {
	attrd_peer_sync(peer);
	}
	} else {
	// Remove all attribute values associated with lost nodes
	attrd_peer_remove(peer->uname, false, "loss");
	gone = true;
	}
	break;
	}

	// Remove votes from cluster nodes that leave, in case election in progress
	if (gone && !is_remote) {
	attrd_remove_voter(peer);
	attrd_remove_peer_protocol_ver(peer->uname);
	attrd_do_not_expect_from_peer(peer->uname);
	}
	}

	static void
	record_peer_nodeid(attribute_value_t v, const char host)
	{
	crm_node_t *known_peer = pcmk__get_node(v->nodeid, host, NULL,
	pcmk__node_search_cluster);

	crm_trace("Learned %s has node id %s", known_peer->uname, known_peer->uuid);
	if (attrd_election_won()) {
	attrd_write_attributes(attrd_write_changed);
	}
	}

	#define readable_value(rv_v) pcmk__s((rv_v)->current, "(unset)")

	#define readable_peer(p) \
	(((p) == NULL)? "all peers" : pcmk__s((p)->uname, "unknown peer"))

	static void
	update_attr_on_host(attribute_t a, const crm_node_t peer, const xmlNode *xml,
	const char attr, const char value, const char *host,
	bool filter)
	{
	int is_remote = 0;
	bool changed = false;
	attribute_value_t *v = NULL;

	// Create entry for value if not already existing
	v = g_hash_table_lookup(a->values, host);
	if (v == NULL) {
	v = calloc(1, sizeof(attribute_value_t));
	CRM_ASSERT(v != NULL);

	pcmk__str_update(&v->nodename, host);
	g_hash_table_replace(a->values, v->nodename, v);
	}

	// If value is for a Pacemaker Remote node, remember that
	crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote);
	if (is_remote) {
	attrd_set_value_flags(v, attrd_value_remote);
	CRM_ASSERT(crm_remote_peer_get(host) != NULL);
	}

	// Check whether the value changed
	changed = !pcmk__str_eq(v->current, value, pcmk__str_casei);

	if (changed && filter && pcmk__str_eq(host, attrd_cluster->uname,
	pcmk__str_casei)) {
	/* Broadcast the local value for an attribute that differs from the
	* value provided in a peer's attribute synchronization response. This
	* ensures a node's values for itself take precedence and all peers are
	* kept in sync.
	*/
	v = g_hash_table_lookup(a->values, attrd_cluster->uname);
	crm_notice("%s[%s]: local value '%s' takes priority over '%s' from %s",
	attr, host, readable_value(v), value, peer->uname);
	attrd_broadcast_value(a, v);

	} else if (changed) {
	crm_notice("Setting %s[%s]%s%s: %s -> %s "
	CRM_XS " from %s with %s write delay",
	attr, host, a->set_type ? " in " : "",
	pcmk__s(a->set_type, ""), readable_value(v),
	pcmk__s(value, "(unset)"), peer->uname,
	(a->timeout_ms == 0)? "no" : pcmk__readable_interval(a->timeout_ms));
	pcmk__str_update(&v->current, value);
	attrd_set_attr_flags(a, attrd_attr_changed);

	if (pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)
	&& pcmk__str_eq(attr, PCMK__NODE_ATTR_SHUTDOWN, pcmk__str_none)) {

	if (!pcmk__str_eq(value, "0", pcmk__str_null_matches)) {
	attrd_set_requesting_shutdown();

	} else {
	attrd_clear_requesting_shutdown();
	}
	}

	// Write out new value or start dampening timer
	if (a->timeout_ms && a->timer) {
	crm_trace("Delaying write of %s %s for dampening",
	attr, pcmk__readable_interval(a->timeout_ms));
	mainloop_timer_start(a->timer);
	} else {
	attrd_write_or_elect_attribute(a);
	}

	} else {
	int is_force_write = 0;

	crm_element_value_int(xml, PCMK__XA_ATTRD_IS_FORCE_WRITE,
	&is_force_write);

	if (is_force_write == 1 && a->timeout_ms && a->timer) {
	/* Save forced writing and set change flag. */
	/* The actual attribute is written by Writer after election. */
	crm_trace("%s[%s] from %s is unchanged (%s), forcing write",
	attr, host, peer->uname, pcmk__s(value, "unset"));
	attrd_set_attr_flags(a, attrd_attr_force_write);
	} else {
	crm_trace("%s[%s] from %s is unchanged (%s)",
	attr, host, peer->uname, pcmk__s(value, "unset"));
	}
	}

	// This allows us to later detect local values that peer doesn't know about
	attrd_set_value_flags(v, attrd_value_from_peer);

	/* If this is a cluster node whose node ID we are learning, remember it */
	if ((v->nodeid == 0) && !pcmk_is_set(v->flags, attrd_value_remote)
	&& (crm_element_value_int(xml, PCMK__XA_ATTR_HOST_ID,
	(int*)&v->nodeid) == 0) && (v->nodeid > 0)) {
	record_peer_nodeid(v, host);
	}
	}

	static void
	attrd_peer_update_one(const crm_node_t peer, xmlNode xml, bool filter)
	{
	attribute_t *a = NULL;
	const char *attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
	const char *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
	const char *host = crm_element_value(xml, PCMK__XA_ATTR_HOST);

	if (attr == NULL) {
	crm_warn("Could not update attribute: peer did not specify name");
	return;
	}

	a = attrd_populate_attribute(xml, attr);
	if (a == NULL) {
	return;
	}

	if (host == NULL) {
	// If no host was specified, update all hosts
	GHashTableIter vIter;

	crm_debug("Setting %s for all hosts to %s", attr, value);
	xml_remove_prop(xml, PCMK__XA_ATTR_HOST_ID);
	g_hash_table_iter_init(&vIter, a->values);

	while (g_hash_table_iter_next(&vIter, (gpointer *) & host, NULL)) {
	update_attr_on_host(a, peer, xml, attr, value, host, filter);
	}

	} else {
	// Update attribute value for the given host
	update_attr_on_host(a, peer, xml, attr, value, host, filter);
	}

	/* If this is a message from some attrd instance broadcasting its protocol
	* version, check to see if it's a new minimum version.
	*/
	if (pcmk__str_eq(attr, CRM_ATTR_PROTOCOL, pcmk__str_none)) {
	attrd_update_minimum_protocol_ver(peer->uname, value);
	}
	}

	static void
	broadcast_unseen_local_values(void)
	{
	GHashTableIter aIter;
	GHashTableIter vIter;
	attribute_t *a = NULL;
	attribute_value_t *v = NULL;
	xmlNode *sync = NULL;

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {

	g_hash_table_iter_init(&vIter, a->values);
	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {

	if (!pcmk_is_set(v->flags, attrd_value_from_peer)
	&& pcmk__str_eq(v->nodename, attrd_cluster->uname,
	pcmk__str_casei)) {
	crm_trace("* %s[%s]='%s' is local-only",
	a->id, v->nodename, readable_value(v));
	if (sync == NULL) {
	sync = create_xml_node(NULL, __func__);
	crm_xml_add(sync, PCMK_XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
	}
	attrd_add_value_xml(sync, a, v, a->timeout_ms && a->timer);
	}
	}
	}

	if (sync != NULL) {
	crm_debug("Broadcasting local-only values");
	attrd_send_message(NULL, sync, false);
	free_xml(sync);
	}
	}

	int
	attrd_cluster_connect(void)
	{
	attrd_cluster = pcmk_cluster_new();

	attrd_cluster->destroy = attrd_cpg_destroy;
	attrd_cluster->cpg.cpg_deliver_fn = attrd_cpg_dispatch;
	attrd_cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;

	crm_set_status_callback(&attrd_peer_change_cb);

	if (crm_cluster_connect(attrd_cluster) == FALSE) {
	crm_err("Cluster connection failed");
	return -ENOTCONN;
	}
	return pcmk_ok;
	}

	void
	attrd_peer_clear_failure(pcmk__request_t *request)
	{
	xmlNode *xml = request->xml;
	const char *rsc = crm_element_value(xml, PCMK__XA_ATTR_RESOURCE);
	const char *host = crm_element_value(xml, PCMK__XA_ATTR_HOST);
	const char *op = crm_element_value(xml, PCMK__XA_ATTR_CLEAR_OPERATION);
	const char *interval_spec = crm_element_value(xml,
	PCMK__XA_ATTR_CLEAR_INTERVAL);
	guint interval_ms = 0U;
	char *attr = NULL;
	GHashTableIter iter;
	regex_t regex;

	crm_node_t *peer = pcmk__get_node(0, request->peer, NULL,
	pcmk__node_search_cluster);

	pcmk_parse_interval_spec(interval_spec, &interval_ms);

	if (attrd_failure_regex(&regex, rsc, op, interval_ms) != pcmk_ok) {
	crm_info("Ignoring invalid request to clear failures for %s",
	pcmk__s(rsc, "all resources"));
	return;
	}

	crm_xml_add(xml, PCMK_XA_TASK, PCMK__ATTRD_CMD_UPDATE);

	/* Make sure value is not set, so we delete */
	xml_remove_prop(xml, PCMK__XA_ATTR_VALUE);

	g_hash_table_iter_init(&iter, attributes);
	while (g_hash_table_iter_next(&iter, (gpointer *) &attr, NULL)) {
	if (regexec(&regex, attr, 0, NULL, 0) == 0) {
	crm_trace("Matched %s when clearing %s",
	attr, pcmk__s(rsc, "all resources"));
	crm_xml_add(xml, PCMK__XA_ATTR_NAME, attr);
	attrd_peer_update(peer, xml, host, false);
	}
	}
	regfree(&regex);
	}

	/*!
	* \internal
	* \brief Load attributes from a peer sync response
	*
	* \param[in] peer Peer that sent sync response
	* \param[in] peer_won Whether peer is the attribute writer
	* \param[in,out] xml Request XML
	*/
	void
	attrd_peer_sync_response(const crm_node_t peer, bool peer_won, xmlNode xml)
	{
	crm_info("Processing " PCMK__ATTRD_CMD_SYNC_RESPONSE " from %s",
	peer->uname);

	if (peer_won) {
	/* Initialize the "seen" flag for all attributes to cleared, so we can
	* detect attributes that local node has but the writer doesn't.
	*/
	attrd_clear_value_seen();
	}

	// Process each attribute update in the sync response
	for (xmlNode *child = pcmk__xml_first_child(xml); child != NULL;
	child = pcmk__xml_next(child)) {
	attrd_peer_update(peer, child,
	crm_element_value(child, PCMK__XA_ATTR_HOST), true);
	}

	if (peer_won) {
	/* If any attributes are still not marked as seen, the writer doesn't
	* know about them, so send all peers an update with them.
	*/
	broadcast_unseen_local_values();
	}
	}

	/*!
	* \internal
	* \brief Remove all attributes and optionally peer cache entries for a node
	*
	* \param[in] host Name of node to purge
	* \param[in] uncache If true, remove node from peer caches
	* \param[in] source Who requested removal (only used for logging)
	*/
	void
	attrd_peer_remove(const char host, bool uncache, const char source)
	{
	attribute_t *a = NULL;
	GHashTableIter aIter;

	CRM_CHECK(host != NULL, return);
	crm_notice("Removing all %s attributes for node %s "
	CRM_XS " %s reaping node from cache",
	host, source, (uncache? "and" : "without"));

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
	if(g_hash_table_remove(a->values, host)) {
	crm_debug("Removed %s[%s] for peer %s", a->id, host, source);
	}
	}

	if (uncache) {
	pcmk__purge_node_from_cache(host, 0);
	}
	}

	/*!
	* \internal
	* \brief Send all known attributes and values to a peer
	*
	* \param[in] peer Peer to send sync to (if NULL, broadcast to all peers)
	*/
	void
	attrd_peer_sync(crm_node_t *peer)
	{
	GHashTableIter aIter;
	GHashTableIter vIter;

	attribute_t *a = NULL;
	attribute_value_t *v = NULL;
	xmlNode *sync = create_xml_node(NULL, __func__);

	crm_xml_add(sync, PCMK_XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);

	g_hash_table_iter_init(&aIter, attributes);
	while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
	g_hash_table_iter_init(&vIter, a->values);
	while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
	crm_debug("Syncing %s[%s]='%s' to %s",
	a->id, v->nodename, readable_value(v),
	readable_peer(peer));
	attrd_add_value_xml(sync, a, v, false);
	}
	}

	crm_debug("Syncing values to %s", readable_peer(peer));
	attrd_send_message(peer, sync, false);
	free_xml(sync);
	}

	void
	attrd_peer_update(const crm_node_t peer, xmlNode xml, const char *host,
	bool filter)
	{
	bool handle_sync_point = false;

	CRM_CHECK((peer != NULL) && (xml != NULL), return);
	if (xml->children != NULL) {
	for (xmlNode *child = first_named_child(xml, PCMK_XE_OP); child != NULL;
	child = crm_next_same_xml(child)) {
	attrd_copy_xml_attributes(xml, child);
	attrd_peer_update_one(peer, child, filter);

	if (attrd_request_has_sync_point(child)) {
	handle_sync_point = true;
	}
	}

	} else {
	attrd_peer_update_one(peer, xml, filter);

	if (attrd_request_has_sync_point(xml)) {
	handle_sync_point = true;
	}
	}

	/* If the update XML specified that the client wanted to wait for a sync
	* point, process that now.
	*/
	if (handle_sync_point) {
	crm_trace("Hit local sync point for attribute update");
	attrd_ack_waitlist_clients(attrd_sync_point_local, xml);
	}
	}
	diff --git a/daemons/based/pacemaker-based.c b/daemons/based/pacemaker-based.c
	index 25a4c13f38..d304d21774 100644
	--- a/daemons/based/pacemaker-based.c
	+++ b/daemons/based/pacemaker-based.c
	@@ -1,447 +1,447 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <stdlib.h>
	#include <pwd.h>
	#include <grp.h>
	#include <bzlib.h>
	#include <sys/types.h>

	#include <glib.h>
	#include <libxml/tree.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/xml.h>

	#include <pacemaker-based.h>

	#define SUMMARY "daemon for managing the configuration of a Pacemaker cluster"

	extern int init_remote_listener(int port, gboolean encrypted);
	gboolean cib_shutdown_flag = FALSE;
	int cib_status = pcmk_ok;

	crm_cluster_t *crm_cluster = NULL;

	GMainLoop *mainloop = NULL;
	gchar *cib_root = NULL;
	static gboolean preserve_status = FALSE;

	gboolean cib_writes_enabled = TRUE;
	gboolean stand_alone = FALSE;

	int remote_fd = 0;
	int remote_tls_fd = 0;

	GHashTable *config_hash = NULL;
	GHashTable *local_notify_queue = NULL;

	static void cib_init(void);
	void cib_shutdown(int nsig);
	static bool startCib(const char *filename);
	extern int write_cib_contents(gpointer p);

	static crm_exit_t exit_code = CRM_EX_OK;

	static void
	cib_enable_writes(int nsig)
	{
	crm_info("(Re)enabling disk writes");
	cib_writes_enabled = TRUE;
	}

	/*!
	* \internal
	* \brief Set up options, users, and groups for stand-alone mode
	*
	* \param[out] error GLib error object
	*
	* \return Standard Pacemaker return code
	*/
	static int
	setup_stand_alone(GError **error)
	{
	int rc = 0;
	struct passwd *pwentry = NULL;

	preserve_status = TRUE;
	cib_writes_enabled = FALSE;

	errno = 0;
	pwentry = getpwnam(CRM_DAEMON_USER);
	if (pwentry == NULL) {
	exit_code = CRM_EX_FATAL;
	if (errno != 0) {
	g_set_error(error, PCMK__EXITC_ERROR, exit_code,
	"Error getting password DB entry for %s: %s",
	CRM_DAEMON_USER, strerror(errno));
	return errno;
	}
	g_set_error(error, PCMK__EXITC_ERROR, exit_code,
	"Password DB entry for '%s' not found", CRM_DAEMON_USER);
	return ENXIO;
	}

	rc = setgid(pwentry->pw_gid);
	if (rc < 0) {
	exit_code = CRM_EX_FATAL;
	g_set_error(error, PCMK__EXITC_ERROR, exit_code,
	"Could not set group to %d: %s",
	pwentry->pw_gid, strerror(errno));
	return errno;
	}

	rc = initgroups(CRM_DAEMON_USER, pwentry->pw_gid);
	if (rc < 0) {
	exit_code = CRM_EX_FATAL;
	g_set_error(error, PCMK__EXITC_ERROR, exit_code,
	"Could not setup groups for user %d: %s",
	pwentry->pw_uid, strerror(errno));
	return errno;
	}

	rc = setuid(pwentry->pw_uid);
	if (rc < 0) {
	exit_code = CRM_EX_FATAL;
	g_set_error(error, PCMK__EXITC_ERROR, exit_code,
	"Could not set user to %d: %s",
	pwentry->pw_uid, strerror(errno));
	return errno;
	}
	return pcmk_rc_ok;
	}

	static void
	based_metadata(void)
	{
	const char *name = "pacemaker-based";
	const char *desc_short = "Cluster Information Base manager options";
	const char *desc_long = "Cluster options used by Pacemaker's Cluster "
	"Information Base manager";

	char *s = pcmk__cluster_option_metadata(name, desc_short, desc_long,
	pcmk__opt_context_based);

	printf("%s", s);
	free(s);
	}

	static GOptionEntry entries[] = {
	{ "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone,
	"(Advanced use only) Run in stand-alone mode", NULL },

	{ "disk-writes", 'w', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&cib_writes_enabled,
	"(Advanced use only) Enable disk writes (enabled by default unless in "
	"stand-alone mode)", NULL },

	{ "cib-root", 'r', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME, &cib_root,
	"(Advanced use only) Directory where the CIB XML file should be located "
	"(default: " CRM_CONFIG_DIR ")", NULL },

	{ NULL }
	};

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group)
	{
	GOptionContext *context = NULL;

	context = pcmk__build_arg_context(args, "text (default), xml", group,
	"[metadata]");
	pcmk__add_main_args(context, entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	crm_ipc_t *old_instance = NULL;

	pcmk__output_t *out = NULL;

	GError *error = NULL;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "r");
	GOptionContext *context = build_arg_context(args, &output_group);

	crm_log_preinit(NULL, argc, argv);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_ERROR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error creating output format %s: %s",
	args->output_ty, pcmk_rc_str(rc));
	goto done;
	}

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	mainloop_add_signal(SIGTERM, cib_shutdown);
	mainloop_add_signal(SIGPIPE, cib_enable_writes);

	cib_writer = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_contents, NULL);

	if ((g_strv_length(processed_args) >= 2)
	&& pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) {
	based_metadata();
	goto done;
	}

	pcmk__cli_init_logging("pacemaker-based", args->verbosity);
	crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
	crm_notice("Starting Pacemaker CIB manager");

	old_instance = crm_ipc_new(PCMK__SERVER_BASED_RO, 0);
	if (old_instance == NULL) {
	/* crm_ipc_new() will have already logged an error message with
	* crm_err()
	*/
	exit_code = CRM_EX_FATAL;
	goto done;
	}

	if (pcmk__connect_generic_ipc(old_instance) == pcmk_rc_ok) {
	/* IPC end-point already up */
	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);
	crm_err("pacemaker-based is already active, aborting startup");
	goto done;
	} else {
	/* not up or not authentic, we'll proceed either way */
	crm_ipc_destroy(old_instance);
	old_instance = NULL;
	}

	if (stand_alone) {
	rc = setup_stand_alone(&error);
	if (rc != pcmk_rc_ok) {
	goto done;
	}
	}

	if (cib_root == NULL) {
	cib_root = g_strdup(CRM_CONFIG_DIR);
	} else {
	crm_notice("Using custom config location: %s", cib_root);
	}

	if (!pcmk__daemon_can_write(cib_root, NULL)) {
	exit_code = CRM_EX_FATAL;
	crm_err("Terminating due to bad permissions on %s", cib_root);
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Bad permissions on %s (see logs for details)", cib_root);
	goto done;
	}

	crm_peer_init();

	// Read initial CIB, connect to cluster, and start IPC servers
	cib_init();

	// Run the main loop
	mainloop = g_main_loop_new(NULL, FALSE);
	crm_notice("Pacemaker CIB manager successfully started and accepting connections");
	g_main_loop_run(mainloop);

	/* If main loop returned, clean up and exit. We disconnect in case
	* terminate_cib() was called with fast=-1.
	*/
	crm_cluster_disconnect(crm_cluster);
	pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	crm_peer_destroy();

	if (local_notify_queue != NULL) {
	g_hash_table_destroy(local_notify_queue);
	}

	if (config_hash != NULL) {
	g_hash_table_destroy(config_hash);
	}
	pcmk__client_cleanup();
	pcmk_cluster_free(crm_cluster);
	g_free(cib_root);

	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}
	pcmk__unregister_formats();
	crm_exit(exit_code);
	}

	#if SUPPORT_COROSYNC
	static void
	cib_cs_dispatch(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	xmlNode *xml = NULL;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	- xml = string2xml(data);
	+ xml = pcmk__xml_parse_string(data);
	if (xml == NULL) {
	crm_err("Invalid XML: '%.120s'", data);
	free(data);
	return;
	}
	crm_xml_add(xml, PCMK__XA_SRC, from);
	cib_peer_callback(xml, NULL);
	}

	free_xml(xml);
	free(data);
	}

	static void
	cib_cs_destroy(gpointer user_data)
	{
	if (cib_shutdown_flag) {
	crm_info("Corosync disconnection complete");
	} else {
	crm_crit("Lost connection to cluster layer, shutting down");
	terminate_cib(__func__, CRM_EX_DISCONNECT);
	}
	}
	#endif

	static void
	cib_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
	{
	switch (type) {
	case crm_status_processes:
	if (cib_legacy_mode()
	&& !pcmk_is_set(node->processes, crm_get_cluster_proc())) {

	uint32_t old = data? (const uint32_t )data : 0;

	if ((node->processes ^ old) & crm_proc_cpg) {
	crm_info("Attempting to disable legacy mode after %s left the cluster",
	node->uname);
	legacy_mode = FALSE;
	}
	}
	break;

	case crm_status_uname:
	case crm_status_nstate:
	if (cib_shutdown_flag && (crm_active_peers() < 2)
	&& (pcmk__ipc_client_count() == 0)) {

	crm_info("No more peers");
	terminate_cib(__func__, -1);
	}
	break;
	}
	}

	static void
	cib_init(void)
	{
	crm_cluster = pcmk_cluster_new();

	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	crm_cluster->destroy = cib_cs_destroy;
	crm_cluster->cpg.cpg_deliver_fn = cib_cs_dispatch;
	crm_cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
	}
	#endif // SUPPORT_COROSYNC

	config_hash = pcmk__strkey_table(free, free);

	if (startCib("cib.xml") == FALSE) {
	crm_crit("Cannot start CIB... terminating");
	crm_exit(CRM_EX_NOINPUT);
	}

	if (!stand_alone) {
	crm_set_status_callback(&cib_peer_update_callback);

	if (!crm_cluster_connect(crm_cluster)) {
	crm_crit("Cannot sign in to the cluster... terminating");
	crm_exit(CRM_EX_FATAL);
	}
	}

	pcmk__serve_based_ipc(&ipcs_ro, &ipcs_rw, &ipcs_shm, &ipc_ro_callbacks,
	&ipc_rw_callbacks);

	if (stand_alone) {
	based_is_primary = true;
	}
	}

	static bool
	startCib(const char *filename)
	{
	gboolean active = FALSE;
	xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status);

	if (activateCibXml(cib, TRUE, "start") == 0) {
	int port = 0;

	active = TRUE;

	cib_read_config(config_hash, cib);

	pcmk__scan_port(crm_element_value(cib, PCMK_XA_REMOTE_TLS_PORT), &port);
	if (port >= 0) {
	remote_tls_fd = init_remote_listener(port, TRUE);
	}

	pcmk__scan_port(crm_element_value(cib, PCMK_XA_REMOTE_CLEAR_PORT),
	&port);
	if (port >= 0) {
	remote_fd = init_remote_listener(port, FALSE);
	}
	}
	return active;
	}
	diff --git a/daemons/controld/controld_corosync.c b/daemons/controld/controld_corosync.c
	index c8319eacc3..8c2a8a3ebb 100644
	--- a/daemons/controld/controld_corosync.c
	+++ b/daemons/controld/controld_corosync.c
	@@ -1,161 +1,161 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/stat.h>

	#include <crm/crm.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/xml.h>

	#include <pacemaker-controld.h>

	#if SUPPORT_COROSYNC

	extern void post_cache_update(int seq);

	/* A_HA_CONNECT */

	static void
	crmd_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	crm_node_t *peer = NULL;
	- xmlNode *xml = string2xml(data);
	+ xmlNode *xml = pcmk__xml_parse_string(data);

	if (xml == NULL) {
	crm_err("Could not parse message content (%d): %.100s", kind, data);
	free(data);
	return;
	}

	crm_xml_add(xml, PCMK__XA_SRC, from);

	peer = pcmk__get_node(0, from, NULL, pcmk__node_search_cluster);
	if (!pcmk_is_set(peer->processes, crm_proc_cpg)) {
	/* If we can still talk to our peer process on that node,
	* then it must be part of the corosync membership
	*/
	crm_warn("Receiving messages from a node we think is dead: %s[%d]",
	peer->uname, peer->id);
	crm_update_peer_proc(__func__, peer, crm_proc_cpg,
	PCMK_VALUE_ONLINE);
	}
	crmd_ha_msg_filter(xml);
	free_xml(xml);
	} else {
	crm_err("Invalid message class (%d): %.100s", kind, data);
	}
	free(data);
	}

	static gboolean
	crmd_quorum_callback(unsigned long long seq, gboolean quorate)
	{
	crm_update_quorum(quorate, FALSE);
	post_cache_update(seq);
	return TRUE;
	}

	static void
	crmd_cs_destroy(gpointer user_data)
	{
	if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
	crm_crit("Lost connection to cluster layer, shutting down");
	crmd_exit(CRM_EX_DISCONNECT);
	}
	}

	/*!
	* \brief Handle a Corosync notification of a CPG configuration change
	*
	* \param[in] handle CPG connection
	* \param[in] cpg_name CPG group name
	* \param[in] member_list List of current CPG members
	* \param[in] member_list_entries Number of entries in \p member_list
	* \param[in] left_list List of CPG members that left
	* \param[in] left_list_entries Number of entries in \p left_list
	* \param[in] joined_list List of CPG members that joined
	* \param[in] joined_list_entries Number of entries in \p joined_list
	*/
	static void
	cpg_membership_callback(cpg_handle_t handle, const struct cpg_name *cpg_name,
	const struct cpg_address *member_list,
	size_t member_list_entries,
	const struct cpg_address *left_list,
	size_t left_list_entries,
	const struct cpg_address *joined_list,
	size_t joined_list_entries)
	{
	/* When nodes leave CPG, the DC clears their transient node attributes.
	*
	* However if there is no DC, or the DC is among the nodes that left, each
	* remaining node needs to do the clearing, to ensure it gets done.
	* Otherwise, the attributes would persist when the nodes rejoin, which
	* could have serious consequences for unfencing, agents that use attributes
	* for internal logic, etc.
	*
	* Here, we set a global boolean if the DC is among the nodes that left, for
	* use by the peer callback.
	*/
	if (controld_globals.dc_name != NULL) {
	crm_node_t *peer = NULL;

	peer = pcmk__search_node_caches(0, controld_globals.dc_name,
	pcmk__node_search_cluster);
	if (peer != NULL) {
	for (int i = 0; i < left_list_entries; ++i) {
	if (left_list[i].nodeid == peer->id) {
	controld_set_global_flags(controld_dc_left);
	break;
	}
	}
	}
	}

	// Process the change normally, which will call the peer callback as needed
	pcmk_cpg_membership(handle, cpg_name, member_list, member_list_entries,
	left_list, left_list_entries,
	joined_list, joined_list_entries);

	controld_clear_global_flags(controld_dc_left);
	}

	extern gboolean crm_connect_corosync(crm_cluster_t * cluster);

	gboolean
	crm_connect_corosync(crm_cluster_t * cluster)
	{
	if (is_corosync_cluster()) {
	crm_set_status_callback(&peer_update_callback);
	cluster->cpg.cpg_deliver_fn = crmd_cs_dispatch;
	cluster->cpg.cpg_confchg_fn = cpg_membership_callback;
	cluster->destroy = crmd_cs_destroy;

	if (crm_cluster_connect(cluster)) {
	pcmk__corosync_quorum_connect(crmd_quorum_callback,
	crmd_cs_destroy);
	return TRUE;
	}
	}
	return FALSE;
	}

	#endif
	diff --git a/daemons/controld/controld_metadata.c b/daemons/controld/controld_metadata.c
	index f945f3d47f..1df34725d7 100644
	--- a/daemons/controld/controld_metadata.c
	+++ b/daemons/controld/controld_metadata.c
	@@ -1,320 +1,320 @@
	/*
	* Copyright 2017-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <glib.h>
	#include <regex.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>

	#include <pacemaker-controld.h>

	static void
	ra_param_free(void *param)
	{
	if (param) {
	struct ra_param_s p = (struct ra_param_s ) param;

	if (p->rap_name) {
	free(p->rap_name);
	}
	free(param);
	}
	}

	static void
	metadata_free(void *metadata)
	{
	if (metadata) {
	struct ra_metadata_s md = (struct ra_metadata_s ) metadata;

	g_list_free_full(md->ra_params, ra_param_free);
	free(metadata);
	}
	}

	GHashTable *
	metadata_cache_new(void)
	{
	return pcmk__strkey_table(free, metadata_free);
	}

	void
	metadata_cache_free(GHashTable *mdc)
	{
	if (mdc) {
	crm_trace("Destroying metadata cache with %d members", g_hash_table_size(mdc));
	g_hash_table_destroy(mdc);
	}
	}

	void
	metadata_cache_reset(GHashTable *mdc)
	{
	if (mdc) {
	crm_trace("Resetting metadata cache with %d members",
	g_hash_table_size(mdc));
	g_hash_table_remove_all(mdc);
	}
	}

	static struct ra_param_s *
	ra_param_from_xml(xmlNode *param_xml)
	{
	const char *param_name = crm_element_value(param_xml, PCMK_XA_NAME);
	struct ra_param_s *p;

	p = calloc(1, sizeof(struct ra_param_s));
	if (p == NULL) {
	return NULL;
	}

	p->rap_name = strdup(param_name);
	if (p->rap_name == NULL) {
	free(p);
	return NULL;
	}

	if (pcmk__xe_attr_is_true(param_xml, PCMK_XA_RELOADABLE)) {
	controld_set_ra_param_flags(p, ra_param_reloadable);
	}

	if (pcmk__xe_attr_is_true(param_xml, PCMK_XA_UNIQUE)) {
	controld_set_ra_param_flags(p, ra_param_unique);
	}

	if (pcmk__xe_attr_is_true(param_xml, "private")) {
	controld_set_ra_param_flags(p, ra_param_private);
	}
	return p;
	}

	static void
	log_ra_ocf_version(const char ra_key, const char ra_ocf_version)
	{
	if (pcmk__str_empty(ra_ocf_version)) {
	crm_warn("%s does not advertise OCF version supported", ra_key);

	} else if (compare_version(ra_ocf_version, "2") >= 0) {
	crm_warn("%s supports OCF version %s (this Pacemaker version supports "
	PCMK_OCF_VERSION " and might not work properly with agent)",
	ra_key, ra_ocf_version);

	} else if (compare_version(ra_ocf_version, PCMK_OCF_VERSION) > 0) {
	crm_info("%s supports OCF version %s (this Pacemaker version supports "
	PCMK_OCF_VERSION " and might not use all agent features)",
	ra_key, ra_ocf_version);

	} else {
	crm_debug("%s supports OCF version %s", ra_key, ra_ocf_version);
	}
	}

	struct ra_metadata_s *
	controld_cache_metadata(GHashTable mdc, const lrmd_rsc_info_t rsc,
	const char *metadata_str)
	{
	char *key = NULL;
	const char *reason = NULL;
	xmlNode *metadata = NULL;
	xmlNode *match = NULL;
	struct ra_metadata_s *md = NULL;
	bool any_private_params = false;
	bool ocf1_1 = false;

	CRM_CHECK(mdc && rsc && metadata_str, return NULL);

	key = crm_generate_ra_key(rsc->standard, rsc->provider, rsc->type);
	if (!key) {
	reason = "Invalid resource agent standard or type";
	goto err;
	}

	- metadata = string2xml(metadata_str);
	+ metadata = pcmk__xml_parse_string(metadata_str);
	if (!metadata) {
	reason = "Metadata is not valid XML";
	goto err;
	}

	md = calloc(1, sizeof(struct ra_metadata_s));
	if (md == NULL) {
	reason = "Could not allocate memory";
	goto err;
	}

	if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
	xmlChar *content = NULL;
	xmlNode *version_element = first_named_child(metadata, PCMK_XE_VERSION);

	if (version_element != NULL) {
	content = xmlNodeGetContent(version_element);
	}
	log_ra_ocf_version(key, (const char *) content);
	if (content != NULL) {
	ocf1_1 = (compare_version((const char *) content, "1.1") >= 0);
	xmlFree(content);
	}
	}

	// Check supported actions
	match = first_named_child(metadata, PCMK_XE_ACTIONS);
	for (match = first_named_child(match, PCMK_XE_ACTION); match != NULL;
	match = crm_next_same_xml(match)) {

	const char *action_name = crm_element_value(match, PCMK_XA_NAME);

	if (pcmk__str_eq(action_name, PCMK_ACTION_RELOAD_AGENT,
	pcmk__str_none)) {
	if (ocf1_1) {
	controld_set_ra_flags(md, key, ra_supports_reload_agent);
	} else {
	crm_notice("reload-agent action will not be used with %s "
	"because it does not support OCF 1.1 or later", key);
	}

	} else if (!ocf1_1 && pcmk__str_eq(action_name, PCMK_ACTION_RELOAD,
	pcmk__str_casei)) {
	controld_set_ra_flags(md, key, ra_supports_legacy_reload);
	}
	}

	// Build a parameter list
	match = first_named_child(metadata, PCMK_XE_PARAMETERS);
	for (match = first_named_child(match, PCMK_XE_PARAMETER); match != NULL;
	match = crm_next_same_xml(match)) {

	const char *param_name = crm_element_value(match, PCMK_XA_NAME);

	if (param_name == NULL) {
	crm_warn("Metadata for %s:%s:%s has parameter without a name",
	rsc->standard, rsc->provider, rsc->type);
	} else {
	struct ra_param_s *p = ra_param_from_xml(match);

	if (p == NULL) {
	reason = "Could not allocate memory";
	goto err;
	}
	if (pcmk_is_set(p->rap_flags, ra_param_private)) {
	any_private_params = true;
	}
	md->ra_params = g_list_prepend(md->ra_params, p);
	}
	}

	/* Newer resource agents support the "private" parameter attribute to
	* indicate sensitive parameters. For backward compatibility with older
	* agents, implicitly treat a few common names as private when the agent
	* doesn't specify any explicitly.
	*/
	if (!any_private_params) {
	for (GList *iter = md->ra_params; iter != NULL; iter = iter->next) {
	struct ra_param_s *p = iter->data;

	if (pcmk__str_any_of(p->rap_name, "password", "passwd", "user",
	NULL)) {
	controld_set_ra_param_flags(p, ra_param_private);
	}
	}
	}

	g_hash_table_replace(mdc, key, md);
	free_xml(metadata);
	return md;

	err:
	crm_warn("Unable to update metadata for %s (%s%s%s:%s): %s",
	rsc->id, rsc->standard, ((rsc->provider == NULL)? "" : ":"),
	pcmk__s(rsc->provider, ""), rsc->type, reason);
	free(key);
	free_xml(metadata);
	metadata_free(md);
	return NULL;
	}

	/*!
	* \internal
	* \brief Get meta-data for a resource
	*
	* \param[in,out] lrm_state Use meta-data cache from this executor connection
	* \param[in] rsc Resource to get meta-data for
	* \param[in] source Allowed meta-data sources (bitmask of
	* enum controld_metadata_source_e values)
	*
	* \return Meta-data cache entry for given resource, or NULL if not available
	*/
	struct ra_metadata_s *
	controld_get_rsc_metadata(lrm_state_t lrm_state, const lrmd_rsc_info_t rsc,
	uint32_t source)
	{
	struct ra_metadata_s *metadata = NULL;
	char *metadata_str = NULL;
	char *key = NULL;
	int rc = pcmk_ok;

	CRM_CHECK((lrm_state != NULL) && (rsc != NULL), return NULL);

	if (pcmk_is_set(source, controld_metadata_from_cache)) {
	key = crm_generate_ra_key(rsc->standard, rsc->provider, rsc->type);
	if (key != NULL) {
	metadata = g_hash_table_lookup(lrm_state->metadata_cache, key);
	free(key);
	}
	if (metadata != NULL) {
	crm_debug("Retrieved metadata for %s (%s%s%s:%s) from cache",
	rsc->id, rsc->standard,
	((rsc->provider == NULL)? "" : ":"),
	((rsc->provider == NULL)? "" : rsc->provider),
	rsc->type);
	return metadata;
	}
	}

	if (!pcmk_is_set(source, controld_metadata_from_agent)) {
	return NULL;
	}

	/* For most actions, metadata was cached asynchronously before action
	* execution (via metadata_complete()).
	*
	* However if that failed, and for other actions, retrieve the metadata now
	* via a local, synchronous, direct execution of the agent.
	*
	* This has multiple issues, which is why this is just a fallback: the
	* executor should execute agents, not the controller; metadata for
	* Pacemaker Remote nodes should be collected on those nodes, not locally;
	* the metadata call shouldn't eat into the timeout of the real action being
	* performed; and the synchronous call blocks the controller (which also
	* means that if the metadata action tries to contact the controller,
	* everything will hang until the timeout).
	*/
	crm_debug("Retrieving metadata for %s (%s%s%s:%s) synchronously",
	rsc->id, rsc->standard,
	((rsc->provider == NULL)? "" : ":"),
	((rsc->provider == NULL)? "" : rsc->provider),
	rsc->type);
	rc = lrm_state_get_metadata(lrm_state, rsc->standard, rsc->provider,
	rsc->type, &metadata_str, 0);
	if (rc != pcmk_ok) {
	crm_warn("Failed to get metadata for %s (%s%s%s:%s): %s",
	rsc->id, rsc->standard,
	((rsc->provider == NULL)? "" : ":"),
	((rsc->provider == NULL)? "" : rsc->provider),
	rsc->type, pcmk_strerror(rc));
	return NULL;
	}

	metadata = controld_cache_metadata(lrm_state->metadata_cache, rsc,
	metadata_str);
	free(metadata_str);
	return metadata;
	}
	diff --git a/daemons/fenced/fenced_commands.c b/daemons/fenced/fenced_commands.c
	index 9511c3873b..f18974a02f 100644
	--- a/daemons/fenced/fenced_commands.c
	+++ b/daemons/fenced/fenced_commands.c
	@@ -1,3712 +1,3712 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>

	#include <crm/crm.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/cluster/internal.h>
	#include <crm/common/mainloop.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <pacemaker-fenced.h>

	GHashTable *device_list = NULL;
	GHashTable *topology = NULL;
	static GList *cmd_list = NULL;

	static GHashTable *fenced_handlers = NULL;

	struct device_search_s {
	/* target of fence action */
	char *host;
	/* requested fence action */
	char *action;
	/* timeout to use if a device is queried dynamically for possible targets */
	int per_device_timeout;
	/* number of registered fencing devices at time of request */
	int replies_needed;
	/* number of device replies received so far */
	int replies_received;
	/* whether the target is eligible to perform requested action (or off) */
	bool allow_suicide;

	/* private data to pass to search callback function */
	void *user_data;
	/* function to call when all replies have been received */
	void (callback) (GList devices, void *user_data);
	/* devices capable of performing requested action (or off if remapping) */
	GList *capable;
	/* Whether to perform searches that support the action */
	uint32_t support_action_only;
	};

	static gboolean stonith_device_dispatch(gpointer user_data);
	static void st_child_done(int pid, const pcmk__action_result_t *result,
	void *user_data);

	static void search_devices_record_result(struct device_search_s search, const char device,
	gboolean can_fence);

	static int get_agent_metadata(const char agent, xmlNode *metadata);
	static void read_action_metadata(stonith_device_t *device);
	static enum fenced_target_by unpack_level_kind(const xmlNode *level);

	typedef struct async_command_s {

	int id;
	int pid;
	int fd_stdout;
	int options;
	int default_timeout; /* seconds */
	int timeout; /* seconds */

	int start_delay; // seconds (-1 means disable static/random fencing delays)
	int delay_id;

	char *op;
	char *origin;
	char *client;
	char *client_name;
	char *remote_op_id;

	char *target;
	uint32_t target_nodeid;
	char *action;
	char *device;

	GList *device_list;
	GList *next_device_iter; // device_list entry for next device to execute

	void *internal_user_data;
	void (done_cb) (int pid, const pcmk__action_result_t result,
	void *user_data);
	guint timer_sigterm;
	guint timer_sigkill;
	/*! If the operation timed out, this is the last signal
	* we sent to the process to get it to terminate */
	int last_timeout_signo;

	stonith_device_t *active_on;
	stonith_device_t *activating_on;
	} async_command_t;

	static xmlNode construct_async_reply(const async_command_t cmd,
	const pcmk__action_result_t *result);

	static gboolean
	is_action_required(const char action, const stonith_device_t device)
	{
	return (device != NULL) && device->automatic_unfencing
	&& pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none);
	}

	static int
	get_action_delay_max(const stonith_device_t device, const char action)
	{
	const char *value = NULL;
	guint delay_max = 0U;

	if (!pcmk__is_fencing_action(action)) {
	return 0;
	}

	value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_MAX);
	if (value) {
	pcmk_parse_interval_spec(value, &delay_max);
	delay_max /= 1000;
	}

	return (int) delay_max;
	}

	static int
	get_action_delay_base(const stonith_device_t device, const char action,
	const char *target)
	{
	char *hash_value = NULL;
	guint delay_base = 0U;

	if (!pcmk__is_fencing_action(action)) {
	return 0;
	}

	hash_value = g_hash_table_lookup(device->params, PCMK_STONITH_DELAY_BASE);

	if (hash_value) {
	char *value = strdup(hash_value);
	char *valptr = value;

	CRM_ASSERT(value != NULL);

	if (target != NULL) {
	for (char *val = strtok(value, "; \t"); val != NULL; val = strtok(NULL, "; \t")) {
	char *mapval = strchr(val, ':');

	if (mapval == NULL \|\| mapval[1] == 0) {
	crm_err("pcmk_delay_base: empty value in mapping", val);
	continue;
	}

	if (mapval != val && strncasecmp(target, val, (size_t)(mapval - val)) == 0) {
	value = mapval + 1;
	crm_debug("pcmk_delay_base mapped to %s for %s",
	value, target);
	break;
	}
	}
	}

	if (strchr(value, ':') == 0) {
	pcmk_parse_interval_spec(value, &delay_base);
	delay_base /= 1000;
	}

	free(valptr);
	}

	return (int) delay_base;
	}

	/*!
	* \internal
	* \brief Override STONITH timeout with pcmk_*_timeout if available
	*
	* \param[in] device STONITH device to use
	* \param[in] action STONITH action name
	* \param[in] default_timeout Timeout to use if device does not have
	* a pcmk_*_timeout parameter for action
	*
	* \return Value of pcmk_(action)_timeout if available, otherwise default_timeout
	* \note For consistency, it would be nice if reboot/off/on timeouts could be
	* set the same way as start/stop/monitor timeouts, i.e. with an
	* <operation> entry in the fencing resource configuration. However that
	* is insufficient because fencing devices may be registered directly via
	* the fencer's register_device() API instead of going through the CIB
	* (e.g. stonith_admin uses it for its -R option, and the executor uses it
	* to ensure a device is registered when a command is issued). As device
	* properties, pcmk_*_timeout parameters can be grabbed by the fencer when
	* the device is registered, whether by CIB change or API call.
	*/
	static int
	get_action_timeout(const stonith_device_t device, const char action,
	int default_timeout)
	{
	if (action && device && device->params) {
	char buffer[64] = { 0, };
	const char *value = NULL;

	/* If "reboot" was requested but the device does not support it,
	* we will remap to "off", so check timeout for "off" instead
	*/
	if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)
	&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {
	crm_trace("%s doesn't support reboot, using timeout for off instead",
	device->id);
	action = PCMK_ACTION_OFF;
	}

	/* If the device config specified an action-specific timeout, use it */
	snprintf(buffer, sizeof(buffer), "pcmk_%s_timeout", action);
	value = g_hash_table_lookup(device->params, buffer);
	if (value) {
	return atoi(value);
	}
	}
	return default_timeout;
	}

	/*!
	* \internal
	* \brief Get the currently executing device for a fencing operation
	*
	* \param[in] cmd Fencing operation to check
	*
	* \return Currently executing device for \p cmd if any, otherwise NULL
	*/
	static stonith_device_t *
	cmd_device(const async_command_t *cmd)
	{
	if ((cmd == NULL) \|\| (cmd->device == NULL) \|\| (device_list == NULL)) {
	return NULL;
	}
	return g_hash_table_lookup(device_list, cmd->device);
	}

	/*!
	* \internal
	* \brief Return the configured reboot action for a given device
	*
	* \param[in] device_id Device ID
	*
	* \return Configured reboot action for \p device_id
	*/
	const char *
	fenced_device_reboot_action(const char *device_id)
	{
	const char *action = NULL;

	if ((device_list != NULL) && (device_id != NULL)) {
	stonith_device_t *device = g_hash_table_lookup(device_list, device_id);

	if ((device != NULL) && (device->params != NULL)) {
	action = g_hash_table_lookup(device->params, "pcmk_reboot_action");
	}
	}
	return pcmk__s(action, PCMK_ACTION_REBOOT);
	}

	/*!
	* \internal
	* \brief Check whether a given device supports the "on" action
	*
	* \param[in] device_id Device ID
	*
	* \return true if \p device_id supports "on", otherwise false
	*/
	bool
	fenced_device_supports_on(const char *device_id)
	{
	if ((device_list != NULL) && (device_id != NULL)) {
	stonith_device_t *device = g_hash_table_lookup(device_list, device_id);

	if (device != NULL) {
	return pcmk_is_set(device->flags, st_device_supports_on);
	}
	}
	return false;
	}

	static void
	free_async_command(async_command_t * cmd)
	{
	if (!cmd) {
	return;
	}

	if (cmd->delay_id) {
	g_source_remove(cmd->delay_id);
	}

	cmd_list = g_list_remove(cmd_list, cmd);

	g_list_free_full(cmd->device_list, free);
	free(cmd->device);
	free(cmd->action);
	free(cmd->target);
	free(cmd->remote_op_id);
	free(cmd->client);
	free(cmd->client_name);
	free(cmd->origin);
	free(cmd->op);
	free(cmd);
	}

	/*!
	* \internal
	* \brief Create a new asynchronous fencing operation from request XML
	*
	* \param[in] msg Fencing request XML (from IPC or CPG)
	*
	* \return Newly allocated fencing operation on success, otherwise NULL
	*
	* \note This asserts on memory errors, so a NULL return indicates an
	* unparseable message.
	*/
	static async_command_t *
	create_async_command(xmlNode *msg)
	{
	xmlNode *op = NULL;
	async_command_t *cmd = NULL;

	if (msg == NULL) {
	return NULL;
	}

	op = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, msg, LOG_ERR);
	if (op == NULL) {
	return NULL;
	}

	cmd = calloc(1, sizeof(async_command_t));
	CRM_ASSERT(cmd != NULL);

	// All messages must include these
	cmd->action = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ACTION);
	cmd->op = crm_element_value_copy(msg, PCMK__XA_ST_OP);
	cmd->client = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTID);
	if ((cmd->action == NULL) \|\| (cmd->op == NULL) \|\| (cmd->client == NULL)) {
	free_async_command(cmd);
	return NULL;
	}

	crm_element_value_int(msg, PCMK__XA_ST_CALLID, &(cmd->id));
	crm_element_value_int(msg, PCMK__XA_ST_CALLOPT, &(cmd->options));
	crm_element_value_int(msg, PCMK__XA_ST_DELAY, &(cmd->start_delay));
	crm_element_value_int(msg, PCMK__XA_ST_TIMEOUT, &(cmd->default_timeout));
	cmd->timeout = cmd->default_timeout;

	cmd->origin = crm_element_value_copy(msg, PCMK__XA_SRC);
	cmd->remote_op_id = crm_element_value_copy(msg, PCMK__XA_ST_REMOTE_OP);
	cmd->client_name = crm_element_value_copy(msg, PCMK__XA_ST_CLIENTNAME);
	cmd->target = crm_element_value_copy(op, PCMK__XA_ST_TARGET);
	cmd->device = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ID);

	cmd->done_cb = st_child_done;

	// Track in global command list
	cmd_list = g_list_append(cmd_list, cmd);

	return cmd;
	}

	static int
	get_action_limit(stonith_device_t * device)
	{
	const char *value = NULL;
	int action_limit = 1;

	value = g_hash_table_lookup(device->params, PCMK_STONITH_ACTION_LIMIT);
	if ((value == NULL)
	\|\| (pcmk__scan_min_int(value, &action_limit, INT_MIN) != pcmk_rc_ok)
	\|\| (action_limit == 0)) {
	action_limit = 1;
	}
	return action_limit;
	}

	static int
	get_active_cmds(stonith_device_t * device)
	{
	int counter = 0;
	GList *gIter = NULL;
	GList *gIterNext = NULL;

	CRM_CHECK(device != NULL, return 0);

	for (gIter = cmd_list; gIter != NULL; gIter = gIterNext) {
	async_command_t *cmd = gIter->data;

	gIterNext = gIter->next;

	if (cmd->active_on == device) {
	counter++;
	}
	}

	return counter;
	}

	static void
	fork_cb(int pid, void *user_data)
	{
	async_command_t cmd = (async_command_t ) user_data;
	stonith_device_t * device =
	/* in case of a retry we've done the move from
	activating_on to active_on already
	*/
	cmd->activating_on?cmd->activating_on:cmd->active_on;

	CRM_ASSERT(device);
	crm_debug("Operation '%s' [%d]%s%s using %s now running with %ds timeout",
	cmd->action, pid,
	((cmd->target == NULL)? "" : " targeting "),
	pcmk__s(cmd->target, ""), device->id, cmd->timeout);
	cmd->active_on = device;
	cmd->activating_on = NULL;
	}

	static int
	get_agent_metadata_cb(gpointer data) {
	stonith_device_t *device = data;
	guint period_ms;

	switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
	case pcmk_rc_ok:
	if (device->agent_metadata) {
	read_action_metadata(device);
	stonith__device_parameter_flags(&(device->flags), device->id,
	device->agent_metadata);
	}
	return G_SOURCE_REMOVE;

	case EAGAIN:
	period_ms = pcmk__mainloop_timer_get_period(device->timer);
	if (period_ms < 160 * 1000) {
	mainloop_timer_set_period(device->timer, 2 * period_ms);
	}
	return G_SOURCE_CONTINUE;

	default:
	return G_SOURCE_REMOVE;
	}
	}

	/*!
	* \internal
	* \brief Call a command's action callback for an internal (not library) result
	*
	* \param[in,out] cmd Command to report result for
	* \param[in] execution_status Execution status to use for result
	* \param[in] exit_status Exit status to use for result
	* \param[in] exit_reason Exit reason to use for result
	*/
	static void
	report_internal_result(async_command_t *cmd, int exit_status,
	int execution_status, const char *exit_reason)
	{
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	pcmk__set_result(&result, exit_status, execution_status, exit_reason);
	cmd->done_cb(0, &result, cmd);
	pcmk__reset_result(&result);
	}

	static gboolean
	stonith_device_execute(stonith_device_t * device)
	{
	int exec_rc = 0;
	const char *action_str = NULL;
	const char *host_arg = NULL;
	async_command_t *cmd = NULL;
	stonith_action_t *action = NULL;
	int active_cmds = 0;
	int action_limit = 0;
	GList *gIter = NULL;
	GList *gIterNext = NULL;

	CRM_CHECK(device != NULL, return FALSE);

	active_cmds = get_active_cmds(device);
	action_limit = get_action_limit(device);
	if (action_limit > -1 && active_cmds >= action_limit) {
	crm_trace("%s is over its action limit of %d (%u active action%s)",
	device->id, action_limit, active_cmds,
	pcmk__plural_s(active_cmds));
	return TRUE;
	}

	for (gIter = device->pending_ops; gIter != NULL; gIter = gIterNext) {
	async_command_t *pending_op = gIter->data;

	gIterNext = gIter->next;

	if (pending_op && pending_op->delay_id) {
	crm_trace("Operation '%s'%s%s using %s was asked to run too early, "
	"waiting for start delay of %ds",
	pending_op->action,
	((pending_op->target == NULL)? "" : " targeting "),
	pcmk__s(pending_op->target, ""),
	device->id, pending_op->start_delay);
	continue;
	}

	device->pending_ops = g_list_remove_link(device->pending_ops, gIter);
	g_list_free_1(gIter);

	cmd = pending_op;
	break;
	}

	if (cmd == NULL) {
	crm_trace("No actions using %s are needed", device->id);
	return TRUE;
	}

	if (pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
	STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
	if (pcmk__is_fencing_action(cmd->action)) {
	if (node_does_watchdog_fencing(stonith_our_uname)) {
	pcmk__panic(__func__);
	goto done;
	}
	} else {
	crm_info("Faking success for %s watchdog operation", cmd->action);
	report_internal_result(cmd, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	goto done;
	}
	}

	#if SUPPORT_CIBSECRETS
	exec_rc = pcmk__substitute_secrets(device->id, device->params);
	if (exec_rc != pcmk_rc_ok) {
	if (pcmk__str_eq(cmd->action, PCMK_ACTION_STOP, pcmk__str_none)) {
	crm_info("Proceeding with stop operation for %s "
	"despite being unable to load CIB secrets (%s)",
	device->id, pcmk_rc_str(exec_rc));
	} else {
	crm_err("Considering %s unconfigured "
	"because unable to load CIB secrets: %s",
	device->id, pcmk_rc_str(exec_rc));
	report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_SECRETS,
	"Failed to get CIB secrets");
	goto done;
	}
	}
	#endif

	action_str = cmd->action;
	if (pcmk__str_eq(cmd->action, PCMK_ACTION_REBOOT, pcmk__str_none)
	&& !pcmk_is_set(device->flags, st_device_supports_reboot)) {

	crm_notice("Remapping 'reboot' action%s%s using %s to 'off' "
	"because agent '%s' does not support reboot",
	((cmd->target == NULL)? "" : " targeting "),
	pcmk__s(cmd->target, ""), device->id, device->agent);
	action_str = PCMK_ACTION_OFF;
	}

	if (pcmk_is_set(device->flags, st_device_supports_parameter_port)) {
	host_arg = "port";

	} else if (pcmk_is_set(device->flags, st_device_supports_parameter_plug)) {
	host_arg = "plug";
	}

	action = stonith__action_create(device->agent, action_str, cmd->target,
	cmd->target_nodeid, cmd->timeout,
	device->params, device->aliases, host_arg);

	/* for async exec, exec_rc is negative for early error exit
	otherwise handling of success/errors is done via callbacks */
	cmd->activating_on = device;
	exec_rc = stonith__execute_async(action, (void *)cmd, cmd->done_cb,
	fork_cb);
	if (exec_rc < 0) {
	cmd->activating_on = NULL;
	cmd->done_cb(0, stonith__action_result(action), cmd);
	stonith__destroy_action(action);
	}

	done:
	/* Device might get triggered to work by multiple fencing commands
	* simultaneously. Trigger the device again to make sure any
	* remaining concurrent commands get executed. */
	if (device->pending_ops) {
	mainloop_set_trigger(device->work);
	}
	return TRUE;
	}

	static gboolean
	stonith_device_dispatch(gpointer user_data)
	{
	return stonith_device_execute(user_data);
	}

	static gboolean
	start_delay_helper(gpointer data)
	{
	async_command_t *cmd = data;
	stonith_device_t *device = cmd_device(cmd);

	cmd->delay_id = 0;
	if (device) {
	mainloop_set_trigger(device->work);
	}

	return FALSE;
	}

	static void
	schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
	{
	int delay_max = 0;
	int delay_base = 0;
	int requested_delay = cmd->start_delay;

	CRM_CHECK(cmd != NULL, return);
	CRM_CHECK(device != NULL, return);

	if (cmd->device) {
	free(cmd->device);
	}

	if (device->include_nodeid && (cmd->target != NULL)) {
	crm_node_t *node = pcmk__get_node(0, cmd->target, NULL,
	pcmk__node_search_cluster);

	cmd->target_nodeid = node->id;
	}

	cmd->device = strdup(device->id);
	cmd->timeout = get_action_timeout(device, cmd->action, cmd->default_timeout);

	if (cmd->remote_op_id) {
	crm_debug("Scheduling '%s' action%s%s using %s for remote peer %s "
	"with op id %.8s and timeout %ds",
	cmd->action,
	(cmd->target == NULL)? "" : " targeting ",
	pcmk__s(cmd->target, ""),
	device->id, cmd->origin, cmd->remote_op_id, cmd->timeout);
	} else {
	crm_debug("Scheduling '%s' action%s%s using %s for %s with timeout %ds",
	cmd->action,
	(cmd->target == NULL)? "" : " targeting ",
	pcmk__s(cmd->target, ""),
	device->id, cmd->client, cmd->timeout);
	}

	device->pending_ops = g_list_append(device->pending_ops, cmd);
	mainloop_set_trigger(device->work);

	// Value -1 means disable any static/random fencing delays
	if (requested_delay < 0) {
	return;
	}

	delay_max = get_action_delay_max(device, cmd->action);
	delay_base = get_action_delay_base(device, cmd->action, cmd->target);
	if (delay_max == 0) {
	delay_max = delay_base;
	}
	if (delay_max < delay_base) {
	crm_warn(PCMK_STONITH_DELAY_BASE " (%ds) is larger than "
	PCMK_STONITH_DELAY_MAX " (%ds) for %s using %s "
	"(limiting to maximum delay)",
	delay_base, delay_max, cmd->action, device->id);
	delay_base = delay_max;
	}
	if (delay_max > 0) {
	// coverity[dontcall] It doesn't matter here if rand() is predictable
	cmd->start_delay +=
	((delay_max != delay_base)?(rand() % (delay_max - delay_base)):0)
	+ delay_base;
	}

	if (cmd->start_delay > 0) {
	crm_notice("Delaying '%s' action%s%s using %s for %ds " CRM_XS
	" timeout=%ds requested_delay=%ds base=%ds max=%ds",
	cmd->action,
	(cmd->target == NULL)? "" : " targeting ",
	pcmk__s(cmd->target, ""),
	device->id, cmd->start_delay, cmd->timeout,
	requested_delay, delay_base, delay_max);
	cmd->delay_id =
	g_timeout_add_seconds(cmd->start_delay, start_delay_helper, cmd);
	}
	}

	static void
	free_device(gpointer data)
	{
	GList *gIter = NULL;
	stonith_device_t *device = data;

	g_hash_table_destroy(device->params);
	g_hash_table_destroy(device->aliases);

	for (gIter = device->pending_ops; gIter != NULL; gIter = gIter->next) {
	async_command_t *cmd = gIter->data;

	crm_warn("Removal of device '%s' purged operation '%s'", device->id, cmd->action);
	report_internal_result(cmd, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"Device was removed before action could be executed");
	}
	g_list_free(device->pending_ops);

	g_list_free_full(device->targets, free);

	if (device->timer) {
	mainloop_timer_stop(device->timer);
	mainloop_timer_del(device->timer);
	}

	mainloop_destroy_trigger(device->work);

	free_xml(device->agent_metadata);
	free(device->namespace);
	if (device->on_target_actions != NULL) {
	g_string_free(device->on_target_actions, TRUE);
	}
	free(device->agent);
	free(device->id);
	free(device);
	}

	void free_device_list(void)
	{
	if (device_list != NULL) {
	g_hash_table_destroy(device_list);
	device_list = NULL;
	}
	}

	void
	init_device_list(void)
	{
	if (device_list == NULL) {
	device_list = pcmk__strkey_table(NULL, free_device);
	}
	}

	static GHashTable *
	build_port_aliases(const char hostmap, GList * targets)
	{
	char *name = NULL;
	int last = 0, lpc = 0, max = 0, added = 0;
	GHashTable *aliases = pcmk__strikey_table(free, free);

	if (hostmap == NULL) {
	return aliases;
	}

	max = strlen(hostmap);
	for (; lpc <= max; lpc++) {
	switch (hostmap[lpc]) {
	/* Skip escaped chars */
	case '\\':
	lpc++;
	break;

	/* Assignment chars */
	case '=':
	case ':':
	if (lpc > last) {
	free(name);
	name = calloc(1, 1 + lpc - last);
	memcpy(name, hostmap + last, lpc - last);
	}
	last = lpc + 1;
	break;

	/* Delimeter chars */
	/* case ',': Potentially used to specify multiple ports */
	case 0:
	case ';':
	case ' ':
	case '\t':
	if (name) {
	char *value = NULL;
	int k = 0;

	value = calloc(1, 1 + lpc - last);
	memcpy(value, hostmap + last, lpc - last);

	for (int i = 0; value[i] != '\0'; i++) {
	if (value[i] != '\\') {
	value[k++] = value[i];
	}
	}
	value[k] = '\0';

	crm_debug("Adding alias '%s'='%s'", name, value);
	g_hash_table_replace(aliases, name, value);
	if (targets) {
	targets = g_list_append(targets, strdup(value));
	}
	value = NULL;
	name = NULL;
	added++;

	} else if (lpc > last) {
	crm_debug("Parse error at offset %d near '%s'", lpc - last, hostmap + last);
	}

	last = lpc + 1;
	break;
	}

	if (hostmap[lpc] == 0) {
	break;
	}
	}

	if (added == 0) {
	crm_info("No host mappings detected in '%s'", hostmap);
	}

	free(name);
	return aliases;
	}

	GHashTable *metadata_cache = NULL;

	void
	free_metadata_cache(void) {
	if (metadata_cache != NULL) {
	g_hash_table_destroy(metadata_cache);
	metadata_cache = NULL;
	}
	}

	static void
	init_metadata_cache(void) {
	if (metadata_cache == NULL) {
	metadata_cache = pcmk__strkey_table(free, free);
	}
	}

	int
	get_agent_metadata(const char agent, xmlNode * metadata)
	{
	char *buffer = NULL;

	if (metadata == NULL) {
	return EINVAL;
	}
	*metadata = NULL;
	if (pcmk__str_eq(agent, STONITH_WATCHDOG_AGENT_INTERNAL, pcmk__str_none)) {
	return pcmk_rc_ok;
	}
	init_metadata_cache();
	buffer = g_hash_table_lookup(metadata_cache, agent);
	if (buffer == NULL) {
	stonith_t *st = stonith_api_new();
	int rc;

	if (st == NULL) {
	crm_warn("Could not get agent meta-data: "
	"API memory allocation failed");
	return EAGAIN;
	}
	rc = st->cmds->metadata(st, st_opt_sync_call, agent,
	NULL, &buffer, 10);
	stonith_api_delete(st);
	if (rc \|\| !buffer) {
	crm_err("Could not retrieve metadata for fencing agent %s", agent);
	return EAGAIN;
	}
	g_hash_table_replace(metadata_cache, strdup(agent), buffer);
	}

	- *metadata = string2xml(buffer);
	+ *metadata = pcmk__xml_parse_string(buffer);
	return pcmk_rc_ok;
	}

	static gboolean
	is_nodeid_required(xmlNode * xml)
	{
	xmlXPathObjectPtr xpath = NULL;

	if (stand_alone) {
	return FALSE;
	}

	if (!xml) {
	return FALSE;
	}

	xpath = xpath_search(xml,
	"//" PCMK_XE_PARAMETER "[@" PCMK_XA_NAME "='nodeid']");
	if (numXpathResults(xpath) <= 0) {
	freeXpathObject(xpath);
	return FALSE;
	}

	freeXpathObject(xpath);
	return TRUE;
	}

	static void
	read_action_metadata(stonith_device_t *device)
	{
	xmlXPathObjectPtr xpath = NULL;
	int max = 0;
	int lpc = 0;

	if (device->agent_metadata == NULL) {
	return;
	}

	xpath = xpath_search(device->agent_metadata, "//action");
	max = numXpathResults(xpath);

	if (max <= 0) {
	freeXpathObject(xpath);
	return;
	}

	for (lpc = 0; lpc < max; lpc++) {
	const char *action = NULL;
	xmlNode *match = getXpathResult(xpath, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match == NULL) { continue; };

	action = crm_element_value(match, PCMK_XA_NAME);

	if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_list);
	} else if (pcmk__str_eq(action, PCMK_ACTION_STATUS, pcmk__str_none)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_status);
	} else if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_reboot);
	} else if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)) {
	/* PCMK_XA_AUTOMATIC means the cluster will unfence a node when it
	* joins.
	*
	* @COMPAT PCMK__XA_REQUIRED is a deprecated synonym for
	* PCMK_XA_AUTOMATIC.
	*/
	if (pcmk__xe_attr_is_true(match, PCMK_XA_AUTOMATIC)
	\|\| pcmk__xe_attr_is_true(match, PCMK__XA_REQUIRED)) {
	device->automatic_unfencing = TRUE;
	}
	stonith__set_device_flags(device->flags, device->id,
	st_device_supports_on);
	}

	if ((action != NULL)
	&& pcmk__xe_attr_is_true(match, PCMK_XA_ON_TARGET)) {

	pcmk__add_word(&(device->on_target_actions), 64, action);
	}
	}

	freeXpathObject(xpath);
	}

	/*!
	* \internal
	* \brief Set a pcmk_*_action parameter if not already set
	*
	* \param[in,out] params Device parameters
	* \param[in] action Name of action
	* \param[in] value Value to use if action is not already set
	*/
	static void
	map_action(GHashTable params, const char action, const char *value)
	{
	char *key = crm_strdup_printf("pcmk_%s_action", action);

	if (g_hash_table_lookup(params, key)) {
	crm_warn("Ignoring %s='%s', see %s instead",
	STONITH_ATTR_ACTION_OP, value, key);
	free(key);
	} else {
	crm_warn("Mapping %s='%s' to %s='%s'",
	STONITH_ATTR_ACTION_OP, value, key, value);
	g_hash_table_insert(params, key, strdup(value));
	}
	}

	/*!
	* \internal
	* \brief Create device parameter table from XML
	*
	* \param[in] name Device name (used for logging only)
	* \param[in] dev XML containing device parameters
	*/
	static GHashTable *
	xml2device_params(const char name, const xmlNode dev)
	{
	GHashTable *params = xml2list(dev);
	const char *value;

	/* Action should never be specified in the device configuration,
	* but we support it for users who are familiar with other software
	* that worked that way.
	*/
	value = g_hash_table_lookup(params, STONITH_ATTR_ACTION_OP);
	if (value != NULL) {
	crm_warn("%s has '%s' parameter, which should never be specified in configuration",
	name, STONITH_ATTR_ACTION_OP);

	if (*value == '\0') {
	crm_warn("Ignoring empty '%s' parameter", STONITH_ATTR_ACTION_OP);

	} else if (strcmp(value, PCMK_ACTION_REBOOT) == 0) {
	crm_warn("Ignoring %s='reboot' (see " PCMK_OPT_STONITH_ACTION
	" cluster property instead)",
	STONITH_ATTR_ACTION_OP);

	} else if (strcmp(value, PCMK_ACTION_OFF) == 0) {
	map_action(params, PCMK_ACTION_REBOOT, value);

	} else {
	map_action(params, PCMK_ACTION_OFF, value);
	map_action(params, PCMK_ACTION_REBOOT, value);
	}

	g_hash_table_remove(params, STONITH_ATTR_ACTION_OP);
	}

	return params;
	}

	static const char *
	target_list_type(stonith_device_t * dev)
	{
	const char *check_type = NULL;

	check_type = g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK);

	if (check_type == NULL) {

	if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_LIST)) {
	check_type = "static-list";
	} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)) {
	check_type = "static-list";
	} else if (pcmk_is_set(dev->flags, st_device_supports_list)) {
	check_type = "dynamic-list";
	} else if (pcmk_is_set(dev->flags, st_device_supports_status)) {
	check_type = "status";
	} else {
	check_type = PCMK_VALUE_NONE;
	}
	}

	return check_type;
	}

	static stonith_device_t *
	build_device_from_xml(xmlNode *dev)
	{
	const char *value;
	stonith_device_t *device = NULL;
	char *agent = crm_element_value_copy(dev, PCMK_XA_AGENT);

	CRM_CHECK(agent != NULL, return device);

	device = calloc(1, sizeof(stonith_device_t));

	CRM_CHECK(device != NULL, {free(agent); return device;});

	device->id = crm_element_value_copy(dev, PCMK_XA_ID);
	device->agent = agent;
	device->namespace = crm_element_value_copy(dev, PCMK__XA_NAMESPACE);
	device->params = xml2device_params(device->id, dev);

	value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_LIST);
	if (value) {
	device->targets = stonith__parse_targets(value);
	}

	value = g_hash_table_lookup(device->params, PCMK_STONITH_HOST_MAP);
	device->aliases = build_port_aliases(value, &(device->targets));

	value = target_list_type(device);
	if (!pcmk__str_eq(value, "static-list", pcmk__str_casei) && device->targets) {
	/* Other than "static-list", dev-> targets is unnecessary. */
	g_list_free_full(device->targets, free);
	device->targets = NULL;
	}
	switch (get_agent_metadata(device->agent, &device->agent_metadata)) {
	case pcmk_rc_ok:
	if (device->agent_metadata) {
	read_action_metadata(device);
	stonith__device_parameter_flags(&(device->flags), device->id,
	device->agent_metadata);
	}
	break;

	case EAGAIN:
	if (device->timer == NULL) {
	device->timer = mainloop_timer_add("get_agent_metadata", 10 * 1000,
	TRUE, get_agent_metadata_cb, device);
	}
	if (!mainloop_timer_running(device->timer)) {
	mainloop_timer_start(device->timer);
	}
	break;

	default:
	break;
	}

	value = g_hash_table_lookup(device->params, "nodeid");
	if (!value) {
	device->include_nodeid = is_nodeid_required(device->agent_metadata);
	}

	value = crm_element_value(dev, PCMK__XA_RSC_PROVIDES);
	if (pcmk__str_eq(value, PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
	device->automatic_unfencing = TRUE;
	}

	if (is_action_required(PCMK_ACTION_ON, device)) {
	crm_info("Fencing device '%s' requires unfencing", device->id);
	}

	if (device->on_target_actions != NULL) {
	crm_info("Fencing device '%s' requires actions (%s) to be executed "
	"on target", device->id,
	(const char *) device->on_target_actions->str);
	}

	device->work = mainloop_add_trigger(G_PRIORITY_HIGH, stonith_device_dispatch, device);
	/* TODO: Hook up priority */

	return device;
	}

	static void
	schedule_internal_command(const char *origin,
	stonith_device_t * device,
	const char *action,
	const char *target,
	int timeout,
	void *internal_user_data,
	void (*done_cb) (int pid,
	const pcmk__action_result_t *result,
	void *user_data))
	{
	async_command_t *cmd = NULL;

	cmd = calloc(1, sizeof(async_command_t));

	cmd->id = -1;
	cmd->default_timeout = timeout ? timeout : 60;
	cmd->timeout = cmd->default_timeout;
	cmd->action = strdup(action);
	pcmk__str_update(&cmd->target, target);
	cmd->device = strdup(device->id);
	cmd->origin = strdup(origin);
	cmd->client = strdup(crm_system_name);
	cmd->client_name = strdup(crm_system_name);

	cmd->internal_user_data = internal_user_data;
	cmd->done_cb = done_cb; /* cmd, not internal_user_data, is passed to 'done_cb' as the userdata */

	schedule_stonith_command(cmd, device);
	}

	// Fence agent status commands use custom exit status codes
	enum fence_status_code {
	fence_status_invalid = -1,
	fence_status_active = 0,
	fence_status_unknown = 1,
	fence_status_inactive = 2,
	};

	static void
	status_search_cb(int pid, const pcmk__action_result_t result, void user_data)
	{
	async_command_t *cmd = user_data;
	struct device_search_s *search = cmd->internal_user_data;
	stonith_device_t *dev = cmd_device(cmd);
	gboolean can = FALSE;

	free_async_command(cmd);

	if (!dev) {
	search_devices_record_result(search, NULL, FALSE);
	return;
	}

	mainloop_set_trigger(dev->work);

	if (result->execution_status != PCMK_EXEC_DONE) {
	crm_warn("Assuming %s cannot fence %s "
	"because status could not be executed: %s%s%s%s",
	dev->id, search->host,
	pcmk_exec_status_str(result->execution_status),
	((result->exit_reason == NULL)? "" : " ("),
	((result->exit_reason == NULL)? "" : result->exit_reason),
	((result->exit_reason == NULL)? "" : ")"));
	search_devices_record_result(search, dev->id, FALSE);
	return;
	}

	switch (result->exit_status) {
	case fence_status_unknown:
	crm_trace("%s reported it cannot fence %s", dev->id, search->host);
	break;

	case fence_status_active:
	case fence_status_inactive:
	crm_trace("%s reported it can fence %s", dev->id, search->host);
	can = TRUE;
	break;

	default:
	crm_warn("Assuming %s cannot fence %s "
	"(status returned unknown code %d)",
	dev->id, search->host, result->exit_status);
	break;
	}
	search_devices_record_result(search, dev->id, can);
	}

	static void
	dynamic_list_search_cb(int pid, const pcmk__action_result_t *result,
	void *user_data)
	{
	async_command_t *cmd = user_data;
	struct device_search_s *search = cmd->internal_user_data;
	stonith_device_t *dev = cmd_device(cmd);
	gboolean can_fence = FALSE;

	free_async_command(cmd);

	/* Host/alias must be in the list output to be eligible to be fenced
	*
	* Will cause problems if down'd nodes aren't listed or (for virtual nodes)
	* if the guest is still listed despite being moved to another machine
	*/
	if (!dev) {
	search_devices_record_result(search, NULL, FALSE);
	return;
	}

	mainloop_set_trigger(dev->work);

	if (pcmk__result_ok(result)) {
	crm_info("Refreshing target list for %s", dev->id);
	g_list_free_full(dev->targets, free);
	dev->targets = stonith__parse_targets(result->action_stdout);
	dev->targets_age = time(NULL);

	} else if (dev->targets != NULL) {
	if (result->execution_status == PCMK_EXEC_DONE) {
	crm_info("Reusing most recent target list for %s "
	"because list returned error code %d",
	dev->id, result->exit_status);
	} else {
	crm_info("Reusing most recent target list for %s "
	"because list could not be executed: %s%s%s%s",
	dev->id, pcmk_exec_status_str(result->execution_status),
	((result->exit_reason == NULL)? "" : " ("),
	((result->exit_reason == NULL)? "" : result->exit_reason),
	((result->exit_reason == NULL)? "" : ")"));
	}

	} else { // We have never successfully executed list
	if (result->execution_status == PCMK_EXEC_DONE) {
	crm_warn("Assuming %s cannot fence %s "
	"because list returned error code %d",
	dev->id, search->host, result->exit_status);
	} else {
	crm_warn("Assuming %s cannot fence %s "
	"because list could not be executed: %s%s%s%s",
	dev->id, search->host,
	pcmk_exec_status_str(result->execution_status),
	((result->exit_reason == NULL)? "" : " ("),
	((result->exit_reason == NULL)? "" : result->exit_reason),
	((result->exit_reason == NULL)? "" : ")"));
	}

	/* Fall back to pcmk_host_check="status" if the user didn't explicitly
	* specify "dynamic-list".
	*/
	if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_CHECK) == NULL) {
	crm_notice("Switching to pcmk_host_check='status' for %s", dev->id);
	pcmk__insert_dup(dev->params, PCMK_STONITH_HOST_CHECK, "status");
	}
	}

	if (dev->targets) {
	const char *alias = g_hash_table_lookup(dev->aliases, search->host);

	if (!alias) {
	alias = search->host;
	}
	if (pcmk__str_in_list(alias, dev->targets, pcmk__str_casei)) {
	can_fence = TRUE;
	}
	}
	search_devices_record_result(search, dev->id, can_fence);
	}

	/*!
	* \internal
	* \brief Returns true if any key in first is not in second or second has a different value for key
	*/
	static int
	device_params_diff(GHashTable first, GHashTable second) {
	char *key = NULL;
	char *value = NULL;
	GHashTableIter gIter;

	g_hash_table_iter_init(&gIter, first);
	while (g_hash_table_iter_next(&gIter, (void )&key, (void )&value)) {

	if(strstr(key, "CRM_meta") == key) {
	continue;
	} else if (strcmp(key, PCMK_XA_CRM_FEATURE_SET) == 0) {
	continue;
	} else {
	char *other_value = g_hash_table_lookup(second, key);

	if (!other_value \|\| !pcmk__str_eq(other_value, value, pcmk__str_casei)) {
	crm_trace("Different value for %s: %s != %s", key, other_value, value);
	return 1;
	}
	}
	}

	return 0;
	}

	/*!
	* \internal
	* \brief Checks to see if an identical device already exists in the device_list
	*/
	static stonith_device_t *
	device_has_duplicate(const stonith_device_t *device)
	{
	stonith_device_t *dup = g_hash_table_lookup(device_list, device->id);

	if (!dup) {
	crm_trace("No match for %s", device->id);
	return NULL;

	} else if (!pcmk__str_eq(dup->agent, device->agent, pcmk__str_casei)) {
	crm_trace("Different agent: %s != %s", dup->agent, device->agent);
	return NULL;
	}

	/* Use calculate_operation_digest() here? */
	if (device_params_diff(device->params, dup->params) \|\|
	device_params_diff(dup->params, device->params)) {
	return NULL;
	}

	crm_trace("Match");
	return dup;
	}

	int
	stonith_device_register(xmlNode *dev, gboolean from_cib)
	{
	stonith_device_t *dup = NULL;
	stonith_device_t *device = build_device_from_xml(dev);
	guint ndevices = 0;
	int rv = pcmk_ok;

	CRM_CHECK(device != NULL, return -ENOMEM);

	/* do we have a watchdog-device? */
	if (pcmk__str_eq(device->id, STONITH_WATCHDOG_ID, pcmk__str_none) \|\|
	pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
	STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) do {
	if (stonith_watchdog_timeout_ms <= 0) {
	crm_err("Ignoring watchdog fence device without "
	PCMK_OPT_STONITH_WATCHDOG_TIMEOUT " set.");
	rv = -ENODEV;
	/* fall through to cleanup & return */
	} else if (!pcmk__str_any_of(device->agent, STONITH_WATCHDOG_AGENT,
	STONITH_WATCHDOG_AGENT_INTERNAL, NULL)) {
	crm_err("Ignoring watchdog fence device with unknown "
	"agent '%s' unequal '" STONITH_WATCHDOG_AGENT "'.",
	device->agent?device->agent:"");
	rv = -ENODEV;
	/* fall through to cleanup & return */
	} else if (!pcmk__str_eq(device->id, STONITH_WATCHDOG_ID,
	pcmk__str_none)) {
	crm_err("Ignoring watchdog fence device "
	"named %s !='"STONITH_WATCHDOG_ID"'.",
	device->id?device->id:"");
	rv = -ENODEV;
	/* fall through to cleanup & return */
	} else {
	if (pcmk__str_eq(device->agent, STONITH_WATCHDOG_AGENT,
	pcmk__str_none)) {
	/* this either has an empty list or the targets
	configured for watchdog-fencing
	*/
	g_list_free_full(stonith_watchdog_targets, free);
	stonith_watchdog_targets = device->targets;
	device->targets = NULL;
	}
	if (node_does_watchdog_fencing(stonith_our_uname)) {
	g_list_free_full(device->targets, free);
	device->targets = stonith__parse_targets(stonith_our_uname);
	pcmk__insert_dup(device->params,
	PCMK_STONITH_HOST_LIST, stonith_our_uname);
	/* proceed as with any other stonith-device */
	break;
	}

	crm_debug("Skip registration of watchdog fence device on node not in host-list.");
	/* cleanup and fall through to more cleanup and return */
	device->targets = NULL;
	stonith_device_remove(device->id, from_cib);
	}
	free_device(device);
	return rv;
	} while (0);

	dup = device_has_duplicate(device);
	if (dup) {
	ndevices = g_hash_table_size(device_list);
	crm_debug("Device '%s' already in device list (%d active device%s)",
	device->id, ndevices, pcmk__plural_s(ndevices));
	free_device(device);
	device = dup;
	dup = g_hash_table_lookup(device_list, device->id);
	dup->dirty = FALSE;

	} else {
	stonith_device_t *old = g_hash_table_lookup(device_list, device->id);

	if (from_cib && old && old->api_registered) {
	/* If the cib is writing over an entry that is shared with a stonith client,
	* copy any pending ops that currently exist on the old entry to the new one.
	* Otherwise the pending ops will be reported as failures
	*/
	crm_info("Overwriting existing entry for %s from CIB", device->id);
	device->pending_ops = old->pending_ops;
	device->api_registered = TRUE;
	old->pending_ops = NULL;
	if (device->pending_ops) {
	mainloop_set_trigger(device->work);
	}
	}
	g_hash_table_replace(device_list, device->id, device);

	ndevices = g_hash_table_size(device_list);
	crm_notice("Added '%s' to device list (%d active device%s)",
	device->id, ndevices, pcmk__plural_s(ndevices));
	}

	if (from_cib) {
	device->cib_registered = TRUE;
	} else {
	device->api_registered = TRUE;
	}

	return pcmk_ok;
	}

	void
	stonith_device_remove(const char *id, bool from_cib)
	{
	stonith_device_t *device = g_hash_table_lookup(device_list, id);
	guint ndevices = 0;

	if (!device) {
	ndevices = g_hash_table_size(device_list);
	crm_info("Device '%s' not found (%d active device%s)",
	id, ndevices, pcmk__plural_s(ndevices));
	return;
	}

	if (from_cib) {
	device->cib_registered = FALSE;
	} else {
	device->verified = FALSE;
	device->api_registered = FALSE;
	}

	if (!device->cib_registered && !device->api_registered) {
	g_hash_table_remove(device_list, id);
	ndevices = g_hash_table_size(device_list);
	crm_info("Removed '%s' from device list (%d active device%s)",
	id, ndevices, pcmk__plural_s(ndevices));
	} else {
	crm_trace("Not removing '%s' from device list (%d active) because "
	"still registered via:%s%s",
	id, g_hash_table_size(device_list),
	(device->cib_registered? " cib" : ""),
	(device->api_registered? " api" : ""));
	}
	}

	/*!
	* \internal
	* \brief Return the number of stonith levels registered for a node
	*
	* \param[in] tp Node's topology table entry
	*
	* \return Number of non-NULL levels in topology entry
	* \note This function is used only for log messages.
	*/
	static int
	count_active_levels(const stonith_topology_t *tp)
	{
	int lpc = 0;
	int count = 0;

	for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if (tp->levels[lpc] != NULL) {
	count++;
	}
	}
	return count;
	}

	static void
	free_topology_entry(gpointer data)
	{
	stonith_topology_t *tp = data;

	int lpc = 0;

	for (lpc = 0; lpc < ST_LEVEL_MAX; lpc++) {
	if (tp->levels[lpc] != NULL) {
	g_list_free_full(tp->levels[lpc], free);
	}
	}
	free(tp->target);
	free(tp->target_value);
	free(tp->target_pattern);
	free(tp->target_attribute);
	free(tp);
	}

	void
	free_topology_list(void)
	{
	if (topology != NULL) {
	g_hash_table_destroy(topology);
	topology = NULL;
	}
	}

	void
	init_topology_list(void)
	{
	if (topology == NULL) {
	topology = pcmk__strkey_table(NULL, free_topology_entry);
	}
	}

	char *
	stonith_level_key(const xmlNode *level, enum fenced_target_by mode)
	{
	if (mode == fenced_target_by_unknown) {
	mode = unpack_level_kind(level);
	}
	switch (mode) {
	case fenced_target_by_name:
	return crm_element_value_copy(level, PCMK_XA_TARGET);

	case fenced_target_by_pattern:
	return crm_element_value_copy(level, PCMK_XA_TARGET_PATTERN);

	case fenced_target_by_attribute:
	return crm_strdup_printf("%s=%s",
	crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE),
	crm_element_value(level, PCMK_XA_TARGET_VALUE));

	default:
	return crm_strdup_printf("unknown-%s", pcmk__xe_id(level));
	}
	}

	/*!
	* \internal
	* \brief Parse target identification from topology level XML
	*
	* \param[in] level Topology level XML to parse
	*
	* \return How to identify target of \p level
	*/
	static enum fenced_target_by
	unpack_level_kind(const xmlNode *level)
	{
	if (crm_element_value(level, PCMK_XA_TARGET) != NULL) {
	return fenced_target_by_name;
	}
	if (crm_element_value(level, PCMK_XA_TARGET_PATTERN) != NULL) {
	return fenced_target_by_pattern;
	}
	if (!stand_alone /* if standalone, there's no attribute manager */
	&& (crm_element_value(level, PCMK_XA_TARGET_ATTRIBUTE) != NULL)
	&& (crm_element_value(level, PCMK_XA_TARGET_VALUE) != NULL)) {
	return fenced_target_by_attribute;
	}
	return fenced_target_by_unknown;
	}

	static stonith_key_value_t *
	parse_device_list(const char *devices)
	{
	int lpc = 0;
	int max = 0;
	int last = 0;
	stonith_key_value_t *output = NULL;

	if (devices == NULL) {
	return output;
	}

	max = strlen(devices);
	for (lpc = 0; lpc <= max; lpc++) {
	if (devices[lpc] == ',' \|\| devices[lpc] == 0) {
	char *line = strndup(devices + last, lpc - last);

	output = stonith_key_value_add(output, NULL, line);
	free(line);

	last = lpc + 1;
	}
	}

	return output;
	}

	/*!
	* \internal
	* \brief Unpack essential information from topology request XML
	*
	* \param[in] xml Request XML to search
	* \param[out] mode If not NULL, where to store level kind
	* \param[out] target If not NULL, where to store representation of target
	* \param[out] id If not NULL, where to store level number
	* \param[out] desc If not NULL, where to store log-friendly level description
	*
	* \return Topology level XML from within \p xml, or NULL if not found
	* \note The caller is responsible for freeing \p target and \p desc if set.
	*/
	static xmlNode *
	unpack_level_request(xmlNode xml, enum fenced_target_by mode, char **target,
	int id, char *desc)
	{
	enum fenced_target_by local_mode = fenced_target_by_unknown;
	char *local_target = NULL;
	int local_id = 0;

	/* The level element can be the top element or lower. If top level, don't
	* search by xpath, because it might give multiple hits if the XML is the
	* CIB.
	*/
	if ((xml != NULL) && !pcmk__xe_is(xml, PCMK_XE_FENCING_LEVEL)) {
	xml = get_xpath_object("//" PCMK_XE_FENCING_LEVEL, xml, LOG_WARNING);
	}

	if (xml == NULL) {
	if (desc != NULL) {
	*desc = crm_strdup_printf("missing");
	}
	} else {
	local_mode = unpack_level_kind(xml);
	local_target = stonith_level_key(xml, local_mode);
	crm_element_value_int(xml, PCMK_XA_INDEX, &local_id);
	if (desc != NULL) {
	*desc = crm_strdup_printf("%s[%d]", local_target, local_id);
	}
	}

	if (mode != NULL) {
	*mode = local_mode;
	}
	if (id != NULL) {
	*id = local_id;
	}

	if (target != NULL) {
	*target = local_target;
	} else {
	free(local_target);
	}

	return xml;
	}

	/*!
	* \internal
	* \brief Register a fencing topology level for a target
	*
	* Given an XML request specifying the target name, level index, and device IDs
	* for the level, this will create an entry for the target in the global topology
	* table if one does not already exist, then append the specified device IDs to
	* the entry's device list for the specified level.
	*
	* \param[in] msg XML request for STONITH level registration
	* \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]"
	* \param[out] result Where to set result of registration
	*/
	void
	fenced_register_level(xmlNode msg, char desc, pcmk__action_result_t result)
	{
	int id = 0;
	xmlNode *level;
	enum fenced_target_by mode;
	char *target;

	stonith_topology_t *tp;
	stonith_key_value_t *dIter = NULL;
	stonith_key_value_t *devices = NULL;

	CRM_CHECK((msg != NULL) && (result != NULL), return);

	level = unpack_level_request(msg, &mode, &target, &id, desc);
	if (level == NULL) {
	fenced_set_protocol_error(result);
	return;
	}

	// Ensure an ID was given (even the client API adds an ID)
	if (pcmk__str_empty(pcmk__xe_id(level))) {
	crm_warn("Ignoring registration for topology level without ID");
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Topology level is invalid without ID");
	return;
	}

	// Ensure a valid target was specified
	if (mode == fenced_target_by_unknown) {
	crm_warn("Ignoring registration for topology level '%s' "
	"without valid target", pcmk__xe_id(level));
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Invalid target for topology level '%s'",
	pcmk__xe_id(level));
	return;
	}

	// Ensure level ID is in allowed range
	if ((id <= 0) \|\| (id >= ST_LEVEL_MAX)) {
	crm_warn("Ignoring topology registration for %s with invalid level %d",
	target, id);
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Invalid level number '%s' for topology level '%s'",
	pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
	""),
	pcmk__xe_id(level));
	return;
	}

	/* Find or create topology table entry */
	tp = g_hash_table_lookup(topology, target);
	if (tp == NULL) {
	tp = calloc(1, sizeof(stonith_topology_t));
	if (tp == NULL) {
	pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
	strerror(ENOMEM));
	free(target);
	return;
	}
	tp->kind = mode;
	tp->target = target;
	tp->target_value = crm_element_value_copy(level, PCMK_XA_TARGET_VALUE);
	tp->target_pattern = crm_element_value_copy(level,
	PCMK_XA_TARGET_PATTERN);
	tp->target_attribute = crm_element_value_copy(level,
	PCMK_XA_TARGET_ATTRIBUTE);

	g_hash_table_replace(topology, tp->target, tp);
	crm_trace("Added %s (%d) to the topology (%d active entries)",
	target, (int) mode, g_hash_table_size(topology));
	} else {
	free(target);
	}

	if (tp->levels[id] != NULL) {
	crm_info("Adding to the existing %s[%d] topology entry",
	tp->target, id);
	}

	devices = parse_device_list(crm_element_value(level, PCMK_XA_DEVICES));
	for (dIter = devices; dIter; dIter = dIter->next) {
	const char *device = dIter->value;

	crm_trace("Adding device '%s' for %s[%d]", device, tp->target, id);
	tp->levels[id] = g_list_append(tp->levels[id], strdup(device));
	}
	stonith_key_value_freeall(devices, 1, 1);

	{
	int nlevels = count_active_levels(tp);

	crm_info("Target %s has %d active fencing level%s",
	tp->target, nlevels, pcmk__plural_s(nlevels));
	}

	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	}

	/*!
	* \internal
	* \brief Unregister a fencing topology level for a target
	*
	* Given an XML request specifying the target name and level index (or 0 for all
	* levels), this will remove any corresponding entry for the target from the
	* global topology table.
	*
	* \param[in] msg XML request for STONITH level registration
	* \param[out] desc If not NULL, set to string representation "TARGET[LEVEL]"
	* \param[out] result Where to set result of unregistration
	*/
	void
	fenced_unregister_level(xmlNode msg, char *desc,
	pcmk__action_result_t *result)
	{
	int id = -1;
	stonith_topology_t *tp;
	char *target;
	xmlNode *level = NULL;

	CRM_CHECK(result != NULL, return);

	level = unpack_level_request(msg, NULL, &target, &id, desc);
	if (level == NULL) {
	fenced_set_protocol_error(result);
	return;
	}

	// Ensure level ID is in allowed range
	if ((id < 0) \|\| (id >= ST_LEVEL_MAX)) {
	crm_warn("Ignoring topology unregistration for %s with invalid level %d",
	target, id);
	free(target);
	crm_log_xml_trace(level, "Bad level");
	pcmk__format_result(result, CRM_EX_INVALID_PARAM, PCMK_EXEC_INVALID,
	"Invalid level number '%s' for topology level %s",
	pcmk__s(crm_element_value(level, PCMK_XA_INDEX),
	"<null>"),

	// Client API doesn't add ID to unregistration XML
	pcmk__s(pcmk__xe_id(level), ""));
	return;
	}

	tp = g_hash_table_lookup(topology, target);
	if (tp == NULL) {
	guint nentries = g_hash_table_size(topology);

	crm_info("No fencing topology found for %s (%d active %s)",
	target, nentries,
	pcmk__plural_alt(nentries, "entry", "entries"));

	} else if (id == 0 && g_hash_table_remove(topology, target)) {
	guint nentries = g_hash_table_size(topology);

	crm_info("Removed all fencing topology entries related to %s "
	"(%d active %s remaining)", target, nentries,
	pcmk__plural_alt(nentries, "entry", "entries"));

	} else if (tp->levels[id] != NULL) {
	guint nlevels;

	g_list_free_full(tp->levels[id], free);
	tp->levels[id] = NULL;

	nlevels = count_active_levels(tp);
	crm_info("Removed level %d from fencing topology for %s "
	"(%d active level%s remaining)",
	id, target, nlevels, pcmk__plural_s(nlevels));
	}

	free(target);
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	}

	static char *
	list_to_string(GList list, const char delim, gboolean terminate_with_delim)
	{
	int max = g_list_length(list);
	size_t delim_len = delim?strlen(delim):0;
	size_t alloc_size = 1 + (max?((max-1+(terminate_with_delim?1:0))*delim_len):0);
	char *rv;
	GList *gIter;

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	const char value = (const char ) gIter->data;

	alloc_size += strlen(value);
	}
	rv = calloc(alloc_size, sizeof(char));
	if (rv) {
	char *pos = rv;
	const char *lead_delim = "";

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	const char value = (const char ) gIter->data;

	pos = &pos[sprintf(pos, "%s%s", lead_delim, value)];
	lead_delim = delim;
	}
	if (max && terminate_with_delim) {
	sprintf(pos, "%s", delim);
	}
	}
	return rv;
	}

	/*!
	* \internal
	* \brief Execute a fence agent action directly (and asynchronously)
	*
	* Handle a STONITH_OP_EXEC API message by scheduling a requested agent action
	* directly on a specified device. Only list, monitor, and status actions are
	* expected to use this call, though it should work with any agent command.
	*
	* \param[in] msg Request XML specifying action
	* \param[out] result Where to store result of action
	*
	* \note If the action is monitor, the device must be registered via the API
	* (CIB registration is not sufficient), because monitor should not be
	* possible unless the device is "started" (API registered).
	*/
	static void
	execute_agent_action(xmlNode msg, pcmk__action_result_t result)
	{
	xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, msg, LOG_ERR);
	xmlNode *op = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, msg,
	LOG_ERR);
	const char *id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
	const char *action = crm_element_value(op, PCMK__XA_ST_DEVICE_ACTION);
	async_command_t *cmd = NULL;
	stonith_device_t *device = NULL;

	if ((id == NULL) \|\| (action == NULL)) {
	crm_info("Malformed API action request: device %s, action %s",
	(id? id : "not specified"),
	(action? action : "not specified"));
	fenced_set_protocol_error(result);
	return;
	}

	if (pcmk__str_eq(id, STONITH_WATCHDOG_ID, pcmk__str_none)) {
	// Watchdog agent actions are implemented internally
	if (stonith_watchdog_timeout_ms <= 0) {
	pcmk__set_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"Watchdog fence device not configured");
	return;

	} else if (pcmk__str_eq(action, PCMK_ACTION_LIST, pcmk__str_none)) {
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	pcmk__set_result_output(result,
	list_to_string(stonith_watchdog_targets,
	"\n", TRUE),
	NULL);
	return;

	} else if (pcmk__str_eq(action, PCMK_ACTION_MONITOR, pcmk__str_none)) {
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return;
	}
	}

	device = g_hash_table_lookup(device_list, id);
	if (device == NULL) {
	crm_info("Ignoring API '%s' action request because device %s not found",
	action, id);
	pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"'%s' not found", id);
	return;

	} else if (!device->api_registered
	&& (strcmp(action, PCMK_ACTION_MONITOR) == 0)) {
	// Monitors may run only on "started" (API-registered) devices
	crm_info("Ignoring API '%s' action request because device %s not active",
	action, id);
	pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"'%s' not active", id);
	return;
	}

	cmd = create_async_command(msg);
	if (cmd == NULL) {
	crm_log_xml_warn(msg, "invalid");
	fenced_set_protocol_error(result);
	return;
	}

	schedule_stonith_command(cmd, device);
	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
	}

	static void
	search_devices_record_result(struct device_search_s search, const char device, gboolean can_fence)
	{
	search->replies_received++;
	if (can_fence && device) {
	if (search->support_action_only != st_device_supports_none) {
	stonith_device_t *dev = g_hash_table_lookup(device_list, device);
	if (dev && !pcmk_is_set(dev->flags, search->support_action_only)) {
	return;
	}
	}
	search->capable = g_list_append(search->capable, strdup(device));
	}

	if (search->replies_needed == search->replies_received) {

	guint ndevices = g_list_length(search->capable);

	crm_debug("Search found %d device%s that can perform '%s' targeting %s",
	ndevices, pcmk__plural_s(ndevices),
	(search->action? search->action : "unknown action"),
	(search->host? search->host : "any node"));

	search->callback(search->capable, search->user_data);
	free(search->host);
	free(search->action);
	free(search);
	}
	}

	/*!
	* \internal
	* \brief Check whether the local host is allowed to execute a fencing action
	*
	* \param[in] device Fence device to check
	* \param[in] action Fence action to check
	* \param[in] target Hostname of fence target
	* \param[in] allow_suicide Whether self-fencing is allowed for this operation
	*
	* \return TRUE if local host is allowed to execute action, FALSE otherwise
	*/
	static gboolean
	localhost_is_eligible(const stonith_device_t device, const char action,
	const char *target, gboolean allow_suicide)
	{
	gboolean localhost_is_target = pcmk__str_eq(target, stonith_our_uname,
	pcmk__str_casei);

	if ((device != NULL) && (action != NULL)
	&& (device->on_target_actions != NULL)
	&& (strstr((const char*) device->on_target_actions->str,
	action) != NULL)) {

	if (!localhost_is_target) {
	crm_trace("Operation '%s' using %s can only be executed for local "
	"host, not %s", action, device->id, target);
	return FALSE;
	}

	} else if (localhost_is_target && !allow_suicide) {
	crm_trace("'%s' operation does not support self-fencing", action);
	return FALSE;
	}
	return TRUE;
	}

	/*!
	* \internal
	* \brief Check if local node is allowed to execute (possibly remapped) action
	*
	* \param[in] device Fence device to check
	* \param[in] action Fence action to check
	* \param[in] target Node name of fence target
	* \param[in] allow_self Whether self-fencing is allowed for this operation
	*
	* \return true if local node is allowed to execute \p action or any actions it
	* might be remapped to, otherwise false
	*/
	static bool
	localhost_is_eligible_with_remap(const stonith_device_t *device,
	const char action, const char target,
	gboolean allow_self)
	{
	// Check exact action
	if (localhost_is_eligible(device, action, target, allow_self)) {
	return true;
	}

	// Check potential remaps

	if (pcmk__str_eq(action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
	/* "reboot" might get remapped to "off" then "on", so even if reboot is
	* disallowed, return true if either of those is allowed. We'll report
	* the disallowed actions with the results. We never allow self-fencing
	* for remapped "on" actions because the target is off at that point.
	*/
	if (localhost_is_eligible(device, PCMK_ACTION_OFF, target, allow_self)
	\|\| localhost_is_eligible(device, PCMK_ACTION_ON, target, FALSE)) {
	return true;
	}
	}

	return false;
	}

	static void
	can_fence_host_with_device(stonith_device_t *dev,
	struct device_search_s *search)
	{
	gboolean can = FALSE;
	const char *check_type = "Internal bug";
	const char *target = NULL;
	const char *alias = NULL;
	const char *dev_id = "Unspecified device";
	const char *action = (search == NULL)? NULL : search->action;

	CRM_CHECK((dev != NULL) && (action != NULL), goto search_report_results);

	if (dev->id != NULL) {
	dev_id = dev->id;
	}

	target = search->host;
	if (target == NULL) {
	can = TRUE;
	check_type = "No target";
	goto search_report_results;
	}

	/* Answer immediately if the device does not support the action
	* or the local node is not allowed to perform it
	*/
	if (pcmk__str_eq(action, PCMK_ACTION_ON, pcmk__str_none)
	&& !pcmk_is_set(dev->flags, st_device_supports_on)) {
	check_type = "Agent does not support 'on'";
	goto search_report_results;

	} else if (!localhost_is_eligible_with_remap(dev, action, target,
	search->allow_suicide)) {
	check_type = "This node is not allowed to execute action";
	goto search_report_results;
	}

	// Check eligibility as specified by pcmk_host_check
	check_type = target_list_type(dev);
	alias = g_hash_table_lookup(dev->aliases, target);
	if (pcmk__str_eq(check_type, PCMK_VALUE_NONE, pcmk__str_casei)) {
	can = TRUE;

	} else if (pcmk__str_eq(check_type, "static-list", pcmk__str_casei)) {
	if (pcmk__str_in_list(target, dev->targets, pcmk__str_casei)) {
	can = TRUE;
	} else if (g_hash_table_lookup(dev->params, PCMK_STONITH_HOST_MAP)
	&& g_hash_table_lookup(dev->aliases, target)) {
	can = TRUE;
	}

	} else if (pcmk__str_eq(check_type, "dynamic-list", pcmk__str_casei)) {
	time_t now = time(NULL);

	if (dev->targets == NULL \|\| dev->targets_age + 60 < now) {
	int device_timeout = get_action_timeout(dev, PCMK_ACTION_LIST,
	search->per_device_timeout);

	if (device_timeout > search->per_device_timeout) {
	crm_notice("Since the pcmk_list_timeout (%ds) parameter of %s "
	"is larger than " PCMK_OPT_STONITH_TIMEOUT
	" (%ds), timeout may occur",
	device_timeout, dev_id, search->per_device_timeout);
	}

	crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
	check_type, dev_id, target, action);

	schedule_internal_command(__func__, dev, PCMK_ACTION_LIST, NULL,
	search->per_device_timeout, search, dynamic_list_search_cb);

	/* we'll respond to this search request async in the cb */
	return;
	}

	if (pcmk__str_in_list(((alias == NULL)? target : alias), dev->targets,
	pcmk__str_casei)) {
	can = TRUE;
	}

	} else if (pcmk__str_eq(check_type, "status", pcmk__str_casei)) {
	int device_timeout = get_action_timeout(dev, check_type, search->per_device_timeout);

	if (device_timeout > search->per_device_timeout) {
	crm_notice("Since the pcmk_status_timeout (%ds) parameter of %s is "
	"larger than " PCMK_OPT_STONITH_TIMEOUT " (%ds), "
	"timeout may occur",
	device_timeout, dev_id, search->per_device_timeout);
	}

	crm_trace("Running '%s' to check whether %s is eligible to fence %s (%s)",
	check_type, dev_id, target, action);
	schedule_internal_command(__func__, dev, PCMK_ACTION_STATUS, target,
	search->per_device_timeout, search, status_search_cb);
	/* we'll respond to this search request async in the cb */
	return;
	} else {
	crm_err("Invalid value for " PCMK_STONITH_HOST_CHECK ": %s", check_type);
	check_type = "Invalid " PCMK_STONITH_HOST_CHECK;
	}

	search_report_results:
	crm_info("%s is%s eligible to fence (%s) %s%s%s%s: %s",
	dev_id, (can? "" : " not"), pcmk__s(action, "unspecified action"),
	pcmk__s(target, "unspecified target"),
	(alias == NULL)? "" : " (as '", pcmk__s(alias, ""),
	(alias == NULL)? "" : "')", check_type);
	search_devices_record_result(search, ((dev == NULL)? NULL : dev_id), can);
	}

	static void
	search_devices(gpointer key, gpointer value, gpointer user_data)
	{
	stonith_device_t *dev = value;
	struct device_search_s *search = user_data;

	can_fence_host_with_device(dev, search);
	}

	#define DEFAULT_QUERY_TIMEOUT 20
	static void
	get_capable_devices(const char host, const char action, int timeout, bool suicide, void *user_data,
	void (callback) (GList devices, void *user_data), uint32_t support_action_only)
	{
	struct device_search_s *search;
	guint ndevices = g_hash_table_size(device_list);

	if (ndevices == 0) {
	callback(NULL, user_data);
	return;
	}

	search = calloc(1, sizeof(struct device_search_s));
	if (!search) {
	crm_crit("Cannot search for capable fence devices: %s",
	strerror(ENOMEM));
	callback(NULL, user_data);
	return;
	}

	pcmk__str_update(&search->host, host);
	pcmk__str_update(&search->action, action);
	search->per_device_timeout = timeout;
	search->allow_suicide = suicide;
	search->callback = callback;
	search->user_data = user_data;
	search->support_action_only = support_action_only;

	/* We are guaranteed this many replies, even if a device is
	* unregistered while the search is in progress.
	*/
	search->replies_needed = ndevices;

	crm_debug("Searching %d device%s to see which can execute '%s' targeting %s",
	ndevices, pcmk__plural_s(ndevices),
	(search->action? search->action : "unknown action"),
	(search->host? search->host : "any node"));
	g_hash_table_foreach(device_list, search_devices, search);
	}

	struct st_query_data {
	xmlNode *reply;
	char *remote_peer;
	char *client_id;
	char *target;
	char *action;
	int call_options;
	};

	/*!
	* \internal
	* \brief Add action-specific attributes to query reply XML
	*
	* \param[in,out] xml XML to add attributes to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	*/
	static void
	add_action_specific_attributes(xmlNode xml, const char action,
	const stonith_device_t *device,
	const char *target)
	{
	int action_specific_timeout;
	int delay_max;
	int delay_base;

	CRM_CHECK(xml && action && device, return);

	// PCMK__XA_ST_REQUIRED is currently used only for unfencing
	if (is_action_required(action, device)) {
	crm_trace("Action '%s' is required using %s", action, device->id);
	crm_xml_add_int(xml, PCMK__XA_ST_REQUIRED, 1);
	}

	// pcmk_<action>_timeout if configured
	action_specific_timeout = get_action_timeout(device, action, 0);
	if (action_specific_timeout) {
	crm_trace("Action '%s' has timeout %dms using %s",
	action, action_specific_timeout, device->id);
	crm_xml_add_int(xml, PCMK__XA_ST_ACTION_TIMEOUT,
	action_specific_timeout);
	}

	delay_max = get_action_delay_max(device, action);
	if (delay_max > 0) {
	crm_trace("Action '%s' has maximum random delay %ds using %s",
	action, delay_max, device->id);
	crm_xml_add_int(xml, PCMK__XA_ST_DELAY_MAX, delay_max);
	}

	delay_base = get_action_delay_base(device, action, target);
	if (delay_base > 0) {
	crm_xml_add_int(xml, PCMK__XA_ST_DELAY_BASE, delay_base);
	}

	if ((delay_max > 0) && (delay_base == 0)) {
	crm_trace("Action '%s' has maximum random delay %ds using %s",
	action, delay_max, device->id);
	} else if ((delay_max == 0) && (delay_base > 0)) {
	crm_trace("Action '%s' has a static delay of %ds using %s",
	action, delay_base, device->id);
	} else if ((delay_max > 0) && (delay_base > 0)) {
	crm_trace("Action '%s' has a minimum delay of %ds and a randomly chosen "
	"maximum delay of %ds using %s",
	action, delay_base, delay_max, device->id);
	}
	}

	/*!
	* \internal
	* \brief Add "disallowed" attribute to query reply XML if appropriate
	*
	* \param[in,out] xml XML to add attribute to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	* \param[in] allow_suicide Whether self-fencing is allowed
	*/
	static void
	add_disallowed(xmlNode xml, const char action, const stonith_device_t *device,
	const char *target, gboolean allow_suicide)
	{
	if (!localhost_is_eligible(device, action, target, allow_suicide)) {
	crm_trace("Action '%s' using %s is disallowed for local host",
	action, device->id);
	pcmk__xe_set_bool_attr(xml, PCMK__XA_ST_ACTION_DISALLOWED, true);
	}
	}

	/*!
	* \internal
	* \brief Add child element with action-specific values to query reply XML
	*
	* \param[in,out] xml XML to add attribute to
	* \param[in] action Fence action
	* \param[in] device Fence device
	* \param[in] target Fence target
	* \param[in] allow_suicide Whether self-fencing is allowed
	*/
	static void
	add_action_reply(xmlNode xml, const char action,
	const stonith_device_t device, const char target,
	gboolean allow_suicide)
	{
	xmlNode *child = create_xml_node(xml, PCMK__XE_ST_DEVICE_ACTION);

	crm_xml_add(child, PCMK_XA_ID, action);
	add_action_specific_attributes(child, action, device, target);
	add_disallowed(child, action, device, target, allow_suicide);
	}

	/*!
	* \internal
	* \brief Send a reply to a CPG peer or IPC client
	*
	* \param[in] reply XML reply to send
	* \param[in] call_options Send synchronously if st_opt_sync_call is set
	* \param[in] remote_peer If not NULL, name of peer node to send CPG reply
	* \param[in,out] client If not NULL, client to send IPC reply
	*/
	static void
	stonith_send_reply(const xmlNode *reply, int call_options,
	const char remote_peer, pcmk__client_t client)
	{
	CRM_CHECK((reply != NULL) && ((remote_peer != NULL) \|\| (client != NULL)),
	return);

	if (remote_peer == NULL) {
	do_local_reply(reply, client, call_options);
	} else {
	send_cluster_message(pcmk__get_node(0, remote_peer, NULL,
	pcmk__node_search_cluster),
	crm_msg_stonith_ng, reply, FALSE);
	}
	}

	static void
	stonith_query_capable_device_cb(GList * devices, void *user_data)
	{
	struct st_query_data *query = user_data;
	int available_devices = 0;
	xmlNode *dev = NULL;
	xmlNode *list = NULL;
	GList *lpc = NULL;
	pcmk__client_t *client = NULL;

	if (query->client_id != NULL) {
	client = pcmk__find_client_by_id(query->client_id);
	if ((client == NULL) && (query->remote_peer == NULL)) {
	crm_trace("Skipping reply to %s: no longer a client",
	query->client_id);
	goto done;
	}
	}

	/* Pack the results into XML */
	list = create_xml_node(NULL, __func__);
	crm_xml_add(list, PCMK__XA_ST_TARGET, query->target);
	for (lpc = devices; lpc != NULL; lpc = lpc->next) {
	stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
	const char *action = query->action;

	if (!device) {
	/* It is possible the device got unregistered while
	* determining who can fence the target */
	continue;
	}

	available_devices++;

	dev = create_xml_node(list, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(dev, PCMK_XA_ID, device->id);
	crm_xml_add(dev, PCMK__XA_NAMESPACE, device->namespace);
	crm_xml_add(dev, PCMK_XA_AGENT, device->agent);

	// Has had successful monitor, list, or status on this node
	crm_xml_add_int(dev, PCMK__XA_ST_MONITOR_VERIFIED, device->verified);

	crm_xml_add_int(dev, PCMK__XA_ST_DEVICE_SUPPORT_FLAGS, device->flags);

	/* If the originating fencer wants to reboot the node, and we have a
	* capable device that doesn't support "reboot", remap to "off" instead.
	*/
	if (!pcmk_is_set(device->flags, st_device_supports_reboot)
	&& pcmk__str_eq(query->action, PCMK_ACTION_REBOOT,
	pcmk__str_none)) {
	crm_trace("%s doesn't support reboot, using values for off instead",
	device->id);
	action = PCMK_ACTION_OFF;
	}

	/* Add action-specific values if available */
	add_action_specific_attributes(dev, action, device, query->target);
	if (pcmk__str_eq(query->action, PCMK_ACTION_REBOOT, pcmk__str_none)) {
	/* A "reboot" might get remapped to "off" then "on", so after
	* sending the "reboot"-specific values in the main element, we add
	* sub-elements for "off" and "on" values.
	*
	* We short-circuited earlier if "reboot", "off" and "on" are all
	* disallowed for the local host. However if only one or two are
	* disallowed, we send back the results and mark which ones are
	* disallowed. If "reboot" is disallowed, this might cause problems
	* with older fencer versions, which won't check for it. Older
	* versions will ignore "off" and "on", so they are not a problem.
	*/
	add_disallowed(dev, action, device, query->target,
	pcmk_is_set(query->call_options, st_opt_allow_suicide));
	add_action_reply(dev, PCMK_ACTION_OFF, device, query->target,
	pcmk_is_set(query->call_options, st_opt_allow_suicide));
	add_action_reply(dev, PCMK_ACTION_ON, device, query->target, FALSE);
	}

	/* A query without a target wants device parameters */
	if (query->target == NULL) {
	xmlNode *attrs = create_xml_node(dev, PCMK__XE_ATTRIBUTES);

	g_hash_table_foreach(device->params, hash2field, attrs);
	}
	}

	crm_xml_add_int(list, PCMK__XA_ST_AVAILABLE_DEVICES, available_devices);
	if (query->target) {
	crm_debug("Found %d matching device%s for target '%s'",
	available_devices, pcmk__plural_s(available_devices),
	query->target);
	} else {
	crm_debug("%d device%s installed",
	available_devices, pcmk__plural_s(available_devices));
	}

	if (list != NULL) {
	crm_log_xml_trace(list, "Add query results");
	add_message_xml(query->reply, PCMK__XA_ST_CALLDATA, list);
	}

	stonith_send_reply(query->reply, query->call_options, query->remote_peer,
	client);

	done:
	free_xml(query->reply);
	free(query->remote_peer);
	free(query->client_id);
	free(query->target);
	free(query->action);
	free(query);
	free_xml(list);
	g_list_free_full(devices, free);
	}

	/*!
	* \internal
	* \brief Log the result of an asynchronous command
	*
	* \param[in] cmd Command the result is for
	* \param[in] result Result of command
	* \param[in] pid Process ID of command, if available
	* \param[in] next Alternate device that will be tried if command failed
	* \param[in] op_merged Whether this command was merged with an earlier one
	*/
	static void
	log_async_result(const async_command_t *cmd,
	const pcmk__action_result_t *result,
	int pid, const char *next, bool op_merged)
	{
	int log_level = LOG_ERR;
	int output_log_level = LOG_NEVER;
	guint devices_remaining = g_list_length(cmd->next_device_iter);

	GString *msg = g_string_sized_new(80); // Reasonable starting size

	// Choose log levels appropriately if we have a result
	if (pcmk__result_ok(result)) {
	log_level = (cmd->target == NULL)? LOG_DEBUG : LOG_NOTICE;
	if ((result->action_stdout != NULL)
	&& !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
	pcmk__str_none)) {
	output_log_level = LOG_DEBUG;
	}
	next = NULL;
	} else {
	log_level = (cmd->target == NULL)? LOG_NOTICE : LOG_ERR;
	if ((result->action_stdout != NULL)
	&& !pcmk__str_eq(cmd->action, PCMK_ACTION_METADATA,
	pcmk__str_none)) {
	output_log_level = LOG_WARNING;
	}
	}

	// Build the log message piece by piece
	pcmk__g_strcat(msg, "Operation '", cmd->action, "' ", NULL);
	if (pid != 0) {
	g_string_append_printf(msg, "[%d] ", pid);
	}
	if (cmd->target != NULL) {
	pcmk__g_strcat(msg, "targeting ", cmd->target, " ", NULL);
	}
	if (cmd->device != NULL) {
	pcmk__g_strcat(msg, "using ", cmd->device, " ", NULL);
	}

	// Add exit status or execution status as appropriate
	if (result->execution_status == PCMK_EXEC_DONE) {
	g_string_append_printf(msg, "returned %d", result->exit_status);
	} else {
	pcmk__g_strcat(msg, "could not be executed: ",
	pcmk_exec_status_str(result->execution_status), NULL);
	}

	// Add exit reason and next device if appropriate
	if (result->exit_reason != NULL) {
	pcmk__g_strcat(msg, " (", result->exit_reason, ")", NULL);
	}
	if (next != NULL) {
	pcmk__g_strcat(msg, ", retrying with ", next, NULL);
	}
	if (devices_remaining > 0) {
	g_string_append_printf(msg, " (%u device%s remaining)",
	(unsigned int) devices_remaining,
	pcmk__plural_s(devices_remaining));
	}
	g_string_append_printf(msg, " " CRM_XS " %scall %d from %s",
	(op_merged? "merged " : ""), cmd->id,
	cmd->client_name);

	// Log the result
	do_crm_log(log_level, "%s", msg->str);
	g_string_free(msg, TRUE);

	// Log the output (which may have multiple lines), if appropriate
	if (output_log_level != LOG_NEVER) {
	char *prefix = crm_strdup_printf("%s[%d]", cmd->device, pid);

	crm_log_output(output_log_level, prefix, result->action_stdout);
	free(prefix);
	}
	}

	/*!
	* \internal
	* \brief Reply to requester after asynchronous command completion
	*
	* \param[in] cmd Command that completed
	* \param[in] result Result of command
	* \param[in] pid Process ID of command, if available
	* \param[in] merged If true, command was merged with another, not executed
	*/
	static void
	send_async_reply(const async_command_t cmd, const pcmk__action_result_t result,
	int pid, bool merged)
	{
	xmlNode *reply = NULL;
	pcmk__client_t *client = NULL;

	CRM_CHECK((cmd != NULL) && (result != NULL), return);

	log_async_result(cmd, result, pid, NULL, merged);

	if (cmd->client != NULL) {
	client = pcmk__find_client_by_id(cmd->client);
	if ((client == NULL) && (cmd->origin == NULL)) {
	crm_trace("Skipping reply to %s: no longer a client", cmd->client);
	return;
	}
	}

	reply = construct_async_reply(cmd, result);
	if (merged) {
	pcmk__xe_set_bool_attr(reply, PCMK__XA_ST_OP_MERGED, true);
	}

	if (!stand_alone && pcmk__is_fencing_action(cmd->action)
	&& pcmk__str_eq(cmd->origin, cmd->target, pcmk__str_casei)) {
	/* The target was also the originator, so broadcast the result on its
	* behalf (since it will be unable to).
	*/
	crm_trace("Broadcast '%s' result for %s (target was also originator)",
	cmd->action, cmd->target);
	crm_xml_add(reply, PCMK__XA_SUBT, PCMK__VALUE_BROADCAST);
	crm_xml_add(reply, PCMK__XA_ST_OP, STONITH_OP_NOTIFY);
	send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
	} else {
	// Reply only to the originator
	stonith_send_reply(reply, cmd->options, cmd->origin, client);
	}

	crm_log_xml_trace(reply, "Reply");
	free_xml(reply);

	if (stand_alone) {
	/* Do notification with a clean data object */
	xmlNode *notify_data = create_xml_node(NULL, PCMK__XE_ST_NOTIFY_FENCE);

	stonith__xe_set_result(notify_data, result);
	crm_xml_add(notify_data, PCMK__XA_ST_TARGET, cmd->target);
	crm_xml_add(notify_data, PCMK__XA_ST_OP, cmd->op);
	crm_xml_add(notify_data, PCMK__XA_ST_DELEGATE, "localhost");
	crm_xml_add(notify_data, PCMK__XA_ST_DEVICE_ID, cmd->device);
	crm_xml_add(notify_data, PCMK__XA_ST_REMOTE_OP, cmd->remote_op_id);
	crm_xml_add(notify_data, PCMK__XA_ST_ORIGIN, cmd->client);

	fenced_send_notification(PCMK__VALUE_ST_NOTIFY_FENCE, result,
	notify_data);
	fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY, NULL, NULL);
	}
	}

	static void
	cancel_stonith_command(async_command_t * cmd)
	{
	stonith_device_t *device = cmd_device(cmd);

	if (device) {
	crm_trace("Cancel scheduled '%s' action using %s",
	cmd->action, device->id);
	device->pending_ops = g_list_remove(device->pending_ops, cmd);
	}
	}

	/*!
	* \internal
	* \brief Cancel and reply to any duplicates of a just-completed operation
	*
	* Check whether any fencing operations are scheduled to do the same thing as
	* one that just succeeded. If so, rather than performing the same operation
	* twice, return the result of this operation for all matching pending commands.
	*
	* \param[in,out] cmd Fencing operation that just succeeded
	* \param[in] result Result of \p cmd
	* \param[in] pid If nonzero, process ID of agent invocation (for logs)
	*
	* \note Duplicate merging will do the right thing for either type of remapped
	* reboot. If the executing fencer remapped an unsupported reboot to off,
	* then cmd->action will be "reboot" and will be merged with any other
	* reboot requests. If the originating fencer remapped a topology reboot
	* to off then on, we will get here once with cmd->action "off" and once
	* with "on", and they will be merged separately with similar requests.
	*/
	static void
	reply_to_duplicates(async_command_t cmd, const pcmk__action_result_t result,
	int pid)
	{
	GList *next = NULL;

	for (GList *iter = cmd_list; iter != NULL; iter = next) {
	async_command_t *cmd_other = iter->data;

	next = iter->next; // We might delete this entry, so grab next now

	if (cmd == cmd_other) {
	continue;
	}

	/* A pending operation matches if:
	* 1. The client connections are different.
	* 2. The target is the same.
	* 3. The fencing action is the same.
	* 4. The device scheduled to execute the action is the same.
	*/
	if (pcmk__str_eq(cmd->client, cmd_other->client, pcmk__str_casei) \|\|
	!pcmk__str_eq(cmd->target, cmd_other->target, pcmk__str_casei) \|\|
	!pcmk__str_eq(cmd->action, cmd_other->action, pcmk__str_none) \|\|
	!pcmk__str_eq(cmd->device, cmd_other->device, pcmk__str_casei)) {

	continue;
	}

	crm_notice("Merging fencing action '%s'%s%s originating from "
	"client %s with identical fencing request from client %s",
	cmd_other->action,
	(cmd_other->target == NULL)? "" : " targeting ",
	pcmk__s(cmd_other->target, ""), cmd_other->client_name,
	cmd->client_name);

	// Stop tracking the duplicate, send its result, and cancel it
	cmd_list = g_list_remove_link(cmd_list, iter);
	send_async_reply(cmd_other, result, pid, true);
	cancel_stonith_command(cmd_other);

	free_async_command(cmd_other);
	g_list_free_1(iter);
	}
	}

	/*!
	* \internal
	* \brief Return the next required device (if any) for an operation
	*
	* \param[in,out] cmd Fencing operation that just succeeded
	*
	* \return Next device required for action if any, otherwise NULL
	*/
	static stonith_device_t *
	next_required_device(async_command_t *cmd)
	{
	for (GList *iter = cmd->next_device_iter; iter != NULL; iter = iter->next) {
	stonith_device_t *next_device = g_hash_table_lookup(device_list,
	iter->data);

	if (is_action_required(cmd->action, next_device)) {
	/* This is only called for successful actions, so it's OK to skip
	* non-required devices.
	*/
	cmd->next_device_iter = iter->next;
	return next_device;
	}
	}
	return NULL;
	}

	static void
	st_child_done(int pid, const pcmk__action_result_t result, void user_data)
	{
	async_command_t *cmd = user_data;

	stonith_device_t *device = NULL;
	stonith_device_t *next_device = NULL;

	CRM_CHECK(cmd != NULL, return);

	device = cmd_device(cmd);
	cmd->active_on = NULL;

	/* The device is ready to do something else now */
	if (device) {
	if (!device->verified && pcmk__result_ok(result)
	&& pcmk__strcase_any_of(cmd->action, PCMK_ACTION_LIST,
	PCMK_ACTION_MONITOR, PCMK_ACTION_STATUS,
	NULL)) {

	device->verified = TRUE;
	}

	mainloop_set_trigger(device->work);
	}

	if (pcmk__result_ok(result)) {
	next_device = next_required_device(cmd);

	} else if ((cmd->next_device_iter != NULL)
	&& !is_action_required(cmd->action, device)) {
	/* if this device didn't work out, see if there are any others we can try.
	* if the failed device was 'required', we can't pick another device. */
	next_device = g_hash_table_lookup(device_list,
	cmd->next_device_iter->data);
	cmd->next_device_iter = cmd->next_device_iter->next;
	}

	if (next_device == NULL) {
	send_async_reply(cmd, result, pid, false);
	if (pcmk__result_ok(result)) {
	reply_to_duplicates(cmd, result, pid);
	}
	free_async_command(cmd);

	} else { // This operation requires more fencing
	log_async_result(cmd, result, pid, next_device->id, false);
	schedule_stonith_command(cmd, next_device);
	}
	}

	static gint
	sort_device_priority(gconstpointer a, gconstpointer b)
	{
	const stonith_device_t *dev_a = a;
	const stonith_device_t *dev_b = b;

	if (dev_a->priority > dev_b->priority) {
	return -1;
	} else if (dev_a->priority < dev_b->priority) {
	return 1;
	}
	return 0;
	}

	static void
	stonith_fence_get_devices_cb(GList * devices, void *user_data)
	{
	async_command_t *cmd = user_data;
	stonith_device_t *device = NULL;
	guint ndevices = g_list_length(devices);

	crm_info("Found %d matching device%s for target '%s'",
	ndevices, pcmk__plural_s(ndevices), cmd->target);

	if (devices != NULL) {
	/* Order based on priority */
	devices = g_list_sort(devices, sort_device_priority);
	device = g_hash_table_lookup(device_list, devices->data);
	}

	if (device == NULL) { // No device found
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	pcmk__format_result(&result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"No device configured for target '%s'",
	cmd->target);
	send_async_reply(cmd, &result, 0, false);
	pcmk__reset_result(&result);
	free_async_command(cmd);
	g_list_free_full(devices, free);

	} else { // Device found, schedule it for fencing
	cmd->device_list = devices;
	cmd->next_device_iter = devices->next;
	schedule_stonith_command(cmd, device);
	}
	}

	/*!
	* \internal
	* \brief Execute a fence action via the local node
	*
	* \param[in] msg Fencing request
	* \param[out] result Where to store result of fence action
	*/
	static void
	fence_locally(xmlNode msg, pcmk__action_result_t result)
	{
	const char *device_id = NULL;
	stonith_device_t *device = NULL;
	async_command_t *cmd = NULL;
	xmlNode *dev = NULL;

	CRM_CHECK((msg != NULL) && (result != NULL), return);

	dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, msg, LOG_ERR);

	cmd = create_async_command(msg);
	if (cmd == NULL) {
	crm_log_xml_warn(msg, "invalid");
	fenced_set_protocol_error(result);
	return;
	}

	device_id = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);
	if (device_id != NULL) {
	device = g_hash_table_lookup(device_list, device_id);
	if (device == NULL) {
	crm_err("Requested device '%s' is not available", device_id);
	pcmk__format_result(result, CRM_EX_ERROR, PCMK_EXEC_NO_FENCE_DEVICE,
	"Requested device '%s' not found", device_id);
	return;
	}
	schedule_stonith_command(cmd, device);

	} else {
	const char *host = crm_element_value(dev, PCMK__XA_ST_TARGET);

	if (pcmk_is_set(cmd->options, st_opt_cs_nodeid)) {
	int nodeid = 0;
	crm_node_t *node = NULL;

	pcmk__scan_min_int(host, &nodeid, 0);
	node = pcmk__search_node_caches(nodeid, NULL,
	pcmk__node_search_any
	\|pcmk__node_search_known);
	if (node != NULL) {
	host = node->uname;
	}
	}

	/* If we get to here, then self-fencing is implicitly allowed */
	get_capable_devices(host, cmd->action, cmd->default_timeout,
	TRUE, cmd, stonith_fence_get_devices_cb,
	fenced_support_flag(cmd->action));
	}

	pcmk__set_result(result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
	}

	/*!
	* \internal
	* \brief Build an XML reply for a fencing operation
	*
	* \param[in] request Request that reply is for
	* \param[in] data If not NULL, add to reply as call data
	* \param[in] result Full result of fencing operation
	*
	* \return Newly created XML reply
	* \note The caller is responsible for freeing the result.
	* \note This has some overlap with construct_async_reply(), but that copies
	* values from an async_command_t, whereas this one copies them from the
	* request.
	*/
	xmlNode *
	fenced_construct_reply(const xmlNode request, xmlNode data,
	const pcmk__action_result_t *result)
	{
	xmlNode *reply = NULL;

	reply = create_xml_node(NULL, PCMK__XE_ST_REPLY);

	crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	stonith__xe_set_result(reply, result);

	if (request == NULL) {
	/* Most likely, this is the result of a stonith operation that was
	* initiated before we came up. Unfortunately that means we lack enough
	* information to provide clients with a full result.
	*
	* @TODO Maybe synchronize this information at start-up?
	*/
	crm_warn("Missing request information for client notifications for "
	"operation with result '%s' (initiated before we came up?)",
	pcmk_exec_status_str(result->execution_status));

	} else {
	const char *name = NULL;
	const char *value = NULL;

	// Attributes to copy from request to reply
	const char *names[] = {
	PCMK__XA_ST_OP,
	PCMK__XA_ST_CALLID,
	PCMK__XA_ST_CLIENTID,
	PCMK__XA_ST_CLIENTNAME,
	PCMK__XA_ST_REMOTE_OP,
	PCMK__XA_ST_CALLOPT,
	};

	for (int lpc = 0; lpc < PCMK__NELEM(names); lpc++) {
	name = names[lpc];
	value = crm_element_value(request, name);
	crm_xml_add(reply, name, value);
	}
	if (data != NULL) {
	add_message_xml(reply, PCMK__XA_ST_CALLDATA, data);
	}
	}
	return reply;
	}

	/*!
	* \internal
	* \brief Build an XML reply to an asynchronous fencing command
	*
	* \param[in] cmd Fencing command that reply is for
	* \param[in] result Command result
	*/
	static xmlNode *
	construct_async_reply(const async_command_t *cmd,
	const pcmk__action_result_t *result)
	{
	xmlNode *reply = create_xml_node(NULL, PCMK__XE_ST_REPLY);

	crm_xml_add(reply, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(reply, PCMK__XA_ST_OP, cmd->op);
	crm_xml_add(reply, PCMK__XA_ST_DEVICE_ID, cmd->device);
	crm_xml_add(reply, PCMK__XA_ST_REMOTE_OP, cmd->remote_op_id);
	crm_xml_add(reply, PCMK__XA_ST_CLIENTID, cmd->client);
	crm_xml_add(reply, PCMK__XA_ST_CLIENTNAME, cmd->client_name);
	crm_xml_add(reply, PCMK__XA_ST_TARGET, cmd->target);
	crm_xml_add(reply, PCMK__XA_ST_DEVICE_ACTION, cmd->op);
	crm_xml_add(reply, PCMK__XA_ST_ORIGIN, cmd->origin);
	crm_xml_add_int(reply, PCMK__XA_ST_CALLID, cmd->id);
	crm_xml_add_int(reply, PCMK__XA_ST_CALLOPT, cmd->options);

	stonith__xe_set_result(reply, result);
	return reply;
	}

	bool fencing_peer_active(crm_node_t *peer)
	{
	if (peer == NULL) {
	return FALSE;
	} else if (peer->uname == NULL) {
	return FALSE;
	} else if (pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
	return TRUE;
	}
	return FALSE;
	}

	void
	set_fencing_completed(remote_fencing_op_t *op)
	{
	struct timespec tv;

	qb_util_timespec_from_epoch_get(&tv);
	op->completed = tv.tv_sec;
	op->completed_nsec = tv.tv_nsec;
	}

	/*!
	* \internal
	* \brief Look for alternate node needed if local node shouldn't fence target
	*
	* \param[in] target Node that must be fenced
	*
	* \return Name of an alternate node that should fence \p target if any,
	* or NULL otherwise
	*/
	static const char *
	check_alternate_host(const char *target)
	{
	if (pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
	GHashTableIter gIter;
	crm_node_t *entry = NULL;

	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
	if (fencing_peer_active(entry)
	&& !pcmk__str_eq(entry->uname, target, pcmk__str_casei)) {
	crm_notice("Forwarding self-fencing request to %s",
	entry->uname);
	return entry->uname;
	}
	}
	crm_warn("Will handle own fencing because no peer can");
	}
	return NULL;
	}

	static void
	remove_relay_op(xmlNode * request)
	{
	xmlNode *dev = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, request,
	LOG_TRACE);
	const char *relay_op_id = NULL;
	const char *op_id = NULL;
	const char *client_name = NULL;
	const char *target = NULL;
	remote_fencing_op_t *relay_op = NULL;

	if (dev) {
	target = crm_element_value(dev, PCMK__XA_ST_TARGET);
	}

	relay_op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP_RELAY);
	op_id = crm_element_value(request, PCMK__XA_ST_REMOTE_OP);
	client_name = crm_element_value(request, PCMK__XA_ST_CLIENTNAME);

	/* Delete RELAY operation. */
	if (relay_op_id && target && pcmk__str_eq(target, stonith_our_uname, pcmk__str_casei)) {
	relay_op = g_hash_table_lookup(stonith_remote_op_list, relay_op_id);

	if (relay_op) {
	GHashTableIter iter;
	remote_fencing_op_t *list_op = NULL;
	g_hash_table_iter_init(&iter, stonith_remote_op_list);

	/* If the operation to be deleted is registered as a duplicate, delete the registration. */
	while (g_hash_table_iter_next(&iter, NULL, (void **)&list_op)) {
	GList *dup_iter = NULL;
	if (list_op != relay_op) {
	for (dup_iter = list_op->duplicates; dup_iter != NULL; dup_iter = dup_iter->next) {
	remote_fencing_op_t *other = dup_iter->data;
	if (other == relay_op) {
	other->duplicates = g_list_remove(other->duplicates, relay_op);
	break;
	}
	}
	}
	}
	crm_debug("Deleting relay op %s ('%s'%s%s for %s), "
	"replaced by op %s ('%s'%s%s for %s)",
	relay_op->id, relay_op->action,
	(relay_op->target == NULL)? "" : " targeting ",
	pcmk__s(relay_op->target, ""),
	relay_op->client_name, op_id, relay_op->action,
	(target == NULL)? "" : " targeting ", pcmk__s(target, ""),
	client_name);

	g_hash_table_remove(stonith_remote_op_list, relay_op_id);
	}
	}
	}

	/*!
	* \internal
	* \brief Check whether an API request was sent by a privileged user
	*
	* API commands related to fencing configuration may be done only by privileged
	* IPC users (i.e. root or hacluster), because all other users should go through
	* the CIB to have ACLs applied. If no client was given, this is a peer request,
	* which is always allowed.
	*
	* \param[in] c IPC client that sent request (or NULL if sent by CPG peer)
	* \param[in] op Requested API operation (for logging only)
	*
	* \return true if sender is peer or privileged client, otherwise false
	*/
	static inline bool
	is_privileged(const pcmk__client_t c, const char op)
	{
	if ((c == NULL) \|\| pcmk_is_set(c->flags, pcmk__client_privileged)) {
	return true;
	} else {
	crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
	pcmk__s(op, ""), pcmk__client_name(c));
	return false;
	}
	}

	// CRM_OP_REGISTER
	static xmlNode *
	handle_register_request(pcmk__request_t *request)
	{
	xmlNode *reply = create_xml_node(NULL, "reply");

	CRM_ASSERT(request->ipc_client != NULL);
	crm_xml_add(reply, PCMK__XA_ST_OP, CRM_OP_REGISTER);
	crm_xml_add(reply, PCMK__XA_ST_CLIENTID, request->ipc_client->id);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	pcmk__set_request_flags(request, pcmk__request_reuse_options);
	return reply;
	}

	// STONITH_OP_EXEC
	static xmlNode *
	handle_agent_request(pcmk__request_t *request)
	{
	execute_agent_action(request->xml, &request->result);
	if (request->result.execution_status == PCMK_EXEC_PENDING) {
	return NULL;
	}
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_TIMEOUT_UPDATE
	static xmlNode *
	handle_update_timeout_request(pcmk__request_t *request)
	{
	const char *call_id = crm_element_value(request->xml, PCMK__XA_ST_CALLID);
	const char *client_id = crm_element_value(request->xml,
	PCMK__XA_ST_CLIENTID);
	int op_timeout = 0;

	crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &op_timeout);
	do_stonith_async_timeout_update(client_id, call_id, op_timeout);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return NULL;
	}

	// STONITH_OP_QUERY
	static xmlNode *
	handle_query_request(pcmk__request_t *request)
	{
	int timeout = 0;
	xmlNode *dev = NULL;
	const char *action = NULL;
	const char *target = NULL;
	const char *client_id = crm_element_value(request->xml,
	PCMK__XA_ST_CLIENTID);
	struct st_query_data *query = NULL;

	if (request->peer != NULL) {
	// Record it for the future notification
	create_remote_stonith_op(client_id, request->xml, TRUE);
	}

	/* Delete the DC node RELAY operation. */
	remove_relay_op(request->xml);

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);

	dev = get_xpath_object("//@" PCMK__XE_ST_DEVICE_ACTION, request->xml,
	LOG_NEVER);
	if (dev != NULL) {
	const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);

	if (pcmk__str_eq(device, "manual_ack", pcmk__str_casei)) {
	return NULL; // No query or reply necessary
	}
	target = crm_element_value(dev, PCMK__XA_ST_TARGET);
	action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
	}

	crm_log_xml_trace(request->xml, "Query");

	query = calloc(1, sizeof(struct st_query_data));
	CRM_ASSERT(query != NULL);

	query->reply = fenced_construct_reply(request->xml, NULL, &request->result);
	pcmk__str_update(&query->remote_peer, request->peer);
	pcmk__str_update(&query->client_id, client_id);
	pcmk__str_update(&query->target, target);
	pcmk__str_update(&query->action, action);
	query->call_options = request->call_options;

	crm_element_value_int(request->xml, PCMK__XA_ST_TIMEOUT, &timeout);
	get_capable_devices(target, action, timeout,
	pcmk_is_set(query->call_options, st_opt_allow_suicide),
	query, stonith_query_capable_device_cb, st_device_supports_none);
	return NULL;
	}

	// STONITH_OP_NOTIFY
	static xmlNode *
	handle_notify_request(pcmk__request_t *request)
	{
	const char *flag_name = NULL;

	CRM_ASSERT(request->ipc_client != NULL);
	flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_ACTIVATE);
	if (flag_name != NULL) {
	crm_debug("Enabling %s callbacks for client %s",
	flag_name, pcmk__request_origin(request));
	pcmk__set_client_flags(request->ipc_client, get_stonith_flag(flag_name));
	}

	flag_name = crm_element_value(request->xml, PCMK__XA_ST_NOTIFY_DEACTIVATE);
	if (flag_name != NULL) {
	crm_debug("Disabling %s callbacks for client %s",
	flag_name, pcmk__request_origin(request));
	pcmk__clear_client_flags(request->ipc_client,
	get_stonith_flag(flag_name));
	}

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	pcmk__set_request_flags(request, pcmk__request_reuse_options);

	return pcmk__ipc_create_ack(request->ipc_flags, PCMK__XE_ACK, NULL,
	CRM_EX_OK);
	}

	// STONITH_OP_RELAY
	static xmlNode *
	handle_relay_request(pcmk__request_t *request)
	{
	xmlNode *dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, request->xml,
	LOG_TRACE);

	crm_notice("Received forwarded fencing request from "
	"%s %s to fence (%s) peer %s",
	pcmk__request_origin_type(request),
	pcmk__request_origin(request),
	crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION),
	crm_element_value(dev, PCMK__XA_ST_TARGET));

	if (initiate_remote_stonith_op(NULL, request->xml, FALSE) == NULL) {
	fenced_set_protocol_error(&request->result);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING, NULL);
	return NULL;
	}

	// STONITH_OP_FENCE
	static xmlNode *
	handle_fence_request(pcmk__request_t *request)
	{
	if ((request->peer != NULL) \|\| stand_alone) {
	fence_locally(request->xml, &request->result);

	} else if (pcmk_is_set(request->call_options, st_opt_manual_ack)) {
	switch (fenced_handle_manual_confirmation(request->ipc_client,
	request->xml)) {
	case pcmk_rc_ok:
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
	NULL);
	break;
	case EINPROGRESS:
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
	NULL);
	break;
	default:
	fenced_set_protocol_error(&request->result);
	break;
	}

	} else {
	const char *alternate_host = NULL;
	xmlNode *dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, request->xml,
	LOG_TRACE);
	const char *target = crm_element_value(dev, PCMK__XA_ST_TARGET);
	const char *action = crm_element_value(dev, PCMK__XA_ST_DEVICE_ACTION);
	const char *device = crm_element_value(dev, PCMK__XA_ST_DEVICE_ID);

	if (request->ipc_client != NULL) {
	int tolerance = 0;

	crm_notice("Client %s wants to fence (%s) %s using %s",
	pcmk__request_origin(request), action,
	target, (device? device : "any device"));
	crm_element_value_int(dev, PCMK__XA_ST_TOLERANCE, &tolerance);
	if (stonith_check_fence_tolerance(tolerance, target, action)) {
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE,
	NULL);
	return fenced_construct_reply(request->xml, NULL,
	&request->result);
	}
	alternate_host = check_alternate_host(target);

	} else {
	crm_notice("Peer %s wants to fence (%s) '%s' with device '%s'",
	request->peer, action, target,
	(device == NULL)? "(any)" : device);
	}

	if (alternate_host != NULL) {
	const char *client_id = NULL;
	remote_fencing_op_t *op = NULL;

	if (request->ipc_client->id == 0) {
	client_id = crm_element_value(request->xml,
	PCMK__XA_ST_CLIENTID);
	} else {
	client_id = request->ipc_client->id;
	}

	/* Create a duplicate fencing operation to relay with the client ID.
	* When a query response is received, this operation should be
	* deleted to avoid keeping the duplicate around.
	*/
	op = create_remote_stonith_op(client_id, request->xml, FALSE);

	crm_xml_add(request->xml, PCMK__XA_ST_OP, STONITH_OP_RELAY);
	crm_xml_add(request->xml, PCMK__XA_ST_CLIENTID,
	request->ipc_client->id);
	crm_xml_add(request->xml, PCMK__XA_ST_REMOTE_OP, op->id);
	send_cluster_message(pcmk__get_node(0, alternate_host, NULL,
	pcmk__node_search_cluster),
	crm_msg_stonith_ng, request->xml, FALSE);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
	NULL);

	} else if (initiate_remote_stonith_op(request->ipc_client, request->xml,
	FALSE) == NULL) {
	fenced_set_protocol_error(&request->result);

	} else {
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_PENDING,
	NULL);
	}
	}

	if (request->result.execution_status == PCMK_EXEC_PENDING) {
	return NULL;
	}
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_FENCE_HISTORY
	static xmlNode *
	handle_history_request(pcmk__request_t *request)
	{
	xmlNode *reply = NULL;
	xmlNode *data = NULL;

	stonith_fence_history(request->xml, &data, request->peer,
	request->call_options);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	if (!pcmk_is_set(request->call_options, st_opt_discard_reply)) {
	/* When the local node broadcasts its history, it sets
	* st_opt_discard_reply and doesn't need a reply.
	*/
	reply = fenced_construct_reply(request->xml, data, &request->result);
	}
	free_xml(data);
	return reply;
	}

	// STONITH_OP_DEVICE_ADD
	static xmlNode *
	handle_device_add_request(pcmk__request_t *request)
	{
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);
	xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, request->xml,
	LOG_ERR);

	if (is_privileged(request->ipc_client, op)) {
	int rc = stonith_device_register(dev, FALSE);

	pcmk__set_result(&request->result,
	((rc == pcmk_ok)? CRM_EX_OK : CRM_EX_ERROR),
	stonith__legacy2status(rc),
	((rc == pcmk_ok)? NULL : pcmk_strerror(rc)));
	} else {
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must register device via CIB");
	}
	fenced_send_config_notification(op, &request->result,
	(dev == NULL)? NULL : pcmk__xe_id(dev));
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_DEVICE_DEL
	static xmlNode *
	handle_device_delete_request(pcmk__request_t *request)
	{
	xmlNode *dev = get_xpath_object("//" PCMK__XE_ST_DEVICE_ID, request->xml,
	LOG_ERR);
	const char *device_id = crm_element_value(dev, PCMK_XA_ID);
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);

	if (is_privileged(request->ipc_client, op)) {
	stonith_device_remove(device_id, false);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	} else {
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must delete device via CIB");
	}
	fenced_send_config_notification(op, &request->result, device_id);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_LEVEL_ADD
	static xmlNode *
	handle_level_add_request(pcmk__request_t *request)
	{
	char *desc = NULL;
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);

	if (is_privileged(request->ipc_client, op)) {
	fenced_register_level(request->xml, &desc, &request->result);
	} else {
	unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must add level via CIB");
	}
	fenced_send_config_notification(op, &request->result, desc);
	free(desc);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// STONITH_OP_LEVEL_DEL
	static xmlNode *
	handle_level_delete_request(pcmk__request_t *request)
	{
	char *desc = NULL;
	const char *op = crm_element_value(request->xml, PCMK__XA_ST_OP);

	if (is_privileged(request->ipc_client, op)) {
	fenced_unregister_level(request->xml, &desc, &request->result);
	} else {
	unpack_level_request(request->xml, NULL, NULL, NULL, &desc);
	pcmk__set_result(&request->result, CRM_EX_INSUFFICIENT_PRIV,
	PCMK_EXEC_INVALID,
	"Unprivileged users must delete level via CIB");
	}
	fenced_send_config_notification(op, &request->result, desc);
	free(desc);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	// CRM_OP_RM_NODE_CACHE
	static xmlNode *
	handle_cache_request(pcmk__request_t *request)
	{
	int node_id = 0;
	const char *name = NULL;

	crm_element_value_int(request->xml, PCMK_XA_ID, &node_id);
	name = crm_element_value(request->xml, PCMK_XA_UNAME);
	reap_crm_member(node_id, name);
	pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
	return NULL;
	}

	static xmlNode *
	handle_unknown_request(pcmk__request_t *request)
	{
	crm_err("Unknown IPC request %s from %s %s",
	request->op, pcmk__request_origin_type(request),
	pcmk__request_origin(request));
	pcmk__format_result(&request->result, CRM_EX_PROTOCOL, PCMK_EXEC_INVALID,
	"Unknown IPC request type '%s' (bug?)", request->op);
	return fenced_construct_reply(request->xml, NULL, &request->result);
	}

	static void
	fenced_register_handlers(void)
	{
	pcmk__server_command_t handlers[] = {
	{ CRM_OP_REGISTER, handle_register_request },
	{ STONITH_OP_EXEC, handle_agent_request },
	{ STONITH_OP_TIMEOUT_UPDATE, handle_update_timeout_request },
	{ STONITH_OP_QUERY, handle_query_request },
	{ STONITH_OP_NOTIFY, handle_notify_request },
	{ STONITH_OP_RELAY, handle_relay_request },
	{ STONITH_OP_FENCE, handle_fence_request },
	{ STONITH_OP_FENCE_HISTORY, handle_history_request },
	{ STONITH_OP_DEVICE_ADD, handle_device_add_request },
	{ STONITH_OP_DEVICE_DEL, handle_device_delete_request },
	{ STONITH_OP_LEVEL_ADD, handle_level_add_request },
	{ STONITH_OP_LEVEL_DEL, handle_level_delete_request },
	{ CRM_OP_RM_NODE_CACHE, handle_cache_request },
	{ NULL, handle_unknown_request },
	};

	fenced_handlers = pcmk__register_handlers(handlers);
	}

	void
	fenced_unregister_handlers(void)
	{
	if (fenced_handlers != NULL) {
	g_hash_table_destroy(fenced_handlers);
	fenced_handlers = NULL;
	}
	}

	static void
	handle_request(pcmk__request_t *request)
	{
	xmlNode *reply = NULL;
	const char *reason = NULL;

	if (fenced_handlers == NULL) {
	fenced_register_handlers();
	}
	reply = pcmk__process_request(request, fenced_handlers);
	if (reply != NULL) {
	if (pcmk_is_set(request->flags, pcmk__request_reuse_options)
	&& (request->ipc_client != NULL)) {
	/* Certain IPC-only commands must reuse the call options from the
	* original request rather than the ones set by stonith_send_reply()
	* -> do_local_reply().
	*/
	pcmk__ipc_send_xml(request->ipc_client, request->ipc_id, reply,
	request->ipc_flags);
	request->ipc_client->request_id = 0;
	} else {
	stonith_send_reply(reply, request->call_options,
	request->peer, request->ipc_client);
	}
	free_xml(reply);
	}

	reason = request->result.exit_reason;
	crm_debug("Processed %s request from %s %s: %s%s%s%s",
	request->op, pcmk__request_origin_type(request),
	pcmk__request_origin(request),
	pcmk_exec_status_str(request->result.execution_status),
	(reason == NULL)? "" : " (",
	(reason == NULL)? "" : reason,
	(reason == NULL)? "" : ")");
	}

	static void
	handle_reply(pcmk__client_t client, xmlNode request, const char *remote_peer)
	{
	// Copy, because request might be freed before we want to log this
	char *op = crm_element_value_copy(request, PCMK__XA_ST_OP);

	if (pcmk__str_eq(op, STONITH_OP_QUERY, pcmk__str_none)) {
	process_remote_stonith_query(request);

	} else if (pcmk__str_any_of(op, STONITH_OP_NOTIFY, STONITH_OP_FENCE,
	NULL)) {
	fenced_process_fencing_reply(request);

	} else {
	crm_err("Ignoring unknown %s reply from %s %s",
	pcmk__s(op, "untyped"), ((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));
	crm_log_xml_warn(request, "UnknownOp");
	free(op);
	return;
	}
	crm_debug("Processed %s reply from %s %s",
	op, ((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));
	free(op);
	}

	/*!
	* \internal
	* \brief Handle a message from an IPC client or CPG peer
	*
	* \param[in,out] client If not NULL, IPC client that sent message
	* \param[in] id If from IPC client, IPC message ID
	* \param[in] flags Message flags
	* \param[in,out] message Message XML
	* \param[in] remote_peer If not NULL, CPG peer that sent message
	*/
	void
	stonith_command(pcmk__client_t *client, uint32_t id, uint32_t flags,
	xmlNode message, const char remote_peer)
	{
	int call_options = st_opt_none;
	bool is_reply = false;

	CRM_CHECK(message != NULL, return);

	if (get_xpath_object("//" PCMK__XE_ST_REPLY, message, LOG_NEVER) != NULL) {
	is_reply = true;
	}
	crm_element_value_int(message, PCMK__XA_ST_CALLOPT, &call_options);
	crm_debug("Processing %ssynchronous %s %s %u from %s %s",
	pcmk_is_set(call_options, st_opt_sync_call)? "" : "a",
	crm_element_value(message, PCMK__XA_ST_OP),
	(is_reply? "reply" : "request"), id,
	((client == NULL)? "peer" : "client"),
	((client == NULL)? remote_peer : pcmk__client_name(client)));

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(client == NULL \|\| client->request_id == id);
	}

	if (is_reply) {
	handle_reply(client, message, remote_peer);
	} else {
	pcmk__request_t request = {
	.ipc_client = client,
	.ipc_id = id,
	.ipc_flags = flags,
	.peer = remote_peer,
	.xml = message,
	.call_options = call_options,
	.result = PCMK__UNKNOWN_RESULT,
	};

	request.op = crm_element_value_copy(request.xml, PCMK__XA_ST_OP);
	CRM_CHECK(request.op != NULL, return);

	if (pcmk_is_set(request.call_options, st_opt_sync_call)) {
	pcmk__set_request_flags(&request, pcmk__request_sync);
	}

	handle_request(&request);
	pcmk__reset_request(&request);
	}
	}
	diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
	index f4325e5236..b8d21be591 100644
	--- a/daemons/fenced/pacemaker-fenced.c
	+++ b/daemons/fenced/pacemaker-fenced.c
	@@ -1,967 +1,967 @@
	/*
	* Copyright 2009-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <inttypes.h> // PRIu32, PRIx32

	#include <crm/crm.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/output_internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib/internal.h>

	#include <pacemaker-fenced.h>

	#define SUMMARY "daemon for executing fencing devices in a Pacemaker cluster"

	char *stonith_our_uname = NULL;
	long long stonith_watchdog_timeout_ms = 0;
	GList *stonith_watchdog_targets = NULL;

	static GMainLoop *mainloop = NULL;

	gboolean stand_alone = FALSE;
	gboolean stonith_shutdown_flag = FALSE;

	static qb_ipcs_service_t *ipcs = NULL;
	static pcmk__output_t *out = NULL;

	pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	static struct {
	bool no_cib_connect;
	gchar **log_files;
	} options;

	crm_exit_t exit_code = CRM_EX_OK;

	static void stonith_cleanup(void);

	static int32_t
	st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (stonith_shutdown_flag) {
	crm_info("Ignoring new client [%d] during shutdown",
	pcmk__client_pid(c));
	return -ECONNREFUSED;
	}

	if (pcmk__new_client(c, uid, gid) == NULL) {
	return -ENOMEM;
	}
	return 0;
	}

	/* Exit code means? */
	static int32_t
	st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	xmlNode *request = NULL;
	pcmk__client_t *c = pcmk__find_client(qbc);
	const char *op = NULL;

	if (c == NULL) {
	crm_info("Invalid client: %p", qbc);
	return 0;
	}

	request = pcmk__client_data2xml(c, data, &id, &flags);
	if (request == NULL) {
	pcmk__ipc_send_ack(c, id, flags, PCMK__XE_NACK, NULL, CRM_EX_PROTOCOL);
	return 0;
	}


	op = crm_element_value(request, PCMK__XA_CRM_TASK);
	if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
	crm_xml_add(request, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(request, PCMK__XA_ST_OP, op);
	crm_xml_add(request, PCMK__XA_ST_CLIENTID, c->id);
	crm_xml_add(request, PCMK__XA_ST_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, PCMK__XA_ST_CLIENTNODE, stonith_our_uname);

	send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE);
	free_xml(request);
	return 0;
	}

	if (c->name == NULL) {
	const char *value = crm_element_value(request, PCMK__XA_ST_CLIENTNAME);

	c->name = crm_strdup_printf("%s.%u", pcmk__s(value, "unknown"), c->pid);
	}

	crm_element_value_int(request, PCMK__XA_ST_CALLOPT, &call_options);
	crm_trace("Flags %#08" PRIx32 "/%#08x for command %" PRIu32
	" from client %s", flags, call_options, id, pcmk__client_name(c));

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(c->request_id == 0); /* This means the client has two synchronous events in-flight */
	c->request_id = id; /* Reply only to the last one */
	}

	crm_xml_add(request, PCMK__XA_ST_CLIENTID, c->id);
	crm_xml_add(request, PCMK__XA_ST_CLIENTNAME, pcmk__client_name(c));
	crm_xml_add(request, PCMK__XA_ST_CLIENTNODE, stonith_our_uname);

	crm_log_xml_trace(request, "ipc-received");
	stonith_command(c, id, flags, request, NULL);

	free_xml(request);
	return 0;
	}

	/* Error code means? */
	static int32_t
	st_ipc_closed(qb_ipcs_connection_t * c)
	{
	pcmk__client_t *client = pcmk__find_client(c);

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p closed", c);
	pcmk__free_client(client);

	/* 0 means: yes, go ahead and destroy the connection */
	return 0;
	}

	static void
	st_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p destroyed", c);
	st_ipc_closed(c);
	}

	static void
	stonith_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *remote_peer = crm_element_value(msg, PCMK__XA_SRC);
	const char *op = crm_element_value(msg, PCMK__XA_ST_OP);

	if (pcmk__str_eq(op, STONITH_OP_POKE, pcmk__str_none)) {
	return;
	}

	crm_log_xml_trace(msg, "Peer[inbound]");
	stonith_command(NULL, 0, 0, msg, remote_peer);
	}

	#if SUPPORT_COROSYNC
	static void
	stonith_peer_ais_callback(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	xmlNode *xml = NULL;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	- xml = string2xml(data);
	+ xml = pcmk__xml_parse_string(data);
	if (xml == NULL) {
	crm_err("Invalid XML: '%.120s'", data);
	free(data);
	return;
	}
	crm_xml_add(xml, PCMK__XA_SRC, from);
	stonith_peer_callback(xml, NULL);
	}

	free_xml(xml);
	free(data);
	return;
	}

	static void
	stonith_peer_cs_destroy(gpointer user_data)
	{
	crm_crit("Lost connection to cluster layer, shutting down");
	stonith_shutdown(0);
	}
	#endif

	void
	do_local_reply(const xmlNode notify_src, pcmk__client_t client,
	int call_options)
	{
	/* send callback to originating child */
	int local_rc = pcmk_rc_ok;
	int rid = 0;
	uint32_t ipc_flags = crm_ipc_server_event;

	if (pcmk_is_set(call_options, st_opt_sync_call)) {
	CRM_LOG_ASSERT(client->request_id);
	rid = client->request_id;
	client->request_id = 0;
	ipc_flags = crm_ipc_flags_none;
	}

	local_rc = pcmk__ipc_send_xml(client, rid, notify_src, ipc_flags);
	if (local_rc == pcmk_rc_ok) {
	crm_trace("Sent response %d to client %s",
	rid, pcmk__client_name(client));
	} else {
	crm_warn("%synchronous reply to client %s failed: %s",
	(pcmk_is_set(call_options, st_opt_sync_call)? "S" : "As"),
	pcmk__client_name(client), pcmk_rc_str(local_rc));
	}
	}

	uint64_t
	get_stonith_flag(const char *name)
	{
	if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_FENCE, pcmk__str_none)) {
	return st_callback_notify_fence;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) {
	return st_callback_device_add;

	} else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) {
	return st_callback_device_del;

	} else if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_HISTORY,
	pcmk__str_none)) {
	return st_callback_notify_history;

	} else if (pcmk__str_eq(name, PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED,
	pcmk__str_none)) {
	return st_callback_notify_history_synced;

	}
	return st_callback_unknown;
	}

	static void
	stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
	{

	const xmlNode *update_msg = user_data;
	pcmk__client_t *client = value;
	const char *type = NULL;

	CRM_CHECK(client != NULL, return);
	CRM_CHECK(update_msg != NULL, return);

	type = crm_element_value(update_msg, PCMK__XA_SUBT);
	CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);

	if (client->ipcs == NULL) {
	crm_trace("Skipping client with NULL channel");
	return;
	}

	if (pcmk_is_set(client->flags, get_stonith_flag(type))) {
	int rc = pcmk__ipc_send_xml(client, 0, update_msg,
	crm_ipc_server_event);

	if (rc != pcmk_rc_ok) {
	crm_warn("%s notification of client %s failed: %s "
	CRM_XS " id=%.8s rc=%d", type, pcmk__client_name(client),
	pcmk_rc_str(rc), client->id, rc);
	} else {
	crm_trace("Sent %s notification to client %s",
	type, pcmk__client_name(client));
	}
	}
	}

	void
	do_stonith_async_timeout_update(const char client_id, const char call_id, int timeout)
	{
	pcmk__client_t *client = NULL;
	xmlNode *notify_data = NULL;

	if (!timeout \|\| !call_id \|\| !client_id) {
	return;
	}

	client = pcmk__find_client_by_id(client_id);
	if (!client) {
	return;
	}

	notify_data = create_xml_node(NULL, PCMK__XE_ST_ASYNC_TIMEOUT_VALUE);
	crm_xml_add(notify_data, PCMK__XA_T, PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE);
	crm_xml_add(notify_data, PCMK__XA_ST_CALLID, call_id);
	crm_xml_add_int(notify_data, PCMK__XA_ST_TIMEOUT, timeout);

	crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);

	if (client) {
	pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event);
	}

	free_xml(notify_data);
	}

	/*!
	* \internal
	* \brief Notify relevant IPC clients of a fencing operation result
	*
	* \param[in] type Notification type
	* \param[in] result Result of fencing operation (assume success if NULL)
	* \param[in] data If not NULL, add to notification as call data
	*/
	void
	fenced_send_notification(const char type, const pcmk__action_result_t result,
	xmlNode *data)
	{
	/* TODO: Standardize the contents of data */
	xmlNode *update_msg = create_xml_node(NULL, PCMK__XE_NOTIFY);

	CRM_LOG_ASSERT(type != NULL);

	crm_xml_add(update_msg, PCMK__XA_T, PCMK__VALUE_ST_NOTIFY);
	crm_xml_add(update_msg, PCMK__XA_SUBT, type);
	crm_xml_add(update_msg, PCMK__XA_ST_OP, type);
	stonith__xe_set_result(update_msg, result);

	if (data != NULL) {
	add_message_xml(update_msg, PCMK__XA_ST_CALLDATA, data);
	}

	crm_trace("Notifying clients");
	pcmk__foreach_ipc_client(stonith_notify_client, update_msg);
	free_xml(update_msg);
	crm_trace("Notify complete");
	}

	/*!
	* \internal
	* \brief Send notifications for a configuration change to subscribed clients
	*
	* \param[in] op Notification type (\c STONITH_OP_DEVICE_ADD,
	* \c STONITH_OP_DEVICE_DEL, \c STONITH_OP_LEVEL_ADD, or
	* \c STONITH_OP_LEVEL_DEL)
	* \param[in] result Operation result
	* \param[in] desc Description of what changed (either device ID or string
	* representation of level
	* (<tt><target>[<level_index>]</tt>))
	*/
	void
	fenced_send_config_notification(const char *op,
	const pcmk__action_result_t *result,
	const char *desc)
	{
	xmlNode *notify_data = create_xml_node(NULL, op);

	CRM_CHECK(notify_data != NULL, return);

	crm_xml_add(notify_data, PCMK__XA_ST_DEVICE_ID, desc);

	fenced_send_notification(op, result, notify_data);
	free_xml(notify_data);
	}

	/*!
	* \internal
	* \brief Check whether a node does watchdog-fencing
	*
	* \param[in] node Name of node to check
	*
	* \return TRUE if node found in stonith_watchdog_targets
	* or stonith_watchdog_targets is empty indicating
	* all nodes are doing watchdog-fencing
	*/
	gboolean
	node_does_watchdog_fencing(const char *node)
	{
	return ((stonith_watchdog_targets == NULL) \|\|
	pcmk__str_in_list(node, stonith_watchdog_targets, pcmk__str_casei));
	}

	void
	stonith_shutdown(int nsig)
	{
	crm_info("Terminating with %d clients", pcmk__ipc_client_count());
	stonith_shutdown_flag = TRUE;
	if (mainloop != NULL && g_main_loop_is_running(mainloop)) {
	g_main_loop_quit(mainloop);
	}
	}

	static void
	stonith_cleanup(void)
	{
	fenced_cib_cleanup();
	if (ipcs) {
	qb_ipcs_destroy(ipcs);
	}

	crm_peer_destroy();
	pcmk__client_cleanup();
	free_stonith_remote_op_list();
	free_topology_list();
	free_device_list();
	free_metadata_cache();
	fenced_unregister_handlers();

	free(stonith_our_uname);
	stonith_our_uname = NULL;
	}

	static gboolean
	stand_alone_cpg_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	stand_alone = FALSE;
	options.no_cib_connect = true;
	return TRUE;
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = st_ipc_accept,
	.connection_created = NULL,
	.msg_process = st_ipc_dispatch,
	.connection_closed = st_ipc_closed,
	.connection_destroyed = st_ipc_destroy
	};

	/*!
	* \internal
	* \brief Callback for peer status changes
	*
	* \param[in] type What changed
	* \param[in] node What peer had the change
	* \param[in] data Previous value of what changed
	*/
	static void
	st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
	{
	if ((type != crm_status_processes)
	&& !pcmk_is_set(node->flags, crm_remote_node)) {
	/*
	* This is a hack until we can send to a nodeid and/or we fix node name lookups
	* These messages are ignored in stonith_peer_callback()
	*/
	xmlNode *query = create_xml_node(NULL, PCMK__XE_STONITH_COMMAND);

	crm_xml_add(query, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(query, PCMK__XA_ST_OP, STONITH_OP_POKE);

	crm_debug("Broadcasting our uname because of node %u", node->id);
	send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);

	free_xml(query);
	}
	}

	static pcmk__cluster_option_t fencer_options[] = {
	/* name, old name, type, allowed values,
	* default value, validator,
	* context,
	* short description,
	* long description
	*/
	{
	PCMK_STONITH_HOST_ARGUMENT, NULL, "string", NULL,
	"port", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"An alternate parameter to supply instead of 'port'"),
	N_("Some devices do not support the standard 'port' parameter or may "
	"provide additional ones. Use this to specify an alternate, device-"
	"specific, parameter that should indicate the machine to be "
	"fenced. A value of \"none\" can be used to tell the cluster not "
	"to supply any additional parameters."),
	},
	{
	PCMK_STONITH_HOST_MAP, NULL, "string", NULL,
	"", NULL,
	pcmk__opt_context_none,
	N_("A mapping of node names to port numbers for devices that do not "
	"support node names."),
	N_("For example, \"node1:1;node2:2,3\" would tell the cluster to use "
	"port 1 for node1 and ports 2 and 3 for node2."),
	},
	{
	PCMK_STONITH_HOST_LIST, NULL, "string", NULL,
	"", NULL,
	pcmk__opt_context_none,
	N_("A list of nodes that can be targeted by this device (optional "
	"unless pcmk_host_list=\"static-list\")"),
	N_("For example, \"node1,node2,node3\"."),
	},
	{
	PCMK_STONITH_HOST_CHECK, NULL, "select",
	"dynamic-list, static-list, status, none",
	"dynamic-list", NULL,
	pcmk__opt_context_none,
	N_("How to determine which nodes can be targeted by the device"),
	N_("Use \"dynamic-list\" to query the device via the 'list' command; "
	"\"static-list\" to check the pcmk_host_list attribute; "
	"\"status\" to query the device via the 'status' command; or "
	"\"none\" to assume every device can fence every node."),
	},
	{
	PCMK_STONITH_DELAY_MAX, NULL, "time", NULL,
	"0s", NULL,
	pcmk__opt_context_none,
	N_("Enable a base delay for fencing actions and specify base delay "
	"value."),
	N_("Enable a delay of no more than the time specified before executing "
	"fencing actions. Pacemaker derives the overall delay by taking "
	"the value of pcmk_delay_base and adding a random delay value such "
	"that the sum is kept below this maximum."),
	},
	{
	PCMK_STONITH_DELAY_BASE, NULL, "string", NULL,
	"0s", NULL,
	pcmk__opt_context_none,
	N_("Enable a base delay for fencing actions and specify base delay "
	"value."),
	N_("This enables a static delay for fencing actions, which can help "
	"avoid \"death matches\" where two nodes try to fence each other "
	"at the same time. If pcmk_delay_max is also used, a random delay "
	"will be added such that the total delay is kept below that value. "
	"This can be set to a single time value to apply to any node "
	"targeted by this device (useful if a separate device is "
	"configured for each target), or to a node map (for example, "
	"\"node1:1s;node2:5\") to set a different value for each target."),
	},
	{
	PCMK_STONITH_ACTION_LIMIT, NULL, "integer", NULL,
	"1", NULL,
	pcmk__opt_context_none,
	N_("The maximum number of actions can be performed in parallel on this "
	"device"),
	N_("Cluster property concurrent-fencing=\"true\" needs to be "
	"configured first. Then use this to specify the maximum number of "
	"actions can be performed in parallel on this device. A value of "
	"-1 means an unlimited number of actions can be performed in "
	"parallel."),
	},
	{
	"pcmk_reboot_action", NULL, "string", NULL,
	PCMK_ACTION_REBOOT, NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"An alternate command to run instead of 'reboot'"),
	N_("Some devices do not support the standard commands or may provide "
	"additional ones. Use this to specify an alternate, device-"
	"specific, command that implements the 'reboot' action."),
	},
	{
	"pcmk_reboot_timeout", NULL, "time", NULL,
	"60s", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"Specify an alternate timeout to use for 'reboot' actions instead "
	"of stonith-timeout"),
	N_("Some devices need much more/less time to complete than normal. "
	"Use this to specify an alternate, device-specific, timeout for "
	"'reboot' actions."),
	},
	{
	"pcmk_reboot_retries", NULL, "integer", NULL,
	"2", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"The maximum number of times to try the 'reboot' command within "
	"the timeout period"),
	N_("Some devices do not support multiple connections. Operations may "
	"\"fail\" if the device is busy with another task. In that case, "
	"Pacemaker will automatically retry the operation if there is time "
	"remaining. Use this option to alter the number of times Pacemaker "
	"tries a 'reboot' action before giving up."),
	},
	{
	"pcmk_off_action", NULL, "string", NULL,
	PCMK_ACTION_OFF, NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"An alternate command to run instead of 'off'"),
	N_("Some devices do not support the standard commands or may provide "
	"additional ones. Use this to specify an alternate, device-"
	"specific, command that implements the 'off' action."),
	},
	{
	"pcmk_off_timeout", NULL, "time", NULL,
	"60s", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"Specify an alternate timeout to use for 'off' actions instead of "
	"stonith-timeout"),
	N_("Some devices need much more/less time to complete than normal. "
	"Use this to specify an alternate, device-specific, timeout for "
	"'off' actions."),
	},
	{
	"pcmk_off_retries", NULL, "integer", NULL,
	"2", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"The maximum number of times to try the 'off' command within the "
	"timeout period"),
	N_("Some devices do not support multiple connections. Operations may "
	"\"fail\" if the device is busy with another task. In that case, "
	"Pacemaker will automatically retry the operation if there is time "
	"remaining. Use this option to alter the number of times Pacemaker "
	"tries a 'off' action before giving up."),
	},
	{
	"pcmk_on_action", NULL, "string", NULL,
	PCMK_ACTION_ON, NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"An alternate command to run instead of 'on'"),
	N_("Some devices do not support the standard commands or may provide "
	"additional ones. Use this to specify an alternate, device-"
	"specific, command that implements the 'on' action."),
	},
	{
	"pcmk_on_timeout", NULL, "time", NULL,
	"60s", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"Specify an alternate timeout to use for 'on' actions instead of "
	"stonith-timeout"),
	N_("Some devices need much more/less time to complete than normal. "
	"Use this to specify an alternate, device-specific, timeout for "
	"'on' actions."),
	},
	{
	"pcmk_on_retries", NULL, "integer", NULL,
	"2", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"The maximum number of times to try the 'on' command within the "
	"timeout period"),
	N_("Some devices do not support multiple connections. Operations may "
	"\"fail\" if the device is busy with another task. In that case, "
	"Pacemaker will automatically retry the operation if there is time "
	"remaining. Use this option to alter the number of times Pacemaker "
	"tries a 'on' action before giving up."),
	},
	{
	"pcmk_list_action", NULL, "string", NULL,
	PCMK_ACTION_LIST, NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"An alternate command to run instead of 'list'"),
	N_("Some devices do not support the standard commands or may provide "
	"additional ones. Use this to specify an alternate, device-"
	"specific, command that implements the 'list' action."),
	},
	{
	"pcmk_list_timeout", NULL, "time", NULL,
	"60s", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"Specify an alternate timeout to use for 'list' actions instead of "
	"stonith-timeout"),
	N_("Some devices need much more/less time to complete than normal. "
	"Use this to specify an alternate, device-specific, timeout for "
	"'list' actions."),
	},
	{
	"pcmk_list_retries", NULL, "integer", NULL,
	"2", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"The maximum number of times to try the 'list' command within the "
	"timeout period"),
	N_("Some devices do not support multiple connections. Operations may "
	"\"fail\" if the device is busy with another task. In that case, "
	"Pacemaker will automatically retry the operation if there is time "
	"remaining. Use this option to alter the number of times Pacemaker "
	"tries a 'list' action before giving up."),
	},
	{
	"pcmk_monitor_action", NULL, "string", NULL,
	PCMK_ACTION_MONITOR, NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"An alternate command to run instead of 'monitor'"),
	N_("Some devices do not support the standard commands or may provide "
	"additional ones. Use this to specify an alternate, device-"
	"specific, command that implements the 'monitor' action."),
	},
	{
	"pcmk_monitor_timeout", NULL, "time", NULL,
	"60s", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"Specify an alternate timeout to use for 'monitor' actions instead "
	"of stonith-timeout"),
	N_("Some devices need much more/less time to complete than normal. "
	"Use this to specify an alternate, device-specific, timeout for "
	"'monitor' actions."),
	},
	{
	"pcmk_monitor_retries", NULL, "integer", NULL,
	"2", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"The maximum number of times to try the 'monitor' command within "
	"the timeout period"),
	N_("Some devices do not support multiple connections. Operations may "
	"\"fail\" if the device is busy with another task. In that case, "
	"Pacemaker will automatically retry the operation if there is time "
	"remaining. Use this option to alter the number of times Pacemaker "
	"tries a 'monitor' action before giving up."),
	},
	{
	"pcmk_status_action", NULL, "string", NULL,
	PCMK_ACTION_STATUS, NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"An alternate command to run instead of 'status'"),
	N_("Some devices do not support the standard commands or may provide "
	"additional ones. Use this to specify an alternate, device-"
	"specific, command that implements the 'status' action."),
	},
	{
	"pcmk_status_timeout", NULL, "time", NULL,
	"60s", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"Specify an alternate timeout to use for 'status' actions instead "
	"of stonith-timeout"),
	N_("Some devices need much more/less time to complete than normal. "
	"Use this to specify an alternate, device-specific, timeout for "
	"'status' actions."),
	},
	{
	"pcmk_status_retries", NULL, "integer", NULL,
	"2", NULL,
	pcmk__opt_context_none,
	N_("* Advanced Use Only * "
	"The maximum number of times to try the 'status' command within "
	"the timeout period"),
	N_("Some devices do not support multiple connections. Operations may "
	"\"fail\" if the device is busy with another task. In that case, "
	"Pacemaker will automatically retry the operation if there is time "
	"remaining. Use this option to alter the number of times Pacemaker "
	"tries a 'status' action before giving up."),
	},
	};

	void
	fencer_metadata(void)
	{
	const char *name = "pacemaker-fenced";
	const char *desc_short = N_("Instance attributes available for all "
	"\"stonith\"-class resources");
	const char *desc_long = N_("Instance attributes available for all "
	"\"stonith\"-class resources and used by "
	"Pacemaker's fence daemon, formerly known as "
	"stonithd");

	char *s = pcmk__format_option_metadata(name, desc_short, desc_long,
	pcmk__opt_context_none,
	fencer_options,
	PCMK__NELEM(fencer_options));
	printf("%s", s);
	free(s);
	}

	static GOptionEntry entries[] = {
	{ "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone,
	N_("Deprecated (will be removed in a future release)"), NULL },

	{ "stand-alone-w-cpg", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	stand_alone_cpg_cb, N_("Intended for use in regression testing only"), NULL },

	{ "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
	&options.log_files, N_("Send logs to the additional named logfile"), NULL },

	{ NULL }
	};

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group)
	{
	GOptionContext *context = NULL;

	context = pcmk__build_arg_context(args, "text (default), xml", group,
	"[metadata]");
	pcmk__add_main_args(context, entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	crm_cluster_t *cluster = NULL;
	crm_ipc_t *old_instance = NULL;

	GError *error = NULL;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "l");
	GOptionContext *context = build_arg_context(args, &output_group);

	crm_log_preinit(NULL, argc, argv);

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_ERROR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error creating output format %s: %s",
	args->output_ty, pcmk_rc_str(rc));
	goto done;
	}

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	if ((g_strv_length(processed_args) >= 2)
	&& pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) {
	fencer_metadata();
	goto done;
	}

	// Open additional log files
	pcmk__add_logfiles(options.log_files, out);

	crm_log_init(NULL, LOG_INFO + args->verbosity, TRUE,
	(args->verbosity > 0), argc, argv, FALSE);

	crm_notice("Starting Pacemaker fencer");

	old_instance = crm_ipc_new("stonith-ng", 0);
	if (old_instance == NULL) {
	/* crm_ipc_new() will have already logged an error message with
	* crm_err()
	*/
	exit_code = CRM_EX_FATAL;
	goto done;
	}

	if (pcmk__connect_generic_ipc(old_instance) == pcmk_rc_ok) {
	// IPC endpoint already up
	crm_ipc_close(old_instance);
	crm_ipc_destroy(old_instance);
	crm_err("pacemaker-fenced is already active, aborting startup");
	goto done;
	} else {
	// Not up or not authentic, we'll proceed either way
	crm_ipc_destroy(old_instance);
	old_instance = NULL;
	}

	mainloop_add_signal(SIGTERM, stonith_shutdown);

	crm_peer_init();

	rc = fenced_scheduler_init();
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_FATAL;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Error initializing scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}

	cluster = pcmk_cluster_new();

	if (!stand_alone) {
	#if SUPPORT_COROSYNC
	if (is_corosync_cluster()) {
	cluster->destroy = stonith_peer_cs_destroy;
	cluster->cpg.cpg_deliver_fn = stonith_peer_ais_callback;
	cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
	}
	#endif // SUPPORT_COROSYNC

	crm_set_status_callback(&st_peer_update_callback);

	if (crm_cluster_connect(cluster) == FALSE) {
	exit_code = CRM_EX_FATAL;
	crm_crit("Cannot sign in to the cluster... terminating");
	goto done;
	}
	pcmk__str_update(&stonith_our_uname, cluster->uname);

	if (!options.no_cib_connect) {
	setup_cib();
	}

	} else {
	pcmk__str_update(&stonith_our_uname, "localhost");
	crm_warn("Stand-alone mode is deprecated and will be removed "
	"in a future release");
	}

	init_device_list();
	init_topology_list();

	pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks);

	// Create the mainloop and run it...
	mainloop = g_main_loop_new(NULL, FALSE);
	crm_notice("Pacemaker fencer successfully started and accepting connections");
	g_main_loop_run(mainloop);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	g_strfreev(options.log_files);

	stonith_cleanup();
	pcmk_cluster_free(cluster);
	fenced_scheduler_cleanup();

	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}

	pcmk__unregister_formats();
	crm_exit(exit_code);
	}
	diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h
	index e03dbb36eb..a1f9a204d0 100644
	--- a/include/crm/common/xml_internal.h
	+++ b/include/crm/common/xml_internal.h
	@@ -1,483 +1,484 @@
	/*
	* Copyright 2017-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#ifndef PCMK__XML_INTERNAL__H
	# define PCMK__XML_INTERNAL__H

	/*
	* Internal-only wrappers for and extensions to libxml2 (libxslt)
	*/

	# include <stdlib.h>
	# include <stdio.h>
	# include <string.h>

	# include <crm/crm.h> /* transitively imports qblog.h */
	# include <crm/common/output_internal.h>

	# include <libxml/relaxng.h>

	/*!
	* \brief Base for directing lib{xml2,xslt} log into standard libqb backend
	*
	* This macro implements the core of what can be needed for directing
	* libxml2 or libxslt error messaging into standard, preconfigured
	* libqb-backed log stream.
	*
	* It's a bit unfortunate that libxml2 (and more sparsely, also libxslt)
	* emits a single message by chunks (location is emitted separatedly from
	* the message itself), so we have to take the effort to combine these
	* chunks back to single message. Whether to do this or not is driven
	* with \p dechunk toggle.
	*
	* The form of a macro was chosen for implicit deriving of __FILE__, etc.
	* and also because static dechunking buffer should be differentiated per
	* library (here we assume different functions referring to this macro
	* will not ever be using both at once), preferably also per-library
	* context of use to avoid clashes altogether.
	*
	* Note that we cannot use qb_logt, because callsite data have to be known
	* at the moment of compilation, which it is not always the case -- xml_log
	* (and unfortunately there's no clear explanation of the fail to compile).
	*
	* Also note that there's no explicit guard against said libraries producing
	* never-newline-terminated chunks (which would just keep consuming memory),
	* as it's quite improbable. Termination of the program in between the
	* same-message chunks will raise a flag with valgrind and the likes, though.
	*
	* And lastly, regarding how dechunking combines with other non-message
	* parameters -- for \p priority, most important running specification
	* wins (possibly elevated to LOG_ERR in case of nonconformance with the
	* newline-termination "protocol"), \p dechunk is expected to always be
	* on once it was at the start, and the rest (\p postemit and \p prefix)
	* are picked directly from the last chunk entry finalizing the message
	* (also reasonable to always have it the same with all related entries).
	*
	* \param[in] priority Syslog priority for the message to be logged
	* \param[in] dechunk Whether to dechunk new-line terminated message
	* \param[in] postemit Code to be executed once message is sent out
	* \param[in] prefix How to prefix the message or NULL for raw passing
	* \param[in] fmt Format string as with printf-like functions
	* \param[in] ap Variable argument list to supplement \p fmt format string
	*/
	#define PCMK__XML_LOG_BASE(priority, dechunk, postemit, prefix, fmt, ap) \
	do { \
	if (!(dechunk) && (prefix) == NULL) { /* quick pass */ \
	qb_log_from_external_source_va(__func__, __FILE__, (fmt), \
	(priority), __LINE__, 0, (ap)); \
	(void) (postemit); \
	} else { \
	int CXLB_len = 0; \
	char *CXLB_buf = NULL; \
	static int CXLB_buffer_len = 0; \
	static char *CXLB_buffer = NULL; \
	static uint8_t CXLB_priority = 0; \
	\
	CXLB_len = vasprintf(&CXLB_buf, (fmt), (ap)); \
	\
	if (CXLB_len <= 0 \|\| CXLB_buf[CXLB_len - 1] == '\n' \|\| !(dechunk)) { \
	if (CXLB_len < 0) { \
	CXLB_buf = (char ) "LOG CORRUPTION HAZARD"; /we don't modify*/\
	CXLB_priority = QB_MIN(CXLB_priority, LOG_ERR); \
	} else if (CXLB_len > 0 /* && (dechunk) */ \
	&& CXLB_buf[CXLB_len - 1] == '\n') { \
	CXLB_buf[CXLB_len - 1] = '\0'; \
	} \
	if (CXLB_buffer) { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s%s", \
	CXLB_priority, __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buffer, CXLB_buf); \
	free(CXLB_buffer); \
	} else { \
	qb_log_from_external_source(__func__, __FILE__, "%s%s", \
	(priority), __LINE__, 0, \
	(prefix) != NULL ? (prefix) : "", \
	CXLB_buf); \
	} \
	if (CXLB_len < 0) { \
	CXLB_buf = NULL; /* restore temporary override */ \
	} \
	CXLB_buffer = NULL; \
	CXLB_buffer_len = 0; \
	(void) (postemit); \
	\
	} else if (CXLB_buffer == NULL) { \
	CXLB_buffer_len = CXLB_len; \
	CXLB_buffer = CXLB_buf; \
	CXLB_buf = NULL; \
	CXLB_priority = (priority); /* remember as a running severest */ \
	\
	} else { \
	CXLB_buffer = realloc(CXLB_buffer, 1 + CXLB_buffer_len + CXLB_len); \
	memcpy(CXLB_buffer + CXLB_buffer_len, CXLB_buf, CXLB_len); \
	CXLB_buffer_len += CXLB_len; \
	CXLB_buffer[CXLB_buffer_len] = '\0'; \
	CXLB_priority = QB_MIN(CXLB_priority, (priority)); /* severest? */ \
	} \
	free(CXLB_buf); \
	} \
	} while (0)

	/*
	* \enum pcmk__xml_fmt_options
	* \brief Bit flags to control format in XML logs and dumps
	*/
	enum pcmk__xml_fmt_options {
	//! Exclude certain XML attributes (for calculating digests)
	pcmk__xml_fmt_filtered = (1 << 0),

	//! Include indentation and newlines
	pcmk__xml_fmt_pretty = (1 << 1),

	//! Include the opening tag of an XML element, and include XML comments
	pcmk__xml_fmt_open = (1 << 3),

	//! Include the children of an XML element
	pcmk__xml_fmt_children = (1 << 4),

	//! Include the closing tag of an XML element
	pcmk__xml_fmt_close = (1 << 5),

	//! Include XML text nodes
	pcmk__xml_fmt_text = (1 << 6),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a created XML subtree
	pcmk__xml_fmt_diff_plus = (1 << 7),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a removed XML subtree
	pcmk__xml_fmt_diff_minus = (1 << 8),

	// @COMPAT Remove when v1 patchsets are removed
	//! Log a minimal version of an XML diff (only showing the changes)
	pcmk__xml_fmt_diff_short = (1 << 9),
	};

	int pcmk__xml_show(pcmk__output_t out, const char prefix, const xmlNode *data,
	int depth, uint32_t options);
	int pcmk__xml_show_changes(pcmk__output_t out, const xmlNode xml);

	/* XML search strings for guest, remote and pacemaker_remote nodes */

	/* search string to find CIB resources entries for cluster nodes */
	#define PCMK__XP_MEMBER_NODE_CONFIG \
	"//" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_NODES \
	"/" PCMK_XE_NODE "[not(@type) or @type='member']"

	/* search string to find CIB resources entries for guest nodes */
	#define PCMK__XP_GUEST_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" PCMK_XE_PRIMITIVE \
	"//" PCMK_XE_META_ATTRIBUTES "//" PCMK_XE_NVPAIR \
	"[@name='" PCMK_META_REMOTE_NODE "']"

	/* search string to find CIB resources entries for remote nodes */
	#define PCMK__XP_REMOTE_NODE_CONFIG \
	"//" PCMK_XE_CIB "//" PCMK_XE_CONFIGURATION "//" PCMK_XE_PRIMITIVE \
	"[@type='remote'][@provider='pacemaker']"

	/* search string to find CIB node status entries for pacemaker_remote nodes */
	#define PCMK__XP_REMOTE_NODE_STATUS \
	"//" PCMK_XE_CIB "//" PCMK_XE_STATUS "//" PCMK__XE_NODE_STATE \
	"[@" PCMK_XA_REMOTE_NODE "='true']"
	/*!
	* \internal
	* \brief Serialize XML (using libxml) into provided descriptor
	*
	* \param[in] fd File descriptor to (piece-wise) write to
	* \param[in] cur XML subtree to proceed
	*
	* \return a standard Pacemaker return code
	*/
	int pcmk__xml2fd(int fd, xmlNode *cur);

	enum pcmk__xml_artefact_ns {
	pcmk__xml_artefact_ns_legacy_rng = 1,
	pcmk__xml_artefact_ns_legacy_xslt,
	pcmk__xml_artefact_ns_base_rng,
	pcmk__xml_artefact_ns_base_xslt,
	};

	void pcmk__strip_xml_text(xmlNode *xml);
	const char pcmk__xe_add_last_written(xmlNode xe);

	xmlNode pcmk__xe_match(const xmlNode parent, const char *node_name,
	const char attr_n, const char attr_v);

	void pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data);

	GString pcmk__element_xpath(const xmlNode xml);

	bool pcmk__xml_needs_escape(const char *text, bool escape_quote);
	char pcmk__xml_escape(const char text, bool escape_quote);

	/*!
	* \internal
	* \brief Get the root directory to scan XML artefacts of given kind for
	*
	* \param[in] ns governs the hierarchy nesting against the inherent root dir
	*
	* \return root directory to scan XML artefacts of given kind for
	*/
	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns);

	/*!
	* \internal
	* \brief Get the fully unwrapped path to particular XML artifact (RNG/XSLT)
	*
	* \param[in] ns denotes path forming details (parent dir, suffix)
	* \param[in] filespec symbolic file specification to be combined with
	* #artefact_ns to form the final path
	* \return unwrapped path to particular XML artifact (RNG/XSLT)
	*/
	char *pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns,
	const char *filespec);

	/*!
	* \internal
	* \brief Retrieve the value of the \c PCMK_XA_ID XML attribute
	*
	* \param[in] xml XML element to check
	*
	* \return Value of the \c PCMK_XA_ID attribute (may be \c NULL)
	*/
	static inline const char *
	pcmk__xe_id(const xmlNode *xml)
	{
	return crm_element_value(xml, PCMK_XA_ID);
	}

	/*!
	* \internal
	* \brief Check whether an XML element is of a particular type
	*
	* \param[in] xml XML element to compare
	* \param[in] name XML element name to compare
	*
	* \return \c true if \p xml is of type \p name, otherwise \c false
	*/
	static inline bool
	pcmk__xe_is(const xmlNode xml, const char name)
	{
	return (xml != NULL) && (xml->name != NULL) && (name != NULL)
	&& (strcmp((const char *) xml->name, name) == 0);
	}

	/*!
	* \internal
	* \brief Return first non-text child node of an XML node
	*
	* \param[in] parent XML node to check
	*
	* \return First non-text child node of \p parent (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_first_child(const xmlNode *parent)
	{
	xmlNode *child = (parent? parent->children : NULL);

	while (child && (child->type == XML_TEXT_NODE)) {
	child = child->next;
	}
	return child;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling node of an XML node
	*
	* \param[in] child XML node to check
	*
	* \return Next non-text sibling of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xml_next(const xmlNode *child)
	{
	xmlNode *next = (child? child->next : NULL);

	while (next && (next->type == XML_TEXT_NODE)) {
	next = next->next;
	}
	return next;
	}

	/*!
	* \internal
	* \brief Return first non-text child element of an XML node
	*
	* \param[in] parent XML node to check
	*
	* \return First child element of \p parent (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xe_first_child(const xmlNode *parent)
	{
	xmlNode *child = (parent? parent->children : NULL);

	while (child && (child->type != XML_ELEMENT_NODE)) {
	child = child->next;
	}
	return child;
	}

	/*!
	* \internal
	* \brief Return next non-text sibling element of an XML element
	*
	* \param[in] child XML element to check
	*
	* \return Next sibling element of \p child (or NULL if none)
	*/
	static inline xmlNode *
	pcmk__xe_next(const xmlNode *child)
	{
	xmlNode *next = child? child->next : NULL;

	while (next && (next->type != XML_ELEMENT_NODE)) {
	next = next->next;
	}
	return next;
	}

	xmlNode pcmk__xml_copy(xmlNode parent, xmlNode *src);

	void pcmk__xe_set_content(xmlNode node, const char content);

	/*!
	* \internal
	* \brief Like pcmk__xe_set_props, but takes a va_list instead of
	* arguments directly.
	*
	* \param[in,out] node XML to add attributes to
	* \param[in] pairs NULL-terminated list of name/value pairs to add
	*/
	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs);

	/*!
	* \internal
	* \brief Add a NULL-terminated list of name/value pairs to the given
	* XML node as properties.
	*
	* \param[in,out] node XML node to add properties to
	* \param[in] ... NULL-terminated list of name/value pairs
	*
	* \note A NULL name terminates the arguments; a NULL value will be skipped.
	*/
	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	G_GNUC_NULL_TERMINATED;

	/*!
	* \internal
	* \brief Get first attribute of an XML element
	*
	* \param[in] xe XML element to check
	*
	* \return First attribute of \p xe (or NULL if \p xe is NULL or has none)
	*/
	static inline xmlAttr *
	pcmk__xe_first_attr(const xmlNode *xe)
	{
	return (xe == NULL)? NULL : xe->properties;
	}

	/*!
	* \internal
	* \brief Extract the ID attribute from an XML element
	*
	* \param[in] xpath String to search
	* \param[in] node Node to get the ID for
	*
	* \return ID attribute of \p node in xpath string \p xpath
	*/
	char *
	pcmk__xpath_node_id(const char xpath, const char node);

	/* internal XML-related utilities */

	enum xml_private_flags {
	pcmk__xf_none = 0x0000,
	pcmk__xf_dirty = 0x0001,
	pcmk__xf_deleted = 0x0002,
	pcmk__xf_created = 0x0004,
	pcmk__xf_modified = 0x0008,

	pcmk__xf_tracking = 0x0010,
	pcmk__xf_processed = 0x0020,
	pcmk__xf_skip = 0x0040,
	pcmk__xf_moved = 0x0080,

	pcmk__xf_acl_enabled = 0x0100,
	pcmk__xf_acl_read = 0x0200,
	pcmk__xf_acl_write = 0x0400,
	pcmk__xf_acl_deny = 0x0800,

	pcmk__xf_acl_create = 0x1000,
	pcmk__xf_acl_denied = 0x2000,
	pcmk__xf_lazy = 0x4000,
	};

	void pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag);

	/*!
	* \internal
	* \brief Iterate over child elements of \p xml
	*
	* This function iterates over the children of \p xml, performing the
	* callback function \p handler on each node. If the callback returns
	* a value other than pcmk_rc_ok, the iteration stops and the value is
	* returned. It is therefore possible that not all children will be
	* visited.
	*
	* \param[in,out] xml The starting XML node. Can be NULL.
	* \param[in] child_element_name The name that the node must match in order
	* for \p handler to be run. If NULL, all
	* child elements will match.
	* \param[in] handler The callback function.
	* \param[in,out] userdata User data to pass to the callback function.
	* Can be NULL.
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__xe_foreach_child(xmlNode xml, const char child_element_name,
	int (handler)(xmlNode xml, void *userdata),
	void *userdata);

	static inline const char *
	pcmk__xml_attr_value(const xmlAttr *attr)
	{
	return ((attr == NULL) \|\| (attr->children == NULL))? NULL
	: (const char *) attr->children->content;
	}


	gboolean pcmk__validate_xml(xmlNode xml_blob, const char validation,
	xmlRelaxNGValidityErrorFunc error_handler,
	void *error_handler_context);

	void pcmk__log_known_schemas(void);
	const char *pcmk__remote_schema_dir(void);
	void pcmk__sort_schemas(void);


	/*
	* I/O
	*/
	xmlNode pcmk__xml_parse_file(const char filename);
	+xmlNode pcmk__xml_parse_string(const char input);

	// @COMPAT Remove when v1 patchsets are removed
	xmlNode pcmk__diff_v1_xml_object(xmlNode left, xmlNode *right, bool suppress);

	#endif // PCMK__XML_INTERNAL__H
	diff --git a/lib/cib/cib_native.c b/lib/cib/cib_native.c
	index ce9d68f58b..0a719f86a8 100644
	--- a/lib/cib/cib_native.c
	+++ b/lib/cib/cib_native.c
	@@ -1,508 +1,508 @@
	/*
	* Copyright 2004 International Business Machines
	* Later changes copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <errno.h>
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>

	#include <crm/common/mainloop.h>
	#include <crm/common/xml.h>

	typedef struct cib_native_opaque_s {
	char *token;
	crm_ipc_t *ipc;
	void (*dnotify_fn) (gpointer user_data);
	mainloop_io_t *source;
	} cib_native_opaque_t;

	static int
	cib_native_perform_op_delegate(cib_t cib, const char op, const char *host,
	const char section, xmlNode data,
	xmlNode **output_data, int call_options,
	const char *user_name)
	{
	int rc = pcmk_ok;
	int reply_id = 0;
	enum crm_ipc_flags ipc_flags = crm_ipc_flags_none;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	cib_native_opaque_t *native = cib->variant_opaque;

	if (cib->state == cib_disconnected) {
	return -ENOTCONN;
	}

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	if (call_options & cib_sync_call) {
	pcmk__set_ipc_flags(ipc_flags, "client", crm_ipc_client_response);
	}

	rc = cib__create_op(cib, op, host, section, data, call_options, user_name,
	NULL, &op_msg);
	if (rc != pcmk_ok) {
	return rc;
	}

	if (pcmk_is_set(call_options, cib_transaction)) {
	rc = cib__extend_transaction(cib, op_msg);
	goto done;
	}

	crm_trace("Sending %s message to the CIB manager (timeout=%ds)", op, cib->call_timeout);
	rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, cib->call_timeout * 1000, &op_reply);

	if (rc < 0) {
	crm_err("Couldn't perform %s operation (timeout=%ds): %s (%d)", op,
	cib->call_timeout, pcmk_strerror(rc), rc);
	rc = -ECOMM;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (!(call_options & cib_sync_call)) {
	crm_trace("Async call, returning %d", cib->call_id);
	CRM_CHECK(cib->call_id != 0, return -ENOMSG);
	free_xml(op_reply);
	return cib->call_id;
	}

	rc = pcmk_ok;
	crm_element_value_int(op_reply, PCMK__XA_CIB_CALLID, &reply_id);
	if (reply_id == cib->call_id) {
	xmlNode *tmp = get_message_xml(op_reply, PCMK__XA_CIB_CALLDATA);

	crm_trace("Synchronous reply %d received", reply_id);
	if (crm_element_value_int(op_reply, PCMK__XA_CIB_RC, &rc) != 0) {
	rc = -EPROTO;
	}

	if (output_data == NULL \|\| (call_options & cib_discard_reply)) {
	crm_trace("Discarding reply");
	} else {
	*output_data = pcmk__xml_copy(NULL, tmp);
	}

	} else if (reply_id <= 0) {
	crm_err("Received bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	rc = -ENOMSG;
	goto done;

	} else {
	crm_err("Received bad reply: %d (wanted %d)", reply_id, cib->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	rc = -ENOMSG;
	goto done;
	}

	if (op_reply == NULL && cib->state == cib_disconnected) {
	rc = -ENOTCONN;

	} else if (rc == pcmk_ok && op_reply == NULL) {
	rc = -ETIME;
	}

	switch (rc) {
	case pcmk_ok:
	case -EPERM:
	break;

	/* This is an internal value that clients do not and should not care about */
	case -pcmk_err_diff_resync:
	rc = pcmk_ok;
	break;

	/* These indicate internal problems */
	case -EPROTO:
	case -ENOMSG:
	crm_err("Call failed: %s", pcmk_strerror(rc));
	if (op_reply) {
	crm_log_xml_err(op_reply, "Invalid reply");
	}
	break;

	default:
	if (!pcmk__str_eq(op, PCMK__CIB_REQUEST_QUERY, pcmk__str_none)) {
	crm_warn("Call failed: %s", pcmk_strerror(rc));
	}
	}

	done:
	if (!crm_ipc_connected(native->ipc)) {
	crm_err("The CIB manager disconnected");
	cib->state = cib_disconnected;
	}

	free_xml(op_msg);
	free_xml(op_reply);
	return rc;
	}

	static int
	cib_native_dispatch_internal(const char *buffer, ssize_t length,
	gpointer userdata)
	{
	const char *type = NULL;
	xmlNode *msg = NULL;

	cib_t *cib = userdata;

	crm_trace("dispatching %p", userdata);

	if (cib == NULL) {
	crm_err("No CIB!");
	return 0;
	}

	- msg = string2xml(buffer);
	+ msg = pcmk__xml_parse_string(buffer);

	if (msg == NULL) {
	crm_warn("Received a NULL message from the CIB manager");
	return 0;
	}

	/* do callbacks */
	type = crm_element_value(msg, PCMK__XA_T);
	crm_trace("Activating %s callbacks...", type);
	crm_log_xml_explicit(msg, "cib-reply");

	if (pcmk__str_eq(type, PCMK__VALUE_CIB, pcmk__str_none)) {
	cib_native_callback(cib, msg, 0, 0);

	} else if (pcmk__str_eq(type, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) {
	g_list_foreach(cib->notify_list, cib_native_notify, msg);

	} else {
	crm_err("Unknown message type: %s", type);
	}

	free_xml(msg);
	return 0;
	}

	static void
	cib_native_destroy(void *userdata)
	{
	cib_t *cib = userdata;
	cib_native_opaque_t *native = cib->variant_opaque;

	crm_trace("destroying %p", userdata);
	cib->state = cib_disconnected;
	native->source = NULL;
	native->ipc = NULL;

	if (native->dnotify_fn) {
	native->dnotify_fn(userdata);
	}
	}

	static int
	cib_native_signoff(cib_t *cib)
	{
	cib_native_opaque_t *native = cib->variant_opaque;

	crm_debug("Disconnecting from the CIB manager");

	cib_free_notify(cib);
	remove_cib_op_callback(0, TRUE);

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;

	} else if (native->ipc) {
	/* Not attached to mainloop */
	crm_ipc_t *ipc = native->ipc;

	native->ipc = NULL;
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	}

	cib->cmds->end_transaction(cib, false, cib_none);
	cib->state = cib_disconnected;
	cib->type = cib_no_connection;

	return pcmk_ok;
	}

	static int
	cib_native_signon_raw(cib_t cib, const char name, enum cib_conn_type type,
	int *async_fd)
	{
	int rc = pcmk_ok;
	const char *channel = NULL;
	cib_native_opaque_t *native = cib->variant_opaque;
	xmlNode *hello = NULL;

	struct ipc_client_callbacks cib_callbacks = {
	.dispatch = cib_native_dispatch_internal,
	.destroy = cib_native_destroy
	};

	cib->call_timeout = PCMK__IPC_TIMEOUT;

	if (type == cib_command) {
	cib->state = cib_connected_command;
	channel = PCMK__SERVER_BASED_RW;

	} else if (type == cib_command_nonblocking) {
	cib->state = cib_connected_command;
	channel = PCMK__SERVER_BASED_SHM;

	} else if (type == cib_query) {
	cib->state = cib_connected_query;
	channel = PCMK__SERVER_BASED_RO;

	} else {
	return -ENOTCONN;
	}

	crm_trace("Connecting %s channel", channel);

	if (async_fd != NULL) {
	native->ipc = crm_ipc_new(channel, 0);
	if (native->ipc != NULL) {
	rc = pcmk__connect_generic_ipc(native->ipc);
	if (rc == pcmk_rc_ok) {
	rc = pcmk__ipc_fd(native->ipc, async_fd);
	if (rc != pcmk_rc_ok) {
	crm_info("Couldn't get file descriptor for %s IPC",
	channel);
	}
	}
	rc = pcmk_rc2legacy(rc);
	}

	} else {
	native->source =
	mainloop_add_ipc_client(channel, G_PRIORITY_HIGH, 512 * 1024 /* 512k */ , cib,
	&cib_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (rc != pcmk_ok \|\| native->ipc == NULL \|\| !crm_ipc_connected(native->ipc)) {
	crm_info("Could not connect to CIB manager for %s", name);
	rc = -ENOTCONN;
	}

	if (rc == pcmk_ok) {
	rc = cib__create_op(cib, CRM_OP_REGISTER, NULL, NULL, NULL,
	cib_sync_call, NULL, name, &hello);
	}

	if (rc == pcmk_ok) {
	xmlNode *reply = NULL;

	if (crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1,
	&reply) > 0) {
	const char *msg_type = crm_element_value(reply, PCMK__XA_CIB_OP);

	crm_log_xml_trace(reply, "reg-reply");

	if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
	crm_info("Reply to CIB registration message has unknown type "
	"'%s'",
	msg_type);
	rc = -EPROTO;

	} else {
	native->token = crm_element_value_copy(reply,
	PCMK__XA_CIB_CLIENTID);
	if (native->token == NULL) {
	rc = -EPROTO;
	}
	}
	free_xml(reply);

	} else {
	rc = -ECOMM;
	}
	free_xml(hello);
	}

	if (rc == pcmk_ok) {
	crm_info("Successfully connected to CIB manager for %s", name);
	return pcmk_ok;
	}

	crm_info("Connection to CIB manager for %s failed: %s",
	name, pcmk_strerror(rc));
	cib_native_signoff(cib);
	return rc;
	}

	static int
	cib_native_signon(cib_t cib, const char name, enum cib_conn_type type)
	{
	return cib_native_signon_raw(cib, name, type, NULL);
	}

	static int
	cib_native_free(cib_t *cib)
	{
	int rc = pcmk_ok;

	if (cib->state != cib_disconnected) {
	rc = cib_native_signoff(cib);
	}

	if (cib->state == cib_disconnected) {
	cib_native_opaque_t *native = cib->variant_opaque;

	free(native->token);
	free(cib->variant_opaque);
	free(cib->cmds);
	free(cib->user);
	free(cib);
	}

	return rc;
	}

	static int
	cib_native_register_notification(cib_t cib, const char callback, int enabled)
	{
	int rc = pcmk_ok;
	xmlNode *notify_msg = create_xml_node(NULL, PCMK__XE_CIB_CALLBACK);
	cib_native_opaque_t *native = cib->variant_opaque;

	if (cib->state != cib_disconnected) {
	crm_xml_add(notify_msg, PCMK__XA_CIB_OP, PCMK__VALUE_CIB_NOTIFY);
	crm_xml_add(notify_msg, PCMK__XA_CIB_NOTIFY_TYPE, callback);
	crm_xml_add_int(notify_msg, PCMK__XA_CIB_NOTIFY_ACTIVATE, enabled);
	rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response,
	1000 * cib->call_timeout, NULL);
	if (rc <= 0) {
	crm_trace("Notification not registered: %d", rc);
	rc = -ECOMM;
	}
	}

	free_xml(notify_msg);
	return rc;
	}

	static int
	cib_native_set_connection_dnotify(cib_t *cib,
	void (*dnotify) (gpointer user_data))
	{
	cib_native_opaque_t *native = NULL;

	if (cib == NULL) {
	crm_err("No CIB!");
	return FALSE;
	}

	native = cib->variant_opaque;
	native->dnotify_fn = dnotify;

	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Get the given CIB connection's unique client identifier
	*
	* These can be used to check whether this client requested the action that
	* triggered a CIB notification.
	*
	* \param[in] cib CIB connection
	* \param[out] async_id If not \p NULL, where to store asynchronous client ID
	* \param[out] sync_id If not \p NULL, where to store synchronous client ID
	*
	* \return Legacy Pacemaker return code (specifically, \p pcmk_ok)
	*
	* \note This is the \p cib_native variant implementation of
	* \p cib_api_operations_t:client_id().
	* \note For \p cib_native objects, \p async_id and \p sync_id are the same.
	* \note The client ID is assigned during CIB sign-on.
	*/
	static int
	cib_native_client_id(const cib_t cib, const char *async_id,
	const char **sync_id)
	{
	cib_native_opaque_t *native = cib->variant_opaque;

	if (async_id != NULL) {
	*async_id = native->token;
	}
	if (sync_id != NULL) {
	*sync_id = native->token;
	}
	return pcmk_ok;
	}

	cib_t *
	cib_native_new(void)
	{
	cib_native_opaque_t *native = NULL;
	cib_t *cib = cib_new_variant();

	if (cib == NULL) {
	return NULL;
	}

	native = calloc(1, sizeof(cib_native_opaque_t));

	if (native == NULL) {
	free(cib);
	return NULL;
	}

	cib->variant = cib_native;
	cib->variant_opaque = native;

	native->ipc = NULL;
	native->source = NULL;
	native->dnotify_fn = NULL;

	/* assign variant specific ops */
	cib->delegate_fn = cib_native_perform_op_delegate;
	cib->cmds->signon = cib_native_signon;
	cib->cmds->signon_raw = cib_native_signon_raw;
	cib->cmds->signoff = cib_native_signoff;
	cib->cmds->free = cib_native_free;

	cib->cmds->register_notification = cib_native_register_notification;
	cib->cmds->set_connection_dnotify = cib_native_set_connection_dnotify;

	cib->cmds->client_id = cib_native_client_id;

	return cib;
	}
	diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
	index 99c95e9fec..b492f3c6ec 100644
	--- a/lib/common/ipc_client.c
	+++ b/lib/common/ipc_client.c
	@@ -1,1690 +1,1690 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#if defined(HAVE_UCRED) \|\| defined(HAVE_SOCKPEERCRED)
	# ifdef HAVE_UCRED
	# ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	# endif
	# endif
	# include <sys/socket.h>
	#elif defined(HAVE_GETPEERUCRED)
	# include <ucred.h>
	#endif

	#include <stdio.h>
	#include <sys/types.h>
	#include <errno.h>
	#include <bzlib.h>

	#include <crm/crm.h> /* indirectly: pcmk_err_generic */
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include "crmcommon_private.h"

	static int is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
	uid_t refuid, gid_t refgid, pid_t *gotpid,
	uid_t gotuid, gid_t gotgid);

	/*!
	* \brief Create a new object for using Pacemaker daemon IPC
	*
	* \param[out] api Where to store new IPC object
	* \param[in] server Which Pacemaker daemon the object is for
	*
	* \return Standard Pacemaker result code
	*
	* \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
	* \note This is intended to supersede crm_ipc_new() but currently only
	* supports the controller, pacemakerd, and schedulerd IPC API.
	*/
	int
	pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
	{
	if (api == NULL) {
	return EINVAL;
	}

	*api = calloc(1, sizeof(pcmk_ipc_api_t));
	if (*api == NULL) {
	return errno;
	}

	(*api)->server = server;
	if (pcmk_ipc_name(*api, false) == NULL) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return EOPNOTSUPP;
	}

	(*api)->ipc_size_max = 0;

	// Set server methods and max_size (if not default)
	switch (server) {
	case pcmk_ipc_attrd:
	(*api)->cmds = pcmk__attrd_api_methods();
	break;

	case pcmk_ipc_based:
	(api)->ipc_size_max = 512 1024; // 512KB
	break;

	case pcmk_ipc_controld:
	(*api)->cmds = pcmk__controld_api_methods();
	break;

	case pcmk_ipc_execd:
	break;

	case pcmk_ipc_fenced:
	break;

	case pcmk_ipc_pacemakerd:
	(*api)->cmds = pcmk__pacemakerd_api_methods();
	break;

	case pcmk_ipc_schedulerd:
	(*api)->cmds = pcmk__schedulerd_api_methods();
	// @TODO max_size could vary by client, maybe take as argument?
	(api)->ipc_size_max = 5 1024 * 1024; // 5MB
	break;
	}
	if ((*api)->cmds == NULL) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return ENOMEM;
	}

	(api)->ipc = crm_ipc_new(pcmk_ipc_name(api, false),
	(*api)->ipc_size_max);
	if ((*api)->ipc == NULL) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return ENOMEM;
	}

	// If daemon API has its own data to track, allocate it
	if ((*api)->cmds->new_data != NULL) {
	if ((api)->cmds->new_data(api) != pcmk_rc_ok) {
	pcmk_free_ipc_api(*api);
	*api = NULL;
	return ENOMEM;
	}
	}
	crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true));
	return pcmk_rc_ok;
	}

	static void
	free_daemon_specific_data(pcmk_ipc_api_t *api)
	{
	if ((api != NULL) && (api->cmds != NULL)) {
	if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) {
	api->cmds->free_data(api->api_data);
	api->api_data = NULL;
	}
	free(api->cmds);
	api->cmds = NULL;
	}
	}

	/*!
	* \internal
	* \brief Call an IPC API event callback, if one is registed
	*
	* \param[in,out] api IPC API connection
	* \param[in] event_type The type of event that occurred
	* \param[in] status Event status
	* \param[in,out] event_data Event-specific data
	*/
	void
	pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
	crm_exit_t status, void *event_data)
	{
	if ((api != NULL) && (api->cb != NULL)) {
	api->cb(api, event_type, status, event_data, api->user_data);
	}
	}

	/*!
	* \internal
	* \brief Clean up after an IPC disconnect
	*
	* \param[in,out] user_data IPC API connection that disconnected
	*
	* \note This function can be used as a main loop IPC destroy callback.
	*/
	static void
	ipc_post_disconnect(gpointer user_data)
	{
	pcmk_ipc_api_t *api = user_data;

	crm_info("Disconnected from %s", pcmk_ipc_name(api, true));

	// Perform any daemon-specific handling needed
	if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) {
	api->cmds->post_disconnect(api);
	}

	// Call client's registered event callback
	pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT,
	NULL);

	/* If this is being called from a running main loop, mainloop_gio_destroy()
	* will free ipc and mainloop_io immediately after calling this function.
	* If this is called from a stopped main loop, these will leak, so the best
	* practice is to close the connection before stopping the main loop.
	*/
	api->ipc = NULL;
	api->mainloop_io = NULL;

	if (api->free_on_disconnect) {
	/* pcmk_free_ipc_api() has already been called, but did not free api
	* or api->cmds because this function needed them. Do that now.
	*/
	free_daemon_specific_data(api);
	crm_trace("Freeing IPC API object after disconnect");
	free(api);
	}
	}

	/*!
	* \brief Free the contents of an IPC API object
	*
	* \param[in,out] api IPC API object to free
	*/
	void
	pcmk_free_ipc_api(pcmk_ipc_api_t *api)
	{
	bool free_on_disconnect = false;

	if (api == NULL) {
	return;
	}
	crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true));

	if (api->ipc != NULL) {
	if (api->mainloop_io != NULL) {
	/* We need to keep the api pointer itself around, because it is the
	* user data for the IPC client destroy callback. That will be
	* triggered by the pcmk_disconnect_ipc() call below, but it might
	* happen later in the main loop (if still running).
	*
	* This flag tells the destroy callback to free the object. It can't
	* do that unconditionally, because the application might call this
	* function after a disconnect that happened by other means.
	*/
	free_on_disconnect = api->free_on_disconnect = true;
	}
	pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true
	}
	if (!free_on_disconnect) {
	free_daemon_specific_data(api);
	crm_trace("Freeing IPC API object");
	free(api);
	}
	}

	/*!
	* \brief Get the IPC name used with an IPC API connection
	*
	* \param[in] api IPC API connection
	* \param[in] for_log If true, return human-friendly name instead of IPC name
	*
	* \return IPC API's human-friendly or connection name, or if none is available,
	* "Pacemaker" if for_log is true and NULL if for_log is false
	*/
	const char *
	pcmk_ipc_name(const pcmk_ipc_api_t *api, bool for_log)
	{
	if (api == NULL) {
	return for_log? "Pacemaker" : NULL;
	}
	switch (api->server) {
	case pcmk_ipc_attrd:
	return for_log? "attribute manager" : PCMK__VALUE_ATTRD;

	case pcmk_ipc_based:
	return for_log? "CIB manager" : NULL /* PCMK__SERVER_BASED_RW */;

	case pcmk_ipc_controld:
	return for_log? "controller" : CRM_SYSTEM_CRMD;

	case pcmk_ipc_execd:
	return for_log? "executor" : NULL /* CRM_SYSTEM_LRMD */;

	case pcmk_ipc_fenced:
	return for_log? "fencer" : NULL /* "stonith-ng" */;

	case pcmk_ipc_pacemakerd:
	return for_log? "launcher" : CRM_SYSTEM_MCP;

	case pcmk_ipc_schedulerd:
	return for_log? "scheduler" : CRM_SYSTEM_PENGINE;

	default:
	return for_log? "Pacemaker" : NULL;
	}
	}

	/*!
	* \brief Check whether an IPC API connection is active
	*
	* \param[in,out] api IPC API connection
	*
	* \return true if IPC is connected, false otherwise
	*/
	bool
	pcmk_ipc_is_connected(pcmk_ipc_api_t *api)
	{
	return (api != NULL) && crm_ipc_connected(api->ipc);
	}

	/*!
	* \internal
	* \brief Call the daemon-specific API's dispatch function
	*
	* Perform daemon-specific handling of IPC reply dispatch. It is the daemon
	* method's responsibility to call the client's registered event callback, as
	* well as allocate and free any event data.
	*
	* \param[in,out] api IPC API connection
	* \param[in,out] message IPC reply XML to dispatch
	*/
	static bool
	call_api_dispatch(pcmk_ipc_api_t api, xmlNode message)
	{
	crm_log_xml_trace(message, "ipc-received");
	if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) {
	return api->cmds->dispatch(api, message);
	}

	return false;
	}

	/*!
	* \internal
	* \brief Dispatch previously read IPC data
	*
	* \param[in] buffer Data read from IPC
	* \param[in,out] api IPC object
	*
	* \return Standard Pacemaker return code. In particular:
	*
	* pcmk_rc_ok: There are no more messages expected from the server. Quit
	* reading.
	* EINPROGRESS: There are more messages expected from the server. Keep reading.
	*
	* All other values indicate an error.
	*/
	static int
	dispatch_ipc_data(const char buffer, pcmk_ipc_api_t api)
	{
	bool more = false;
	xmlNode *msg;

	if (buffer == NULL) {
	crm_warn("Empty message received from %s IPC",
	pcmk_ipc_name(api, true));
	return ENOMSG;
	}

	- msg = string2xml(buffer);
	+ msg = pcmk__xml_parse_string(buffer);
	if (msg == NULL) {
	crm_warn("Malformed message received from %s IPC",
	pcmk_ipc_name(api, true));
	return EPROTO;
	}

	more = call_api_dispatch(api, msg);
	free_xml(msg);

	if (more) {
	return EINPROGRESS;
	} else {
	return pcmk_rc_ok;
	}
	}

	/*!
	* \internal
	* \brief Dispatch data read from IPC source
	*
	* \param[in] buffer Data read from IPC
	* \param[in] length Number of bytes of data in buffer (ignored)
	* \param[in,out] user_data IPC object
	*
	* \return Always 0 (meaning connection is still required)
	*
	* \note This function can be used as a main loop IPC dispatch callback.
	*/
	static int
	dispatch_ipc_source_data(const char *buffer, ssize_t length, gpointer user_data)
	{
	pcmk_ipc_api_t *api = user_data;

	CRM_CHECK(api != NULL, return 0);
	dispatch_ipc_data(buffer, api);
	return 0;
	}

	/*!
	* \brief Check whether an IPC connection has data available (without main loop)
	*
	* \param[in] api IPC API connection
	* \param[in] timeout_ms If less than 0, poll indefinitely; if 0, poll once
	* and return immediately; otherwise, poll for up to
	* this many milliseconds
	*
	* \return Standard Pacemaker return code
	*
	* \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
	* this function to check whether IPC data is available. Return values of
	* interest include pcmk_rc_ok meaning data is available, and EAGAIN
	* meaning no data is available; all other values indicate errors.
	* \todo This does not allow the caller to poll multiple file descriptors at
	* once. If there is demand for that, we could add a wrapper for
	* pcmk__ipc_fd(api->ipc), so the caller can call poll() themselves.
	*/
	int
	pcmk_poll_ipc(const pcmk_ipc_api_t *api, int timeout_ms)
	{
	int rc;
	struct pollfd pollfd = { 0, };

	if ((api == NULL) \|\| (api->dispatch_type != pcmk_ipc_dispatch_poll)) {
	return EINVAL;
	}

	rc = pcmk__ipc_fd(api->ipc, &(pollfd.fd));
	if (rc != pcmk_rc_ok) {
	crm_debug("Could not obtain file descriptor for %s IPC: %s",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc));
	return rc;
	}

	pollfd.events = POLLIN;
	rc = poll(&pollfd, 1, timeout_ms);
	if (rc < 0) {
	/* Some UNIX systems return negative and set EAGAIN for failure to
	* allocate memory; standardize the return code in that case
	*/
	return (errno == EAGAIN)? ENOMEM : errno;
	} else if (rc == 0) {
	return EAGAIN;
	}
	return pcmk_rc_ok;
	}

	/*!
	* \brief Dispatch available messages on an IPC connection (without main loop)
	*
	* \param[in,out] api IPC API connection
	*
	* \return Standard Pacemaker return code
	*
	* \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
	* this function when IPC data is available.
	*/
	void
	pcmk_dispatch_ipc(pcmk_ipc_api_t *api)
	{
	if (api == NULL) {
	return;
	}
	while (crm_ipc_ready(api->ipc) > 0) {
	if (crm_ipc_read(api->ipc) > 0) {
	dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);
	}
	}
	}

	// \return Standard Pacemaker return code
	static int
	connect_with_main_loop(pcmk_ipc_api_t *api)
	{
	int rc;

	struct ipc_client_callbacks callbacks = {
	.dispatch = dispatch_ipc_source_data,
	.destroy = ipc_post_disconnect,
	};

	rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api,
	&callbacks, &(api->mainloop_io));
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	crm_debug("Connected to %s IPC (attached to main loop)",
	pcmk_ipc_name(api, true));
	/* After this point, api->mainloop_io owns api->ipc, so api->ipc
	* should not be explicitly freed.
	*/
	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	connect_without_main_loop(pcmk_ipc_api_t *api)
	{
	int rc = pcmk__connect_generic_ipc(api->ipc);

	if (rc != pcmk_rc_ok) {
	crm_ipc_close(api->ipc);
	} else {
	crm_debug("Connected to %s IPC (without main loop)",
	pcmk_ipc_name(api, true));
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Connect to a Pacemaker daemon via IPC (retrying after soft errors)
	*
	* \param[in,out] api IPC API instance
	* \param[in] dispatch_type How IPC replies should be dispatched
	* \param[in] attempts How many times to try (in case of soft error)
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type,
	int attempts)
	{
	int rc = pcmk_rc_ok;

	if ((api == NULL) \|\| (attempts < 1)) {
	return EINVAL;
	}

	if (api->ipc == NULL) {
	api->ipc = crm_ipc_new(pcmk_ipc_name(api, false), api->ipc_size_max);
	if (api->ipc == NULL) {
	return ENOMEM;
	}
	}

	if (crm_ipc_connected(api->ipc)) {
	crm_trace("Already connected to %s", pcmk_ipc_name(api, true));
	return pcmk_rc_ok;
	}

	api->dispatch_type = dispatch_type;

	crm_debug("Attempting connection to %s (up to %d time%s)",
	pcmk_ipc_name(api, true), attempts, pcmk__plural_s(attempts));
	for (int remaining = attempts - 1; remaining >= 0; --remaining) {
	switch (dispatch_type) {
	case pcmk_ipc_dispatch_main:
	rc = connect_with_main_loop(api);
	break;

	case pcmk_ipc_dispatch_sync:
	case pcmk_ipc_dispatch_poll:
	rc = connect_without_main_loop(api);
	break;
	}

	if ((remaining == 0) \|\| ((rc != EAGAIN) && (rc != EALREADY))) {
	break; // Result is final
	}

	// Retry after soft error (interrupted by signal, etc.)
	pcmk__sleep_ms((attempts - remaining) * 500);
	crm_debug("Re-attempting connection to %s (%d attempt%s remaining)",
	pcmk_ipc_name(api, true), remaining,
	pcmk__plural_s(remaining));
	}

	if (rc != pcmk_rc_ok) {
	return rc;
	}

	if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) {
	rc = api->cmds->post_connect(api);
	if (rc != pcmk_rc_ok) {
	crm_ipc_close(api->ipc);
	}
	}
	return rc;
	}

	/*!
	* \brief Connect to a Pacemaker daemon via IPC
	*
	* \param[in,out] api IPC API instance
	* \param[in] dispatch_type How IPC replies should be dispatched
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type)
	{
	int rc = pcmk__connect_ipc(api, dispatch_type, 2);

	if (rc != pcmk_rc_ok) {
	crm_err("Connection to %s failed: %s",
	pcmk_ipc_name(api, true), pcmk_rc_str(rc));
	}
	return rc;
	}

	/*!
	* \brief Disconnect an IPC API instance
	*
	* \param[in,out] api IPC API connection
	*
	* \return Standard Pacemaker return code
	*
	* \note If the connection is attached to a main loop, this function should be
	* called before quitting the main loop, to ensure that all memory is
	* freed.
	*/
	void
	pcmk_disconnect_ipc(pcmk_ipc_api_t *api)
	{
	if ((api == NULL) \|\| (api->ipc == NULL)) {
	return;
	}
	switch (api->dispatch_type) {
	case pcmk_ipc_dispatch_main:
	{
	mainloop_io_t *mainloop_io = api->mainloop_io;

	// Make sure no code with access to api can use these again
	api->mainloop_io = NULL;
	api->ipc = NULL;

	mainloop_del_ipc_client(mainloop_io);
	// After this point api might have already been freed
	}
	break;

	case pcmk_ipc_dispatch_poll:
	case pcmk_ipc_dispatch_sync:
	{
	crm_ipc_t *ipc = api->ipc;

	// Make sure no code with access to api can use ipc again
	api->ipc = NULL;

	// This should always be the case already, but to be safe
	api->free_on_disconnect = false;

	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	ipc_post_disconnect(api);
	}
	break;
	}
	}

	/*!
	* \brief Register a callback for IPC API events
	*
	* \param[in,out] api IPC API connection
	* \param[in] callback Callback to register
	* \param[in] userdata Caller data to pass to callback
	*
	* \note This function may be called multiple times to update the callback
	* and/or user data. The caller remains responsible for freeing
	* userdata in any case (after the IPC is disconnected, if the
	* user data is still registered with the IPC).
	*/
	void
	pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
	void *user_data)
	{
	if (api == NULL) {
	return;
	}
	api->cb = cb;
	api->user_data = user_data;
	}

	/*!
	* \internal
	* \brief Send an XML request across an IPC API connection
	*
	* \param[in,out] api IPC API connection
	* \param[in] request XML request to send
	*
	* \return Standard Pacemaker return code
	*
	* \note Daemon-specific IPC API functions should call this function to send
	* requests, because it handles different dispatch types appropriately.
	*/
	int
	pcmk__send_ipc_request(pcmk_ipc_api_t api, const xmlNode request)
	{
	int rc;
	xmlNode *reply = NULL;
	enum crm_ipc_flags flags = crm_ipc_flags_none;

	if ((api == NULL) \|\| (api->ipc == NULL) \|\| (request == NULL)) {
	return EINVAL;
	}
	crm_log_xml_trace(request, "ipc-sent");

	// Synchronous dispatch requires waiting for a reply
	if ((api->dispatch_type == pcmk_ipc_dispatch_sync)
	&& (api->cmds != NULL)
	&& (api->cmds->reply_expected != NULL)
	&& (api->cmds->reply_expected(api, request))) {
	flags = crm_ipc_client_response;
	}

	// The 0 here means a default timeout of 5 seconds
	rc = crm_ipc_send(api->ipc, request, flags, 0, &reply);

	if (rc < 0) {
	return pcmk_legacy2rc(rc);
	} else if (rc == 0) {
	return ENODATA;
	}

	// With synchronous dispatch, we dispatch any reply now
	if (reply != NULL) {
	bool more = call_api_dispatch(api, reply);

	free_xml(reply);

	while (more) {
	rc = crm_ipc_read(api->ipc);

	if (rc == -EAGAIN) {
	continue;
	} else if (rc == -ENOMSG \|\| rc == pcmk_ok) {
	return pcmk_rc_ok;
	} else if (rc < 0) {
	return -rc;
	}

	rc = dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);

	if (rc == pcmk_rc_ok) {
	more = false;
	} else if (rc == EINPROGRESS) {
	more = true;
	} else {
	continue;
	}
	}
	}
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Create the XML for an IPC request to purge a node from the peer cache
	*
	* \param[in] api IPC API connection
	* \param[in] node_name If not NULL, name of node to purge
	* \param[in] nodeid If not 0, node ID of node to purge
	*
	* \return Newly allocated IPC request XML
	*
	* \note The controller, fencer, and pacemakerd use the same request syntax, but
	* the attribute manager uses a different one. The CIB manager doesn't
	* have any syntax for it. The executor and scheduler don't connect to the
	* cluster layer and thus don't have or need any syntax for it.
	*
	* \todo Modify the attribute manager to accept the common syntax (as well
	* as its current one, for compatibility with older clients). Modify
	* the CIB manager to accept and honor the common syntax. Modify the
	* executor and scheduler to accept the syntax (immediately returning
	* success), just for consistency. Modify this function to use the
	* common syntax with all daemons if their version supports it.
	*/
	static xmlNode *
	create_purge_node_request(const pcmk_ipc_api_t api, const char node_name,
	uint32_t nodeid)
	{
	xmlNode *request = NULL;
	const char *client = crm_system_name? crm_system_name : "client";

	switch (api->server) {
	case pcmk_ipc_attrd:
	request = create_xml_node(NULL, __func__);
	crm_xml_add(request, PCMK__XA_T, PCMK__VALUE_ATTRD);
	crm_xml_add(request, PCMK__XA_SRC, crm_system_name);
	crm_xml_add(request, PCMK_XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE);
	pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, true);
	pcmk__xe_add_node(request, node_name, nodeid);
	break;

	case pcmk_ipc_controld:
	case pcmk_ipc_fenced:
	case pcmk_ipc_pacemakerd:
	request = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL,
	pcmk_ipc_name(api, false), client, NULL);
	if (nodeid > 0) {
	crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
	}
	crm_xml_add(request, PCMK_XA_UNAME, node_name);
	break;

	case pcmk_ipc_based:
	case pcmk_ipc_execd:
	case pcmk_ipc_schedulerd:
	break;
	}
	return request;
	}

	/*!
	* \brief Ask a Pacemaker daemon to purge a node from its peer cache
	*
	* \param[in,out] api IPC API connection
	* \param[in] node_name If not NULL, name of node to purge
	* \param[in] nodeid If not 0, node ID of node to purge
	*
	* \return Standard Pacemaker return code
	*
	* \note At least one of node_name or nodeid must be specified.
	*/
	int
	pcmk_ipc_purge_node(pcmk_ipc_api_t api, const char node_name, uint32_t nodeid)
	{
	int rc = 0;
	xmlNode *request = NULL;

	if (api == NULL) {
	return EINVAL;
	}
	if ((node_name == NULL) && (nodeid == 0)) {
	return EINVAL;
	}

	request = create_purge_node_request(api, node_name, nodeid);
	if (request == NULL) {
	return EOPNOTSUPP;
	}
	rc = pcmk__send_ipc_request(api, request);
	free_xml(request);

	crm_debug("%s peer cache purge of node %s[%lu]: rc=%d",
	pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc);
	return rc;
	}

	/*
	* Generic IPC API (to eventually be deprecated as public API and made internal)
	*/

	struct crm_ipc_s {
	struct pollfd pfd;
	unsigned int max_buf_size; // maximum bytes we can send or receive over IPC
	unsigned int buf_size; // size of allocated buffer
	int msg_size;
	int need_reply;
	char *buffer;
	char *server_name; // server IPC name being connected to
	qb_ipcc_connection_t *ipc;
	};

	/*!
	* \brief Create a new (legacy) object for using Pacemaker daemon IPC
	*
	* \param[in] name IPC system name to connect to
	* \param[in] max_size Use a maximum IPC buffer size of at least this size
	*
	* \return Newly allocated IPC object on success, NULL otherwise
	*
	* \note The caller is responsible for freeing the result using
	* crm_ipc_destroy().
	* \note This should be considered deprecated for use with daemons supported by
	* pcmk_new_ipc_api().
	*/
	crm_ipc_t *
	crm_ipc_new(const char *name, size_t max_size)
	{
	crm_ipc_t *client = NULL;

	client = calloc(1, sizeof(crm_ipc_t));
	if (client == NULL) {
	crm_err("Could not create IPC connection: %s", strerror(errno));
	return NULL;
	}

	client->server_name = strdup(name);
	if (client->server_name == NULL) {
	crm_err("Could not create %s IPC connection: %s",
	name, strerror(errno));
	free(client);
	return NULL;
	}
	client->buf_size = pcmk__ipc_buffer_size(max_size);
	client->buffer = malloc(client->buf_size);
	if (client->buffer == NULL) {
	crm_err("Could not create %s IPC connection: %s",
	name, strerror(errno));
	free(client->server_name);
	free(client);
	return NULL;
	}

	/* Clients initiating connection pick the max buf size */
	client->max_buf_size = client->buf_size;

	client->pfd.fd = -1;
	client->pfd.events = POLLIN;
	client->pfd.revents = 0;

	return client;
	}

	/*!
	* \internal
	* \brief Connect a generic (not daemon-specific) IPC object
	*
	* \param[in,out] ipc Generic IPC object to connect
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__connect_generic_ipc(crm_ipc_t *ipc)
	{
	uid_t cl_uid = 0;
	gid_t cl_gid = 0;
	pid_t found_pid = 0;
	uid_t found_uid = 0;
	gid_t found_gid = 0;
	int rc = pcmk_rc_ok;

	if (ipc == NULL) {
	return EINVAL;
	}

	ipc->need_reply = FALSE;
	ipc->ipc = qb_ipcc_connect(ipc->server_name, ipc->buf_size);
	if (ipc->ipc == NULL) {
	return errno;
	}

	rc = qb_ipcc_fd_get(ipc->ipc, &ipc->pfd.fd);
	if (rc < 0) { // -errno
	crm_ipc_close(ipc);
	return -rc;
	}

	rc = pcmk_daemon_user(&cl_uid, &cl_gid);
	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	crm_ipc_close(ipc);
	return rc;
	}

	rc = is_ipc_provider_expected(ipc->ipc, ipc->pfd.fd, cl_uid, cl_gid,
	&found_pid, &found_uid, &found_gid);
	if (rc != pcmk_rc_ok) {
	if (rc == pcmk_rc_ipc_unauthorized) {
	crm_info("%s IPC provider authentication failed: process %lld has "
	"uid %lld (expected %lld) and gid %lld (expected %lld)",
	ipc->server_name,
	(long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) cl_uid,
	(long long) found_gid, (long long) cl_gid);
	}
	crm_ipc_close(ipc);
	return rc;
	}

	ipc->max_buf_size = qb_ipcc_get_buffer_size(ipc->ipc);
	if (ipc->max_buf_size > ipc->buf_size) {
	free(ipc->buffer);
	ipc->buffer = calloc(ipc->max_buf_size, sizeof(char));
	if (ipc->buffer == NULL) {
	rc = errno;
	crm_ipc_close(ipc);
	return rc;
	}
	ipc->buf_size = ipc->max_buf_size;
	}

	return pcmk_rc_ok;
	}

	/*!
	* \brief Establish an IPC connection to a Pacemaker component
	*
	* \param[in,out] client Connection instance obtained from crm_ipc_new()
	*
	* \return true on success, false otherwise (in which case errno will be set;
	* specifically, in case of discovering the remote side is not
	* authentic, its value is set to ECONNABORTED).
	*/
	bool
	crm_ipc_connect(crm_ipc_t *client)
	{
	int rc = pcmk__connect_generic_ipc(client);

	if (rc == pcmk_rc_ok) {
	return true;
	}
	if ((client != NULL) && (client->ipc == NULL)) {
	errno = (rc > 0)? rc : ENOTCONN;
	crm_debug("Could not establish %s IPC connection: %s (%d)",
	client->server_name, pcmk_rc_str(errno), errno);
	} else if (rc == pcmk_rc_ipc_unauthorized) {
	crm_err("%s IPC provider authentication failed",
	(client == NULL)? "Pacemaker" : client->server_name);
	errno = ECONNABORTED;
	} else {
	crm_perror(LOG_ERR,
	"Could not verify authenticity of %s IPC provider",
	(client == NULL)? "Pacemaker" : client->server_name);
	errno = ENOTCONN;
	}
	return false;
	}

	void
	crm_ipc_close(crm_ipc_t * client)
	{
	if (client) {
	if (client->ipc) {
	qb_ipcc_connection_t *ipc = client->ipc;

	client->ipc = NULL;
	qb_ipcc_disconnect(ipc);
	}
	}
	}

	void
	crm_ipc_destroy(crm_ipc_t * client)
	{
	if (client) {
	if (client->ipc && qb_ipcc_is_connected(client->ipc)) {
	crm_notice("Destroying active %s IPC connection",
	client->server_name);
	/* The next line is basically unsafe
	*
	* If this connection was attached to mainloop and mainloop is active,
	* the 'disconnected' callback will end up back here and we'll end
	* up free'ing the memory twice - something that can still happen
	* even without this if we destroy a connection and it closes before
	* we call exit
	*/
	/* crm_ipc_close(client); */
	} else {
	crm_trace("Destroying inactive %s IPC connection",
	client->server_name);
	}
	free(client->buffer);
	free(client->server_name);
	free(client);
	}
	}

	/*!
	* \internal
	* \brief Get the file descriptor for a generic IPC object
	*
	* \param[in,out] ipc Generic IPC object to get file descriptor for
	* \param[out] fd Where to store file descriptor
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__ipc_fd(crm_ipc_t ipc, int fd)
	{
	if ((ipc == NULL) \|\| (fd == NULL)) {
	return EINVAL;
	}
	if ((ipc->ipc == NULL) \|\| (ipc->pfd.fd < 0)) {
	return ENOTCONN;
	}
	*fd = ipc->pfd.fd;
	return pcmk_rc_ok;
	}

	int
	crm_ipc_get_fd(crm_ipc_t * client)
	{
	int fd = -1;

	if (pcmk__ipc_fd(client, &fd) != pcmk_rc_ok) {
	crm_err("Could not obtain file descriptor for %s IPC",
	((client == NULL)? "unspecified" : client->server_name));
	errno = EINVAL;
	return -EINVAL;
	}
	return fd;
	}

	bool
	crm_ipc_connected(crm_ipc_t * client)
	{
	bool rc = FALSE;

	if (client == NULL) {
	crm_trace("No client");
	return FALSE;

	} else if (client->ipc == NULL) {
	crm_trace("No connection");
	return FALSE;

	} else if (client->pfd.fd < 0) {
	crm_trace("Bad descriptor");
	return FALSE;
	}

	rc = qb_ipcc_is_connected(client->ipc);
	if (rc == FALSE) {
	client->pfd.fd = -EINVAL;
	}
	return rc;
	}

	/*!
	* \brief Check whether an IPC connection is ready to be read
	*
	* \param[in,out] client Connection to check
	*
	* \return Positive value if ready to be read, 0 if not ready, -errno on error
	*/
	int
	crm_ipc_ready(crm_ipc_t *client)
	{
	int rc;

	CRM_ASSERT(client != NULL);

	if (!crm_ipc_connected(client)) {
	return -ENOTCONN;
	}

	client->pfd.revents = 0;
	rc = poll(&(client->pfd), 1, 0);
	return (rc < 0)? -errno : rc;
	}

	// \return Standard Pacemaker return code
	static int
	crm_ipc_decompress(crm_ipc_t * client)
	{
	pcmk__ipc_header_t header = (pcmk__ipc_header_t )(void*)client->buffer;

	if (header->size_compressed) {
	int rc = 0;
	unsigned int size_u = 1 + header->size_uncompressed;
	/* never let buf size fall below our max size required for ipc reads. */
	unsigned int new_buf_size = QB_MAX((sizeof(pcmk__ipc_header_t) + size_u), client->max_buf_size);
	char *uncompressed = calloc(1, new_buf_size);

	crm_trace("Decompressing message data %u bytes into %u bytes",
	header->size_compressed, size_u);

	rc = BZ2_bzBuffToBuffDecompress(uncompressed + sizeof(pcmk__ipc_header_t), &size_u,
	client->buffer + sizeof(pcmk__ipc_header_t), header->size_compressed, 1, 0);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_err("Decompression failed: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	free(uncompressed);
	return rc;
	}

	/*
	* This assert no longer holds true. For an identical msg, some clients may
	* require compression, and others may not. If that same msg (event) is sent
	* to multiple clients, it could result in some clients receiving a compressed
	* msg even though compression was not explicitly required for them.
	*
	* CRM_ASSERT((header->size_uncompressed + sizeof(pcmk__ipc_header_t)) >= ipc_buffer_max);
	*/
	CRM_ASSERT(size_u == header->size_uncompressed);

	memcpy(uncompressed, client->buffer, sizeof(pcmk__ipc_header_t)); /* Preserve the header */
	header = (pcmk__ipc_header_t )(void)uncompressed;

	free(client->buffer);
	client->buf_size = new_buf_size;
	client->buffer = uncompressed;
	}

	CRM_ASSERT(client->buffer[sizeof(pcmk__ipc_header_t) + header->size_uncompressed - 1] == 0);
	return pcmk_rc_ok;
	}

	long
	crm_ipc_read(crm_ipc_t * client)
	{
	pcmk__ipc_header_t *header = NULL;

	CRM_ASSERT(client != NULL);
	CRM_ASSERT(client->ipc != NULL);
	CRM_ASSERT(client->buffer != NULL);

	client->buffer[0] = 0;
	client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer,
	client->buf_size, 0);
	if (client->msg_size >= 0) {
	int rc = crm_ipc_decompress(client);

	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}

	header = (pcmk__ipc_header_t )(void)client->buffer;
	if (!pcmk__valid_ipc_header(header)) {
	return -EBADMSG;
	}

	crm_trace("Received %s IPC event %d size=%u rc=%d text='%.100s'",
	client->server_name, header->qb.id, header->qb.size,
	client->msg_size,
	client->buffer + sizeof(pcmk__ipc_header_t));

	} else {
	crm_trace("No message received from %s IPC: %s",
	client->server_name, pcmk_strerror(client->msg_size));

	if (client->msg_size == -EAGAIN) {
	return -EAGAIN;
	}
	}

	if (!crm_ipc_connected(client) \|\| client->msg_size == -ENOTCONN) {
	crm_err("Connection to %s IPC failed", client->server_name);
	}

	if (header) {
	/* Data excluding the header */
	return header->size_uncompressed;
	}
	return -ENOMSG;
	}

	const char *
	crm_ipc_buffer(crm_ipc_t * client)
	{
	CRM_ASSERT(client != NULL);
	return client->buffer + sizeof(pcmk__ipc_header_t);
	}

	uint32_t
	crm_ipc_buffer_flags(crm_ipc_t * client)
	{
	pcmk__ipc_header_t *header = NULL;

	CRM_ASSERT(client != NULL);
	if (client->buffer == NULL) {
	return 0;
	}

	header = (pcmk__ipc_header_t )(void)client->buffer;
	return header->flags;
	}

	const char *
	crm_ipc_name(crm_ipc_t * client)
	{
	CRM_ASSERT(client != NULL);
	return client->server_name;
	}

	// \return Standard Pacemaker return code
	static int
	internal_ipc_get_reply(crm_ipc_t *client, int request_id, int ms_timeout,
	ssize_t *bytes)
	{
	time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);
	int rc = pcmk_rc_ok;

	/* get the reply */
	crm_trace("Waiting on reply to %s IPC message %d",
	client->server_name, request_id);
	do {

	*bytes = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000);
	if (*bytes > 0) {
	pcmk__ipc_header_t *hdr = NULL;

	rc = crm_ipc_decompress(client);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	hdr = (pcmk__ipc_header_t )(void)client->buffer;
	if (hdr->qb.id == request_id) {
	/* Got it */
	break;
	} else if (hdr->qb.id < request_id) {
	- xmlNode *bad = string2xml(crm_ipc_buffer(client));
	+ xmlNode *bad = pcmk__xml_parse_string(crm_ipc_buffer(client));

	crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id);
	crm_log_xml_notice(bad, "OldIpcReply");

	} else {
	- xmlNode *bad = string2xml(crm_ipc_buffer(client));
	+ xmlNode *bad = pcmk__xml_parse_string(crm_ipc_buffer(client));

	crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id);
	crm_log_xml_notice(bad, "ImpossibleReply");
	CRM_ASSERT(hdr->qb.id <= request_id);
	}
	} else if (!crm_ipc_connected(client)) {
	crm_err("%s IPC provider disconnected while waiting for message %d",
	client->server_name, request_id);
	break;
	}

	} while (time(NULL) < timeout);

	if (*bytes < 0) {
	rc = (int) -*bytes; // System errno
	}
	return rc;
	}

	/*!
	* \brief Send an IPC XML message
	*
	* \param[in,out] client Connection to IPC server
	* \param[in] message XML message to send
	* \param[in] flags Bitmask of crm_ipc_flags
	* \param[in] ms_timeout Give up if not sent within this much time
	* (5 seconds if 0, or no timeout if negative)
	* \param[out] reply Reply from server (or NULL if none)
	*
	* \return Negative errno on error, otherwise size of reply received in bytes
	* if reply was needed, otherwise number of bytes sent
	*/
	int
	crm_ipc_send(crm_ipc_t client, const xmlNode message,
	enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply)
	{
	int rc = 0;
	ssize_t qb_rc = 0;
	ssize_t bytes = 0;
	struct iovec *iov;
	static uint32_t id = 0;
	static int factor = 8;
	pcmk__ipc_header_t *header;

	if (client == NULL) {
	crm_notice("Can't send IPC request without connection (bug?): %.100s",
	message);
	return -ENOTCONN;

	} else if (!crm_ipc_connected(client)) {
	/* Don't even bother */
	crm_notice("Can't send %s IPC requests: Connection closed",
	client->server_name);
	return -ENOTCONN;
	}

	if (ms_timeout == 0) {
	ms_timeout = 5000;
	}

	if (client->need_reply) {
	qb_rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout);
	if (qb_rc < 0) {
	crm_warn("Sending %s IPC disabled until pending reply received",
	client->server_name);
	return -EALREADY;

	} else {
	crm_notice("Sending %s IPC re-enabled after pending reply received",
	client->server_name);
	client->need_reply = FALSE;
	}
	}

	id++;
	CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */
	rc = pcmk__ipc_prepare_iov(id, message, client->max_buf_size, &iov, &bytes);
	if (rc != pcmk_rc_ok) {
	crm_warn("Couldn't prepare %s IPC request: %s " CRM_XS " rc=%d",
	client->server_name, pcmk_rc_str(rc), rc);
	return pcmk_rc2legacy(rc);
	}

	header = iov[0].iov_base;
	pcmk__set_ipc_flags(header->flags, client->server_name, flags);

	if (pcmk_is_set(flags, crm_ipc_proxied)) {
	/* Don't look for a synchronous response */
	pcmk__clear_ipc_flags(flags, "client", crm_ipc_client_response);
	}

	if(header->size_compressed) {
	if(factor < 10 && (client->max_buf_size / 10) < (bytes / factor)) {
	crm_notice("Compressed message exceeds %d0%% of configured IPC "
	"limit (%u bytes); consider setting PCMK_ipc_buffer to "
	"%u or higher",
	factor, client->max_buf_size, 2 * client->max_buf_size);
	factor++;
	}
	}

	crm_trace("Sending %s IPC request %d of %u bytes using %dms timeout",
	client->server_name, header->qb.id, header->qb.size, ms_timeout);

	if ((ms_timeout > 0) \|\| !pcmk_is_set(flags, crm_ipc_client_response)) {

	time_t timeout = time(NULL) + 1 + (ms_timeout / 1000);

	do {
	/* @TODO Is this check really needed? Won't qb_ipcc_sendv() return
	* an error if it's not connected?
	*/
	if (!crm_ipc_connected(client)) {
	goto send_cleanup;
	}

	qb_rc = qb_ipcc_sendv(client->ipc, iov, 2);
	} while ((qb_rc == -EAGAIN) && (time(NULL) < timeout));

	rc = (int) qb_rc; // Negative of system errno, or bytes sent
	if (qb_rc <= 0) {
	goto send_cleanup;

	} else if (!pcmk_is_set(flags, crm_ipc_client_response)) {
	crm_trace("Not waiting for reply to %s IPC request %d",
	client->server_name, header->qb.id);
	goto send_cleanup;
	}

	rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout, &bytes);
	if (rc != pcmk_rc_ok) {
	/* We didn't get the reply in time, so disable future sends for now.
	* The only alternative would be to close the connection since we
	* don't know how to detect and discard out-of-sequence replies.
	*
	* @TODO Implement out-of-sequence detection
	*/
	client->need_reply = TRUE;
	}
	rc = (int) bytes; // Negative system errno, or size of reply received

	} else {
	// No timeout, and client response needed
	do {
	qb_rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer,
	client->buf_size, -1);
	} while ((qb_rc == -EAGAIN) && crm_ipc_connected(client));
	rc = (int) qb_rc; // Negative system errno, or size of reply received
	}

	if (rc > 0) {
	pcmk__ipc_header_t hdr = (pcmk__ipc_header_t )(void*)client->buffer;

	crm_trace("Received %d-byte reply %d to %s IPC %d: %.100s",
	rc, hdr->qb.id, client->server_name, header->qb.id,
	crm_ipc_buffer(client));

	if (reply) {
	- *reply = string2xml(crm_ipc_buffer(client));
	+ *reply = pcmk__xml_parse_string(crm_ipc_buffer(client));
	}

	} else {
	crm_trace("No reply to %s IPC %d: rc=%d",
	client->server_name, header->qb.id, rc);
	}

	send_cleanup:
	if (!crm_ipc_connected(client)) {
	crm_notice("Couldn't send %s IPC request %d: Connection closed "
	CRM_XS " rc=%d", client->server_name, header->qb.id, rc);

	} else if (rc == -ETIMEDOUT) {
	crm_warn("%s IPC request %d failed: %s after %dms " CRM_XS " rc=%d",
	client->server_name, header->qb.id, pcmk_strerror(rc),
	ms_timeout, rc);
	crm_write_blackbox(0, NULL);

	} else if (rc <= 0) {
	crm_warn("%s IPC request %d failed: %s " CRM_XS " rc=%d",
	client->server_name, header->qb.id,
	((rc == 0)? "No bytes sent" : pcmk_strerror(rc)), rc);
	}

	pcmk_free_ipc_event(iov);
	return rc;
	}

	/*!
	* \brief Ensure an IPC provider has expected user or group
	*
	* \param[in] qb_ipc libqb client connection if available
	* \param[in] sock Connected Unix socket for IPC
	* \param[in] refuid Expected user ID
	* \param[in] refgid Expected group ID
	* \param[out] gotpid If not NULL, where to store provider's actual process ID
	* (or 1 on platforms where ID is not available)
	* \param[out] gotuid If not NULL, where to store provider's actual user ID
	* \param[out] gotgid If not NULL, where to store provider's actual group ID
	*
	* \return Standard Pacemaker return code
	* \note An actual user ID of 0 (root) will always be considered authorized,
	* regardless of the expected values provided. The caller can use the
	* output arguments to be stricter than this function.
	*/
	static int
	is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
	uid_t refuid, gid_t refgid,
	pid_t gotpid, uid_t gotuid, gid_t *gotgid)
	{
	int rc = EOPNOTSUPP;
	pid_t found_pid = 0;
	uid_t found_uid = 0;
	gid_t found_gid = 0;

	#ifdef HAVE_QB_IPCC_AUTH_GET
	if (qb_ipc != NULL) {
	rc = qb_ipcc_auth_get(qb_ipc, &found_pid, &found_uid, &found_gid);
	rc = -rc; // libqb returns 0 or -errno
	if (rc == pcmk_rc_ok) {
	goto found;
	}
	}
	#endif

	#ifdef HAVE_UCRED
	{
	struct ucred ucred;
	socklen_t ucred_len = sizeof(ucred);

	if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &ucred_len) < 0) {
	rc = errno;
	} else if (ucred_len != sizeof(ucred)) {
	rc = EOPNOTSUPP;
	} else {
	found_pid = ucred.pid;
	found_uid = ucred.uid;
	found_gid = ucred.gid;
	goto found;
	}
	}
	#endif

	#ifdef HAVE_SOCKPEERCRED
	{
	struct sockpeercred sockpeercred;
	socklen_t sockpeercred_len = sizeof(sockpeercred);

	if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED,
	&sockpeercred, &sockpeercred_len) < 0) {
	rc = errno;
	} else if (sockpeercred_len != sizeof(sockpeercred)) {
	rc = EOPNOTSUPP;
	} else {
	found_pid = sockpeercred.pid;
	found_uid = sockpeercred.uid;
	found_gid = sockpeercred.gid;
	goto found;
	}
	}
	#endif

	#ifdef HAVE_GETPEEREID // For example, FreeBSD
	if (getpeereid(sock, &found_uid, &found_gid) < 0) {
	rc = errno;
	} else {
	found_pid = PCMK__SPECIAL_PID;
	goto found;
	}
	#endif

	#ifdef HAVE_GETPEERUCRED
	{
	ucred_t *ucred = NULL;

	if (getpeerucred(sock, &ucred) < 0) {
	rc = errno;
	} else {
	found_pid = ucred_getpid(ucred);
	found_uid = ucred_geteuid(ucred);
	found_gid = ucred_getegid(ucred);
	ucred_free(ucred);
	goto found;
	}
	}
	#endif

	return rc; // If we get here, nothing succeeded

	found:
	if (gotpid != NULL) {
	*gotpid = found_pid;
	}
	if (gotuid != NULL) {
	*gotuid = found_uid;
	}
	if (gotgid != NULL) {
	*gotgid = found_gid;
	}
	if ((found_uid != 0) && (found_uid != refuid) && (found_gid != refgid)) {
	return pcmk_rc_ipc_unauthorized;
	}
	return pcmk_rc_ok;
	}

	int
	crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid,
	pid_t gotpid, uid_t gotuid, gid_t *gotgid)
	{
	int ret = is_ipc_provider_expected(NULL, sock, refuid, refgid,
	gotpid, gotuid, gotgid);

	/* The old function had some very odd return codes*/
	if (ret == 0) {
	return 1;
	} else if (ret == pcmk_rc_ipc_unauthorized) {
	return 0;
	} else {
	return pcmk_rc2legacy(ret);
	}
	}

	int
	pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid,
	gid_t refgid, pid_t *gotpid)
	{
	static char last_asked_name[PATH_MAX / 2] = ""; /* log spam prevention */
	int fd;
	int rc = pcmk_rc_ipc_unresponsive;
	int auth_rc = 0;
	int32_t qb_rc;
	pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0;
	qb_ipcc_connection_t *c;
	#ifdef HAVE_QB_IPCC_CONNECT_ASYNC
	struct pollfd pollfd = { 0, };
	int poll_rc;

	c = qb_ipcc_connect_async(name, 0,
	&(pollfd.fd));
	#else
	c = qb_ipcc_connect(name, 0);
	#endif
	if (c == NULL) {
	crm_info("Could not connect to %s IPC: %s", name, strerror(errno));
	rc = pcmk_rc_ipc_unresponsive;
	goto bail;
	}
	#ifdef HAVE_QB_IPCC_CONNECT_ASYNC
	pollfd.events = POLLIN;
	do {
	poll_rc = poll(&pollfd, 1, 2000);
	} while ((poll_rc == -1) && (errno == EINTR));
	if ((poll_rc <= 0) \|\| (qb_ipcc_connect_continue(c) != 0)) {
	crm_info("Could not connect to %s IPC: %s", name,
	(poll_rc == 0)?"timeout":strerror(errno));
	rc = pcmk_rc_ipc_unresponsive;
	if (poll_rc > 0) {
	c = NULL; // qb_ipcc_connect_continue cleaned up for us
	}
	goto bail;
	}
	#endif

	qb_rc = qb_ipcc_fd_get(c, &fd);
	if (qb_rc != 0) {
	rc = (int) -qb_rc; // System errno
	crm_err("Could not get fd from %s IPC: %s " CRM_XS " rc=%d",
	name, pcmk_rc_str(rc), rc);
	goto bail;
	}

	auth_rc = is_ipc_provider_expected(c, fd, refuid, refgid,
	&found_pid, &found_uid, &found_gid);
	if (auth_rc == pcmk_rc_ipc_unauthorized) {
	crm_err("Daemon (IPC %s) effectively blocked with unauthorized"
	" process %lld (uid: %lld, gid: %lld)",
	name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
	(long long) found_uid, (long long) found_gid);
	rc = pcmk_rc_ipc_unauthorized;
	goto bail;
	}

	if (auth_rc != pcmk_rc_ok) {
	rc = auth_rc;
	crm_err("Could not get peer credentials from %s IPC: %s "
	CRM_XS " rc=%d", name, pcmk_rc_str(rc), rc);
	goto bail;
	}

	if (gotpid != NULL) {
	*gotpid = found_pid;
	}

	rc = pcmk_rc_ok;
	if ((found_uid != refuid \|\| found_gid != refgid)
	&& strncmp(last_asked_name, name, sizeof(last_asked_name))) {
	if ((found_uid == 0) && (refuid != 0)) {
	crm_warn("Daemon (IPC %s) runs as root, whereas the expected"
	" credentials are %lld:%lld, hazard of violating"
	" the least privilege principle",
	name, (long long) refuid, (long long) refgid);
	} else {
	crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the"
	" expected credentials are %lld:%lld, which may"
	" mean a different set of privileges than expected",
	name, (long long) found_uid, (long long) found_gid,
	(long long) refuid, (long long) refgid);
	}
	memccpy(last_asked_name, name, '\0', sizeof(last_asked_name));
	}

	bail:
	if (c != NULL) {
	qb_ipcc_disconnect(c);
	}
	return rc;
	}
	diff --git a/lib/common/ipc_server.c b/lib/common/ipc_server.c
	index b1c0b90fe0..5fb30cb595 100644
	--- a/lib/common/ipc_server.c
	+++ b/lib/common/ipc_server.c
	@@ -1,1014 +1,1014 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <errno.h>
	#include <bzlib.h>
	#include <sys/stat.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipc_internal.h>
	#include "crmcommon_private.h"

	/* Evict clients whose event queue grows this large (by default) */
	#define PCMK_IPC_DEFAULT_QUEUE_MAX 500

	static GHashTable *client_connections = NULL;

	/*!
	* \internal
	* \brief Count IPC clients
	*
	* \return Number of active IPC client connections
	*/
	guint
	pcmk__ipc_client_count(void)
	{
	return client_connections? g_hash_table_size(client_connections) : 0;
	}

	/*!
	* \internal
	* \brief Execute a function for each active IPC client connection
	*
	* \param[in] func Function to call
	* \param[in,out] user_data Pointer to pass to function
	*
	* \note The parameters are the same as for g_hash_table_foreach().
	*/
	void
	pcmk__foreach_ipc_client(GHFunc func, gpointer user_data)
	{
	if ((func != NULL) && (client_connections != NULL)) {
	g_hash_table_foreach(client_connections, func, user_data);
	}
	}

	pcmk__client_t *
	pcmk__find_client(const qb_ipcs_connection_t *c)
	{
	if (client_connections) {
	return g_hash_table_lookup(client_connections, c);
	}

	crm_trace("No client found for %p", c);
	return NULL;
	}

	pcmk__client_t *
	pcmk__find_client_by_id(const char *id)
	{
	if ((client_connections != NULL) && (id != NULL)) {
	gpointer key;
	pcmk__client_t *client = NULL;
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, client_connections);
	while (g_hash_table_iter_next(&iter, &key, (gpointer *) & client)) {
	if (strcmp(client->id, id) == 0) {
	return client;
	}
	}
	}
	crm_trace("No client found with id='%s'", pcmk__s(id, ""));
	return NULL;
	}

	/*!
	* \internal
	* \brief Get a client identifier for use in log messages
	*
	* \param[in] c Client
	*
	* \return Client's name, client's ID, or a string literal, as available
	* \note This is intended to be used in format strings like "client %s".
	*/
	const char *
	pcmk__client_name(const pcmk__client_t *c)
	{
	if (c == NULL) {
	return "(unspecified)";

	} else if (c->name != NULL) {
	return c->name;

	} else if (c->id != NULL) {
	return c->id;

	} else {
	return "(unidentified)";
	}
	}

	void
	pcmk__client_cleanup(void)
	{
	if (client_connections != NULL) {
	int active = g_hash_table_size(client_connections);

	if (active > 0) {
	crm_warn("Exiting with %d active IPC client%s",
	active, pcmk__plural_s(active));
	}
	g_hash_table_destroy(client_connections);
	client_connections = NULL;
	}
	}

	void
	pcmk__drop_all_clients(qb_ipcs_service_t *service)
	{
	qb_ipcs_connection_t *c = NULL;

	if (service == NULL) {
	return;
	}

	c = qb_ipcs_connection_first_get(service);

	while (c != NULL) {
	qb_ipcs_connection_t *last = c;

	c = qb_ipcs_connection_next_get(service, last);

	/* There really shouldn't be anyone connected at this point */
	crm_notice("Disconnecting client %p, pid=%d...",
	last, pcmk__client_pid(last));
	qb_ipcs_disconnect(last);
	qb_ipcs_connection_unref(last);
	}
	}

	/*!
	* \internal
	* \brief Allocate a new pcmk__client_t object based on an IPC connection
	*
	* \param[in] c IPC connection (NULL to allocate generic client)
	* \param[in] key Connection table key (NULL to use sane default)
	* \param[in] uid_client UID corresponding to c (ignored if c is NULL)
	*
	* \return Pointer to new pcmk__client_t (or NULL on error)
	*/
	static pcmk__client_t *
	client_from_connection(qb_ipcs_connection_t c, void key, uid_t uid_client)
	{
	pcmk__client_t *client = calloc(1, sizeof(pcmk__client_t));

	if (client == NULL) {
	crm_perror(LOG_ERR, "Allocating client");
	return NULL;
	}

	if (c) {
	client->user = pcmk__uid2username(uid_client);
	if (client->user == NULL) {
	client->user = strdup("#unprivileged");
	CRM_CHECK(client->user != NULL, free(client); return NULL);
	crm_err("Unable to enforce ACLs for user ID %d, assuming unprivileged",
	uid_client);
	}
	client->ipcs = c;
	pcmk__set_client_flags(client, pcmk__client_ipc);
	client->pid = pcmk__client_pid(c);
	if (key == NULL) {
	key = c;
	}
	}

	client->id = crm_generate_uuid();
	if (key == NULL) {
	key = client->id;
	}
	if (client_connections == NULL) {
	crm_trace("Creating IPC client table");
	client_connections = g_hash_table_new(g_direct_hash, g_direct_equal);
	}
	g_hash_table_insert(client_connections, key, client);
	return client;
	}

	/*!
	* \brief Allocate a new pcmk__client_t object and generate its ID
	*
	* \param[in] key What to use as connections hash table key (NULL to use ID)
	*
	* \return Pointer to new pcmk__client_t (asserts on failure)
	*/
	pcmk__client_t *
	pcmk__new_unauth_client(void *key)
	{
	pcmk__client_t *client = client_from_connection(NULL, key, 0);

	CRM_ASSERT(client != NULL);
	return client;
	}

	pcmk__client_t *
	pcmk__new_client(qb_ipcs_connection_t *c, uid_t uid_client, gid_t gid_client)
	{
	gid_t uid_cluster = 0;
	gid_t gid_cluster = 0;

	pcmk__client_t *client = NULL;

	CRM_CHECK(c != NULL, return NULL);

	if (pcmk_daemon_user(&uid_cluster, &gid_cluster) < 0) {
	static bool need_log = TRUE;

	if (need_log) {
	crm_warn("Could not find user and group IDs for user %s",
	CRM_DAEMON_USER);
	need_log = FALSE;
	}
	}

	if (uid_client != 0) {
	crm_trace("Giving group %u access to new IPC connection", gid_cluster);
	/* Passing -1 to chown(2) means don't change */
	qb_ipcs_connection_auth_set(c, -1, gid_cluster, S_IRUSR \| S_IWUSR \| S_IRGRP \| S_IWGRP);
	}

	/* TODO: Do our own auth checking, return NULL if unauthorized */
	client = client_from_connection(c, NULL, uid_client);
	if (client == NULL) {
	return NULL;
	}

	if ((uid_client == 0) \|\| (uid_client == uid_cluster)) {
	/* Remember when a connection came from root or hacluster */
	pcmk__set_client_flags(client, pcmk__client_privileged);
	}

	crm_debug("New IPC client %s for PID %u with uid %d and gid %d",
	client->id, client->pid, uid_client, gid_client);
	return client;
	}

	static struct iovec *
	pcmk__new_ipc_event(void)
	{
	struct iovec *iov = calloc(2, sizeof(struct iovec));

	CRM_ASSERT(iov != NULL);
	return iov;
	}

	/*!
	* \brief Free an I/O vector created by pcmk__ipc_prepare_iov()
	*
	* \param[in,out] event I/O vector to free
	*/
	void
	pcmk_free_ipc_event(struct iovec *event)
	{
	if (event != NULL) {
	free(event[0].iov_base);
	free(event[1].iov_base);
	free(event);
	}
	}

	static void
	free_event(gpointer data)
	{
	pcmk_free_ipc_event((struct iovec *) data);
	}

	static void
	add_event(pcmk__client_t c, struct iovec iov)
	{
	if (c->event_queue == NULL) {
	c->event_queue = g_queue_new();
	}
	g_queue_push_tail(c->event_queue, iov);
	}

	void
	pcmk__free_client(pcmk__client_t *c)
	{
	if (c == NULL) {
	return;
	}

	if (client_connections) {
	if (c->ipcs) {
	crm_trace("Destroying %p/%p (%d remaining)",
	c, c->ipcs, g_hash_table_size(client_connections) - 1);
	g_hash_table_remove(client_connections, c->ipcs);

	} else {
	crm_trace("Destroying remote connection %p (%d remaining)",
	c, g_hash_table_size(client_connections) - 1);
	g_hash_table_remove(client_connections, c->id);
	}
	}

	if (c->event_timer) {
	g_source_remove(c->event_timer);
	}

	if (c->event_queue) {
	crm_debug("Destroying %d events", g_queue_get_length(c->event_queue));
	g_queue_free_full(c->event_queue, free_event);
	}

	free(c->id);
	free(c->name);
	free(c->user);
	if (c->remote) {
	if (c->remote->auth_timeout) {
	g_source_remove(c->remote->auth_timeout);
	}
	free(c->remote->buffer);
	free(c->remote);
	}
	free(c);
	}

	/*!
	* \internal
	* \brief Raise IPC eviction threshold for a client, if allowed
	*
	* \param[in,out] client Client to modify
	* \param[in] qmax New threshold (as non-NULL string)
	*
	* \return true if change was allowed, false otherwise
	*/
	bool
	pcmk__set_client_queue_max(pcmk__client_t client, const char qmax)
	{
	if (pcmk_is_set(client->flags, pcmk__client_privileged)) {
	long long qmax_ll;

	if ((pcmk__scan_ll(qmax, &qmax_ll, 0LL) == pcmk_rc_ok)
	&& (qmax_ll > 0LL) && (qmax_ll <= UINT_MAX)) {
	client->queue_max = (unsigned int) qmax_ll;
	return true;
	}
	}
	return false;
	}

	int
	pcmk__client_pid(qb_ipcs_connection_t *c)
	{
	struct qb_ipcs_connection_stats stats;

	stats.client_pid = 0;
	qb_ipcs_connection_stats_get(c, &stats, 0);
	return stats.client_pid;
	}

	/*!
	* \internal
	* \brief Retrieve message XML from data read from client IPC
	*
	* \param[in,out] c IPC client connection
	* \param[in] data Data read from client connection
	* \param[out] id Where to store message ID from libqb header
	* \param[out] flags Where to store flags from libqb header
	*
	* \return Message XML on success, NULL otherwise
	*/
	xmlNode *
	pcmk__client_data2xml(pcmk__client_t c, void data, uint32_t *id,
	uint32_t *flags)
	{
	xmlNode *xml = NULL;
	char *uncompressed = NULL;
	char text = ((char )data) + sizeof(pcmk__ipc_header_t);
	pcmk__ipc_header_t *header = data;

	if (!pcmk__valid_ipc_header(header)) {
	return NULL;
	}

	if (id) {
	id = ((struct qb_ipc_response_header )data)->id;
	}
	if (flags) {
	*flags = header->flags;
	}

	if (pcmk_is_set(header->flags, crm_ipc_proxied)) {
	/* Mark this client as being the endpoint of a proxy connection.
	* Proxy connections responses are sent on the event channel, to avoid
	* blocking the controller serving as proxy.
	*/
	pcmk__set_client_flags(c, pcmk__client_proxied);
	}

	if (header->size_compressed) {
	int rc = 0;
	unsigned int size_u = 1 + header->size_uncompressed;
	uncompressed = calloc(1, size_u);

	crm_trace("Decompressing message data %u bytes into %u bytes",
	header->size_compressed, size_u);

	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &size_u, text, header->size_compressed, 1, 0);
	text = uncompressed;

	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_err("Decompression failed: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	free(uncompressed);
	return NULL;
	}
	}

	CRM_ASSERT(text[header->size_uncompressed - 1] == 0);

	- xml = string2xml(text);
	+ xml = pcmk__xml_parse_string(text);
	crm_log_xml_trace(xml, "[IPC received]");

	free(uncompressed);
	return xml;
	}

	static int crm_ipcs_flush_events(pcmk__client_t *c);

	static gboolean
	crm_ipcs_flush_events_cb(gpointer data)
	{
	pcmk__client_t *c = data;

	c->event_timer = 0;
	crm_ipcs_flush_events(c);
	return FALSE;
	}

	/*!
	* \internal
	* \brief Add progressive delay before next event queue flush
	*
	* \param[in,out] c Client connection to add delay to
	* \param[in] queue_len Current event queue length
	*/
	static inline void
	delay_next_flush(pcmk__client_t *c, unsigned int queue_len)
	{
	/* Delay a maximum of 1.5 seconds */
	guint delay = (queue_len < 5)? (1000 + 100 * queue_len) : 1500;

	c->event_timer = g_timeout_add(delay, crm_ipcs_flush_events_cb, c);
	}

	/*!
	* \internal
	* \brief Send client any messages in its queue
	*
	* \param[in,out] c Client to flush
	*
	* \return Standard Pacemaker return value
	*/
	static int
	crm_ipcs_flush_events(pcmk__client_t *c)
	{
	int rc = pcmk_rc_ok;
	ssize_t qb_rc = 0;
	unsigned int sent = 0;
	unsigned int queue_len = 0;

	if (c == NULL) {
	return rc;

	} else if (c->event_timer) {
	/* There is already a timer, wait until it goes off */
	crm_trace("Timer active for %p - %d", c->ipcs, c->event_timer);
	return rc;
	}

	if (c->event_queue) {
	queue_len = g_queue_get_length(c->event_queue);
	}
	while (sent < 100) {
	pcmk__ipc_header_t *header = NULL;
	struct iovec *event = NULL;

	if (c->event_queue) {
	// We don't pop unless send is successful
	event = g_queue_peek_head(c->event_queue);
	}
	if (event == NULL) { // Queue is empty
	break;
	}

	qb_rc = qb_ipcs_event_sendv(c->ipcs, event, 2);
	if (qb_rc < 0) {
	rc = (int) -qb_rc;
	break;
	}
	event = g_queue_pop_head(c->event_queue);

	sent++;
	header = event[0].iov_base;
	if (header->size_compressed) {
	crm_trace("Event %d to %p[%d] (%lld compressed bytes) sent",
	header->qb.id, c->ipcs, c->pid, (long long) qb_rc);
	} else {
	crm_trace("Event %d to %p[%d] (%lld bytes) sent: %.120s",
	header->qb.id, c->ipcs, c->pid, (long long) qb_rc,
	(char *) (event[1].iov_base));
	}
	pcmk_free_ipc_event(event);
	}

	queue_len -= sent;
	if (sent > 0 \|\| queue_len) {
	crm_trace("Sent %d events (%d remaining) for %p[%d]: %s (%lld)",
	sent, queue_len, c->ipcs, c->pid,
	pcmk_rc_str(rc), (long long) qb_rc);
	}

	if (queue_len) {

	/* Allow clients to briefly fall behind on processing incoming messages,
	* but drop completely unresponsive clients so the connection doesn't
	* consume resources indefinitely.
	*/
	if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) {
	if ((c->queue_backlog <= 1) \|\| (queue_len < c->queue_backlog)) {
	/* Don't evict for a new or shrinking backlog */
	crm_warn("Client with process ID %u has a backlog of %u messages "
	CRM_XS " %p", c->pid, queue_len, c->ipcs);
	} else {
	crm_err("Evicting client with process ID %u due to backlog of %u messages "
	CRM_XS " %p", c->pid, queue_len, c->ipcs);
	c->queue_backlog = 0;
	qb_ipcs_disconnect(c->ipcs);
	return rc;
	}
	}

	c->queue_backlog = queue_len;
	delay_next_flush(c, queue_len);

	} else {
	/* Event queue is empty, there is no backlog */
	c->queue_backlog = 0;
	}

	return rc;
	}

	/*!
	* \internal
	* \brief Create an I/O vector for sending an IPC XML message
	*
	* \param[in] request Identifier for libqb response header
	* \param[in] message XML message to send
	* \param[in] max_send_size If 0, default IPC buffer size is used
	* \param[out] result Where to store prepared I/O vector
	* \param[out] bytes Size of prepared data in bytes
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__ipc_prepare_iov(uint32_t request, const xmlNode *message,
	uint32_t max_send_size, struct iovec **result,
	ssize_t *bytes)
	{
	static unsigned int biggest = 0;
	struct iovec *iov;
	unsigned int total = 0;
	char *compressed = NULL;
	char *buffer = NULL;
	pcmk__ipc_header_t *header = NULL;

	if ((message == NULL) \|\| (result == NULL)) {
	return EINVAL;
	}

	header = calloc(1, sizeof(pcmk__ipc_header_t));
	if (header == NULL) {
	return ENOMEM; /* errno mightn't be set by allocator */
	}

	buffer = dump_xml_unformatted(message);

	if (max_send_size == 0) {
	max_send_size = crm_ipc_default_buffer_size();
	}
	CRM_LOG_ASSERT(max_send_size != 0);

	*result = NULL;
	iov = pcmk__new_ipc_event();
	iov[0].iov_len = sizeof(pcmk__ipc_header_t);
	iov[0].iov_base = header;

	header->version = PCMK__IPC_VERSION;
	header->size_uncompressed = 1 + strlen(buffer);
	total = iov[0].iov_len + header->size_uncompressed;

	if (total < max_send_size) {
	iov[1].iov_base = buffer;
	iov[1].iov_len = header->size_uncompressed;

	} else {
	unsigned int new_size = 0;

	if (pcmk__compress(buffer, (unsigned int) header->size_uncompressed,
	(unsigned int) max_send_size, &compressed,
	&new_size) == pcmk_rc_ok) {

	pcmk__set_ipc_flags(header->flags, "send data", crm_ipc_compressed);
	header->size_compressed = new_size;

	iov[1].iov_len = header->size_compressed;
	iov[1].iov_base = compressed;

	free(buffer);

	biggest = QB_MAX(header->size_compressed, biggest);

	} else {
	crm_log_xml_trace(message, "EMSGSIZE");
	biggest = QB_MAX(header->size_uncompressed, biggest);

	crm_err("Could not compress %u-byte message into less than IPC "
	"limit of %u bytes; set PCMK_ipc_buffer to higher value "
	"(%u bytes suggested)",
	header->size_uncompressed, max_send_size, 4 * biggest);

	free(compressed);
	free(buffer);
	pcmk_free_ipc_event(iov);
	return EMSGSIZE;
	}
	}

	header->qb.size = iov[0].iov_len + iov[1].iov_len;
	header->qb.id = (int32_t)request; /* Replying to a specific request */

	*result = iov;
	CRM_ASSERT(header->qb.size > 0);
	if (bytes != NULL) {
	*bytes = header->qb.size;
	}
	return pcmk_rc_ok;
	}

	int
	pcmk__ipc_send_iov(pcmk__client_t c, struct iovec iov, uint32_t flags)
	{
	int rc = pcmk_rc_ok;
	static uint32_t id = 1;
	pcmk__ipc_header_t *header = iov[0].iov_base;

	if (c->flags & pcmk__client_proxied) {
	/* _ALL_ replies to proxied connections need to be sent as events */
	if (!pcmk_is_set(flags, crm_ipc_server_event)) {
	/* The proxied flag lets us know this was originally meant to be a
	* response, even though we're sending it over the event channel.
	*/
	pcmk__set_ipc_flags(flags, "server event",
	crm_ipc_server_event
	\|crm_ipc_proxied_relay_response);
	}
	}

	pcmk__set_ipc_flags(header->flags, "server event", flags);
	if (flags & crm_ipc_server_event) {
	header->qb.id = id++; /* We don't really use it, but doesn't hurt to set one */

	if (flags & crm_ipc_server_free) {
	crm_trace("Sending the original to %p[%d]", c->ipcs, c->pid);
	add_event(c, iov);

	} else {
	struct iovec *iov_copy = pcmk__new_ipc_event();

	crm_trace("Sending a copy to %p[%d]", c->ipcs, c->pid);
	iov_copy[0].iov_len = iov[0].iov_len;
	iov_copy[0].iov_base = malloc(iov[0].iov_len);
	memcpy(iov_copy[0].iov_base, iov[0].iov_base, iov[0].iov_len);

	iov_copy[1].iov_len = iov[1].iov_len;
	iov_copy[1].iov_base = malloc(iov[1].iov_len);
	memcpy(iov_copy[1].iov_base, iov[1].iov_base, iov[1].iov_len);

	add_event(c, iov_copy);
	}

	} else {
	ssize_t qb_rc;

	CRM_LOG_ASSERT(header->qb.id != 0); /* Replying to a specific request */

	qb_rc = qb_ipcs_response_sendv(c->ipcs, iov, 2);
	if (qb_rc < header->qb.size) {
	if (qb_rc < 0) {
	rc = (int) -qb_rc;
	}
	crm_notice("Response %d to pid %d failed: %s "
	CRM_XS " bytes=%u rc=%lld ipcs=%p",
	header->qb.id, c->pid, pcmk_rc_str(rc),
	header->qb.size, (long long) qb_rc, c->ipcs);

	} else {
	crm_trace("Response %d sent, %lld bytes to %p[%d]",
	header->qb.id, (long long) qb_rc, c->ipcs, c->pid);
	}

	if (flags & crm_ipc_server_free) {
	pcmk_free_ipc_event(iov);
	}
	}

	if (flags & crm_ipc_server_event) {
	rc = crm_ipcs_flush_events(c);
	} else {
	crm_ipcs_flush_events(c);
	}

	if ((rc == EPIPE) \|\| (rc == ENOTCONN)) {
	crm_trace("Client %p disconnected", c->ipcs);
	}
	return rc;
	}

	int
	pcmk__ipc_send_xml(pcmk__client_t c, uint32_t request, const xmlNode message,
	uint32_t flags)
	{
	struct iovec *iov = NULL;
	int rc = pcmk_rc_ok;

	if (c == NULL) {
	return EINVAL;
	}
	rc = pcmk__ipc_prepare_iov(request, message, crm_ipc_default_buffer_size(),
	&iov, NULL);
	if (rc == pcmk_rc_ok) {
	pcmk__set_ipc_flags(flags, "send data", crm_ipc_server_free);
	rc = pcmk__ipc_send_iov(c, iov, flags);
	} else {
	pcmk_free_ipc_event(iov);
	crm_notice("IPC message to pid %d failed: %s " CRM_XS " rc=%d",
	c->pid, pcmk_rc_str(rc), rc);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Create an acknowledgement with a status code to send to a client
	*
	* \param[in] function Calling function
	* \param[in] line Source file line within calling function
	* \param[in] flags IPC flags to use when sending
	* \param[in] tag Element name to use for acknowledgement
	* \param[in] ver IPC protocol version (can be NULL)
	* \param[in] status Exit status code to add to ack
	*
	* \return Newly created XML for ack
	* \note The caller is responsible for freeing the return value with free_xml().
	*/
	xmlNode *
	pcmk__ipc_create_ack_as(const char *function, int line, uint32_t flags,
	const char tag, const char ver, crm_exit_t status)
	{
	xmlNode *ack = NULL;

	if (pcmk_is_set(flags, crm_ipc_client_response)) {
	ack = create_xml_node(NULL, tag);
	crm_xml_add(ack, PCMK_XA_FUNCTION, function);
	crm_xml_add_int(ack, PCMK__XA_LINE, line);
	crm_xml_add_int(ack, PCMK_XA_STATUS, (int) status);
	crm_xml_add(ack, PCMK__XA_IPC_PROTO_VERSION, ver);
	}
	return ack;
	}

	/*!
	* \internal
	* \brief Send an acknowledgement with a status code to a client
	*
	* \param[in] function Calling function
	* \param[in] line Source file line within calling function
	* \param[in] c Client to send ack to
	* \param[in] request Request ID being replied to
	* \param[in] flags IPC flags to use when sending
	* \param[in] tag Element name to use for acknowledgement
	* \param[in] ver IPC protocol version (can be NULL)
	* \param[in] status Status code to send with acknowledgement
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__ipc_send_ack_as(const char function, int line, pcmk__client_t c,
	uint32_t request, uint32_t flags, const char *tag,
	const char *ver, crm_exit_t status)
	{
	int rc = pcmk_rc_ok;
	xmlNode *ack = pcmk__ipc_create_ack_as(function, line, flags, tag, ver, status);

	if (ack != NULL) {
	crm_trace("Ack'ing IPC message from client %s as <%s status=%d>",
	pcmk__client_name(c), tag, status);
	crm_log_xml_trace(ack, "sent-ack");
	c->request_id = 0;
	rc = pcmk__ipc_send_xml(c, request, ack, flags);
	free_xml(ack);
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Add an IPC server to the main loop for the pacemaker-based API
	*
	* \param[out] ipcs_ro New IPC server for read-only pacemaker-based API
	* \param[out] ipcs_rw New IPC server for read/write pacemaker-based API
	* \param[out] ipcs_shm New IPC server for shared-memory pacemaker-based API
	* \param[in] ro_cb IPC callbacks for read-only API
	* \param[in] rw_cb IPC callbacks for read/write and shared-memory APIs
	*
	* \note This function exits fatally if unable to create the servers.
	*/
	void pcmk__serve_based_ipc(qb_ipcs_service_t **ipcs_ro,
	qb_ipcs_service_t **ipcs_rw,
	qb_ipcs_service_t **ipcs_shm,
	struct qb_ipcs_service_handlers *ro_cb,
	struct qb_ipcs_service_handlers *rw_cb)
	{
	*ipcs_ro = mainloop_add_ipc_server(PCMK__SERVER_BASED_RO,
	QB_IPC_NATIVE, ro_cb);

	*ipcs_rw = mainloop_add_ipc_server(PCMK__SERVER_BASED_RW,
	QB_IPC_NATIVE, rw_cb);

	*ipcs_shm = mainloop_add_ipc_server(PCMK__SERVER_BASED_SHM,
	QB_IPC_SHM, rw_cb);

	if (ipcs_ro == NULL \|\| ipcs_rw == NULL \|\| *ipcs_shm == NULL) {
	crm_err("Failed to create the CIB manager: exiting and inhibiting respawn");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled");
	crm_exit(CRM_EX_FATAL);
	}
	}

	/*!
	* \internal
	* \brief Destroy IPC servers for pacemaker-based API
	*
	* \param[out] ipcs_ro IPC server for read-only pacemaker-based API
	* \param[out] ipcs_rw IPC server for read/write pacemaker-based API
	* \param[out] ipcs_shm IPC server for shared-memory pacemaker-based API
	*
	* \note This is a convenience function for calling qb_ipcs_destroy() for each
	* argument.
	*/
	void
	pcmk__stop_based_ipc(qb_ipcs_service_t *ipcs_ro,
	qb_ipcs_service_t *ipcs_rw,
	qb_ipcs_service_t *ipcs_shm)
	{
	qb_ipcs_destroy(ipcs_ro);
	qb_ipcs_destroy(ipcs_rw);
	qb_ipcs_destroy(ipcs_shm);
	}

	/*!
	* \internal
	* \brief Add an IPC server to the main loop for the pacemaker-controld API
	*
	* \param[in] cb IPC callbacks
	*
	* \return Newly created IPC server
	*/
	qb_ipcs_service_t *
	pcmk__serve_controld_ipc(struct qb_ipcs_service_handlers *cb)
	{
	return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb);
	}

	/*!
	* \internal
	* \brief Add an IPC server to the main loop for the pacemaker-attrd API
	*
	* \param[out] ipcs Where to store newly created IPC server
	* \param[in] cb IPC callbacks
	*
	* \note This function exits fatally if unable to create the servers.
	*/
	void
	pcmk__serve_attrd_ipc(qb_ipcs_service_t **ipcs,
	struct qb_ipcs_service_handlers *cb)
	{
	*ipcs = mainloop_add_ipc_server(PCMK__VALUE_ATTRD, QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Failed to create pacemaker-attrd server: exiting and inhibiting respawn");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(CRM_EX_FATAL);
	}
	}

	/*!
	* \internal
	* \brief Add an IPC server to the main loop for the pacemaker-fenced API
	*
	* \param[out] ipcs Where to store newly created IPC server
	* \param[in] cb IPC callbacks
	*
	* \note This function exits fatally if unable to create the servers.
	*/
	void
	pcmk__serve_fenced_ipc(qb_ipcs_service_t **ipcs,
	struct qb_ipcs_service_handlers *cb)
	{
	*ipcs = mainloop_add_ipc_server_with_prio("stonith-ng", QB_IPC_NATIVE, cb,
	QB_LOOP_HIGH);

	if (*ipcs == NULL) {
	crm_err("Failed to create fencer: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(CRM_EX_FATAL);
	}
	}

	/*!
	* \internal
	* \brief Add an IPC server to the main loop for the pacemakerd API
	*
	* \param[out] ipcs Where to store newly created IPC server
	* \param[in] cb IPC callbacks
	*
	* \note This function exits with CRM_EX_OSERR if unable to create the servers.
	*/
	void
	pcmk__serve_pacemakerd_ipc(qb_ipcs_service_t **ipcs,
	struct qb_ipcs_service_handlers *cb)
	{
	*ipcs = mainloop_add_ipc_server(CRM_SYSTEM_MCP, QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Couldn't start pacemakerd IPC server");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	/* sub-daemons are observed by pacemakerd. Thus we exit CRM_EX_FATAL
	* if we want to prevent pacemakerd from restarting them.
	* With pacemakerd we leave the exit-code shown to e.g. systemd
	* to what it was prior to moving the code here from pacemakerd.c
	*/
	crm_exit(CRM_EX_OSERR);
	}
	}

	/*!
	* \internal
	* \brief Add an IPC server to the main loop for the pacemaker-schedulerd API
	*
	* \param[in] cb IPC callbacks
	*
	* \return Newly created IPC server
	* \note This function exits fatally if unable to create the servers.
	*/
	qb_ipcs_service_t *
	pcmk__serve_schedulerd_ipc(struct qb_ipcs_service_handlers *cb)
	{
	return mainloop_add_ipc_server(CRM_SYSTEM_PENGINE, QB_IPC_NATIVE, cb);
	}

	/*!
	* \brief Check whether string represents a client name used by cluster daemons
	*
	* \param[in] name String to check
	*
	* \return true if name is standard client name used by daemons, false otherwise
	*
	* \note This is provided by the client, and so cannot be used by itself as a
	* secure means of authentication.
	*/
	bool
	crm_is_daemon_name(const char *name)
	{
	return pcmk__str_any_of(pcmk__message_name(name),
	"attrd",
	CRM_SYSTEM_CIB,
	CRM_SYSTEM_CRMD,
	CRM_SYSTEM_DC,
	CRM_SYSTEM_LRMD,
	CRM_SYSTEM_MCP,
	CRM_SYSTEM_PENGINE,
	CRM_SYSTEM_STONITHD,
	CRM_SYSTEM_TENGINE,
	"pacemaker-remoted",
	"stonith-ng",
	NULL);
	}
	diff --git a/lib/common/remote.c b/lib/common/remote.c
	index 488b73a24c..140783caea 100644
	--- a/lib/common/remote.c
	+++ b/lib/common/remote.c
	@@ -1,1284 +1,1284 @@
	/*
	- * Copyright 2008-2023 the Pacemaker project contributors
	+ * Copyright 2008-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/socket.h>
	#include <arpa/inet.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <netinet/tcp.h>
	#include <netdb.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <inttypes.h> // PRIx32

	#include <glib.h>
	#include <bzlib.h>

	#include <crm/common/ipc_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/remote_internal.h>

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# include <gnutls/gnutls.h>
	#endif

	/* Swab macros from linux/swab.h */
	#ifdef HAVE_LINUX_SWAB_H
	# include <linux/swab.h>
	#else
	/*
	* casts are necessary for constants, because we never know how for sure
	* how U/UL/ULL map to __u16, __u32, __u64. At least not in a portable way.
	*/
	#define __swab16(x) ((uint16_t)( \
	(((uint16_t)(x) & (uint16_t)0x00ffU) << 8) \| \
	(((uint16_t)(x) & (uint16_t)0xff00U) >> 8)))

	#define __swab32(x) ((uint32_t)( \
	(((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) \| \
	(((uint32_t)(x) & (uint32_t)0x0000ff00UL) << 8) \| \
	(((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) \| \
	(((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24)))

	#define __swab64(x) ((uint64_t)( \
	(((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) \| \
	(((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) \| \
	(((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) \| \
	(((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) << 8) \| \
	(((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >> 8) \| \
	(((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) \| \
	(((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) \| \
	(((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56)))
	#endif

	#define REMOTE_MSG_VERSION 1
	#define ENDIAN_LOCAL 0xBADADBBD

	struct remote_header_v0 {
	uint32_t endian; /* Detect messages from hosts with different endian-ness */
	uint32_t version;
	uint64_t id;
	uint64_t flags;
	uint32_t size_total;
	uint32_t payload_offset;
	uint32_t payload_compressed;
	uint32_t payload_uncompressed;

	/* New fields get added here */

	} __attribute__ ((packed));

	/*!
	* \internal
	* \brief Retrieve remote message header, in local endianness
	*
	* Return a pointer to the header portion of a remote connection's message
	* buffer, converting the header to local endianness if needed.
	*
	* \param[in,out] remote Remote connection with new message
	*
	* \return Pointer to message header, localized if necessary
	*/
	static struct remote_header_v0 *
	localized_remote_header(pcmk__remote_t *remote)
	{
	struct remote_header_v0 header = (struct remote_header_v0 )remote->buffer;
	if(remote->buffer_offset < sizeof(struct remote_header_v0)) {
	return NULL;

	} else if(header->endian != ENDIAN_LOCAL) {
	uint32_t endian = __swab32(header->endian);

	CRM_LOG_ASSERT(endian == ENDIAN_LOCAL);
	if(endian != ENDIAN_LOCAL) {
	crm_err("Invalid message detected, endian mismatch: %" PRIx32
	" is neither %" PRIx32 " nor the swab'd %" PRIx32,
	ENDIAN_LOCAL, header->endian, endian);
	return NULL;
	}

	header->id = __swab64(header->id);
	header->flags = __swab64(header->flags);
	header->endian = __swab32(header->endian);

	header->version = __swab32(header->version);
	header->size_total = __swab32(header->size_total);
	header->payload_offset = __swab32(header->payload_offset);
	header->payload_compressed = __swab32(header->payload_compressed);
	header->payload_uncompressed = __swab32(header->payload_uncompressed);
	}

	return header;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H

	int
	pcmk__tls_client_handshake(pcmk__remote_t *remote, int timeout_ms)
	{
	int rc = 0;
	int pollrc = 0;
	time_t time_limit = time(NULL) + timeout_ms / 1000;

	do {
	rc = gnutls_handshake(*remote->tls_session);
	if ((rc == GNUTLS_E_INTERRUPTED) \|\| (rc == GNUTLS_E_AGAIN)) {
	pollrc = pcmk__remote_ready(remote, 1000);
	if ((pollrc != pcmk_rc_ok) && (pollrc != ETIME)) {
	/* poll returned error, there is no hope */
	crm_trace("TLS handshake poll failed: %s (%d)",
	pcmk_strerror(pollrc), pollrc);
	return pcmk_legacy2rc(pollrc);
	}
	} else if (rc < 0) {
	crm_trace("TLS handshake failed: %s (%d)",
	gnutls_strerror(rc), rc);
	return EPROTO;
	} else {
	return pcmk_rc_ok;
	}
	} while (time(NULL) < time_limit);
	return ETIME;
	}

	/*!
	* \internal
	* \brief Set minimum prime size required by TLS client
	*
	* \param[in] session TLS session to affect
	*/
	static void
	set_minimum_dh_bits(const gnutls_session_t *session)
	{
	int dh_min_bits;

	pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MIN_BITS), &dh_min_bits,
	0);

	/* This function is deprecated since GnuTLS 3.1.7, in favor of letting
	* the priority string imply the DH requirements, but this is the only
	* way to give the user control over compatibility with older servers.
	*/
	if (dh_min_bits > 0) {
	crm_info("Requiring server use a Diffie-Hellman prime of at least %d bits",
	dh_min_bits);
	gnutls_dh_set_prime_bits(*session, dh_min_bits);
	}
	}

	static unsigned int
	get_bound_dh_bits(unsigned int dh_bits)
	{
	int dh_min_bits;
	int dh_max_bits;

	pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MIN_BITS), &dh_min_bits,
	0);
	pcmk__scan_min_int(pcmk__env_option(PCMK__ENV_DH_MAX_BITS), &dh_max_bits,
	0);

	if ((dh_max_bits > 0) && (dh_max_bits < dh_min_bits)) {
	crm_warn("Ignoring PCMK_dh_max_bits less than PCMK_dh_min_bits");
	dh_max_bits = 0;
	}
	if ((dh_min_bits > 0) && (dh_bits < dh_min_bits)) {
	return dh_min_bits;
	}
	if ((dh_max_bits > 0) && (dh_bits > dh_max_bits)) {
	return dh_max_bits;
	}
	return dh_bits;
	}

	/*!
	* \internal
	* \brief Initialize a new TLS session
	*
	* \param[in] csock Connected socket for TLS session
	* \param[in] conn_type GNUTLS_SERVER or GNUTLS_CLIENT
	* \param[in] cred_type GNUTLS_CRD_ANON or GNUTLS_CRD_PSK
	* \param[in] credentials TLS session credentials
	*
	* \return Pointer to newly created session object, or NULL on error
	*/
	gnutls_session_t *
	pcmk__new_tls_session(int csock, unsigned int conn_type,
	gnutls_credentials_type_t cred_type, void *credentials)
	{
	int rc = GNUTLS_E_SUCCESS;
	const char *prio_base = NULL;
	char *prio = NULL;
	gnutls_session_t *session = NULL;

	/* Determine list of acceptable ciphers, etc. Pacemaker always adds the
	* values required for its functionality.
	*
	* For an example of anonymous authentication, see:
	* http://www.manpagez.com/info/gnutls/gnutls-2.10.4/gnutls_81.php#Echo-Server-with-anonymous-authentication
	*/

	prio_base = pcmk__env_option(PCMK__ENV_TLS_PRIORITIES);
	if (prio_base == NULL) {
	prio_base = PCMK_GNUTLS_PRIORITIES;
	}
	prio = crm_strdup_printf("%s:%s", prio_base,
	(cred_type == GNUTLS_CRD_ANON)? "+ANON-DH" : "+DHE-PSK:+PSK");

	session = gnutls_malloc(sizeof(gnutls_session_t));
	if (session == NULL) {
	rc = GNUTLS_E_MEMORY_ERROR;
	goto error;
	}

	rc = gnutls_init(session, conn_type);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}

	/* @TODO On the server side, it would be more efficient to cache the
	* priority with gnutls_priority_init2() and set it with
	* gnutls_priority_set() for all sessions.
	*/
	rc = gnutls_priority_set_direct(*session, prio, NULL);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}
	if (conn_type == GNUTLS_CLIENT) {
	set_minimum_dh_bits(session);
	}

	gnutls_transport_set_ptr(*session,
	(gnutls_transport_ptr_t) GINT_TO_POINTER(csock));

	rc = gnutls_credentials_set(*session, cred_type, credentials);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}
	free(prio);
	return session;

	error:
	crm_err("Could not initialize %s TLS %s session: %s "
	CRM_XS " rc=%d priority='%s'",
	(cred_type == GNUTLS_CRD_ANON)? "anonymous" : "PSK",
	(conn_type == GNUTLS_SERVER)? "server" : "client",
	gnutls_strerror(rc), rc, prio);
	free(prio);
	if (session != NULL) {
	gnutls_free(session);
	}
	return NULL;
	}

	/*!
	* \internal
	* \brief Initialize Diffie-Hellman parameters for a TLS server
	*
	* \param[out] dh_params Parameter object to initialize
	*
	* \return Standard Pacemaker return code
	* \todo The current best practice is to allow the client and server to
	* negotiate the Diffie-Hellman parameters via a TLS extension (RFC 7919).
	* However, we have to support both older versions of GnuTLS (<3.6) that
	* don't support the extension on our side, and older Pacemaker versions
	* that don't support the extension on the other side. The next best
	* practice would be to use a known good prime (see RFC 5114 section 2.2),
	* possibly stored in a file distributed with Pacemaker.
	*/
	int
	pcmk__init_tls_dh(gnutls_dh_params_t *dh_params)
	{
	int rc = GNUTLS_E_SUCCESS;
	unsigned int dh_bits = 0;

	rc = gnutls_dh_params_init(dh_params);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}

	dh_bits = gnutls_sec_param_to_pk_bits(GNUTLS_PK_DH,
	GNUTLS_SEC_PARAM_NORMAL);
	if (dh_bits == 0) {
	rc = GNUTLS_E_DH_PRIME_UNACCEPTABLE;
	goto error;
	}
	dh_bits = get_bound_dh_bits(dh_bits);

	crm_info("Generating Diffie-Hellman parameters with %u-bit prime for TLS",
	dh_bits);
	rc = gnutls_dh_params_generate2(*dh_params, dh_bits);
	if (rc != GNUTLS_E_SUCCESS) {
	goto error;
	}

	return pcmk_rc_ok;

	error:
	crm_err("Could not initialize Diffie-Hellman parameters for TLS: %s "
	CRM_XS " rc=%d", gnutls_strerror(rc), rc);
	return EPROTO;
	}

	/*!
	* \internal
	* \brief Process handshake data from TLS client
	*
	* Read as much TLS handshake data as is available.
	*
	* \param[in] client Client connection
	*
	* \return Standard Pacemaker return code (of particular interest, EAGAIN
	* if some data was successfully read but more data is needed)
	*/
	int
	pcmk__read_handshake_data(const pcmk__client_t *client)
	{
	int rc = 0;

	CRM_ASSERT(client && client->remote && client->remote->tls_session);

	do {
	rc = gnutls_handshake(*client->remote->tls_session);
	} while (rc == GNUTLS_E_INTERRUPTED);

	if (rc == GNUTLS_E_AGAIN) {
	/* No more data is available at the moment. This function should be
	* invoked again once the client sends more.
	*/
	return EAGAIN;
	} else if (rc != GNUTLS_E_SUCCESS) {
	crm_err("TLS handshake with remote client failed: %s "
	CRM_XS " rc=%d", gnutls_strerror(rc), rc);
	return EPROTO;
	}
	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	send_tls(gnutls_session_t session, struct iovec iov)
	{
	const char *unsent = iov->iov_base;
	size_t unsent_len = iov->iov_len;
	ssize_t gnutls_rc;

	if (unsent == NULL) {
	return EINVAL;
	}

	crm_trace("Sending TLS message of %llu bytes",
	(unsigned long long) unsent_len);
	while (true) {
	gnutls_rc = gnutls_record_send(*session, unsent, unsent_len);

	if (gnutls_rc == GNUTLS_E_INTERRUPTED \|\| gnutls_rc == GNUTLS_E_AGAIN) {
	crm_trace("Retrying to send %llu bytes remaining",
	(unsigned long long) unsent_len);

	} else if (gnutls_rc < 0) {
	// Caller can log as error if necessary
	crm_info("TLS connection terminated: %s " CRM_XS " rc=%lld",
	gnutls_strerror((int) gnutls_rc),
	(long long) gnutls_rc);
	return ECONNABORTED;

	} else if (gnutls_rc < unsent_len) {
	crm_trace("Sent %lld of %llu bytes remaining",
	(long long) gnutls_rc, (unsigned long long) unsent_len);
	unsent_len -= gnutls_rc;
	unsent += gnutls_rc;
	} else {
	crm_trace("Sent all %lld bytes remaining", (long long) gnutls_rc);
	break;
	}
	}
	return pcmk_rc_ok;
	}
	#endif

	// \return Standard Pacemaker return code
	static int
	send_plaintext(int sock, struct iovec *iov)
	{
	const char *unsent = iov->iov_base;
	size_t unsent_len = iov->iov_len;
	ssize_t write_rc;

	if (unsent == NULL) {
	return EINVAL;
	}

	crm_debug("Sending plaintext message of %llu bytes to socket %d",
	(unsigned long long) unsent_len, sock);
	while (true) {
	write_rc = write(sock, unsent, unsent_len);
	if (write_rc < 0) {
	int rc = errno;

	if ((errno == EINTR) \|\| (errno == EAGAIN)) {
	crm_trace("Retrying to send %llu bytes remaining to socket %d",
	(unsigned long long) unsent_len, sock);
	continue;
	}

	// Caller can log as error if necessary
	crm_info("Could not send message: %s " CRM_XS " rc=%d socket=%d",
	pcmk_rc_str(rc), rc, sock);
	return rc;

	} else if (write_rc < unsent_len) {
	crm_trace("Sent %lld of %llu bytes remaining",
	(long long) write_rc, (unsigned long long) unsent_len);
	unsent += write_rc;
	unsent_len -= write_rc;
	continue;

	} else {
	crm_trace("Sent all %lld bytes remaining: %.100s",
	(long long) write_rc, (char *) (iov->iov_base));
	break;
	}
	}
	return pcmk_rc_ok;
	}

	// \return Standard Pacemaker return code
	static int
	remote_send_iovs(pcmk__remote_t remote, struct iovec iov, int iovs)
	{
	int rc = pcmk_rc_ok;

	for (int lpc = 0; (lpc < iovs) && (rc == pcmk_rc_ok); lpc++) {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (remote->tls_session) {
	rc = send_tls(remote->tls_session, &(iov[lpc]));
	continue;
	}
	#endif
	if (remote->tcp_socket) {
	rc = send_plaintext(remote->tcp_socket, &(iov[lpc]));
	} else {
	rc = ESOCKTNOSUPPORT;
	}
	}
	return rc;
	}

	/*!
	* \internal
	* \brief Send an XML message over a Pacemaker Remote connection
	*
	* \param[in,out] remote Pacemaker Remote connection to use
	* \param[in] msg XML to send
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__remote_send_xml(pcmk__remote_t remote, const xmlNode msg)
	{
	int rc = pcmk_rc_ok;
	static uint64_t id = 0;
	char *xml_text = NULL;

	struct iovec iov[2];
	struct remote_header_v0 *header;

	CRM_CHECK((remote != NULL) && (msg != NULL), return EINVAL);

	xml_text = dump_xml_unformatted(msg);
	CRM_CHECK(xml_text != NULL, return EINVAL);

	header = calloc(1, sizeof(struct remote_header_v0));
	CRM_ASSERT(header != NULL);

	iov[0].iov_base = header;
	iov[0].iov_len = sizeof(struct remote_header_v0);

	iov[1].iov_base = xml_text;
	iov[1].iov_len = 1 + strlen(xml_text);

	id++;
	header->id = id;
	header->endian = ENDIAN_LOCAL;
	header->version = REMOTE_MSG_VERSION;
	header->payload_offset = iov[0].iov_len;
	header->payload_uncompressed = iov[1].iov_len;
	header->size_total = iov[0].iov_len + iov[1].iov_len;

	rc = remote_send_iovs(remote, iov, 2);
	if (rc != pcmk_rc_ok) {
	crm_err("Could not send remote message: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	}

	free(iov[0].iov_base);
	free(iov[1].iov_base);
	return rc;
	}

	/*!
	* \internal
	* \brief Obtain the XML from the currently buffered remote connection message
	*
	* \param[in,out] remote Remote connection possibly with message available
	*
	* \return Newly allocated XML object corresponding to message data, or NULL
	* \note This effectively removes the message from the connection buffer.
	*/
	xmlNode *
	pcmk__remote_message_xml(pcmk__remote_t *remote)
	{
	xmlNode *xml = NULL;
	struct remote_header_v0 *header = localized_remote_header(remote);

	if (header == NULL) {
	return NULL;
	}

	/* Support compression on the receiving end now, in case we ever want to add it later */
	if (header->payload_compressed) {
	int rc = 0;
	unsigned int size_u = 1 + header->payload_uncompressed;
	char *uncompressed = calloc(1, header->payload_offset + size_u);

	crm_trace("Decompressing message data %d bytes into %d bytes",
	header->payload_compressed, size_u);

	rc = BZ2_bzBuffToBuffDecompress(uncompressed + header->payload_offset, &size_u,
	remote->buffer + header->payload_offset,
	header->payload_compressed, 1, 0);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok && header->version > REMOTE_MSG_VERSION) {
	crm_warn("Couldn't decompress v%d message, we only understand v%d",
	header->version, REMOTE_MSG_VERSION);
	free(uncompressed);
	return NULL;

	} else if (rc != pcmk_rc_ok) {
	crm_err("Decompression failed: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	free(uncompressed);
	return NULL;
	}

	CRM_ASSERT(size_u == header->payload_uncompressed);

	memcpy(uncompressed, remote->buffer, header->payload_offset); /* Preserve the header */
	remote->buffer_size = header->payload_offset + size_u;

	free(remote->buffer);
	remote->buffer = uncompressed;
	header = localized_remote_header(remote);
	}

	/* take ownership of the buffer */
	remote->buffer_offset = 0;

	CRM_LOG_ASSERT(remote->buffer[sizeof(struct remote_header_v0) + header->payload_uncompressed - 1] == 0);

	- xml = string2xml(remote->buffer + header->payload_offset);
	+ xml = pcmk__xml_parse_string(remote->buffer + header->payload_offset);
	if (xml == NULL && header->version > REMOTE_MSG_VERSION) {
	crm_warn("Couldn't parse v%d message, we only understand v%d",
	header->version, REMOTE_MSG_VERSION);

	} else if (xml == NULL) {
	crm_err("Couldn't parse: '%.120s'", remote->buffer + header->payload_offset);
	}

	return xml;
	}

	static int
	get_remote_socket(const pcmk__remote_t *remote)
	{
	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (remote->tls_session) {
	void sock_ptr = gnutls_transport_get_ptr(remote->tls_session);

	return GPOINTER_TO_INT(sock_ptr);
	}
	#endif

	if (remote->tcp_socket) {
	return remote->tcp_socket;
	}

	crm_err("Remote connection type undetermined (bug?)");
	return -1;
	}

	/*!
	* \internal
	* \brief Wait for a remote session to have data to read
	*
	* \param[in] remote Connection to check
	* \param[in] timeout_ms Maximum time (in ms) to wait
	*
	* \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
	* there is data ready to be read, and ETIME if there is no data within
	* the specified timeout)
	*/
	int
	pcmk__remote_ready(const pcmk__remote_t *remote, int timeout_ms)
	{
	struct pollfd fds = { 0, };
	int sock = 0;
	int rc = 0;
	time_t start;
	int timeout = timeout_ms;

	sock = get_remote_socket(remote);
	if (sock <= 0) {
	crm_trace("No longer connected");
	return ENOTCONN;
	}

	start = time(NULL);
	errno = 0;
	do {
	fds.fd = sock;
	fds.events = POLLIN;

	/* If we got an EINTR while polling, and we have a
	* specific timeout we are trying to honor, attempt
	* to adjust the timeout to the closest second. */
	if (errno == EINTR && (timeout > 0)) {
	timeout = timeout_ms - ((time(NULL) - start) * 1000);
	if (timeout < 1000) {
	timeout = 1000;
	}
	}

	rc = poll(&fds, 1, timeout);
	} while (rc < 0 && errno == EINTR);

	if (rc < 0) {
	return errno;
	}
	return (rc == 0)? ETIME : pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Read bytes from non-blocking remote connection
	*
	* \param[in,out] remote Remote connection to read
	*
	* \return Standard Pacemaker return code (of particular interest, pcmk_rc_ok if
	* a full message has been received, or EAGAIN for a partial message)
	* \note Use only with non-blocking sockets after polling the socket.
	* \note This function will return when the socket read buffer is empty or an
	* error is encountered.
	*/
	static int
	read_available_remote_data(pcmk__remote_t *remote)
	{
	int rc = pcmk_rc_ok;
	size_t read_len = sizeof(struct remote_header_v0);
	struct remote_header_v0 *header = localized_remote_header(remote);
	bool received = false;
	ssize_t read_rc;

	if(header) {
	/* Stop at the end of the current message */
	read_len = header->size_total;
	}

	/* automatically grow the buffer when needed */
	if(remote->buffer_size < read_len) {
	remote->buffer_size = 2 * read_len;
	crm_trace("Expanding buffer to %llu bytes",
	(unsigned long long) remote->buffer_size);
	remote->buffer = pcmk__realloc(remote->buffer, remote->buffer_size + 1);
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (!received && remote->tls_session) {
	read_rc = gnutls_record_recv(*(remote->tls_session),
	remote->buffer + remote->buffer_offset,
	remote->buffer_size - remote->buffer_offset);
	if (read_rc == GNUTLS_E_INTERRUPTED) {
	rc = EINTR;
	} else if (read_rc == GNUTLS_E_AGAIN) {
	rc = EAGAIN;
	} else if (read_rc < 0) {
	crm_debug("TLS receive failed: %s (%lld)",
	gnutls_strerror(read_rc), (long long) read_rc);
	rc = EIO;
	}
	received = true;
	}
	#endif

	if (!received && remote->tcp_socket) {
	read_rc = read(remote->tcp_socket,
	remote->buffer + remote->buffer_offset,
	remote->buffer_size - remote->buffer_offset);
	if (read_rc < 0) {
	rc = errno;
	}
	received = true;
	}

	if (!received) {
	crm_err("Remote connection type undetermined (bug?)");
	return ESOCKTNOSUPPORT;
	}

	/* process any errors. */
	if (read_rc > 0) {
	remote->buffer_offset += read_rc;
	/* always null terminate buffer, the +1 to alloc always allows for this. */
	remote->buffer[remote->buffer_offset] = '\0';
	crm_trace("Received %lld more bytes (%llu total)",
	(long long) read_rc,
	(unsigned long long) remote->buffer_offset);

	} else if ((rc == EINTR) \|\| (rc == EAGAIN)) {
	crm_trace("No data available for non-blocking remote read: %s (%d)",
	pcmk_rc_str(rc), rc);

	} else if (read_rc == 0) {
	crm_debug("End of remote data encountered after %llu bytes",
	(unsigned long long) remote->buffer_offset);
	return ENOTCONN;

	} else {
	crm_debug("Error receiving remote data after %llu bytes: %s (%d)",
	(unsigned long long) remote->buffer_offset,
	pcmk_rc_str(rc), rc);
	return ENOTCONN;
	}

	header = localized_remote_header(remote);
	if(header) {
	if(remote->buffer_offset < header->size_total) {
	crm_trace("Read partial remote message (%llu of %u bytes)",
	(unsigned long long) remote->buffer_offset,
	header->size_total);
	} else {
	crm_trace("Read full remote message of %llu bytes",
	(unsigned long long) remote->buffer_offset);
	return pcmk_rc_ok;
	}
	}

	return EAGAIN;
	}

	/*!
	* \internal
	* \brief Read one message from a remote connection
	*
	* \param[in,out] remote Remote connection to read
	* \param[in] timeout_ms Fail if message not read in this many milliseconds
	* (10s will be used if 0, and 60s if negative)
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__read_remote_message(pcmk__remote_t *remote, int timeout_ms)
	{
	int rc = pcmk_rc_ok;
	time_t start = time(NULL);
	int remaining_timeout = 0;

	if (timeout_ms == 0) {
	timeout_ms = 10000;
	} else if (timeout_ms < 0) {
	timeout_ms = 60000;
	}

	remaining_timeout = timeout_ms;
	while (remaining_timeout > 0) {

	crm_trace("Waiting for remote data (%d ms of %d ms timeout remaining)",
	remaining_timeout, timeout_ms);
	rc = pcmk__remote_ready(remote, remaining_timeout);

	if (rc == ETIME) {
	crm_err("Timed out (%d ms) while waiting for remote data",
	remaining_timeout);
	return rc;

	} else if (rc != pcmk_rc_ok) {
	crm_debug("Wait for remote data aborted (will retry): %s "
	CRM_XS " rc=%d", pcmk_rc_str(rc), rc);

	} else {
	rc = read_available_remote_data(remote);
	if (rc == pcmk_rc_ok) {
	return rc;
	} else if (rc == EAGAIN) {
	crm_trace("Waiting for more remote data");
	} else {
	crm_debug("Could not receive remote data: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	}
	}

	// Don't waste time retrying after fatal errors
	if ((rc == ENOTCONN) \|\| (rc == ESOCKTNOSUPPORT)) {
	return rc;
	}

	remaining_timeout = timeout_ms - ((time(NULL) - start) * 1000);
	}
	return ETIME;
	}

	struct tcp_async_cb_data {
	int sock;
	int timeout_ms;
	time_t start;
	void *userdata;
	void (callback) (void userdata, int rc, int sock);
	};

	// \return TRUE if timer should be rescheduled, FALSE otherwise
	static gboolean
	check_connect_finished(gpointer userdata)
	{
	struct tcp_async_cb_data *cb_data = userdata;
	int rc;

	fd_set rset, wset;
	struct timeval ts = { 0, };

	if (cb_data->start == 0) {
	// Last connect() returned success immediately
	rc = pcmk_rc_ok;
	goto dispatch_done;
	}

	// If the socket is ready for reading or writing, the connect succeeded
	FD_ZERO(&rset);
	FD_SET(cb_data->sock, &rset);
	wset = rset;
	rc = select(cb_data->sock + 1, &rset, &wset, NULL, &ts);

	if (rc < 0) { // select() error
	rc = errno;
	if ((rc == EINPROGRESS) \|\| (rc == EAGAIN)) {
	if ((time(NULL) - cb_data->start) < (cb_data->timeout_ms / 1000)) {
	return TRUE; // There is time left, so reschedule timer
	} else {
	rc = ETIMEDOUT;
	}
	}
	crm_trace("Could not check socket %d for connection success: %s (%d)",
	cb_data->sock, pcmk_rc_str(rc), rc);

	} else if (rc == 0) { // select() timeout
	if ((time(NULL) - cb_data->start) < (cb_data->timeout_ms / 1000)) {
	return TRUE; // There is time left, so reschedule timer
	}
	crm_debug("Timed out while waiting for socket %d connection success",
	cb_data->sock);
	rc = ETIMEDOUT;

	// select() returned number of file descriptors that are ready

	} else if (FD_ISSET(cb_data->sock, &rset)
	\|\| FD_ISSET(cb_data->sock, &wset)) {

	// The socket is ready; check it for connection errors
	int error = 0;
	socklen_t len = sizeof(error);

	if (getsockopt(cb_data->sock, SOL_SOCKET, SO_ERROR, &error, &len) < 0) {
	rc = errno;
	crm_trace("Couldn't check socket %d for connection errors: %s (%d)",
	cb_data->sock, pcmk_rc_str(rc), rc);
	} else if (error != 0) {
	rc = error;
	crm_trace("Socket %d connected with error: %s (%d)",
	cb_data->sock, pcmk_rc_str(rc), rc);
	} else {
	rc = pcmk_rc_ok;
	}

	} else { // Should not be possible
	crm_trace("select() succeeded, but socket %d not in resulting "
	"read/write sets", cb_data->sock);
	rc = EAGAIN;
	}

	dispatch_done:
	if (rc == pcmk_rc_ok) {
	crm_trace("Socket %d is connected", cb_data->sock);
	} else {
	close(cb_data->sock);
	cb_data->sock = -1;
	}

	if (cb_data->callback) {
	cb_data->callback(cb_data->userdata, rc, cb_data->sock);
	}
	free(cb_data);
	return FALSE; // Do not reschedule timer
	}

	/*!
	* \internal
	* \brief Attempt to connect socket, calling callback when done
	*
	* Set a given socket non-blocking, then attempt to connect to it,
	* retrying periodically until success or a timeout is reached.
	* Call a caller-supplied callback function when completed.
	*
	* \param[in] sock Newly created socket
	* \param[in] addr Socket address information for connect
	* \param[in] addrlen Size of socket address information in bytes
	* \param[in] timeout_ms Fail if not connected within this much time
	* \param[out] timer_id If not NULL, store retry timer ID here
	* \param[in] userdata User data to pass to callback
	* \param[in] callback Function to call when connection attempt completes
	*
	* \return Standard Pacemaker return code
	*/
	static int
	connect_socket_retry(int sock, const struct sockaddr *addr, socklen_t addrlen,
	int timeout_ms, int timer_id, void userdata,
	void (callback) (void userdata, int rc, int sock))
	{
	int rc = 0;
	int interval = 500;
	int timer;
	struct tcp_async_cb_data *cb_data = NULL;

	rc = pcmk__set_nonblocking(sock);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	return rc;
	}

	rc = connect(sock, addr, addrlen);
	if (rc < 0 && (errno != EINPROGRESS) && (errno != EAGAIN)) {
	rc = errno;
	crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	return rc;
	}

	cb_data = calloc(1, sizeof(struct tcp_async_cb_data));
	cb_data->userdata = userdata;
	cb_data->callback = callback;
	cb_data->sock = sock;
	cb_data->timeout_ms = timeout_ms;

	if (rc == 0) {
	/* The connect was successful immediately, we still return to mainloop
	* and let this callback get called later. This avoids the user of this api
	* to have to account for the fact the callback could be invoked within this
	* function before returning. */
	cb_data->start = 0;
	interval = 1;
	} else {
	cb_data->start = time(NULL);
	}

	/* This timer function does a non-blocking poll on the socket to see if we
	* can use it. Once we can, the connect has completed. This method allows us
	* to connect without blocking the mainloop.
	*
	* @TODO Use a mainloop fd callback for this instead of polling. Something
	* about the way mainloop is currently polling prevents this from
	* working at the moment though. (See connect(2) regarding EINPROGRESS
	* for possible new handling needed.)
	*/
	crm_trace("Scheduling check in %dms for whether connect to fd %d finished",
	interval, sock);
	timer = g_timeout_add(interval, check_connect_finished, cb_data);
	if (timer_id) {
	*timer_id = timer;
	}

	// timer callback should be taking care of cb_data
	// cppcheck-suppress memleak
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Attempt once to connect socket and set it non-blocking
	*
	* \param[in] sock Newly created socket
	* \param[in] addr Socket address information for connect
	* \param[in] addrlen Size of socket address information in bytes
	*
	* \return Standard Pacemaker return code
	*/
	static int
	connect_socket_once(int sock, const struct sockaddr *addr, socklen_t addrlen)
	{
	int rc = connect(sock, addr, addrlen);

	if (rc < 0) {
	rc = errno;
	crm_warn("Could not connect socket: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	return rc;
	}

	rc = pcmk__set_nonblocking(sock);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	return rc;
	}

	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Connect to server at specified TCP port
	*
	* \param[in] host Name of server to connect to
	* \param[in] port Server port to connect to
	* \param[in] timeout_ms If asynchronous, fail if not connected in this time
	* \param[out] timer_id If asynchronous and this is non-NULL, retry timer ID
	* will be put here (for ease of cancelling by caller)
	* \param[out] sock_fd Where to store socket file descriptor
	* \param[in] userdata If asynchronous, data to pass to callback
	* \param[in] callback If NULL, attempt a single synchronous connection,
	* otherwise retry asynchronously then call this
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__connect_remote(const char host, int port, int timeout, int timer_id,
	int sock_fd, void userdata,
	void (callback) (void userdata, int rc, int sock))
	{
	char buffer[INET6_ADDRSTRLEN];
	struct addrinfo *res = NULL;
	struct addrinfo *rp = NULL;
	struct addrinfo hints;
	const char *server = host;
	int rc;
	int sock = -1;

	CRM_CHECK((host != NULL) && (sock_fd != NULL), return EINVAL);

	// Get host's IP address(es)
	memset(&hints, 0, sizeof(struct addrinfo));
	hints.ai_family = AF_UNSPEC; /* Allow IPv4 or IPv6 */
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_flags = AI_CANONNAME;

	rc = getaddrinfo(server, NULL, &hints, &res);
	rc = pcmk__gaierror2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_err("Unable to get IP address info for %s: %s",
	server, pcmk_rc_str(rc));
	goto async_cleanup;
	}

	if (!res \|\| !res->ai_addr) {
	crm_err("Unable to get IP address info for %s: no result", server);
	rc = ENOTCONN;
	goto async_cleanup;
	}

	// getaddrinfo() returns a list of host's addresses, try them in order
	for (rp = res; rp != NULL; rp = rp->ai_next) {
	struct sockaddr *addr = rp->ai_addr;

	if (!addr) {
	continue;
	}

	if (rp->ai_canonname) {
	server = res->ai_canonname;
	}
	crm_debug("Got canonical name %s for %s", server, host);

	sock = socket(rp->ai_family, SOCK_STREAM, IPPROTO_TCP);
	if (sock == -1) {
	rc = errno;
	crm_warn("Could not create socket for remote connection to %s:%d: "
	"%s " CRM_XS " rc=%d", server, port, pcmk_rc_str(rc), rc);
	continue;
	}

	/* Set port appropriately for address family */
	/* (void) casts avoid false-positive compiler alignment warnings /
	if (addr->sa_family == AF_INET6) {
	((struct sockaddr_in6 )(void)addr)->sin6_port = htons(port);
	} else {
	((struct sockaddr_in )(void)addr)->sin_port = htons(port);
	}

	memset(buffer, 0, PCMK__NELEM(buffer));
	pcmk__sockaddr2str(addr, buffer);
	crm_info("Attempting remote connection to %s:%d", buffer, port);

	if (callback) {
	if (connect_socket_retry(sock, rp->ai_addr, rp->ai_addrlen, timeout,
	timer_id, userdata, callback) == pcmk_rc_ok) {
	goto async_cleanup; /* Success for now, we'll hear back later in the callback */
	}

	} else if (connect_socket_once(sock, rp->ai_addr,
	rp->ai_addrlen) == pcmk_rc_ok) {
	break; /* Success */
	}

	// Connect failed
	close(sock);
	sock = -1;
	rc = ENOTCONN;
	}

	async_cleanup:

	if (res) {
	freeaddrinfo(res);
	}
	*sock_fd = sock;
	return rc;
	}

	/*!
	* \internal
	* \brief Convert an IP address (IPv4 or IPv6) to a string for logging
	*
	* \param[in] sa Socket address for IP
	* \param[out] s Storage for at least INET6_ADDRSTRLEN bytes
	*
	* \note sa The socket address can be a pointer to struct sockaddr_in (IPv4),
	* struct sockaddr_in6 (IPv6) or struct sockaddr_storage (either),
	* as long as its sa_family member is set correctly.
	*/
	void
	pcmk__sockaddr2str(const void sa, char s)
	{
	switch (((const struct sockaddr *) sa)->sa_family) {
	case AF_INET:
	inet_ntop(AF_INET, &(((const struct sockaddr_in *) sa)->sin_addr),
	s, INET6_ADDRSTRLEN);
	break;

	case AF_INET6:
	inet_ntop(AF_INET6,
	&(((const struct sockaddr_in6 *) sa)->sin6_addr),
	s, INET6_ADDRSTRLEN);
	break;

	default:
	strcpy(s, "<invalid>");
	}
	}

	/*!
	* \internal
	* \brief Accept a client connection on a remote server socket
	*
	* \param[in] ssock Server socket file descriptor being listened on
	* \param[out] csock Where to put new client socket's file descriptor
	*
	* \return Standard Pacemaker return code
	*/
	int
	pcmk__accept_remote_connection(int ssock, int *csock)
	{
	int rc;
	struct sockaddr_storage addr;
	socklen_t laddr = sizeof(addr);
	char addr_str[INET6_ADDRSTRLEN];
	#ifdef TCP_USER_TIMEOUT
	long sbd_timeout = 0;
	#endif

	/* accept the connection */
	memset(&addr, 0, sizeof(addr));
	csock = accept(ssock, (struct sockaddr )&addr, &laddr);
	if (*csock == -1) {
	rc = errno;
	crm_err("Could not accept remote client connection: %s "
	CRM_XS " rc=%d", pcmk_rc_str(rc), rc);
	return rc;
	}
	pcmk__sockaddr2str(&addr, addr_str);
	crm_info("Accepted new remote client connection from %s", addr_str);

	rc = pcmk__set_nonblocking(*csock);
	if (rc != pcmk_rc_ok) {
	crm_err("Could not set socket non-blocking: %s " CRM_XS " rc=%d",
	pcmk_rc_str(rc), rc);
	close(*csock);
	*csock = -1;
	return rc;
	}

	#ifdef TCP_USER_TIMEOUT
	sbd_timeout = pcmk__get_sbd_watchdog_timeout();
	if (sbd_timeout > 0) {
	// Time to fail and retry before watchdog
	long half = sbd_timeout / 2;
	unsigned int optval = (half <= UINT_MAX)? half : UINT_MAX;

	rc = setsockopt(*csock, SOL_TCP, TCP_USER_TIMEOUT,
	&optval, sizeof(optval));
	if (rc < 0) {
	rc = errno;
	crm_err("Could not set TCP timeout to %d ms on remote connection: "
	"%s " CRM_XS " rc=%d", optval, pcmk_rc_str(rc), rc);
	close(*csock);
	*csock = -1;
	return rc;
	}
	}
	#endif

	return rc;
	}

	/*!
	* \brief Get the default remote connection TCP port on this host
	*
	* \return Remote connection TCP port number
	*/
	int
	crm_default_remote_port(void)
	{
	static int port = 0;

	if (port == 0) {
	const char *env = pcmk__env_option(PCMK__ENV_REMOTE_PORT);

	if (env) {
	errno = 0;
	port = strtol(env, NULL, 10);
	if (errno \|\| (port < 1) \|\| (port > 65535)) {
	crm_warn("Environment variable PCMK_" PCMK__ENV_REMOTE_PORT
	" has invalid value '%s', using %d instead",
	env, DEFAULT_REMOTE_PORT);
	port = DEFAULT_REMOTE_PORT;
	}
	} else {
	port = DEFAULT_REMOTE_PORT;
	}
	}
	return port;
	}
	diff --git a/lib/common/schemas.c b/lib/common/schemas.c
	index dc461aa1b0..2994707d3b 100644
	--- a/lib/common/schemas.c
	+++ b/lib/common/schemas.c
	@@ -1,1575 +1,1575 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <dirent.h>
	#include <errno.h>
	#include <sys/stat.h>
	#include <stdarg.h>

	#include <libxml/relaxng.h>
	#include <libxslt/xslt.h>
	#include <libxslt/transform.h>
	#include <libxslt/security.h>
	#include <libxslt/xsltutils.h>

	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h> /* PCMK__XML_LOG_BASE */

	#include "crmcommon_private.h"

	#define SCHEMA_ZERO { .v = { 0, 0 } }

	#define schema_strdup_printf(prefix, version, suffix) \
	crm_strdup_printf(prefix "%u.%u" suffix, (version).v[0], (version).v[1])

	typedef struct {
	xmlRelaxNGPtr rng;
	xmlRelaxNGValidCtxtPtr valid;
	xmlRelaxNGParserCtxtPtr parser;
	} relaxng_ctx_cache_t;

	static GList *known_schemas = NULL;
	static bool silent_logging = FALSE;

	static void G_GNUC_PRINTF(2, 3)
	xml_log(int priority, const char *fmt, ...)
	{
	va_list ap;

	va_start(ap, fmt);
	if (silent_logging == FALSE) {
	/* XXX should not this enable dechunking as well? */
	PCMK__XML_LOG_BASE(priority, FALSE, 0, NULL, fmt, ap);
	}
	va_end(ap);
	}

	static int
	xml_latest_schema_index(GList *schemas)
	{
	// @COMPAT: pacemaker-next is deprecated since 2.1.5
	// FIXME: This function assumes at least three schemas have been added
	// before it has been called for the first time, which is only the case
	// if we are not unit testing.
	#if defined(PCMK__UNIT_TESTING)
	return g_list_length(schemas) - 1; // index from 0
	#else
	return g_list_length(schemas) - 3; // index from 0, ignore "pacemaker-next"/"none"
	#endif
	}

	/* Return the index of the most recent X.0 schema. */
	int
	pcmk__find_x_0_schema_index(GList *schemas)
	{
	/* We can't just use best to determine whether we've found the index
	* or not. What if we have a very long list of schemas all in the
	* same major version series? We'd return 0 for that, which means
	* we would still run this function every time.
	*/
	#if defined(PCMK__UNIT_TESTING)
	/* If we're unit testing, these can't be static because they'll stick
	* around from one test run to the next. They need to be cleared out
	* every time.
	*/
	bool found = false;
	int best = 0;
	#else
	static bool found = false;
	static int best = 0;
	#endif
	int i;
	GList *best_node = NULL;
	pcmk__schema_t *best_schema = NULL;

	if (found) {
	return best;
	}

	CRM_ASSERT(schemas != NULL);

	/* Get the most recent schema so we can look at its version number. */
	best = xml_latest_schema_index(schemas);
	best_node = g_list_nth(schemas, best);
	best_schema = best_node->data;

	/* If we are unit testing, we don't add the pacemaker-next/none schemas
	* to the list because we're not using the standard schema adding
	* functions. Thus, a singleton list means we're done.
	*
	* On the other hand, if we are running as usually, we have those two
	* schemas added to the list. A list of length three actually only has
	* one useful schema. So we're still done.
	*
	* @COMPAT Change this when we stop adding those schemas.
	*/
	#if defined(PCMK__UNIT_TESTING)
	if (pcmk__list_of_1(schemas)) {
	goto done;
	}
	#else
	if (g_list_length(schemas) == 3) {
	goto done;
	}
	#endif

	/* Start comparing the list from the node before the best schema (there's
	* no point in comparing something to itself). Then, 'i' is an index
	* starting at the best schema and will always point at the node after
	* 'iter'. This makes it the value we want to return when we find what
	* we're looking for.
	*/
	i = best;

	for (GList *iter = best_node->prev; iter != NULL; iter = iter->prev) {
	pcmk__schema_t *schema = iter->data;

	/* We've found a schema in an older major version series. Return
	* the index of the first one in the same major version series as
	* the best schema.
	*/
	if (schema->version.v[0] < best_schema->version.v[0]) {
	best = i;
	goto done;

	/* We're out of list to examine. This probably means there was only
	* one major version series, so return index 0.
	*/
	} else if (iter->prev == NULL) {
	best = 0;
	goto done;
	}

	i--;
	}

	done:
	found = true;
	return best;
	}

	const char *
	xml_latest_schema(void)
	{
	return get_schema_name(xml_latest_schema_index(known_schemas));
	}

	static inline bool
	version_from_filename(const char filename, pcmk__schema_version_t version)
	{
	if (pcmk__ends_with(filename, ".rng")) {
	return sscanf(filename, "pacemaker-%hhu.%hhu.rng", &(version->v[0]), &(version->v[1])) == 2;
	} else {
	return sscanf(filename, "pacemaker-%hhu.%hhu", &(version->v[0]), &(version->v[1])) == 2;
	}
	}

	static int
	schema_filter(const struct dirent *a)
	{
	int rc = 0;
	pcmk__schema_version_t version = SCHEMA_ZERO;

	if (strstr(a->d_name, "pacemaker-") != a->d_name) {
	/* crm_trace("%s - wrong prefix", a->d_name); */

	} else if (!pcmk__ends_with_ext(a->d_name, ".rng")) {
	/* crm_trace("%s - wrong suffix", a->d_name); */

	} else if (!version_from_filename(a->d_name, &version)) {
	/* crm_trace("%s - wrong format", a->d_name); */

	} else {
	/* crm_debug("%s - candidate", a->d_name); */
	rc = 1;
	}

	return rc;
	}

	static int
	schema_cmp(pcmk__schema_version_t a_version, pcmk__schema_version_t b_version)
	{
	for (int i = 0; i < 2; ++i) {
	if (a_version.v[i] < b_version.v[i]) {
	return -1;
	} else if (a_version.v[i] > b_version.v[i]) {
	return 1;
	}
	}
	return 0;
	}

	static int
	schema_cmp_directory(const struct dirent a, const struct dirent b)
	{
	pcmk__schema_version_t a_version = SCHEMA_ZERO;
	pcmk__schema_version_t b_version = SCHEMA_ZERO;

	if (!version_from_filename(a[0]->d_name, &a_version)
	\|\| !version_from_filename(b[0]->d_name, &b_version)) {
	// Shouldn't be possible, but makes static analysis happy
	return 0;
	}

	return schema_cmp(a_version, b_version);
	}

	/*!
	* \internal
	* \brief Add given schema + auxiliary data to internal bookkeeping.
	*
	* \note When providing \p version, should not be called directly but
	* through \c add_schema_by_version.
	*/
	static void
	add_schema(enum pcmk__schema_validator validator, const pcmk__schema_version_t *version,
	const char name, const char transform,
	const char *transform_enter, bool transform_onleave)
	{
	pcmk__schema_t *schema = NULL;
	int last = g_list_length(known_schemas);

	schema = calloc(1, sizeof(pcmk__schema_t));
	CRM_ASSERT(schema != NULL);

	schema->validator = validator;
	schema->version.v[0] = version->v[0];
	schema->version.v[1] = version->v[1];
	schema->transform_onleave = transform_onleave;

	if (version->v[0] \|\| version->v[1]) {
	schema->name = schema_strdup_printf("pacemaker-", *version, "");
	} else {
	schema->name = strdup(name);
	CRM_ASSERT(schema->name != NULL);
	}

	if (transform) {
	schema->transform = strdup(transform);
	CRM_ASSERT(schema->transform != NULL);
	}

	if (transform_enter) {
	schema->transform_enter = strdup(transform_enter);
	CRM_ASSERT(schema->transform_enter != NULL);
	}

	known_schemas = g_list_append(known_schemas, schema);

	if (schema->transform != NULL) {
	crm_debug("Added supported schema %d: %s (upgrades with %s.xsl)",
	last, schema->name, schema->transform);

	} else {
	crm_debug("Added supported schema %d: %s", last, schema->name);
	}
	}

	/*!
	* \internal
	* \brief Add version-specified schema + auxiliary data to internal bookkeeping.
	* \return Standard Pacemaker return value (the only possible values are
	* \c ENOENT when no upgrade schema is associated, or \c pcmk_rc_ok otherwise.
	*
	* \note There's no reliance on the particular order of schemas entering here.
	*
	* \par A bit of theory
	* We track 3 XSLT stylesheets that differ per usage:
	* - "upgrade":
	* . sparsely spread over the sequence of all available schemas,
	* as they are only relevant when major version of the schema
	* is getting bumped -- in that case, it MUST be set
	* . name convention: upgrade-X.Y.xsl
	* - "upgrade-enter":
	* . may only accompany "upgrade" occurrence, but doesn't need to
	* be present anytime such one is, i.e., it MAY not be set when
	* "upgrade" is
	* . name convention: upgrade-X.Y-enter.xsl,
	* when not present: upgrade-enter.xsl
	* - "upgrade-leave":
	* . like "upgrade-enter", but SHOULD be present whenever
	* "upgrade-enter" is (and vice versa, but that's only
	* to prevent confusion based on observing the files,
	* it would get ignored regardless)
	* . name convention: (see "upgrade-enter")
	*/
	static int
	add_schema_by_version(const pcmk__schema_version_t *version, bool transform_expected)
	{
	bool transform_onleave = FALSE;
	int rc = pcmk_rc_ok;
	struct stat s;
	char *xslt = NULL,
	*transform_upgrade = NULL,
	*transform_enter = NULL;

	/* prologue for further transform_expected handling */
	if (transform_expected) {
	/* check if there's suitable "upgrade" stylesheet */
	transform_upgrade = schema_strdup_printf("upgrade-", *version, );
	xslt = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_xslt,
	transform_upgrade);
	}

	if (!transform_expected) {
	/* jump directly to the end */

	} else if (stat(xslt, &s) == 0) {
	/* perhaps there's also a targeted "upgrade-enter" stylesheet */
	transform_enter = schema_strdup_printf("upgrade-", *version, "-enter");
	free(xslt);
	xslt = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_xslt,
	transform_enter);
	if (stat(xslt, &s) != 0) {
	/* or initially, at least a generic one */
	crm_debug("Upgrade-enter transform %s.xsl not found", xslt);
	free(xslt);
	free(transform_enter);
	transform_enter = strdup("upgrade-enter");
	xslt = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_xslt,
	transform_enter);
	if (stat(xslt, &s) != 0) {
	crm_debug("Upgrade-enter transform %s.xsl not found, either", xslt);
	free(xslt);
	xslt = NULL;
	}
	}
	/* xslt contains full path to "upgrade-enter" stylesheet */
	if (xslt != NULL) {
	/* then there should be "upgrade-leave" counterpart (enter->leave) */
	memcpy(strrchr(xslt, '-') + 1, "leave", sizeof("leave") - 1);
	transform_onleave = (stat(xslt, &s) == 0);
	free(xslt);
	} else {
	free(transform_enter);
	transform_enter = NULL;
	}

	} else {
	crm_err("Upgrade transform %s not found", xslt);
	free(xslt);
	free(transform_upgrade);
	transform_upgrade = NULL;
	rc = ENOENT;
	}

	add_schema(pcmk__schema_validator_rng, version, NULL,
	transform_upgrade, transform_enter, transform_onleave);

	free(transform_upgrade);
	free(transform_enter);

	return rc;
	}

	static void
	wrap_libxslt(bool finalize)
	{
	static xsltSecurityPrefsPtr secprefs;
	int ret = 0;

	/* security framework preferences */
	if (!finalize) {
	CRM_ASSERT(secprefs == NULL);
	secprefs = xsltNewSecurityPrefs();
	ret = xsltSetSecurityPrefs(secprefs, XSLT_SECPREF_WRITE_FILE,
	xsltSecurityForbid)
	\| xsltSetSecurityPrefs(secprefs, XSLT_SECPREF_CREATE_DIRECTORY,
	xsltSecurityForbid)
	\| xsltSetSecurityPrefs(secprefs, XSLT_SECPREF_READ_NETWORK,
	xsltSecurityForbid)
	\| xsltSetSecurityPrefs(secprefs, XSLT_SECPREF_WRITE_NETWORK,
	xsltSecurityForbid);
	if (ret != 0) {
	return;
	}
	} else {
	xsltFreeSecurityPrefs(secprefs);
	secprefs = NULL;
	}

	/* cleanup only */
	if (finalize) {
	xsltCleanupGlobals();
	}
	}

	void
	pcmk__load_schemas_from_dir(const char *dir)
	{
	int lpc, max;
	struct dirent **namelist = NULL;

	max = scandir(dir, &namelist, schema_filter, schema_cmp_directory);
	if (max < 0) {
	crm_warn("Could not load schemas from %s: %s", dir, strerror(errno));
	return;
	}

	for (lpc = 0; lpc < max; lpc++) {
	bool transform_expected = false;
	pcmk__schema_version_t version = SCHEMA_ZERO;

	if (!version_from_filename(namelist[lpc]->d_name, &version)) {
	// Shouldn't be possible, but makes static analysis happy
	crm_warn("Skipping schema '%s': could not parse version",
	namelist[lpc]->d_name);
	continue;
	}
	if ((lpc + 1) < max) {
	pcmk__schema_version_t next_version = SCHEMA_ZERO;

	if (version_from_filename(namelist[lpc+1]->d_name, &next_version)
	&& (version.v[0] < next_version.v[0])) {
	transform_expected = true;
	}
	}

	if (add_schema_by_version(&version, transform_expected) != pcmk_rc_ok) {
	break;
	}
	}

	for (lpc = 0; lpc < max; lpc++) {
	free(namelist[lpc]);
	}

	free(namelist);
	}

	static gint
	schema_sort_GCompareFunc(gconstpointer a, gconstpointer b)
	{
	const pcmk__schema_t *schema_a = a;
	const pcmk__schema_t *schema_b = b;

	if (pcmk__str_eq(schema_a->name, "pacemaker-next", pcmk__str_none)) {
	if (pcmk__str_eq(schema_b->name, "none", pcmk__str_none)) {
	return -1;
	} else {
	return 1;
	}
	} else if (pcmk__str_eq(schema_a->name, "none", pcmk__str_none)) {
	return 1;
	} else if (pcmk__str_eq(schema_b->name, "pacemaker-next", pcmk__str_none)) {
	return -1;
	} else {
	return schema_cmp(schema_a->version, schema_b->version);
	}
	}

	/*!
	* \internal
	* \brief Sort the list of known schemas such that all pacemaker-X.Y are in
	* version order, then pacemaker-next, then none
	*
	* This function should be called whenever additional schemas are loaded using
	* pcmk__load_schemas_from_dir(), after the initial sets in crm_schema_init().
	*/
	void
	pcmk__sort_schemas(void)
	{
	known_schemas = g_list_sort(known_schemas, schema_sort_GCompareFunc);
	}

	/*!
	* \internal
	* \brief Load pacemaker schemas into cache
	*
	* \note This currently also serves as an entry point for the
	* generic initialization of the libxslt library.
	*/
	void
	crm_schema_init(void)
	{
	const char *remote_schema_dir = pcmk__remote_schema_dir();
	char *base = pcmk__xml_artefact_root(pcmk__xml_artefact_ns_legacy_rng);
	const pcmk__schema_version_t zero = SCHEMA_ZERO;

	wrap_libxslt(false);

	pcmk__load_schemas_from_dir(base);
	pcmk__load_schemas_from_dir(remote_schema_dir);

	// @COMPAT: Deprecated since 2.1.5
	add_schema(pcmk__schema_validator_rng, &zero, "pacemaker-next",
	NULL, NULL, FALSE);

	add_schema(pcmk__schema_validator_none, &zero, PCMK_VALUE_NONE,
	NULL, NULL, FALSE);

	/* This shouldn't be strictly necessary, but we'll do it here just in case
	* there's anything in PCMK__REMOTE_SCHEMA_DIR that messes up the order.
	*/
	pcmk__sort_schemas();
	}

	static gboolean
	validate_with_relaxng(xmlDocPtr doc, xmlRelaxNGValidityErrorFunc error_handler, void error_handler_context, const char relaxng_file,
	relaxng_ctx_cache_t **cached_ctx)
	{
	int rc = 0;
	gboolean valid = TRUE;
	relaxng_ctx_cache_t *ctx = NULL;

	CRM_CHECK(doc != NULL, return FALSE);
	CRM_CHECK(relaxng_file != NULL, return FALSE);

	if (cached_ctx && *cached_ctx) {
	ctx = *cached_ctx;

	} else {
	crm_debug("Creating RNG parser context");
	ctx = calloc(1, sizeof(relaxng_ctx_cache_t));

	ctx->parser = xmlRelaxNGNewParserCtxt(relaxng_file);
	CRM_CHECK(ctx->parser != NULL, goto cleanup);

	if (error_handler) {
	xmlRelaxNGSetParserErrors(ctx->parser,
	(xmlRelaxNGValidityErrorFunc) error_handler,
	(xmlRelaxNGValidityWarningFunc) error_handler,
	error_handler_context);
	} else {
	xmlRelaxNGSetParserErrors(ctx->parser,
	(xmlRelaxNGValidityErrorFunc) fprintf,
	(xmlRelaxNGValidityWarningFunc) fprintf,
	stderr);
	}

	ctx->rng = xmlRelaxNGParse(ctx->parser);
	CRM_CHECK(ctx->rng != NULL,
	crm_err("Could not find/parse %s", relaxng_file);
	goto cleanup);

	ctx->valid = xmlRelaxNGNewValidCtxt(ctx->rng);
	CRM_CHECK(ctx->valid != NULL, goto cleanup);

	if (error_handler) {
	xmlRelaxNGSetValidErrors(ctx->valid,
	(xmlRelaxNGValidityErrorFunc) error_handler,
	(xmlRelaxNGValidityWarningFunc) error_handler,
	error_handler_context);
	} else {
	xmlRelaxNGSetValidErrors(ctx->valid,
	(xmlRelaxNGValidityErrorFunc) fprintf,
	(xmlRelaxNGValidityWarningFunc) fprintf,
	stderr);
	}
	}

	rc = xmlRelaxNGValidateDoc(ctx->valid, doc);
	if (rc > 0) {
	valid = FALSE;

	} else if (rc < 0) {
	crm_err("Internal libxml error during validation");
	}

	cleanup:

	if (cached_ctx) {
	*cached_ctx = ctx;

	} else {
	if (ctx->parser != NULL) {
	xmlRelaxNGFreeParserCtxt(ctx->parser);
	}
	if (ctx->valid != NULL) {
	xmlRelaxNGFreeValidCtxt(ctx->valid);
	}
	if (ctx->rng != NULL) {
	xmlRelaxNGFree(ctx->rng);
	}
	free(ctx);
	}

	return valid;
	}

	static void
	free_schema(gpointer data)
	{
	pcmk__schema_t *schema = data;
	relaxng_ctx_cache_t *ctx = NULL;

	switch (schema->validator) {
	case pcmk__schema_validator_none: // not cached
	break;

	case pcmk__schema_validator_rng: // cached
	ctx = (relaxng_ctx_cache_t *) schema->cache;
	if (ctx == NULL) {
	break;
	}

	if (ctx->parser != NULL) {
	xmlRelaxNGFreeParserCtxt(ctx->parser);
	}

	if (ctx->valid != NULL) {
	xmlRelaxNGFreeValidCtxt(ctx->valid);
	}

	if (ctx->rng != NULL) {
	xmlRelaxNGFree(ctx->rng);
	}

	free(ctx);
	schema->cache = NULL;
	break;
	}

	free(schema->name);
	free(schema->transform);
	free(schema->transform_enter);
	}

	/*!
	* \internal
	* \brief Clean up global memory associated with XML schemas
	*/
	void
	crm_schema_cleanup(void)
	{
	g_list_free_full(known_schemas, free_schema);
	known_schemas = NULL;

	wrap_libxslt(true);
	}

	static gboolean
	validate_with(xmlNode xml, pcmk__schema_t schema, xmlRelaxNGValidityErrorFunc error_handler, void* error_handler_context)
	{
	gboolean valid = FALSE;
	char *file = NULL;
	relaxng_ctx_cache_t **cache = NULL;

	if (schema == NULL) {
	return FALSE;
	}

	if (schema->validator == pcmk__schema_validator_none) {
	return TRUE;
	}

	if (pcmk__str_eq(schema->name, "pacemaker-next", pcmk__str_none)) {
	crm_warn("The pacemaker-next schema is deprecated and will be removed "
	"in a future release.");
	}

	file = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_rng,
	schema->name);

	crm_trace("Validating with %s (type=%d)",
	pcmk__s(file, "missing schema"), schema->validator);
	switch (schema->validator) {
	case pcmk__schema_validator_rng:
	cache = (relaxng_ctx_cache_t **) &(schema->cache);
	valid = validate_with_relaxng(xml->doc, error_handler, error_handler_context, file, cache);
	break;
	default:
	crm_err("Unknown validator type: %d", schema->validator);
	break;
	}

	free(file);
	return valid;
	}

	static bool
	validate_with_silent(xmlNode xml, pcmk__schema_t schema)
	{
	bool rc, sl_backup = silent_logging;
	silent_logging = TRUE;
	rc = validate_with(xml, schema, (xmlRelaxNGValidityErrorFunc) xml_log, GUINT_TO_POINTER(LOG_ERR));
	silent_logging = sl_backup;
	return rc;
	}

	static void
	dump_file(const char *filename)
	{

	FILE *fp = NULL;
	int ch, line = 0;

	CRM_CHECK(filename != NULL, return);

	fp = fopen(filename, "r");
	if (fp == NULL) {
	crm_perror(LOG_ERR, "Could not open %s for reading", filename);
	return;
	}

	fprintf(stderr, "%4d ", ++line);
	do {
	ch = getc(fp);
	if (ch == EOF) {
	putc('\n', stderr);
	break;
	} else if (ch == '\n') {
	fprintf(stderr, "\n%4d ", ++line);
	} else {
	putc(ch, stderr);
	}
	} while (1);

	fclose(fp);
	}

	gboolean
	validate_xml_verbose(const xmlNode *xml_blob)
	{
	int fd = 0;
	xmlDoc *doc = NULL;
	xmlNode *xml = NULL;
	gboolean rc = FALSE;
	char *filename = NULL;

	filename = crm_strdup_printf("%s/cib-invalid.XXXXXX", pcmk__get_tmpdir());

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);
	fd = mkstemp(filename);
	write_xml_fd(xml_blob, filename, fd, FALSE);

	dump_file(filename);

	doc = xmlReadFile(filename, NULL, 0);
	xml = xmlDocGetRootElement(doc);
	rc = validate_xml(xml, NULL, FALSE);
	free_xml(xml);

	unlink(filename);
	free(filename);

	return rc;
	}

	gboolean
	validate_xml(xmlNode xml_blob, const char validation, gboolean to_logs)
	{
	return pcmk__validate_xml(xml_blob, validation, to_logs ? (xmlRelaxNGValidityErrorFunc) xml_log : NULL, GUINT_TO_POINTER(LOG_ERR));
	}

	gboolean
	pcmk__validate_xml(xmlNode xml_blob, const char validation, xmlRelaxNGValidityErrorFunc error_handler, void* error_handler_context)
	{
	int version = 0;

	CRM_CHECK((xml_blob != NULL) && (xml_blob->doc != NULL), return FALSE);

	if (validation == NULL) {
	validation = crm_element_value(xml_blob, PCMK_XA_VALIDATE_WITH);
	}

	if (validation == NULL) {
	bool valid = FALSE;

	for (GList *iter = known_schemas; iter != NULL; iter = iter->next) {
	pcmk__schema_t *schema = iter->data;

	if (validate_with(xml_blob, schema, NULL, NULL)) {
	valid = TRUE;
	crm_xml_add(xml_blob, PCMK_XA_VALIDATE_WITH, schema->name);
	crm_info("XML validated against %s", schema->name);
	}
	}

	return valid;
	}

	version = get_schema_version(validation);
	if (strcmp(validation, PCMK_VALUE_NONE) == 0) {
	return TRUE;
	} else if (version < g_list_length(known_schemas)) {
	pcmk__schema_t *schema = g_list_nth_data(known_schemas, version);
	return validate_with(xml_blob, schema, error_handler,
	error_handler_context);
	}

	crm_err("Unknown validator: %s", validation);
	return FALSE;
	}

	/* With this arrangement, an attempt to identify the message severity
	as explicitly signalled directly from XSLT is performed in rather
	a smart way (no reliance on formatting string + arguments being
	always specified as ["%s", purposeful_string], as it can also be
	["%s: %s", some_prefix, purposeful_string] etc. so every argument
	pertaining %s specifier is investigated), and if such a mark found,
	the respective level is determined and, when the messages are to go
	to the native logs, the mark itself gets dropped
	(by the means of string shift).

	NOTE: whether the native logging is the right sink is decided per
	the ctx parameter -- NULL denotes this case, otherwise it
	carries a pointer to the numeric expression of the desired
	target logging level (messages with higher level will be
	suppressed)

	NOTE: on some architectures, this string shift may not have any
	effect, but that's an acceptable tradeoff

	The logging level for not explicitly designated messages
	(suspicious, likely internal errors or some runaways) is
	LOG_WARNING.
	*/
	static void G_GNUC_PRINTF(2, 3)
	cib_upgrade_err(void ctx, const char fmt, ...)
	{
	va_list ap, aq;
	char *arg_cur;

	bool found = FALSE;
	const char *fmt_iter = fmt;
	uint8_t msg_log_level = LOG_WARNING; /* default for runaway messages */
	const unsigned * log_level = (const unsigned *) ctx;
	enum {
	escan_seennothing,
	escan_seenpercent,
	} scan_state = escan_seennothing;

	va_start(ap, fmt);
	va_copy(aq, ap);

	while (!found && *fmt_iter != '\0') {
	/* while casing schema borrowed from libqb:qb_vsnprintf_serialize */
	switch (*fmt_iter++) {
	case '%':
	if (scan_state == escan_seennothing) {
	scan_state = escan_seenpercent;
	} else if (scan_state == escan_seenpercent) {
	scan_state = escan_seennothing;
	}
	break;
	case 's':
	if (scan_state == escan_seenpercent) {
	scan_state = escan_seennothing;
	arg_cur = va_arg(aq, char *);
	if (arg_cur != NULL) {
	switch (arg_cur[0]) {
	case 'W':
	if (!strncmp(arg_cur, "WARNING: ",
	sizeof("WARNING: ") - 1)) {
	msg_log_level = LOG_WARNING;
	}
	if (ctx == NULL) {
	memmove(arg_cur, arg_cur + sizeof("WARNING: ") - 1,
	strlen(arg_cur + sizeof("WARNING: ") - 1) + 1);
	}
	found = TRUE;
	break;
	case 'I':
	if (!strncmp(arg_cur, "INFO: ",
	sizeof("INFO: ") - 1)) {
	msg_log_level = LOG_INFO;
	}
	if (ctx == NULL) {
	memmove(arg_cur, arg_cur + sizeof("INFO: ") - 1,
	strlen(arg_cur + sizeof("INFO: ") - 1) + 1);
	}
	found = TRUE;
	break;
	case 'D':
	if (!strncmp(arg_cur, "DEBUG: ",
	sizeof("DEBUG: ") - 1)) {
	msg_log_level = LOG_DEBUG;
	}
	if (ctx == NULL) {
	memmove(arg_cur, arg_cur + sizeof("DEBUG: ") - 1,
	strlen(arg_cur + sizeof("DEBUG: ") - 1) + 1);
	}
	found = TRUE;
	break;
	}
	}
	}
	break;
	case '#': case '-': case ' ': case '+': case '\'': case 'I': case '.':
	case '0': case '1': case '2': case '3': case '4':
	case '5': case '6': case '7': case '8': case '9':
	case '*':
	break;
	case 'l':
	case 'z':
	case 't':
	case 'j':
	case 'd': case 'i':
	case 'o':
	case 'u':
	case 'x': case 'X':
	case 'e': case 'E':
	case 'f': case 'F':
	case 'g': case 'G':
	case 'a': case 'A':
	case 'c':
	case 'p':
	if (scan_state == escan_seenpercent) {
	(void) va_arg(aq, void ); / skip forward */
	scan_state = escan_seennothing;
	}
	break;
	default:
	scan_state = escan_seennothing;
	break;
	}
	}

	if (log_level != NULL) {
	/* intention of the following offset is:
	cibadmin -V -> start showing INFO labelled messages */
	if (*log_level + 4 >= msg_log_level) {
	vfprintf(stderr, fmt, ap);
	}
	} else {
	PCMK__XML_LOG_BASE(msg_log_level, TRUE, 0, "CIB upgrade: ", fmt, ap);
	}

	va_end(aq);
	va_end(ap);
	}

	static xmlNode *
	apply_transformation(xmlNode xml, const char transform, gboolean to_logs)
	{
	char *xform = NULL;
	xmlNode *out = NULL;
	xmlDocPtr res = NULL;
	xsltStylesheet *xslt = NULL;

	xform = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_xslt,
	transform);

	/* for capturing, e.g., what's emitted via <xsl:message> */
	if (to_logs) {
	xsltSetGenericErrorFunc(NULL, cib_upgrade_err);
	} else {
	xsltSetGenericErrorFunc(&crm_log_level, cib_upgrade_err);
	}

	xslt = xsltParseStylesheetFile((pcmkXmlStr) xform);
	CRM_CHECK(xslt != NULL, goto cleanup);

	res = xsltApplyStylesheet(xslt, xml->doc, NULL);
	CRM_CHECK(res != NULL, goto cleanup);

	xsltSetGenericErrorFunc(NULL, NULL); /* restore default one */

	out = xmlDocGetRootElement(res);

	cleanup:
	if (xslt) {
	xsltFreeStylesheet(xslt);
	}

	free(xform);

	return out;
	}

	/*!
	* \internal
	* \brief Possibly full enter->upgrade->leave trip per internal bookkeeping.
	*
	* \note Only emits warnings about enter/leave phases in case of issues.
	*/
	static xmlNode *
	apply_upgrade(xmlNode xml, const pcmk__schema_t schema, gboolean to_logs)
	{
	bool transform_onleave = schema->transform_onleave;
	char *transform_leave;
	xmlNode *upgrade = NULL,
	*final = NULL;

	if (schema->transform_enter) {
	crm_debug("Upgrading %s-style configuration, pre-upgrade phase with %s.xsl",
	schema->name, schema->transform_enter);
	upgrade = apply_transformation(xml, schema->transform_enter, to_logs);
	if (upgrade == NULL) {
	crm_warn("Upgrade-enter transformation %s.xsl failed",
	schema->transform_enter);
	transform_onleave = FALSE;
	}
	}
	if (upgrade == NULL) {
	upgrade = xml;
	}

	crm_debug("Upgrading %s-style configuration, main phase with %s.xsl",
	schema->name, schema->transform);
	final = apply_transformation(upgrade, schema->transform, to_logs);
	if (upgrade != xml) {
	free_xml(upgrade);
	upgrade = NULL;
	}

	if (final != NULL && transform_onleave) {
	upgrade = final;
	/* following condition ensured in add_schema_by_version */
	CRM_ASSERT(schema->transform_enter != NULL);
	transform_leave = strdup(schema->transform_enter);
	/* enter -> leave */
	memcpy(strrchr(transform_leave, '-') + 1, "leave", sizeof("leave") - 1);
	crm_debug("Upgrading %s-style configuration, post-upgrade phase with %s.xsl",
	schema->name, transform_leave);
	final = apply_transformation(upgrade, transform_leave, to_logs);
	if (final == NULL) {
	crm_warn("Upgrade-leave transformation %s.xsl failed", transform_leave);
	final = upgrade;
	} else {
	free_xml(upgrade);
	}
	free(transform_leave);
	}

	return final;
	}

	const char *
	get_schema_name(int version)
	{
	pcmk__schema_t *schema = g_list_nth_data(known_schemas, version);

	return (schema != NULL)? schema->name : "unknown";
	}

	int
	get_schema_version(const char *name)
	{
	int lpc = 0;

	if (name == NULL) {
	name = PCMK_VALUE_NONE;
	}

	for (GList *iter = known_schemas; iter != NULL; iter = iter->next) {
	pcmk__schema_t *schema = iter->data;

	if (pcmk__str_eq(name, schema->name, pcmk__str_casei)) {
	return lpc;
	}

	lpc++;
	}

	return -1;
	}

	/* set which validation to use */
	int
	update_validation(xmlNode *xml_blob, int best, int max, gboolean transform,
	gboolean to_logs)
	{
	xmlNode *xml = NULL;
	char *value = NULL;
	int max_stable_schemas = xml_latest_schema_index(known_schemas);
	int lpc = 0, match = -1, rc = pcmk_ok;
	int next = -1; /* -1 denotes "inactive" value */
	xmlRelaxNGValidityErrorFunc error_handler =
	to_logs ? (xmlRelaxNGValidityErrorFunc) xml_log : NULL;

	CRM_CHECK(best != NULL, return -EINVAL);
	*best = 0;

	CRM_CHECK((xml_blob != NULL) && (*xml_blob != NULL)
	&& ((*xml_blob)->doc != NULL),
	return -EINVAL);

	xml = *xml_blob;
	value = crm_element_value_copy(xml, PCMK_XA_VALIDATE_WITH);

	if (value != NULL) {
	match = get_schema_version(value);

	lpc = match;
	if (lpc >= 0 && transform == FALSE) {
	*best = lpc++;

	} else if (lpc < 0) {
	crm_debug("Unknown validation schema");
	lpc = 0;
	}
	}

	if (match >= max_stable_schemas) {
	/* nothing to do */
	free(value);
	*best = match;
	return pcmk_ok;
	}

	while (lpc <= max_stable_schemas) {
	/* FIXME: This will cause us to walk the known_schemas list every time
	* this loop iterates, which is not ideal. However, for now it's a lot
	* easier than trying to get all the loop indices we're using here
	* sorted out and working correctly.
	*/
	pcmk__schema_t *schema = g_list_nth_data(known_schemas, lpc);

	crm_debug("Testing '%s' validation (%d of %d)",
	pcmk__s(schema->name, "<unset>"), lpc, max_stable_schemas);

	if (validate_with(xml, schema, error_handler, GUINT_TO_POINTER(LOG_ERR)) == FALSE) {
	if (next != -1) {
	crm_info("Configuration not valid for schema: %s",
	schema->name);
	next = -1;
	} else {
	crm_trace("%s validation failed", pcmk__s(schema->name, "<unset>"));
	}
	if (*best) {
	/* we've satisfied the validation, no need to check further */
	break;
	}
	rc = -pcmk_err_schema_validation;

	} else {
	if (next != -1) {
	crm_debug("Configuration valid for schema: %s", schema->name);
	next = -1;
	}
	rc = pcmk_ok;
	}

	if (rc == pcmk_ok) {
	*best = lpc;
	}

	if (rc == pcmk_ok && transform) {
	xmlNode *upgrade = NULL;
	pcmk__schema_t *next_schema = NULL;
	next = lpc+1;

	if (next > max_stable_schemas) {
	/* There is no next version */
	crm_trace("Stopping at %s", schema->name);
	break;
	}

	if (max > 0 && (lpc == max \|\| next > max)) {
	crm_trace("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)",
	schema->name, lpc, next, max);
	break;
	}

	next_schema = g_list_nth_data(known_schemas, next);
	CRM_ASSERT(next_schema != NULL);

	if (schema->transform == NULL
	/* possibly avoid transforming when readily valid
	(in general more restricted when crossing the major
	version boundary, as X.0 "transitional" version is
	expected to be more strict than it's successors that
	may re-allow constructs from previous major line) */
	\|\| validate_with_silent(xml, next_schema)) {
	crm_debug("%s-style configuration is also valid for %s",
	schema->name, next_schema->name);

	lpc = next;

	} else {
	crm_debug("Upgrading %s-style configuration to %s with %s.xsl",
	schema->name, next_schema->name, schema->transform);

	upgrade = apply_upgrade(xml, schema, to_logs);
	if (upgrade == NULL) {
	crm_err("Transformation %s.xsl failed", schema->transform);
	rc = -pcmk_err_transform_failed;

	} else if (validate_with(upgrade, next_schema, error_handler,
	GUINT_TO_POINTER(LOG_ERR))) {
	crm_info("Transformation %s.xsl successful", schema->transform);
	lpc = next;
	*best = next;
	free_xml(xml);
	xml = upgrade;
	rc = pcmk_ok;

	} else {
	crm_err("Transformation %s.xsl did not produce a valid configuration",
	schema->transform);
	crm_log_xml_info(upgrade, "transform:bad");
	free_xml(upgrade);
	rc = -pcmk_err_schema_validation;
	}
	next = -1;
	}
	}

	if (transform == FALSE \|\| rc != pcmk_ok) {
	/* we need some progress! */
	lpc++;
	}
	}

	if (best > match && best) {
	pcmk__schema_t best_schema = g_list_nth_data(known_schemas, best);

	crm_info("%s the configuration from %s to %s",
	transform?"Transformed":"Upgraded", pcmk__s(value, "<none>"),
	best_schema->name);
	crm_xml_add(xml, PCMK_XA_VALIDATE_WITH, best_schema->name);
	}

	*xml_blob = xml;
	free(value);
	return rc;
	}

	gboolean
	cli_config_update(xmlNode *xml, int best_version, gboolean to_logs)
	{
	gboolean rc = TRUE;
	const char value = crm_element_value(xml, PCMK_XA_VALIDATE_WITH);
	char *const orig_value = strdup(value == NULL ? "(none)" : value);

	int version = get_schema_version(value);
	int orig_version = version;
	int min_version = pcmk__find_x_0_schema_index(known_schemas);

	if (version < min_version) {
	// Current configuration schema is not acceptable, try to update
	xmlNode *converted = NULL;

	converted = pcmk__xml_copy(NULL, *xml);
	update_validation(&converted, &version, 0, TRUE, to_logs);

	value = crm_element_value(converted, PCMK_XA_VALIDATE_WITH);
	if (version < min_version) {
	// Updated configuration schema is still not acceptable

	if (version < orig_version \|\| orig_version == -1) {
	// We couldn't validate any schema at all
	if (to_logs) {
	pcmk__config_err("Cannot upgrade configuration (claiming "
	"schema %s) to at least %s because it "
	"does not validate with any schema from "
	"%s to %s",
	orig_value,
	get_schema_name(min_version),
	get_schema_name(orig_version),
	xml_latest_schema());
	} else {
	fprintf(stderr, "Cannot upgrade configuration (claiming "
	"schema %s) to at least %s because it "
	"does not validate with any schema from "
	"%s to %s\n",
	orig_value,
	get_schema_name(min_version),
	get_schema_name(orig_version),
	xml_latest_schema());
	}
	} else {
	// We updated configuration successfully, but still too low
	if (to_logs) {
	pcmk__config_err("Cannot upgrade configuration (claiming "
	"schema %s) to at least %s because it "
	"would not upgrade past %s",
	orig_value,
	get_schema_name(min_version),
	pcmk__s(value, "unspecified version"));
	} else {
	fprintf(stderr, "Cannot upgrade configuration (claiming "
	"schema %s) to at least %s because it "
	"would not upgrade past %s\n",
	orig_value,
	get_schema_name(min_version),
	pcmk__s(value, "unspecified version"));
	}
	}

	free_xml(converted);
	converted = NULL;
	rc = FALSE;

	} else {
	// Updated configuration schema is acceptable
	free_xml(*xml);
	*xml = converted;

	if (version < xml_latest_schema_index(known_schemas)) {
	if (to_logs) {
	pcmk__config_warn("Configuration with schema %s was "
	"internally upgraded to acceptable (but "
	"not most recent) %s",
	orig_value, get_schema_name(version));
	}
	} else {
	if (to_logs) {
	crm_info("Configuration with schema %s was internally "
	"upgraded to latest version %s",
	orig_value, get_schema_name(version));
	}
	}
	}

	} else if (version >= get_schema_version(PCMK_VALUE_NONE)) {
	// Schema validation is disabled
	if (to_logs) {
	pcmk__config_warn("Schema validation of configuration is disabled "
	"(enabling is encouraged and prevents common "
	"misconfigurations)");

	} else {
	fprintf(stderr, "Schema validation of configuration is disabled "
	"(enabling is encouraged and prevents common "
	"misconfigurations)\n");
	}
	}

	if (best_version) {
	*best_version = version;
	}

	free(orig_value);
	return rc;
	}

	/*!
	* \internal
	* \brief Return a list of all schema files and any associated XSLT files
	* later than the given one
	* \brief Return a list of all schema versions later than the given one
	*
	* \param[in] schema The schema to compare against (for example,
	* "pacemaker-3.1.rng" or "pacemaker-3.1")
	*
	* \note The caller is responsible for freeing both the returned list and
	* the elements of the list
	*/
	GList *
	pcmk__schema_files_later_than(const char *name)
	{
	GList *lst = NULL;
	pcmk__schema_version_t ver;

	if (!version_from_filename(name, &ver)) {
	return lst;
	}

	for (GList *iter = g_list_nth(known_schemas, xml_latest_schema_index(known_schemas));
	iter != NULL; iter = iter->prev) {
	pcmk__schema_t *schema = iter->data;
	char *s = NULL;

	if (schema_cmp(ver, schema->version) != -1) {
	continue;
	}

	s = crm_strdup_printf("%s.rng", schema->name);
	lst = g_list_prepend(lst, s);

	if (schema->transform != NULL) {
	char *xform = crm_strdup_printf("%s.xsl", schema->transform);
	lst = g_list_prepend(lst, xform);
	}

	if (schema->transform_enter != NULL) {
	char *enter = crm_strdup_printf("%s.xsl", schema->transform_enter);

	lst = g_list_prepend(lst, enter);

	if (schema->transform_onleave) {
	int last_dash = strrchr(enter, '-') - enter;
	char leave = crm_strdup_printf("%.s-leave.xsl", last_dash, enter);

	lst = g_list_prepend(lst, leave);
	}
	}
	}

	return lst;
	}

	static void
	append_href(xmlNode xml, void user_data)
	{
	GList **list = user_data;
	const char *href = crm_element_value(xml, "href");
	char *s = NULL;

	if (href == NULL) {
	return;
	}

	s = strdup(href);
	CRM_ASSERT(s != NULL);
	list = g_list_prepend(list, s);
	}

	static void
	external_refs_in_schema(GList *list, const char contents)
	{
	/* local-name()= is needed to ignore the xmlns= setting at the top of
	* the XML file. Otherwise, the xpath query will always return nothing.
	*/
	const char search = "//[local-name()='externalRef'] \| //*[local-name()='include']";
	- xmlNode *xml = string2xml(contents);
	+ xmlNode *xml = pcmk__xml_parse_string(contents);

	crm_foreach_xpath_result(xml, search, append_href, list);
	free_xml(xml);
	}

	static int
	read_file_contents(const char file, char *contents)
	{
	int rc = pcmk_rc_ok;
	char *path = NULL;

	if (pcmk__ends_with(file, ".rng")) {
	path = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_rng, file);
	} else {
	path = pcmk__xml_artefact_path(pcmk__xml_artefact_ns_legacy_xslt, file);
	}

	rc = pcmk__file_contents(path, contents);

	free(path);
	return rc;
	}

	static void
	add_schema_file_to_xml(xmlNode parent, const char file, GList **already_included)
	{
	char *contents = NULL;
	char *path = NULL;
	xmlNode *file_node = NULL;
	GList *includes = NULL;
	int rc = pcmk_rc_ok;

	/* If we already included this file, don't do so again. */
	if (g_list_find_custom(*already_included, file, (GCompareFunc) strcmp) != NULL) {
	return;
	}

	/* Ensure whatever file we were given has a suffix we know about. If not,
	* just assume it's an RNG file.
	*/
	if (!pcmk__ends_with(file, ".rng") && !pcmk__ends_with(file, ".xsl")) {
	path = crm_strdup_printf("%s.rng", file);
	} else {
	path = strdup(file);
	CRM_ASSERT(path != NULL);
	}

	rc = read_file_contents(path, &contents);
	if (rc != pcmk_rc_ok \|\| contents == NULL) {
	crm_warn("Could not read schema file %s: %s", file, pcmk_rc_str(rc));
	free(path);
	return;
	}

	/* Create a new <file path="..."> node with the contents of the file
	* as a CDATA block underneath it.
	*/
	file_node = create_xml_node(parent, PCMK_XA_FILE);
	if (file_node == NULL) {
	free(contents);
	free(path);
	return;
	}

	crm_xml_add(file_node, PCMK_XA_PATH, path);
	already_included = g_list_prepend(already_included, path);

	xmlAddChild(file_node, xmlNewCDataBlock(parent->doc, (pcmkXmlStr) contents,
	strlen(contents)));

	/* Scan the file for any <externalRef> or <include> nodes and build up
	* a list of the files they reference.
	*/
	external_refs_in_schema(&includes, contents);

	/* For each referenced file, recurse to add it (and potentially anything it
	* references, ...) to the XML.
	*/
	for (GList *iter = includes; iter != NULL; iter = iter->next) {
	add_schema_file_to_xml(parent, iter->data, already_included);
	}

	free(contents);
	g_list_free_full(includes, free);
	}

	/*!
	* \internal
	* \brief Add an XML schema file and all the files it references as children
	* of a given XML node
	*
	* \param[in,out] parent The parent XML node
	* \param[in] name The schema version to compare against
	* (for example, "pacemaker-3.1" or "pacemaker-3.1.rng")
	* \param[in,out] already_included A list of names that have already been added
	* to the parent node.
	*
	* \note The caller is responsible for freeing both the returned list and
	* the elements of the list
	*/
	void
	pcmk__build_schema_xml_node(xmlNode parent, const char name, GList **already_included)
	{
	/* First, create an unattached node to add all the schema files to as children. */
	xmlNode *schema_node = create_xml_node(NULL, PCMK__XA_SCHEMA);

	crm_xml_add(schema_node, PCMK_XA_VERSION, name);
	add_schema_file_to_xml(schema_node, name, already_included);

	/* Then, if we actually added any children, attach the node to parent. If
	* we did not add any children (for instance, name was invalid), this prevents
	* us from returning a document with additional empty children.
	*/
	if (schema_node->children != NULL) {
	xmlAddChild(parent, schema_node);
	} else {
	free_xml(schema_node);
	}
	}

	/*!
	* \internal
	* \brief Return the directory containing any extra schema files that a
	* Pacemaker Remote node fetched from the cluster
	*/
	const char *
	pcmk__remote_schema_dir(void)
	{
	const char *dir = pcmk__env_option(PCMK__ENV_REMOTE_SCHEMA_DIRECTORY);

	if (pcmk__str_empty(dir)) {
	return PCMK__REMOTE_SCHEMA_DIR;
	}

	return dir;
	}

	void
	pcmk__log_known_schemas(void)
	{
	int lpc = 0;

	for (GList *iter = known_schemas; iter != NULL; iter = iter->next) {
	pcmk__schema_t *schema = iter->data;

	if (schema->transform != NULL) {
	crm_debug("known_schemas[%d] => %s (upgrades with %s.xsl)",
	lpc, schema->name, schema->transform);

	} else {
	crm_debug("known_schemas[%d] => %s", lpc, schema->name);
	}

	lpc++;
	}
	}
	diff --git a/lib/common/tests/actions/fix_plus_plus_recursive_test.c b/lib/common/tests/actions/fix_plus_plus_recursive_test.c
	index b3c7cc2e98..d223e3ca77 100644
	--- a/lib/common/tests/actions/fix_plus_plus_recursive_test.c
	+++ b/lib/common/tests/actions/fix_plus_plus_recursive_test.c
	@@ -1,47 +1,47 @@
	/*
	* Copyright 2022 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>

	#include <glib.h>

	static void
	element_nodes(void **state)
	{
	const char *new_value_root;
	const char *new_value_child;
	const char *new_value_grandchild;

	xmlNode *test_xml_root = create_xml_node(NULL, "test_xml_root");
	xmlNode *test_xml_child = create_xml_node(test_xml_root, "test_xml_child");
	xmlNode *test_xml_grandchild = create_xml_node(test_xml_child, "test_xml_grandchild");
	xmlNode *test_xml_text = pcmk_create_xml_text_node(test_xml_root, "text_xml_text", "content");
	- xmlNode *test_xml_comment = string2xml("<!-- a comment -->");
	+ xmlNode *test_xml_comment = pcmk__xml_parse_string("<!-- a comment -->");

	crm_xml_add(test_xml_root, "X", "5");
	crm_xml_add(test_xml_child, "X", "X++");
	crm_xml_add(test_xml_grandchild, "X", "X+=2");
	crm_xml_add(test_xml_text, "X", "X++");

	fix_plus_plus_recursive(test_xml_root);
	fix_plus_plus_recursive(test_xml_comment);

	new_value_root = crm_element_value(test_xml_root, "X");
	new_value_child = crm_element_value(test_xml_child, "X");
	new_value_grandchild = crm_element_value(test_xml_grandchild, "X");

	assert_string_equal(new_value_root, "5");
	assert_string_equal(new_value_child, "1");
	assert_string_equal(new_value_grandchild, "2");
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(element_nodes))
	diff --git a/lib/common/tests/nvpair/pcmk__xe_attr_is_true_test.c b/lib/common/tests/nvpair/pcmk__xe_attr_is_true_test.c
	index 3723707943..ed36f5d9ca 100644
	--- a/lib/common/tests/nvpair/pcmk__xe_attr_is_true_test.c
	+++ b/lib/common/tests/nvpair/pcmk__xe_attr_is_true_test.c
	@@ -1,50 +1,50 @@
	/*
	- * Copyright 2021 the Pacemaker project contributors
	+ * Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>
	#include <crm/common/xml_internal.h>

	static void
	empty_input(void **state)
	{
	- xmlNode *node = string2xml("<node/>");
	+ xmlNode *node = pcmk__xml_parse_string("<node/>");

	assert_false(pcmk__xe_attr_is_true(NULL, NULL));
	assert_false(pcmk__xe_attr_is_true(NULL, "whatever"));
	assert_false(pcmk__xe_attr_is_true(node, NULL));

	free_xml(node);
	}

	static void
	attr_missing(void **state)
	{
	- xmlNode *node = string2xml("<node a=\"true\" b=\"false\"/>");
	+ xmlNode *node = pcmk__xml_parse_string("<node a=\"true\" b=\"false\"/>");

	assert_false(pcmk__xe_attr_is_true(node, "c"));
	free_xml(node);
	}

	static void
	attr_present(void **state)
	{
	- xmlNode *node = string2xml("<node a=\"true\" b=\"false\"/>");
	+ xmlNode *node = pcmk__xml_parse_string("<node a=\"true\" b=\"false\"/>");

	assert_true(pcmk__xe_attr_is_true(node, "a"));
	assert_false(pcmk__xe_attr_is_true(node, "b"));

	free_xml(node);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(empty_input),
	cmocka_unit_test(attr_missing),
	cmocka_unit_test(attr_present))
	diff --git a/lib/common/tests/nvpair/pcmk__xe_get_bool_attr_test.c b/lib/common/tests/nvpair/pcmk__xe_get_bool_attr_test.c
	index 500d8a6675..cd2115e6ba 100644
	--- a/lib/common/tests/nvpair/pcmk__xe_get_bool_attr_test.c
	+++ b/lib/common/tests/nvpair/pcmk__xe_get_bool_attr_test.c
	@@ -1,59 +1,60 @@
	/*
	- * Copyright 2021-2023 the Pacemaker project contributors
	+ * Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>
	#include <crm/common/xml_internal.h>

	static void
	empty_input(void **state)
	{
	- xmlNode *node = string2xml("<node/>");
	+ xmlNode *node = pcmk__xml_parse_string("<node/>");
	bool value;

	assert_int_equal(pcmk__xe_get_bool_attr(NULL, NULL, &value), ENODATA);
	assert_int_equal(pcmk__xe_get_bool_attr(NULL, "whatever", &value), ENODATA);
	assert_int_equal(pcmk__xe_get_bool_attr(node, NULL, &value), EINVAL);
	assert_int_equal(pcmk__xe_get_bool_attr(node, "whatever", NULL), EINVAL);

	free_xml(node);
	}

	static void
	attr_missing(void **state)
	{
	- xmlNode *node = string2xml("<node a=\"true\" b=\"false\"/>");
	+ xmlNode *node = pcmk__xml_parse_string("<node a=\"true\" b=\"false\"/>");
	bool value;

	assert_int_equal(pcmk__xe_get_bool_attr(node, "c", &value), ENODATA);
	free_xml(node);
	}

	static void
	attr_present(void **state)
	{
	- xmlNode *node = string2xml("<node a=\"true\" b=\"false\" c=\"blah\"/>");
	+ xmlNode *node = pcmk__xml_parse_string("<node a=\"true\" b=\"false\" "
	+ "c=\"blah\"/>");
	bool value;

	value = false;
	assert_int_equal(pcmk__xe_get_bool_attr(node, "a", &value), pcmk_rc_ok);
	assert_true(value);
	value = true;
	assert_int_equal(pcmk__xe_get_bool_attr(node, "b", &value), pcmk_rc_ok);
	assert_false(value);
	assert_int_equal(pcmk__xe_get_bool_attr(node, "c", &value), pcmk_rc_bad_input);

	free_xml(node);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(empty_input),
	cmocka_unit_test(attr_missing),
	cmocka_unit_test(attr_present))
	diff --git a/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c b/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c
	index 1fd077fc8a..43cb0cdff9 100644
	--- a/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c
	+++ b/lib/common/tests/nvpair/pcmk__xe_set_bool_attr_test.c
	@@ -1,31 +1,31 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>
	#include <crm/common/xml_internal.h>
	#include <crm/common/xml.h>

	static void
	set_attr(void **state)
	{
	- xmlNode *node = string2xml("<node/>");
	+ xmlNode *node = pcmk__xml_parse_string("<node/>");

	pcmk__xe_set_bool_attr(node, "a", true);
	pcmk__xe_set_bool_attr(node, "b", false);

	assert_string_equal(crm_element_value(node, "a"), PCMK_VALUE_TRUE);
	assert_string_equal(crm_element_value(node, "b"), PCMK_VALUE_FALSE);

	free_xml(node);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(set_attr))
	diff --git a/lib/common/tests/probes/pcmk_xe_is_probe_test.c b/lib/common/tests/probes/pcmk_xe_is_probe_test.c
	index 8abb797c03..e4930bab1c 100644
	--- a/lib/common/tests/probes/pcmk_xe_is_probe_test.c
	+++ b/lib/common/tests/probes/pcmk_xe_is_probe_test.c
	@@ -1,51 +1,54 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>

	static void
	op_is_probe_test(void **state)
	{
	xmlNode *node = NULL;

	assert_false(pcmk_xe_is_probe(NULL));

	- node = string2xml("<" PCMK__XE_LRM_RSC_OP "/>");
	+ node = pcmk__xml_parse_string("<" PCMK__XE_LRM_RSC_OP "/>");
	assert_false(pcmk_xe_is_probe(node));
	free_xml(node);

	- node = string2xml("<" PCMK__XE_LRM_RSC_OP " "
	- PCMK__XA_OPERATION_KEY "=\"blah\" "
	- PCMK_META_INTERVAL "=\"30s\"/>");
	+ node = pcmk__xml_parse_string("<" PCMK__XE_LRM_RSC_OP " "
	+ PCMK__XA_OPERATION_KEY "=\"blah\" "
	+ PCMK_META_INTERVAL "=\"30s\"/>");
	assert_false(pcmk_xe_is_probe(node));
	free_xml(node);

	- node = string2xml("<" PCMK__XE_LRM_RSC_OP " "
	- PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	- PCMK_META_INTERVAL "=\"30s\"/>");
	+ node = pcmk__xml_parse_string("<" PCMK__XE_LRM_RSC_OP " "
	+ PCMK_XA_OPERATION
	+ "=\"" PCMK_ACTION_MONITOR "\" "
	+ PCMK_META_INTERVAL "=\"30s\"/>");
	assert_false(pcmk_xe_is_probe(node));
	free_xml(node);

	- node = string2xml("<" PCMK__XE_LRM_RSC_OP " "
	- PCMK_XA_OPERATION "=\"" PCMK_ACTION_START "\" "
	- PCMK_META_INTERVAL "=\"0\"/>");
	+ node = pcmk__xml_parse_string("<" PCMK__XE_LRM_RSC_OP " "
	+ PCMK_XA_OPERATION
	+ "=\"" PCMK_ACTION_START "\" "
	+ PCMK_META_INTERVAL "=\"0\"/>");
	assert_false(pcmk_xe_is_probe(node));
	free_xml(node);

	- node = string2xml("<" PCMK__XE_LRM_RSC_OP " "
	- PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	- PCMK_META_INTERVAL "=\"0\"/>");
	+ node = pcmk__xml_parse_string("<" PCMK__XE_LRM_RSC_OP " "
	+ PCMK_XA_OPERATION
	+ "=\"" PCMK_ACTION_MONITOR "\" "
	+ PCMK_META_INTERVAL "=\"0\"/>");
	assert_true(pcmk_xe_is_probe(node));
	free_xml(node);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(op_is_probe_test))
	diff --git a/lib/common/tests/probes/pcmk_xe_mask_probe_failure_test.c b/lib/common/tests/probes/pcmk_xe_mask_probe_failure_test.c
	index 6ea4ba3b18..4f977067ca 100644
	--- a/lib/common/tests/probes/pcmk_xe_mask_probe_failure_test.c
	+++ b/lib/common/tests/probes/pcmk_xe_mask_probe_failure_test.c
	@@ -1,331 +1,333 @@
	/*
	* Copyright 2021-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>

	static void
	op_is_not_probe_test(void **state) {
	xmlNode *node = NULL;

	/* Not worth testing this thoroughly since it's just a duplicate of whether
	* pcmk_op_is_probe works or not.
	*/

	- node = string2xml("<" PCMK__XE_LRM_RSC_OP " "
	- PCMK_XA_OPERATION "=\"" PCMK_ACTION_START "\" "
	- PCMK_META_INTERVAL "=\"0\"/>");
	+ node = pcmk__xml_parse_string("<" PCMK__XE_LRM_RSC_OP " "
	+ PCMK_XA_OPERATION
	+ "=\"" PCMK_ACTION_START "\" "
	+ PCMK_META_INTERVAL "=\"0\"/>");
	assert_false(pcmk_xe_mask_probe_failure(node));
	free_xml(node);
	}

	static void
	op_does_not_have_right_values_test(void **state) {
	xmlNode *node = NULL;
	char *s = NULL;

	- node = string2xml("<" PCMK__XE_LRM_RSC_OP " "
	- PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	- PCMK_META_INTERVAL "=\"0\"/>");
	+ node = pcmk__xml_parse_string("<" PCMK__XE_LRM_RSC_OP " "
	+ PCMK_XA_OPERATION
	+ "=\"" PCMK_ACTION_MONITOR "\" "
	+ PCMK_META_INTERVAL "=\"0\"/>");
	assert_false(pcmk_xe_mask_probe_failure(node));
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"\"/>",
	PCMK_OCF_OK);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);
	}

	static void
	check_values_test(void **state) {
	xmlNode *node = NULL;
	char *s = NULL;

	/* PCMK_EXEC_NOT_SUPPORTED */
	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_OK, PCMK_EXEC_NOT_SUPPORTED);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_INSTALLED, PCMK_EXEC_NOT_SUPPORTED);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	/* PCMK_EXEC_DONE */
	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_OK, PCMK_EXEC_DONE);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_INVALID_PARAM, PCMK_EXEC_DONE);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_INSTALLED, PCMK_EXEC_DONE);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_DONE);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_RUNNING, PCMK_EXEC_DONE);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	/* PCMK_EXEC_NOT_INSTALLED */
	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_OK, PCMK_EXEC_NOT_INSTALLED);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_INSTALLED, PCMK_EXEC_NOT_INSTALLED);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	/* PCMK_EXEC_ERROR */
	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_OK, PCMK_EXEC_ERROR);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_INVALID_PARAM, PCMK_EXEC_ERROR);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_INSTALLED, PCMK_EXEC_ERROR);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_ERROR);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_RUNNING, PCMK_EXEC_ERROR);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	/* PCMK_EXEC_ERROR_HARD */
	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_OK, PCMK_EXEC_ERROR_HARD);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_INVALID_PARAM, PCMK_EXEC_ERROR_HARD);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_INSTALLED, PCMK_EXEC_ERROR_HARD);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_ERROR_HARD);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_RUNNING, PCMK_EXEC_ERROR_HARD);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	/* PCMK_EXEC_ERROR_FATAL */
	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_OK, PCMK_EXEC_ERROR_FATAL);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_INVALID_PARAM, PCMK_EXEC_ERROR_FATAL);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_INSTALLED, PCMK_EXEC_ERROR_FATAL);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_true(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_CONFIGURED, PCMK_EXEC_ERROR_FATAL);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);

	s = crm_strdup_printf("<" PCMK__XE_LRM_RSC_OP " "
	PCMK_XA_OPERATION "=\"" PCMK_ACTION_MONITOR "\" "
	PCMK_META_INTERVAL "=\"0\" "
	PCMK__XA_RC_CODE "=\"%d\" "
	PCMK__XA_OP_STATUS "=\"%d\"/>",
	PCMK_OCF_NOT_RUNNING, PCMK_EXEC_ERROR_FATAL);
	- node = string2xml(s);
	+ node = pcmk__xml_parse_string(s);
	assert_false(pcmk_xe_mask_probe_failure(node));
	free(s);
	free_xml(node);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(op_is_not_probe_test),
	cmocka_unit_test(op_does_not_have_right_values_test),
	cmocka_unit_test(check_values_test))
	diff --git a/lib/common/tests/xml/pcmk__xe_foreach_child_test.c b/lib/common/tests/xml/pcmk__xe_foreach_child_test.c
	index ffb9171a43..8dd5504b79 100644
	--- a/lib/common/tests/xml/pcmk__xe_foreach_child_test.c
	+++ b/lib/common/tests/xml/pcmk__xe_foreach_child_test.c
	@@ -1,216 +1,216 @@
	/*
	- * Copyright 2022-2023 the Pacemaker project contributors
	+ * Copyright 2022-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/unittest_internal.h>
	#include <crm/common/xml_internal.h>

	static int compare_name_handler(xmlNode xml, void userdata) {
	function_called();
	assert_string_equal((char ) userdata, (const char ) xml->name);
	return pcmk_rc_ok;
	}

	const char *str1 =
	"<xml>\n"
	" <!-- This is a level 1 node -->\n"
	" <level1>\n"
	" content\n"
	" </level1>\n"
	" <!-- This is a level 1 node -->\n"
	" <level1>\n"
	" content\n"
	" </level1>\n"
	" <!-- This is a level 1 node -->\n"
	" <level1>\n"
	" content\n"
	" </level1>\n"
	"</xml>";

	static void
	bad_input(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);

	pcmk__assert_asserts(pcmk__xe_foreach_child(xml, NULL, NULL, NULL));

	free_xml(xml);
	}

	static void
	name_given_test(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);

	/* The handler should be called once for every <level1> node. */
	expect_function_call(compare_name_handler);
	expect_function_call(compare_name_handler);
	expect_function_call(compare_name_handler);

	pcmk__xe_foreach_child(xml, "level1", compare_name_handler, (void *) "level1");
	free_xml(xml);
	}

	static void
	no_name_given_test(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);

	/* The handler should be called once for every <level1> node. */
	expect_function_call(compare_name_handler);
	expect_function_call(compare_name_handler);
	expect_function_call(compare_name_handler);

	pcmk__xe_foreach_child(xml, NULL, compare_name_handler, (void *) "level1");
	free_xml(xml);
	}

	static void
	name_doesnt_exist_test(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);
	pcmk__xe_foreach_child(xml, "xxx", compare_name_handler, NULL);
	free_xml(xml);
	}

	const char *str2 =
	"<xml>\n"
	" <level1>\n"
	" <!-- Inside a level 1 node -->\n"
	" <level2>\n"
	" <!-- Inside a level 2 node -->\n"
	" </level2>\n"
	" </level1>\n"
	" <level1>\n"
	" <!-- Inside a level 1 node -->\n"
	" <level2>\n"
	" <!-- Inside a level 2 node -->\n"
	" <level3>\n"
	" <!-- Inside a level 3 node -->\n"
	" </level3>\n"
	" </level2>\n"
	" <level2>\n"
	" <!-- Inside a level 2 node -->\n"
	" </level2>\n"
	" </level1>\n"
	"</xml>";

	static void
	multiple_levels_test(void **state) {
	- xmlNode *xml = string2xml(str2);
	+ xmlNode *xml = pcmk__xml_parse_string(str2);

	/* The handler should be called once for every <level1> node. */
	expect_function_call(compare_name_handler);
	expect_function_call(compare_name_handler);

	pcmk__xe_foreach_child(xml, "level1", compare_name_handler, (void *) "level1");
	free_xml(xml);
	}

	static void
	multiple_levels_no_name_test(void **state) {
	- xmlNode *xml = string2xml(str2);
	+ xmlNode *xml = pcmk__xml_parse_string(str2);

	/* The handler should be called once for every <level1> node. */
	expect_function_call(compare_name_handler);
	expect_function_call(compare_name_handler);

	pcmk__xe_foreach_child(xml, NULL, compare_name_handler, (void *) "level1");
	free_xml(xml);
	}

	const char *str3 =
	"<xml>\n"
	" <!-- This is node #1 -->\n"
	" <node1>\n"
	" content\n"
	" </node1>\n"
	" <!-- This is node #2 -->\n"
	" <node2>\n"
	" content\n"
	" </node2>\n"
	" <!-- This is node #3 -->\n"
	" <node3>\n"
	" content\n"
	" </node3>\n"
	"</xml>";

	static int any_of_handler(xmlNode xml, void userdata) {
	function_called();
	assert_true(pcmk__str_any_of((const char *) xml->name,
	"node1", "node2", "node3", NULL));
	return pcmk_rc_ok;
	}

	static void
	any_of_test(void **state) {
	- xmlNode *xml = string2xml(str3);
	+ xmlNode *xml = pcmk__xml_parse_string(str3);

	/* The handler should be called once for every <nodeX> node. */
	expect_function_call(any_of_handler);
	expect_function_call(any_of_handler);
	expect_function_call(any_of_handler);

	pcmk__xe_foreach_child(xml, NULL, any_of_handler, NULL);
	free_xml(xml);
	}

	static int stops_on_first_handler(xmlNode xml, void userdata) {
	function_called();

	if (pcmk__xe_is(xml, "node1")) {
	return pcmk_rc_error;
	} else {
	return pcmk_rc_ok;
	}
	}

	static int stops_on_second_handler(xmlNode xml, void userdata) {
	function_called();

	if (pcmk__xe_is(xml, "node2")) {
	return pcmk_rc_error;
	} else {
	return pcmk_rc_ok;
	}
	}

	static int stops_on_third_handler(xmlNode xml, void userdata) {
	function_called();

	if (pcmk__xe_is(xml, "node3")) {
	return pcmk_rc_error;
	} else {
	return pcmk_rc_ok;
	}
	}

	static void
	one_of_test(void **state) {
	- xmlNode *xml = string2xml(str3);
	+ xmlNode *xml = pcmk__xml_parse_string(str3);

	/* The handler should be called once. */
	expect_function_call(stops_on_first_handler);
	assert_int_equal(pcmk__xe_foreach_child(xml, "node1", stops_on_first_handler, NULL), pcmk_rc_error);

	expect_function_call(stops_on_second_handler);
	assert_int_equal(pcmk__xe_foreach_child(xml, "node2", stops_on_second_handler, NULL), pcmk_rc_error);

	expect_function_call(stops_on_third_handler);
	assert_int_equal(pcmk__xe_foreach_child(xml, "node3", stops_on_third_handler, NULL), pcmk_rc_error);

	free_xml(xml);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(bad_input),
	cmocka_unit_test(name_given_test),
	cmocka_unit_test(no_name_given_test),
	cmocka_unit_test(name_doesnt_exist_test),
	cmocka_unit_test(multiple_levels_test),
	cmocka_unit_test(multiple_levels_no_name_test),
	cmocka_unit_test(any_of_test),
	cmocka_unit_test(one_of_test))
	diff --git a/lib/common/tests/xml/pcmk__xe_match_test.c b/lib/common/tests/xml/pcmk__xe_match_test.c
	index 78dc0aca03..ebad309072 100644
	--- a/lib/common/tests/xml/pcmk__xe_match_test.c
	+++ b/lib/common/tests/xml/pcmk__xe_match_test.c
	@@ -1,106 +1,106 @@
	/*
	* Copyright 2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/xml.h>
	#include <crm/common/unittest_internal.h>
	#include <crm/common/xml_internal.h>

	const char *str1 =
	"<xml>\n"
	" <!-- This is an A node -->\n"
	" <nodeA attrA=\"123\" " PCMK_XA_ID "=\"1\">\n"
	" content\n"
	" </nodeA>\n"
	" <!-- This is an A node -->\n"
	" <nodeA attrA=\"456\" " PCMK_XA_ID "=\"2\">\n"
	" content\n"
	" </nodeA>\n"
	" <!-- This is an A node -->\n"
	" <nodeA attrB=\"XYZ\" " PCMK_XA_ID "=\"3\">\n"
	" content\n"
	" </nodeA>\n"
	" <!-- This is a B node -->\n"
	" <nodeB attrA=\"123\" " PCMK_XA_ID "=\"4\">\n"
	" content\n"
	" </nodeA>\n"
	" <!-- This is a B node -->\n"
	" <nodeB attrB=\"ABC\" " PCMK_XA_ID "=\"5\">\n"
	" content\n"
	" </nodeA>\n"
	"</xml>";

	static void
	bad_input(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);

	assert_null(pcmk__xe_match(NULL, NULL, NULL, NULL));
	assert_null(pcmk__xe_match(NULL, NULL, NULL, "attrX"));

	free_xml(xml);
	}

	static void
	not_found(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);

	/* No node with an attrX attribute */
	assert_null(pcmk__xe_match(xml, NULL, "attrX", NULL));
	/* No nodeX node */
	assert_null(pcmk__xe_match(xml, "nodeX", NULL, NULL));
	/* No nodeA node with attrX */
	assert_null(pcmk__xe_match(xml, "nodeA", "attrX", NULL));
	/* No nodeA node with attrA=XYZ */
	assert_null(pcmk__xe_match(xml, "nodeA", "attrA", "XYZ"));

	free_xml(xml);
	}

	static void
	find_attrB(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);
	xmlNode *result = NULL;

	/* Find the first node with attrB */
	result = pcmk__xe_match(xml, NULL, "attrB", NULL);
	assert_non_null(result);
	assert_string_equal(crm_element_value(result, PCMK_XA_ID), "3");

	/* Find the first nodeB with attrB */
	result = pcmk__xe_match(xml, "nodeB", "attrB", NULL);
	assert_non_null(result);
	assert_string_equal(crm_element_value(result, PCMK_XA_ID), "5");

	free_xml(xml);
	}

	static void
	find_attrA_matching(void **state) {
	- xmlNode *xml = string2xml(str1);
	+ xmlNode *xml = pcmk__xml_parse_string(str1);
	xmlNode *result = NULL;

	/* Find attrA=456 */
	result = pcmk__xe_match(xml, NULL, "attrA", "456");
	assert_non_null(result);
	assert_string_equal(crm_element_value(result, PCMK_XA_ID), "2");

	/* Find a nodeB with attrA=123 */
	result = pcmk__xe_match(xml, "nodeB", "attrA", "123");
	assert_non_null(result);
	assert_string_equal(crm_element_value(result, PCMK_XA_ID), "4");

	free_xml(xml);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(bad_input),
	cmocka_unit_test(not_found),
	cmocka_unit_test(find_attrB),
	cmocka_unit_test(find_attrA_matching));
	diff --git a/lib/common/xml.c b/lib/common/xml.c
	index 913cfd5ec0..75c5e65ef2 100644
	--- a/lib/common/xml.c
	+++ b/lib/common/xml.c
	@@ -1,3047 +1,3036 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <sys/stat.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <time.h>
	#include <string.h>
	#include <stdlib.h>
	#include <stdarg.h>
	#include <bzlib.h>

	#include <libxml/parser.h>
	#include <libxml/tree.h>
	#include <libxml/xmlIO.h> /* xmlAllocOutputBuffer */

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/xml_internal.h> // PCMK__XML_LOG_BASE, etc.
	#include "crmcommon_private.h"

	// Define this as 1 in development to get insanely verbose trace messages
	#ifndef XML_PARSER_DEBUG
	#define XML_PARSER_DEBUG 0
	#endif

	/* @TODO XML_PARSE_RECOVER allows some XML errors to be silently worked around
	* by libxml2, which is potentially ambiguous and dangerous. We should drop it
	* when we can break backward compatibility with configurations that might be
	* relying on it (i.e. pacemaker 3.0.0).
	*
	* It might be a good idea to have a transitional period where we first try
	* parsing without XML_PARSE_RECOVER, and if that fails, try parsing again with
	* it, logging a warning if it succeeds.
	*/
	#define PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER (XML_PARSE_NOBLANKS)
	#define PCMK__XML_PARSE_OPTS_WITH_RECOVER (XML_PARSE_NOBLANKS \| XML_PARSE_RECOVER)

	bool
	pcmk__tracking_xml_changes(xmlNode *xml, bool lazy)
	{
	if(xml == NULL \|\| xml->doc == NULL \|\| xml->doc->_private == NULL) {
	return FALSE;
	} else if (!pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
	pcmk__xf_tracking)) {
	return FALSE;
	} else if (lazy && !pcmk_is_set(((xml_doc_private_t *)xml->doc->_private)->flags,
	pcmk__xf_lazy)) {
	return FALSE;
	}
	return TRUE;
	}

	static inline void
	set_parent_flag(xmlNode *xml, long flag)
	{
	for(; xml; xml = xml->parent) {
	xml_node_private_t *nodepriv = xml->_private;

	if (nodepriv == NULL) {
	/* During calls to xmlDocCopyNode(), _private will be unset for parent nodes */
	} else {
	pcmk__set_xml_flags(nodepriv, flag);
	}
	}
	}

	void
	pcmk__set_xml_doc_flag(xmlNode *xml, enum xml_private_flags flag)
	{
	if(xml && xml->doc && xml->doc->_private){
	/* During calls to xmlDocCopyNode(), xml->doc may be unset */
	xml_doc_private_t *docpriv = xml->doc->_private;

	pcmk__set_xml_flags(docpriv, flag);
	}
	}

	// Mark document, element, and all element's parents as changed
	void
	pcmk__mark_xml_node_dirty(xmlNode *xml)
	{
	pcmk__set_xml_doc_flag(xml, pcmk__xf_dirty);
	set_parent_flag(xml, pcmk__xf_dirty);
	}

	// Clear flags on XML node and its children
	static void
	reset_xml_node_flags(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_node_private_t *nodepriv = xml->_private;

	if (nodepriv) {
	nodepriv->flags = 0;
	}

	for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	reset_xml_node_flags(cIter);
	}
	}

	// Set xpf_created flag on XML node and any children
	void
	pcmk__mark_xml_created(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_node_private_t *nodepriv = NULL;

	CRM_ASSERT(xml != NULL);
	nodepriv = xml->_private;

	if (nodepriv && pcmk__tracking_xml_changes(xml, FALSE)) {
	if (!pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
	pcmk__set_xml_flags(nodepriv, pcmk__xf_created);
	pcmk__mark_xml_node_dirty(xml);
	}
	for (cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	pcmk__mark_xml_created(cIter);
	}
	}
	}

	#define XML_DOC_PRIVATE_MAGIC 0x81726354UL
	#define XML_NODE_PRIVATE_MAGIC 0x54637281UL

	// Free an XML object previously marked as deleted
	static void
	free_deleted_object(void *data)
	{
	if(data) {
	pcmk__deleted_xml_t *deleted_obj = data;

	free(deleted_obj->path);
	free(deleted_obj);
	}
	}

	// Free and NULL user, ACLs, and deleted objects in an XML node's private data
	static void
	reset_xml_private_data(xml_doc_private_t *docpriv)
	{
	if (docpriv != NULL) {
	CRM_ASSERT(docpriv->check == XML_DOC_PRIVATE_MAGIC);

	free(docpriv->user);
	docpriv->user = NULL;

	if (docpriv->acls != NULL) {
	pcmk__free_acls(docpriv->acls);
	docpriv->acls = NULL;
	}

	if(docpriv->deleted_objs) {
	g_list_free_full(docpriv->deleted_objs, free_deleted_object);
	docpriv->deleted_objs = NULL;
	}
	}
	}

	// Free all private data associated with an XML node
	static void
	free_private_data(xmlNode *node)
	{
	/* Note:

	This function frees private data assosciated with an XML node,
	unless the function is being called as a result of internal
	XSLT cleanup.

	That could happen through, for example, the following chain of
	function calls:

	xsltApplyStylesheetInternal
	-> xsltFreeTransformContext
	-> xsltFreeRVTs
	-> xmlFreeDoc

	And in that case, the node would fulfill three conditions:

	1. It would be a standalone document (i.e. it wouldn't be
	part of a document)
	2. It would have a space-prefixed name (for reference, please
	see xsltInternals.h: XSLT_MARK_RES_TREE_FRAG)
	3. It would carry its own payload in the _private field.

	We do not free data in this circumstance to avoid a failed
	assertion on the XML_*_PRIVATE_MAGIC later.

	*/
	if (node->name == NULL \|\| node->name[0] != ' ') {
	if (node->_private) {
	if (node->type == XML_DOCUMENT_NODE) {
	reset_xml_private_data(node->_private);
	} else {
	CRM_ASSERT(((xml_node_private_t *) node->_private)->check
	== XML_NODE_PRIVATE_MAGIC);
	/* nothing dynamically allocated nested */
	}
	free(node->_private);
	node->_private = NULL;
	}
	}
	}

	// Allocate and initialize private data for an XML node
	static void
	new_private_data(xmlNode *node)
	{
	switch (node->type) {
	case XML_DOCUMENT_NODE: {
	xml_doc_private_t *docpriv = NULL;
	docpriv = calloc(1, sizeof(xml_doc_private_t));
	CRM_ASSERT(docpriv != NULL);
	docpriv->check = XML_DOC_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	pcmk__set_xml_flags(docpriv, pcmk__xf_dirty\|pcmk__xf_created);
	node->_private = docpriv;
	break;
	}
	case XML_ELEMENT_NODE:
	case XML_ATTRIBUTE_NODE:
	case XML_COMMENT_NODE: {
	xml_node_private_t *nodepriv = NULL;
	nodepriv = calloc(1, sizeof(xml_node_private_t));
	CRM_ASSERT(nodepriv != NULL);
	nodepriv->check = XML_NODE_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty\|pcmk__xf_created);
	node->_private = nodepriv;
	if (pcmk__tracking_xml_changes(node, FALSE)) {
	/* XML_ELEMENT_NODE doesn't get picked up here, node->doc is
	* not hooked up at the point we are called
	*/
	pcmk__mark_xml_node_dirty(node);
	}
	break;
	}
	case XML_TEXT_NODE:
	case XML_DTD_NODE:
	case XML_CDATA_SECTION_NODE:
	break;
	default:
	/* Ignore */
	crm_trace("Ignoring %p %d", node, node->type);
	CRM_LOG_ASSERT(node->type == XML_ELEMENT_NODE);
	break;
	}
	}

	void
	xml_track_changes(xmlNode * xml, const char user, xmlNode acl_source, bool enforce_acls)
	{
	xml_accept_changes(xml);
	crm_trace("Tracking changes%s to %p", enforce_acls?" with ACLs":"", xml);
	pcmk__set_xml_doc_flag(xml, pcmk__xf_tracking);
	if(enforce_acls) {
	if(acl_source == NULL) {
	acl_source = xml;
	}
	pcmk__set_xml_doc_flag(xml, pcmk__xf_acl_enabled);
	pcmk__unpack_acl(acl_source, xml, user);
	pcmk__apply_acl(xml);
	}
	}

	bool xml_tracking_changes(xmlNode * xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_tracking);
	}

	bool xml_document_dirty(xmlNode *xml)
	{
	return (xml != NULL) && (xml->doc != NULL) && (xml->doc->_private != NULL)
	&& pcmk_is_set(((xml_doc_private_t *)(xml->doc->_private))->flags,
	pcmk__xf_dirty);
	}

	/*!
	* \internal
	* \brief Return ordinal position of an XML node among its siblings
	*
	* \param[in] xml XML node to check
	* \param[in] ignore_if_set Don't count siblings with this flag set
	*
	* \return Ordinal position of \p xml (starting with 0)
	*/
	int
	pcmk__xml_position(const xmlNode *xml, enum xml_private_flags ignore_if_set)
	{
	int position = 0;

	for (const xmlNode *cIter = xml; cIter->prev; cIter = cIter->prev) {
	xml_node_private_t nodepriv = ((xmlNode)cIter->prev)->_private;

	if (!pcmk_is_set(nodepriv->flags, ignore_if_set)) {
	position++;
	}
	}

	return position;
	}

	// Remove all attributes marked as deleted from an XML node
	static void
	accept_attr_deletions(xmlNode *xml)
	{
	// Clear XML node's flags
	((xml_node_private_t *) xml->_private)->flags = pcmk__xf_none;

	// Remove this XML node's attributes that were marked as deleted
	pcmk__xe_remove_matching_attrs(xml, pcmk__marked_as_deleted, NULL);

	// Recursively do the same for this XML node's children
	for (xmlNodePtr cIter = pcmk__xml_first_child(xml); cIter != NULL;
	cIter = pcmk__xml_next(cIter)) {
	accept_attr_deletions(cIter);
	}
	}

	/*!
	* \internal
	* \brief Find first child XML node matching another given XML node
	*
	* \param[in] haystack XML whose children should be checked
	* \param[in] needle XML to match (comment content or element name and ID)
	* \param[in] exact If true and needle is a comment, position must match
	*/
	xmlNode *
	pcmk__xml_match(const xmlNode haystack, const xmlNode needle, bool exact)
	{
	CRM_CHECK(needle != NULL, return NULL);

	if (needle->type == XML_COMMENT_NODE) {
	return pcmk__xc_match(haystack, needle, exact);

	} else {
	const char *id = pcmk__xe_id(needle);
	const char *attr = (id == NULL)? NULL : PCMK_XA_ID;

	return pcmk__xe_match(haystack, (const char *) needle->name, attr, id);
	}
	}

	void
	xml_accept_changes(xmlNode * xml)
	{
	xmlNode *top = NULL;
	xml_doc_private_t *docpriv = NULL;

	if(xml == NULL) {
	return;
	}

	crm_trace("Accepting changes to %p", xml);
	docpriv = xml->doc->_private;
	top = xmlDocGetRootElement(xml->doc);

	reset_xml_private_data(xml->doc->_private);

	if (!pcmk_is_set(docpriv->flags, pcmk__xf_dirty)) {
	docpriv->flags = pcmk__xf_none;
	return;
	}

	docpriv->flags = pcmk__xf_none;
	accept_attr_deletions(top);
	}

	xmlNode *
	find_xml_node(const xmlNode root, const char search_path, gboolean must_find)
	{
	xmlNode *a_child = NULL;
	const char name = (root == NULL)? "<NULL>" : (const char ) root->name;

	if (search_path == NULL) {
	crm_warn("Will never find <NULL>");
	return NULL;
	}

	for (a_child = pcmk__xml_first_child(root); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	if (strcmp((const char *)a_child->name, search_path) == 0) {
	return a_child;
	}
	}

	if (must_find) {
	crm_warn("Could not find %s in %s.", search_path, name);
	} else if (root != NULL) {
	crm_trace("Could not find %s in %s.", search_path, name);
	} else {
	crm_trace("Could not find %s in <NULL>.", search_path);
	}

	return NULL;
	}

	#define attr_matches(c, n, v) pcmk__str_eq(crm_element_value((c), (n)), \
	(v), pcmk__str_none)

	/*!
	* \internal
	* \brief Find first XML child element matching given criteria
	*
	* \param[in] parent XML element to search
	* \param[in] node_name If not NULL, only match children of this type
	* \param[in] attr_n If not NULL, only match children with an attribute
	* of this name.
	* \param[in] attr_v If \p attr_n and this are not NULL, only match children
	* with an attribute named \p attr_n and this value
	*
	* \return Matching XML child element, or NULL if none found
	*/
	xmlNode *
	pcmk__xe_match(const xmlNode parent, const char node_name,
	const char attr_n, const char attr_v)
	{
	CRM_CHECK(parent != NULL, return NULL);
	CRM_CHECK(attr_v == NULL \|\| attr_n != NULL, return NULL);

	for (xmlNode *child = pcmk__xml_first_child(parent); child != NULL;
	child = pcmk__xml_next(child)) {
	if (((node_name == NULL) \|\| pcmk__xe_is(child, node_name))
	&& ((attr_n == NULL) \|\|
	(attr_v == NULL && xmlHasProp(child, (pcmkXmlStr) attr_n)) \|\|
	(attr_v != NULL && attr_matches(child, attr_n, attr_v)))) {
	return child;
	}
	}
	crm_trace("XML child node <%s%s%s%s%s> not found in %s",
	(node_name? node_name : "(any)"),
	(attr_n? " " : ""),
	(attr_n? attr_n : ""),
	(attr_n? "=" : ""),
	(attr_n? attr_v : ""),
	(const char *) parent->name);
	return NULL;
	}

	void
	copy_in_properties(xmlNode target, const xmlNode src)
	{
	if (src == NULL) {
	crm_warn("No node to copy properties from");

	} else if (target == NULL) {
	crm_err("No node to copy properties into");

	} else {
	for (xmlAttrPtr a = pcmk__xe_first_attr(src); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	expand_plus_plus(target, p_name, p_value);
	if (xml_acl_denied(target)) {
	crm_trace("Cannot copy %s=%s to %s", p_name, p_value, target->name);
	return;
	}
	}
	}

	return;
	}

	/*!
	* \brief Parse integer assignment statements on this node and all its child
	* nodes
	*
	* \param[in,out] target Root XML node to be processed
	*
	* \note This function is recursive
	*/
	void
	fix_plus_plus_recursive(xmlNode *target)
	{
	/* TODO: Remove recursion and use xpath searches for value++ */
	xmlNode *child = NULL;

	for (xmlAttrPtr a = pcmk__xe_first_attr(target); a != NULL; a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	expand_plus_plus(target, p_name, p_value);
	}
	for (child = pcmk__xml_first_child(target); child != NULL;
	child = pcmk__xml_next(child)) {
	fix_plus_plus_recursive(child);
	}
	}

	/*!
	* \brief Update current XML attribute value per parsed integer assignment
	statement
	*
	* \param[in,out] target an XML node, containing a XML attribute that is
	* initialized to some numeric value, to be processed
	* \param[in] name name of the XML attribute, e.g. X, whose value
	* should be updated
	* \param[in] value assignment statement, e.g. "X++" or
	* "X+=5", to be applied to the initialized value.
	*
	* \note The original XML attribute value is treated as 0 if non-numeric and
	* truncated to be an integer if decimal-point-containing.
	* \note The final XML attribute value is truncated to not exceed 1000000.
	* \note Undefined behavior if unexpected input.
	*/
	void
	expand_plus_plus(xmlNode * target, const char name, const char value)
	{
	int offset = 1;
	int name_len = 0;
	int int_value = 0;
	int value_len = 0;

	const char *old_value = NULL;

	if (target == NULL \|\| value == NULL \|\| name == NULL) {
	return;
	}

	old_value = crm_element_value(target, name);

	if (old_value == NULL) {
	/* if no previous value, set unexpanded */
	goto set_unexpanded;

	} else if (strstr(value, name) != value) {
	goto set_unexpanded;
	}

	name_len = strlen(name);
	value_len = strlen(value);
	if (value_len < (name_len + 2)
	\|\| value[name_len] != '+' \|\| (value[name_len + 1] != '+' && value[name_len + 1] != '=')) {
	goto set_unexpanded;
	}

	/* if we are expanding ourselves,
	* then no previous value was set and leave int_value as 0
	*/
	if (old_value != value) {
	int_value = char2score(old_value);
	}

	if (value[name_len + 1] != '+') {
	const char *offset_s = value + (name_len + 2);

	offset = char2score(offset_s);
	}
	int_value += offset;

	if (int_value > PCMK_SCORE_INFINITY) {
	int_value = PCMK_SCORE_INFINITY;
	}

	crm_xml_add_int(target, name, int_value);
	return;

	set_unexpanded:
	if (old_value == value) {
	/* the old value is already set, nothing to do */
	return;
	}
	crm_xml_add(target, name, value);
	return;
	}

	/*!
	* \internal
	* \brief Remove an XML element's attributes that match some criteria
	*
	* \param[in,out] element XML element to modify
	* \param[in] match If not NULL, only remove attributes for which
	* this function returns true
	* \param[in,out] user_data Data to pass to \p match
	*/
	void
	pcmk__xe_remove_matching_attrs(xmlNode *element,
	bool (match)(xmlAttrPtr, void ),
	void *user_data)
	{
	xmlAttrPtr next = NULL;

	for (xmlAttrPtr a = pcmk__xe_first_attr(element); a != NULL; a = next) {
	next = a->next; // Grab now because attribute might get removed
	if ((match == NULL) \|\| match(a, user_data)) {
	if (!pcmk__check_acl(element, NULL, pcmk__xf_acl_write)) {
	crm_trace("ACLs prevent removal of attributes (%s and "
	"possibly others) from %s element",
	(const char ) a->name, (const char ) element->name);
	return; // ACLs apply to element, not particular attributes
	}

	if (pcmk__tracking_xml_changes(element, false)) {
	// Leave (marked for removal) until after diff is calculated
	set_parent_flag(element, pcmk__xf_dirty);
	pcmk__set_xml_flags((xml_node_private_t *) a->_private,
	pcmk__xf_deleted);
	} else {
	xmlRemoveProp(a);
	}
	}
	}
	}

	xmlNode *
	create_xml_node(xmlNode * parent, const char *name)
	{
	xmlDoc *doc = NULL;
	xmlNode *node = NULL;

	if (pcmk__str_empty(name)) {
	CRM_CHECK(name != NULL && name[0] == 0, return NULL);
	return NULL;
	}

	if (parent == NULL) {
	// libxml2 supports only XML 1.0
	doc = xmlNewDoc((pcmkXmlStr) "1.0");
	if (doc == NULL) {
	return NULL;
	}

	node = xmlNewDocRawNode(doc, NULL, (pcmkXmlStr) name, NULL);
	if (node == NULL) {
	xmlFreeDoc(doc);
	return NULL;
	}
	xmlDocSetRootElement(doc, node);

	} else {
	node = xmlNewChild(parent, NULL, (pcmkXmlStr) name, NULL);
	if (node == NULL) {
	return NULL;
	}
	}
	pcmk__mark_xml_created(node);
	return node;
	}

	/*!
	* \internal
	* \brief Set a given string as an XML node's content
	*
	* \param[in,out] node Node whose content to set
	* \param[in] content String to set as the content
	*
	* \note \c xmlNodeSetContent() does not escape special characters.
	*/
	void
	pcmk__xe_set_content(xmlNode node, const char content)
	{
	if (node != NULL) {
	char *escaped = pcmk__xml_escape(content, false);

	xmlNodeSetContent(node, (pcmkXmlStr) escaped);
	free(escaped);
	}
	}

	xmlNode *
	pcmk_create_xml_text_node(xmlNode * parent, const char name, const char content)
	{
	xmlNode *node = create_xml_node(parent, name);

	pcmk__xe_set_content(node, content);
	return node;
	}

	xmlNode *
	pcmk_create_html_node(xmlNode * parent, const char element_name, const char id,
	const char class_name, const char text)
	{
	xmlNode *node = pcmk_create_xml_text_node(parent, element_name, text);

	if (class_name != NULL) {
	crm_xml_add(node, PCMK_XA_CLASS, class_name);
	}

	if (id != NULL) {
	crm_xml_add(node, PCMK_XA_ID, id);
	}

	return node;
	}

	/*!
	* Free an XML element and all of its children, removing it from its parent
	*
	* \param[in,out] xml XML element to free
	*/
	void
	pcmk_free_xml_subtree(xmlNode *xml)
	{
	xmlUnlinkNode(xml); // Detaches from parent and siblings
	xmlFreeNode(xml); // Frees
	}

	static void
	free_xml_with_position(xmlNode * child, int position)
	{
	if (child != NULL) {
	xmlNode *top = NULL;
	xmlDoc *doc = child->doc;
	xml_node_private_t *nodepriv = child->_private;
	xml_doc_private_t *docpriv = NULL;

	if (doc != NULL) {
	top = xmlDocGetRootElement(doc);
	}

	if (doc != NULL && top == child) {
	/* Free everything */
	xmlFreeDoc(doc);

	} else if (pcmk__check_acl(child, NULL, pcmk__xf_acl_write) == FALSE) {
	GString *xpath = NULL;

	pcmk__if_tracing({}, return);
	xpath = pcmk__element_xpath(child);
	qb_log_from_external_source(__func__, __FILE__,
	"Cannot remove %s %x", LOG_TRACE,
	__LINE__, 0, (const char *) xpath->str,
	nodepriv->flags);
	g_string_free(xpath, TRUE);
	return;

	} else {
	if (doc && pcmk__tracking_xml_changes(child, FALSE)
	&& !pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {

	GString *xpath = pcmk__element_xpath(child);

	if (xpath != NULL) {
	pcmk__deleted_xml_t *deleted_obj = NULL;

	crm_trace("Deleting %s %p from %p",
	(const char *) xpath->str, child, doc);

	deleted_obj = calloc(1, sizeof(pcmk__deleted_xml_t));
	deleted_obj->path = strdup((const char *) xpath->str);

	CRM_ASSERT(deleted_obj->path != NULL);
	g_string_free(xpath, TRUE);

	deleted_obj->position = -1;
	/* Record the "position" only for XML comments for now */
	if (child->type == XML_COMMENT_NODE) {
	if (position >= 0) {
	deleted_obj->position = position;

	} else {
	deleted_obj->position = pcmk__xml_position(child,
	pcmk__xf_skip);
	}
	}

	docpriv = doc->_private;
	docpriv->deleted_objs = g_list_append(docpriv->deleted_objs, deleted_obj);
	pcmk__set_xml_doc_flag(child, pcmk__xf_dirty);
	}
	}
	pcmk_free_xml_subtree(child);
	}
	}
	}


	void
	free_xml(xmlNode * child)
	{
	free_xml_with_position(child, -1);
	}

	/*!
	* \internal
	* \brief Make a deep copy of an XML node under a given parent
	*
	* \param[in,out] parent XML element that will be the copy's parent (\c NULL
	* to create a new XML document with the copy as root)
	* \param[in] src XML node to copy
	*
	* \return Deep copy of \p src, or \c NULL if \p src is \c NULL
	*/
	xmlNode *
	pcmk__xml_copy(xmlNode parent, xmlNode src)
	{
	xmlNode *copy = NULL;

	if (src == NULL) {
	return NULL;
	}

	if (parent == NULL) {
	xmlDoc *doc = NULL;

	// The copy will be the root element of a new document
	CRM_ASSERT(src->type == XML_ELEMENT_NODE);

	// libxml2 supports only XML 1.0.
	doc = xmlNewDoc((pcmkXmlStr) "1.0");
	CRM_ASSERT(doc != NULL);

	copy = xmlDocCopyNode(src, doc, 1);
	CRM_ASSERT(copy != NULL);

	xmlDocSetRootElement(doc, copy);

	} else {
	copy = xmlDocCopyNode(src, parent->doc, 1);
	CRM_ASSERT(copy != NULL);

	xmlAddChild(parent, copy);
	}

	pcmk__mark_xml_created(copy);
	return copy;
	}

	xmlNode *
	string2xml(const char *input)
	{
	- xmlNode *xml = NULL;
	- xmlDocPtr output = NULL;
	- xmlParserCtxtPtr ctxt = NULL;
	- const xmlError *last_error = NULL;
	-
	- if (input == NULL) {
	- crm_err("Can't parse NULL input");
	- return NULL;
	- }
	-
	- /* create a parser context */
	- ctxt = xmlNewParserCtxt();
	- CRM_CHECK(ctxt != NULL, return NULL);
	-
	- xmlCtxtResetLastError(ctxt);
	- xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err);
	- output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	- PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER);
	-
	- if (output == NULL) {
	- output = xmlCtxtReadDoc(ctxt, (pcmkXmlStr) input, NULL, NULL,
	- PCMK__XML_PARSE_OPTS_WITH_RECOVER);
	- if (output) {
	- crm_warn("Successfully recovered from XML errors "
	- "(note: a future release will treat this as a fatal failure)");
	- }
	- }
	-
	- if (output) {
	- xml = xmlDocGetRootElement(output);
	- }
	- last_error = xmlCtxtGetLastError(ctxt);
	- if (last_error && last_error->code != XML_ERR_OK) {
	- /* crm_abort(__FILE__,__func__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	- /*
	- * http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	- * http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	- */
	- crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
	- last_error->domain, last_error->level, last_error->code, last_error->message);
	-
	- if (last_error->code == XML_ERR_DOCUMENT_EMPTY) {
	- CRM_LOG_ASSERT("Cannot parse an empty string");
	-
	- } else if (last_error->code != XML_ERR_DOCUMENT_END) {
	- crm_err("Couldn't%s parse %d chars: %s", xml ? " fully" : "", (int)strlen(input),
	- input);
	- if (xml != NULL) {
	- crm_log_xml_err(xml, "Partial");
	- }
	-
	- } else {
	- int len = strlen(input);
	- int lpc = 0;
	-
	- while(lpc < len) {
	- crm_warn("Parse error[+%.3d]: %.80s", lpc, input+lpc);
	- lpc += 80;
	- }
	-
	- CRM_LOG_ASSERT("String parsing error");
	- }
	- }
	-
	- xmlFreeParserCtxt(ctxt);
	- return xml;
	+ return pcmk__xml_parse_string(input);
	}

	/*!
	* \internal
	* \brief Read from \c stdin until EOF or error
	*
	* \return Newly allocated string containing the bytes read from \c stdin, or
	* \c NULL on error
	*
	* \note The caller is responsible for freeing the return value using \c free().
	*/
	static char *
	read_stdin(void)
	{
	char *buf = NULL;
	size_t length = 0;

	do {
	size_t bytes_read = 0;

	buf = pcmk__realloc(buf, length + PCMK__BUFFER_SIZE + 1);
	bytes_read = fread(buf + length, 1, PCMK__BUFFER_SIZE, stdin);
	length += bytes_read;
	} while ((feof(stdin) == 0) && (ferror(stdin) == 0));

	if (ferror(stdin) != 0) {
	crm_err("Error reading input from stdin");
	free(buf);
	buf = NULL;
	} else {
	buf[length] = '\0';
	}
	clearerr(stdin);
	return buf;
	}

	/*!
	* \internal
	* \brief Decompress a <tt>bzip2</tt>-compressed file into a string buffer
	*
	* \param[in] filename Name of file to decompress
	*
	* \return Newly allocated string with the decompressed contents of \p filename,
	* or \c NULL on error.
	*
	* \note The caller is responsible for freeing the return value using \c free().
	*/
	static char *
	decompress_file(const char *filename)
	{
	char *buffer = NULL;
	int rc = 0;
	size_t length = 0;
	BZFILE *bz_file = NULL;
	FILE *input = fopen(filename, "r");

	if (input == NULL) {
	crm_perror(LOG_ERR, "Could not open %s for reading", filename);
	return NULL;
	}

	bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);
	if (rc != BZ_OK) {
	rc = pcmk__bzlib2rc(rc);
	crm_err("Could not prepare to read compressed %s: %s "
	CRM_XS " rc=%d", filename, pcmk_rc_str(rc), rc);
	goto done;
	}

	// cppcheck seems not to understand the abort-logic in pcmk__realloc
	// cppcheck-suppress memleak
	do {
	int read_len = 0;

	buffer = pcmk__realloc(buffer, length + PCMK__BUFFER_SIZE + 1);
	read_len = BZ2_bzRead(&rc, bz_file, buffer + length, PCMK__BUFFER_SIZE);

	if ((rc == BZ_OK) \|\| (rc == BZ_STREAM_END)) {
	crm_trace("Read %ld bytes from file: %d", (long) read_len, rc);
	length += read_len;
	}
	} while (rc == BZ_OK);

	if (rc != BZ_STREAM_END) {
	rc = pcmk__bzlib2rc(rc);
	crm_err("Could not read compressed %s: %s " CRM_XS " rc=%d",
	filename, pcmk_rc_str(rc), rc);
	free(buffer);
	buffer = NULL;
	} else {
	buffer[length] = '\0';
	}

	done:
	BZ2_bzReadClose(&rc, bz_file);
	fclose(input);
	return buffer;
	}

	/*!
	* \internal
	* \brief Remove XML text nodes from specified XML and all its children
	*
	* \param[in,out] xml XML to strip text from
	*/
	void
	pcmk__strip_xml_text(xmlNode *xml)
	{
	xmlNode *iter = xml->children;

	while (iter) {
	xmlNode *next = iter->next;

	switch (iter->type) {
	case XML_TEXT_NODE:
	/* Remove it */
	pcmk_free_xml_subtree(iter);
	break;

	case XML_ELEMENT_NODE:
	/* Search it */
	pcmk__strip_xml_text(iter);
	break;

	default:
	/* Leave it */
	break;
	}

	iter = next;
	}
	}

	// @COMPAT Remove macro at 3.0.0 when we drop XML_PARSE_RECOVER
	/*!
	* \internal
	* \brief Try to parse XML first without and then with recovery enabled
	*
	* \param[out] result Where to store the resulting XML doc (<tt>xmlDoc **</tt>)
	* \param[in] fn XML parser function
	* \param[in] ... All arguments for \p fn except the final one (an
	* \c xmlParserOption group)
	*/
	#define parse_xml_recover(result, fn, ...) do { \
	*result = fn(__VA_ARGS__, PCMK__XML_PARSE_OPTS_WITHOUT_RECOVER); \
	if (*result == NULL) { \
	*result = fn(__VA_ARGS__, PCMK__XML_PARSE_OPTS_WITH_RECOVER); \
	\
	if (*result != NULL) { \
	crm_warn("Successfully recovered from XML errors " \
	"(note: a future release will treat this as a " \
	"fatal failure)"); \
	} \
	} \
	} while (0);

	/*!
	* \internal
	* \brief Parse XML from a file
	*
	* \param[in] filename Name of file containing XML (\c NULL or \c "-" for
	* \c stdin); if \p filename ends in \c ".bz2", the file
	* will be decompressed using \c bzip2
	*
	* \return XML tree parsed from the given file; may be \c NULL or only partial
	* on error
	*/
	xmlNode *
	pcmk__xml_parse_file(const char *filename)
	{
	bool use_stdin = pcmk__str_eq(filename, "-", pcmk__str_null_matches);
	xmlNode *xml = NULL;
	xmlDoc *output = NULL;
	xmlParserCtxt *ctxt = NULL;
	const xmlError *last_error = NULL;

	// Create a parser context
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err);

	if (use_stdin) {
	/* @COMPAT After dropping XML_PARSE_RECOVER, we can avoid capturing
	* stdin into a buffer and instead call
	* xmlCtxtReadFd(ctxt, STDIN_FILENO, NULL, NULL, XML_PARSE_NOBLANKS);
	*
	* For now we have to save the input so that we can use it twice.
	*/
	char *input = read_stdin();

	if (input != NULL) {
	parse_xml_recover(&output, xmlCtxtReadDoc, ctxt, (pcmkXmlStr) input,
	NULL, NULL);
	free(input);
	}

	} else if (pcmk__ends_with_ext(filename, ".bz2")) {
	char *input = decompress_file(filename);

	if (input != NULL) {
	parse_xml_recover(&output, xmlCtxtReadDoc, ctxt, (pcmkXmlStr) input,
	NULL, NULL);
	free(input);
	}

	} else {
	parse_xml_recover(&output, xmlCtxtReadFile, ctxt, filename, NULL);
	}

	if (output != NULL) {
	xml = xmlDocGetRootElement(output);
	if (xml != NULL) {
	/* @TODO Should we really be stripping out text? This seems like an
	* overly broad way to get rid of whitespace, if that's the goal.
	* Text nodes may be invalid in most or all Pacemaker inputs, but
	* stripping them in a generic "parse XML from file" function may
	* not be the best way to ignore them.
	*/
	pcmk__strip_xml_text(xml);
	}
	}

	last_error = xmlCtxtGetLastError(ctxt);
	if (last_error != NULL) {
	crm_err("Couldn't %sparse XML from %s: %s "
	CRM_XS " (domain=%d, level=%d, code=%d)",
	((xml != NULL)? "fully " : ""), (use_stdin? "stdin" : filename),
	last_error->message, last_error->domain, last_error->level,
	last_error->code);

	if (xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	+/*!
	+ * \internal
	+ * \brief Parse XML from a string
	+ *
	+ * \param[in] input String to parse
	+ *
	+ * \return XML tree parsed from the given string; may be \c NULL or only partial
	+ * on error
	+ */
	+xmlNode *
	+pcmk__xml_parse_string(const char *input)
	+{
	+ xmlNode *xml = NULL;
	+ xmlDoc *output = NULL;
	+ xmlParserCtxt *ctxt = NULL;
	+ const xmlError *last_error = NULL;
	+
	+ if (input == NULL) {
	+ crm_err("Can't parse NULL input");
	+ return NULL;
	+ }
	+
	+ // Create a parser context
	+ ctxt = xmlNewParserCtxt();
	+ CRM_CHECK(ctxt != NULL, return NULL);
	+
	+ xmlCtxtResetLastError(ctxt);
	+ xmlSetGenericErrorFunc(ctxt, pcmk__log_xmllib_err);
	+
	+ parse_xml_recover(&output, xmlCtxtReadDoc, ctxt, (pcmkXmlStr) input, NULL,
	+ NULL);
	+
	+ if (output != NULL) {
	+ xml = xmlDocGetRootElement(output);
	+ }
	+
	+ last_error = xmlCtxtGetLastError(ctxt);
	+ if (last_error != NULL) {
	+ crm_err("Couldn't %sparse XML from string: %s "
	+ CRM_XS " (domain=%d, level=%d, code=%d)",
	+ ((xml != NULL)? "fully " : ""), last_error->message,
	+ last_error->domain, last_error->level, last_error->code);
	+
	+ crm_info("XML parse error: %s", input);
	+
	+ if (xml != NULL) {
	+ crm_log_xml_info(xml, "Partial");
	+ }
	+ }
	+
	+ xmlFreeParserCtxt(ctxt);
	+ return xml;
	+}
	+
	/*!
	* \internal
	* \brief Add a "last written" attribute to an XML element, set to current time
	*
	* \param[in,out] xe XML element to add attribute to
	*
	* \return Value that was set, or NULL on error
	*/
	const char *
	pcmk__xe_add_last_written(xmlNode *xe)
	{
	char *now_s = pcmk__epoch2str(NULL, 0);
	const char *result = NULL;

	result = crm_xml_add(xe, PCMK_XA_CIB_LAST_WRITTEN,
	pcmk__s(now_s, "Could not determine current time"));
	free(now_s);
	return result;
	}

	/*!
	* \brief Sanitize a string so it is usable as an XML ID
	*
	* \param[in,out] id String to sanitize
	*/
	void
	crm_xml_sanitize_id(char *id)
	{
	char *c;

	for (c = id; *c; ++c) {
	/* @TODO Sanitize more comprehensively */
	switch (*c) {
	case ':':
	case '#':
	*c = '.';
	}
	}
	}

	/*!
	* \brief Set the ID of an XML element using a format
	*
	* \param[in,out] xml XML element
	* \param[in] fmt printf-style format
	* \param[in] ... any arguments required by format
	*/
	void
	crm_xml_set_id(xmlNode xml, const char format, ...)
	{
	va_list ap;
	int len = 0;
	char *id = NULL;

	/* equivalent to crm_strdup_printf() */
	va_start(ap, format);
	len = vasprintf(&id, format, ap);
	va_end(ap);
	CRM_ASSERT(len > 0);

	crm_xml_sanitize_id(id);
	crm_xml_add(xml, PCMK_XA_ID, id);
	free(id);
	}

	/*!
	* \internal
	* \brief Write XML to a file stream
	*
	* \param[in] xml XML to write
	* \param[in] filename Name of file being written (for logging only)
	* \param[in,out] stream Open file stream corresponding to filename
	* \param[in] compress Whether to compress XML before writing
	* \param[out] nbytes Number of bytes written
	*
	* \return Standard Pacemaker return code
	*/
	static int
	write_xml_stream(const xmlNode xml, const char filename, FILE *stream,
	bool compress, unsigned int *nbytes)
	{
	int rc = pcmk_rc_ok;
	char *buffer = NULL;

	*nbytes = 0;
	crm_log_xml_trace(xml, "writing");

	buffer = dump_xml_formatted(xml);
	CRM_CHECK(buffer && strlen(buffer),
	crm_log_xml_warn(xml, "formatting failed");
	rc = pcmk_rc_error;
	goto bail);

	if (compress) {
	unsigned int in = 0;
	BZFILE *bz_file = NULL;

	rc = BZ_OK;
	bz_file = BZ2_bzWriteOpen(&rc, stream, 5, 0, 30);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_warn("Not compressing %s: could not prepare file stream: %s "
	CRM_XS " rc=%d", filename, pcmk_rc_str(rc), rc);
	} else {
	BZ2_bzWrite(&rc, bz_file, buffer, strlen(buffer));
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_warn("Not compressing %s: could not compress data: %s "
	CRM_XS " rc=%d errno=%d",
	filename, pcmk_rc_str(rc), rc, errno);
	}
	}

	if (rc == pcmk_rc_ok) {
	BZ2_bzWriteClose(&rc, bz_file, 0, &in, nbytes);
	rc = pcmk__bzlib2rc(rc);

	if (rc != pcmk_rc_ok) {
	crm_warn("Not compressing %s: could not write compressed data: %s "
	CRM_XS " rc=%d errno=%d",
	filename, pcmk_rc_str(rc), rc, errno);
	*nbytes = 0; // retry without compression
	} else {
	crm_trace("Compressed XML for %s from %u bytes to %u",
	filename, in, *nbytes);
	}
	}
	rc = pcmk_rc_ok; // Either true, or we'll retry without compression
	}

	if (*nbytes == 0) {
	rc = fprintf(stream, "%s", buffer);
	if (rc < 0) {
	rc = errno;
	crm_perror(LOG_ERR, "writing %s", filename);
	} else {
	*nbytes = (unsigned int) rc;
	rc = pcmk_rc_ok;
	}
	}

	bail:

	if (fflush(stream) != 0) {
	rc = errno;
	crm_perror(LOG_ERR, "flushing %s", filename);
	}

	/* Don't report error if the file does not support synchronization */
	if (fsync(fileno(stream)) < 0 && errno != EROFS && errno != EINVAL) {
	rc = errno;
	crm_perror(LOG_ERR, "synchronizing %s", filename);
	}

	fclose(stream);

	crm_trace("Saved %d bytes to %s as XML", *nbytes, filename);
	free(buffer);

	return rc;
	}

	/*!
	* \brief Write XML to a file descriptor
	*
	* \param[in] xml XML to write
	* \param[in] filename Name of file being written (for logging only)
	* \param[in] fd Open file descriptor corresponding to filename
	* \param[in] compress Whether to compress XML before writing
	*
	* \return Number of bytes written on success, -errno otherwise
	*/
	int
	write_xml_fd(const xmlNode xml, const char filename, int fd,
	gboolean compress)
	{
	FILE *stream = NULL;
	unsigned int nbytes = 0;
	int rc = pcmk_rc_ok;

	CRM_CHECK((xml != NULL) && (fd > 0), return -EINVAL);
	stream = fdopen(fd, "w");
	if (stream == NULL) {
	return -errno;
	}
	rc = write_xml_stream(xml, filename, stream, compress, &nbytes);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}
	return (int) nbytes;
	}

	/*!
	* \brief Write XML to a file
	*
	* \param[in] xml XML to write
	* \param[in] filename Name of file to write
	* \param[in] compress Whether to compress XML before writing
	*
	* \return Number of bytes written on success, -errno otherwise
	*/
	int
	write_xml_file(const xmlNode xml, const char filename, gboolean compress)
	{
	FILE *stream = NULL;
	unsigned int nbytes = 0;
	int rc = pcmk_rc_ok;

	CRM_CHECK((xml != NULL) && (filename != NULL), return -EINVAL);
	stream = fopen(filename, "w");
	if (stream == NULL) {
	return -errno;
	}
	rc = write_xml_stream(xml, filename, stream, compress, &nbytes);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}
	return (int) nbytes;
	}

	/*!
	* \internal
	* \brief Get consecutive bytes encoding non-ASCII UTF-8 characters
	*
	* \param[in] text String to check
	*
	* \return Number of non-ASCII UTF-8 bytes at the beginning of \p text
	*/
	static size_t
	utf8_bytes(const char *text)
	{
	// Total number of consecutive bytes containing UTF-8 characters
	size_t c_bytes = 0;

	if (text == NULL) {
	return 0;
	}

	/* UTF-8 uses one to four 8-bit bytes per character. The first byte
	* indicates the width of the character. A byte beginning with a '0' bit is
	* a one-byte ASCII character.
	*
	* A C byte is 8 bits on most systems, but this is not guaranteed.
	*
	* Count until we find an ASCII character or an invalid byte. Check bytes
	* aligned with the C byte boundary.
	*/
	for (const uint8_t utf8_byte = (const uint8_t ) text;
	(*utf8_byte & 0x80) != 0;
	utf8_byte = (const uint8_t *) (text + c_bytes)) {

	size_t utf8_bits = 0;

	if ((*utf8_byte & 0xf0) == 0xf0) {
	// Four-byte character (first byte: 11110xxx)
	utf8_bits = 32;

	} else if ((*utf8_byte & 0xe0) == 0xe0) {
	// Three-byte character (first byte: 1110xxxx)
	utf8_bits = 24;

	} else if ((*utf8_byte & 0xc0) == 0xc0) {
	// Two-byte character (first byte: 110xxxxx)
	utf8_bits = 16;

	} else {
	crm_warn("Found invalid UTF-8 character %.2x",
	(unsigned char) *utf8_byte);
	return c_bytes;
	}

	c_bytes += utf8_bits / CHAR_BIT;

	#if (CHAR_BIT != 8) // Coverity complains about dead code without this CPP guard
	if ((utf8_bits % CHAR_BIT) > 0) {
	c_bytes++;
	}
	#endif // CHAR_BIT != 8
	}

	return c_bytes;
	}

	/*!
	* \internal
	* \brief Replace a character in a dynamically allocated string, reallocating
	* memory
	*
	* \param[in,out] text String to replace a character in
	* \param[in,out] index Index of character to replace with new string; on
	* return, reset to index of end of replacement string
	* \param[in,out] length Length of \p text
	* \param[in] replace String to replace character at \p index with (must
	* not be empty)
	*
	* \return \p text, with the character at \p index replaced by \p replace
	*/
	static char *
	replace_text(char text, size_t index, size_t length, const char replace)
	{
	/* @TODO Replace with GString? Or at least copy char-by-char, escaping
	* characters as needed, instead of shifting characters on every replacement
	*/

	// We have space for 1 char already
	size_t offset = strlen(replace) - 1;

	if (offset > 0) {
	*length += offset;
	text = pcmk__realloc(text, *length + 1);

	// Shift characters to the right to make room for the replacement string
	for (size_t i = length; i > (index + offset); i--) {
	text[i] = text[i - offset];
	}
	}

	// Replace the character at index by the replacement string
	memcpy(text + *index, replace, offset + 1);

	// Reset index to the end of replacement string
	*index += offset;
	return text;
	}

	/*!
	* \internal
	* \brief Check whether a string has XML special characters that must be escaped
	*
	* See \c pcmk__xml_escape() for more details.
	*
	* \param[in] text String to check
	* \param[in] escape_quote If \c true, double quotes must be escaped
	*
	* \return \c true if \p text has special characters that need to be escaped, or
	* \c false otherwise
	*/
	bool
	pcmk__xml_needs_escape(const char *text, bool escape_quote)
	{
	size_t length = 0;

	if (text == NULL) {
	return false;
	}
	length = strlen(text);

	for (size_t index = 0; index < length; index++) {
	// Don't escape any non-ASCII characters
	index += utf8_bytes(&(text[index]));

	switch (text[index]) {
	case '\0':
	// Reached end of string by skipping UTF-8 bytes
	return false;
	case '<':
	return true;
	case '>':
	// Not necessary, but for symmetry with '<'
	return true;
	case '&':
	return true;
	case '"':
	if (escape_quote) {
	return true;
	}
	break;
	case '\n':
	case '\t':
	// Don't escape newline or tab
	break;
	default:
	if ((text[index] < 0x20) \|\| (text[index] >= 0x7f)) {
	// Escape non-printing characters
	return true;
	}
	break;
	}
	}
	return false;
	}

	/*!
	* \internal
	* \brief Replace special characters with their XML escape sequences
	*
	* XML allows the escaping of special characters by replacing them with entity
	* references (for example, <tt>"""</tt>) or character references (for
	* example, <tt>" "</tt>).
	*
	* The special characters <tt>'<'</tt> and <tt>'&'</tt> are not allowed in their
	* literal forms in XML character data. Character data is non-markup text (for
	* example, the content of a text node).
	*
	* Additionally, if an attribute value is delimited by single quotes, then
	* single quotes must be escaped within the value. Similarly, if an attribute
	* value is delimited by double quotes, then double quotes must be escaped
	* within the value.
	*
	* For more details, see the "Character Data and Markup" section of the XML
	* spec, currently section 2.4:
	* https://www.w3.org/TR/xml/#dt-markup
	*
	* Pacemaker always delimits attribute values with double quotes, so this
	* function doesn't escape single quotes.
	*
	* \param[in] text Text to escape
	* \param[in] escape_quote If \c true, escape double quotes (should be enabled
	* for attribute values)
	*
	* \return Newly allocated string equivalent to \p text but with special
	* characters replaced with XML escape sequences (or \c NULL if \p text
	* is \c NULL). If \p text is not \c NULL, the return value is
	* guaranteed not to be \c NULL.
	*
	* \note There are libxml functions that purport to do this:
	* \c xmlEncodeEntitiesReentrant() and \c xmlEncodeSpecialChars().
	* However, their escaping is incomplete. See:
	* https://discourse.gnome.org/t/intended-use-of-xmlencodeentitiesreentrant-vs-xmlencodespecialchars/19252
	*/
	char *
	pcmk__xml_escape(const char *text, bool escape_quote)
	{
	size_t length = 0;
	char *copy = NULL;
	char buf[32] = { '\0', };

	if (text == NULL) {
	return NULL;
	}
	length = strlen(text);
	pcmk__str_update(&copy, text);

	for (size_t index = 0; index < length; index++) {
	// Don't escape any non-ASCII characters
	index += utf8_bytes(&(copy[index]));

	switch (copy[index]) {
	case '\0':
	// Reached end of string by skipping UTF-8 bytes
	break;
	case '<':
	copy = replace_text(copy, &index, &length, "<");
	break;
	case '>':
	// Not necessary, but for symmetry with '<'
	copy = replace_text(copy, &index, &length, ">");
	break;
	case '&':
	copy = replace_text(copy, &index, &length, "&");
	break;
	case '"':
	if (escape_quote) {
	copy = replace_text(copy, &index, &length, """);
	}
	break;
	case '\n':
	case '\t':
	// Don't escape newlines and tabs
	break;
	default:
	if ((copy[index] < 0x20) \|\| (copy[index] >= 0x7f)) {
	// Escape non-printing characters
	snprintf(buf, sizeof(buf), "&#%.2x;", copy[index]);
	copy = replace_text(copy, &index, &length, buf);
	}
	break;
	}
	}
	return copy;
	}

	/*!
	* \internal
	* \brief Append a string representation of an XML element to a buffer
	*
	* \param[in] data XML whose representation to append
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_element(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	bool filtered = pcmk_is_set(options, pcmk__xml_fmt_filtered);
	int spaces = pretty? (2 * depth) : 0;

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "<", data->name, NULL);

	for (const xmlAttr *attr = pcmk__xe_first_attr(data); attr != NULL;
	attr = attr->next) {

	if (!filtered \|\| !pcmk__xa_filterable((const char *) (attr->name))) {
	pcmk__dump_xml_attr(attr, buffer);
	}
	}

	if (data->children == NULL) {
	g_string_append(buffer, "/>");

	} else {
	g_string_append_c(buffer, '>');
	}

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}

	if (data->children) {
	for (const xmlNode *child = data->children; child != NULL;
	child = child->next) {
	pcmk__xml2text(child, options, buffer, depth + 1);
	}

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "</", data->name, ">", NULL);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	}
	}

	/*!
	* \internal
	* \brief Append XML text content to a buffer
	*
	* \param[in] data XML whose content to append
	* \param[in] options Group of \p xml_log_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_text(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	int spaces = pretty? (2 * depth) : 0;
	const char content = (const char ) data->content;
	char *content_esc = NULL;

	if (pcmk__xml_needs_escape(content, false)) {
	content_esc = pcmk__xml_escape(content, false);
	content = content_esc;
	}

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	g_string_append(buffer, content);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	free(content_esc);
	}

	/*!
	* \internal
	* \brief Append XML CDATA content to a buffer
	*
	* \param[in] data XML whose content to append
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_cdata(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	int spaces = pretty? (2 * depth) : 0;

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "<![CDATA[", (const char *) data->content, "]]>",
	NULL);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	}

	/*!
	* \internal
	* \brief Append an XML comment to a buffer
	*
	* \param[in] data XML whose content to append
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to append the content (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	static void
	dump_xml_comment(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	bool pretty = pcmk_is_set(options, pcmk__xml_fmt_pretty);
	int spaces = pretty? (2 * depth) : 0;

	for (int lpc = 0; lpc < spaces; lpc++) {
	g_string_append_c(buffer, ' ');
	}

	pcmk__g_strcat(buffer, "<!--", (const char *) data->content, "-->", NULL);

	if (pretty) {
	g_string_append_c(buffer, '\n');
	}
	}

	/*!
	* \internal
	* \brief Get a string representation of an XML element type
	*
	* \param[in] type XML element type
	*
	* \return String representation of \p type
	*/
	static const char *
	xml_element_type2str(xmlElementType type)
	{
	static const char *const element_type_names[] = {
	[XML_ELEMENT_NODE] = "element",
	[XML_ATTRIBUTE_NODE] = "attribute",
	[XML_TEXT_NODE] = "text",
	[XML_CDATA_SECTION_NODE] = "CDATA section",
	[XML_ENTITY_REF_NODE] = "entity reference",
	[XML_ENTITY_NODE] = "entity",
	[XML_PI_NODE] = "PI",
	[XML_COMMENT_NODE] = "comment",
	[XML_DOCUMENT_NODE] = "document",
	[XML_DOCUMENT_TYPE_NODE] = "document type",
	[XML_DOCUMENT_FRAG_NODE] = "document fragment",
	[XML_NOTATION_NODE] = "notation",
	[XML_HTML_DOCUMENT_NODE] = "HTML document",
	[XML_DTD_NODE] = "DTD",
	[XML_ELEMENT_DECL] = "element declaration",
	[XML_ATTRIBUTE_DECL] = "attribute declaration",
	[XML_ENTITY_DECL] = "entity declaration",
	[XML_NAMESPACE_DECL] = "namespace declaration",
	[XML_XINCLUDE_START] = "XInclude start",
	[XML_XINCLUDE_END] = "XInclude end",
	};

	if ((type < 0) \|\| (type >= PCMK__NELEM(element_type_names))) {
	return "unrecognized type";
	}
	return element_type_names[type];
	}

	/*!
	* \internal
	* \brief Create a text representation of an XML object
	*
	* \param[in] data XML to convert
	* \param[in] options Group of \p pcmk__xml_fmt_options flags
	* \param[in,out] buffer Where to store the text (must not be \p NULL)
	* \param[in] depth Current indentation level
	*/
	void
	pcmk__xml2text(const xmlNode data, uint32_t options, GString buffer,
	int depth)
	{
	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	CRM_ASSERT(buffer != NULL);
	CRM_CHECK(depth >= 0, depth = 0);

	switch(data->type) {
	case XML_ELEMENT_NODE:
	/* Handle below */
	dump_xml_element(data, options, buffer, depth);
	break;
	case XML_TEXT_NODE:
	if (pcmk_is_set(options, pcmk__xml_fmt_text)) {
	dump_xml_text(data, options, buffer, depth);
	}
	break;
	case XML_COMMENT_NODE:
	dump_xml_comment(data, options, buffer, depth);
	break;
	case XML_CDATA_SECTION_NODE:
	dump_xml_cdata(data, options, buffer, depth);
	break;
	default:
	crm_warn("Cannot convert XML %s node to text " CRM_XS " type=%d",
	xml_element_type2str(data->type), data->type);
	break;
	}
	}

	char *
	dump_xml_formatted_with_text(const xmlNode *xml)
	{
	/* libxml's xmlNodeDumpOutput() would work here since we're not specifically
	* filtering out any nodes. However, use pcmk__xml2text() for consistency,
	* to escape attribute values, and to allow a const argument.
	*/
	char *buffer = NULL;
	GString *g_buffer = g_string_sized_new(1024);

	pcmk__xml2text(xml, pcmk__xml_fmt_pretty\|pcmk__xml_fmt_text, g_buffer, 0);

	pcmk__str_update(&buffer, g_buffer->str);
	g_string_free(g_buffer, TRUE);
	return buffer;
	}

	char *
	dump_xml_formatted(const xmlNode *xml)
	{
	char *buffer = NULL;
	GString *g_buffer = g_string_sized_new(1024);

	pcmk__xml2text(xml, pcmk__xml_fmt_pretty, g_buffer, 0);

	pcmk__str_update(&buffer, g_buffer->str);
	g_string_free(g_buffer, TRUE);
	return buffer;
	}

	char *
	dump_xml_unformatted(const xmlNode *xml)
	{
	char *buffer = NULL;
	GString *g_buffer = g_string_sized_new(1024);

	pcmk__xml2text(xml, 0, g_buffer, 0);

	pcmk__str_update(&buffer, g_buffer->str);
	g_string_free(g_buffer, TRUE);
	return buffer;
	}

	int
	pcmk__xml2fd(int fd, xmlNode *cur)
	{
	bool success;

	xmlOutputBuffer *fd_out = xmlOutputBufferCreateFd(fd, NULL);
	CRM_ASSERT(fd_out != NULL);
	xmlNodeDumpOutput(fd_out, cur->doc, cur, 0, pcmk__xml_fmt_pretty, NULL);

	success = xmlOutputBufferWrite(fd_out, sizeof("\n") - 1, "\n") != -1;

	success = xmlOutputBufferClose(fd_out) != -1 && success;

	if (!success) {
	return EIO;
	}

	fsync(fd);
	return pcmk_rc_ok;
	}

	void
	xml_remove_prop(xmlNode * obj, const char *name)
	{
	if (crm_element_value(obj, name) == NULL) {
	return;
	}

	if (pcmk__check_acl(obj, NULL, pcmk__xf_acl_write) == FALSE) {
	crm_trace("Cannot remove %s from %s", name, obj->name);

	} else if (pcmk__tracking_xml_changes(obj, FALSE)) {
	/* Leave in place (marked for removal) until after the diff is calculated */
	xmlAttr *attr = xmlHasProp(obj, (pcmkXmlStr) name);
	xml_node_private_t *nodepriv = attr->_private;

	set_parent_flag(obj, pcmk__xf_dirty);
	pcmk__set_xml_flags(nodepriv, pcmk__xf_deleted);
	} else {
	xmlUnsetProp(obj, (pcmkXmlStr) name);
	}
	}

	void
	save_xml_to_file(const xmlNode xml, const char desc, const char *filename)
	{
	char *f = NULL;

	if (filename == NULL) {
	char *uuid = crm_generate_uuid();

	f = crm_strdup_printf("%s/%s", pcmk__get_tmpdir(), uuid);
	filename = f;
	free(uuid);
	}

	crm_info("Saving %s to %s", desc, filename);
	write_xml_file(xml, filename, FALSE);
	free(f);
	}

	/*!
	* \internal
	* \brief Set a flag on all attributes of an XML element
	*
	* \param[in,out] xml XML node to set flags on
	* \param[in] flag XML private flag to set
	*/
	static void
	set_attrs_flag(xmlNode *xml, enum xml_private_flags flag)
	{
	for (xmlAttr *attr = pcmk__xe_first_attr(xml); attr; attr = attr->next) {
	pcmk__set_xml_flags((xml_node_private_t *) (attr->_private), flag);
	}
	}

	/*!
	* \internal
	* \brief Add an XML attribute to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when an attribute has been
	* deleted. Add the attribute back to the new XML, so that we can check the
	* removal against ACLs, and mark it as deleted for later removal after
	* differences have been calculated.
	*
	* \param[in,out] new_xml XML to modify
	* \param[in] element Name of XML element that changed (for logging)
	* \param[in] attr_name Name of attribute that was deleted
	* \param[in] old_value Value of attribute that was deleted
	*/
	static void
	mark_attr_deleted(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	xml_doc_private_t *docpriv = new_xml->doc->_private;
	xmlAttr *attr = NULL;
	xml_node_private_t *nodepriv;

	// Prevent the dirty flag being set recursively upwards
	pcmk__clear_xml_flags(docpriv, pcmk__xf_tracking);

	// Restore the old value (and the tracking flag)
	attr = xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);
	pcmk__set_xml_flags(docpriv, pcmk__xf_tracking);

	// Reset flags (so the attribute doesn't appear as newly created)
	nodepriv = attr->_private;
	nodepriv->flags = 0;

	// Check ACLs and mark restored value for later removal
	xml_remove_prop(new_xml, attr_name);

	crm_trace("XML attribute %s=%s was removed from %s",
	attr_name, old_value, element);
	}

	/*
	* \internal
	* \brief Check ACLs for a changed XML attribute
	*/
	static void
	mark_attr_changed(xmlNode new_xml, const char element, const char *attr_name,
	const char *old_value)
	{
	char *vcopy = crm_element_value_copy(new_xml, attr_name);

	crm_trace("XML attribute %s was changed from '%s' to '%s' in %s",
	attr_name, old_value, vcopy, element);

	// Restore the original value
	xmlSetProp(new_xml, (pcmkXmlStr) attr_name, (pcmkXmlStr) old_value);

	// Change it back to the new value, to check ACLs
	crm_xml_add(new_xml, attr_name, vcopy);
	free(vcopy);
	}

	/*!
	* \internal
	* \brief Mark an XML attribute as having changed position
	*
	* \param[in,out] new_xml XML to modify
	* \param[in] element Name of XML element that changed (for logging)
	* \param[in,out] old_attr Attribute that moved, in original XML
	* \param[in,out] new_attr Attribute that moved, in \p new_xml
	* \param[in] p_old Ordinal position of \p old_attr in original XML
	* \param[in] p_new Ordinal position of \p new_attr in \p new_xml
	*/
	static void
	mark_attr_moved(xmlNode new_xml, const char element, xmlAttr *old_attr,
	xmlAttr *new_attr, int p_old, int p_new)
	{
	xml_node_private_t *nodepriv = new_attr->_private;

	crm_trace("XML attribute %s moved from position %d to %d in %s",
	old_attr->name, p_old, p_new, element);

	// Mark document, element, and all element's parents as changed
	pcmk__mark_xml_node_dirty(new_xml);

	// Mark attribute as changed
	pcmk__set_xml_flags(nodepriv, pcmk__xf_dirty\|pcmk__xf_moved);

	nodepriv = (p_old > p_new)? old_attr->_private : new_attr->_private;
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	}

	/*!
	* \internal
	* \brief Calculate differences in all previously existing XML attributes
	*
	* \param[in,out] old_xml Original XML to compare
	* \param[in,out] new_xml New XML to compare
	*/
	static void
	xml_diff_old_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(old_xml);

	while (attr_iter != NULL) {
	const char name = (const char ) attr_iter->name;
	xmlAttr *old_attr = attr_iter;
	xmlAttr *new_attr = xmlHasProp(new_xml, attr_iter->name);
	const char *old_value = pcmk__xml_attr_value(attr_iter);

	attr_iter = attr_iter->next;
	if (new_attr == NULL) {
	mark_attr_deleted(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else {
	xml_node_private_t *nodepriv = new_attr->_private;
	int new_pos = pcmk__xml_position((xmlNode*) new_attr,
	pcmk__xf_skip);
	int old_pos = pcmk__xml_position((xmlNode*) old_attr,
	pcmk__xf_skip);
	const char *new_value = crm_element_value(new_xml, name);

	// This attribute isn't new
	pcmk__clear_xml_flags(nodepriv, pcmk__xf_created);

	if (strcmp(new_value, old_value) != 0) {
	mark_attr_changed(new_xml, (const char *) old_xml->name, name,
	old_value);

	} else if ((old_pos != new_pos)
	&& !pcmk__tracking_xml_changes(new_xml, TRUE)) {
	mark_attr_moved(new_xml, (const char *) old_xml->name,
	old_attr, new_attr, old_pos, new_pos);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Check all attributes in new XML for creation
	*
	* For each of a given XML element's attributes marked as newly created, accept
	* (and mark as dirty) or reject the creation according to ACLs.
	*
	* \param[in,out] new_xml XML to check
	*/
	static void
	mark_created_attrs(xmlNode *new_xml)
	{
	xmlAttr *attr_iter = pcmk__xe_first_attr(new_xml);

	while (attr_iter != NULL) {
	xmlAttr *new_attr = attr_iter;
	xml_node_private_t *nodepriv = attr_iter->_private;

	attr_iter = attr_iter->next;
	if (pcmk_is_set(nodepriv->flags, pcmk__xf_created)) {
	const char attr_name = (const char ) new_attr->name;

	crm_trace("Created new attribute %s=%s in %s",
	attr_name, pcmk__xml_attr_value(new_attr),
	new_xml->name);

	/* Check ACLs (we can't use the remove-then-create trick because it
	* would modify the attribute position).
	*/
	if (pcmk__check_acl(new_xml, attr_name, pcmk__xf_acl_write)) {
	pcmk__mark_xml_attr_dirty(new_attr);
	} else {
	// Creation was not allowed, so remove the attribute
	xmlUnsetProp(new_xml, new_attr->name);
	}
	}
	}
	}

	/*!
	* \internal
	* \brief Calculate differences in attributes between two XML nodes
	*
	* \param[in,out] old_xml Original XML to compare
	* \param[in,out] new_xml New XML to compare
	*/
	static void
	xml_diff_attrs(xmlNode old_xml, xmlNode new_xml)
	{
	set_attrs_flag(new_xml, pcmk__xf_created); // cleared later if not really new
	xml_diff_old_attrs(old_xml, new_xml);
	mark_created_attrs(new_xml);
	}

	/*!
	* \internal
	* \brief Add an XML child element to a node, marked as deleted
	*
	* When calculating XML changes, we need to know when a child element has been
	* deleted. Add the child back to the new XML, so that we can check the removal
	* against ACLs, and mark it as deleted for later removal after differences have
	* been calculated.
	*
	* \param[in,out] old_child Child element from original XML
	* \param[in,out] new_parent New XML to add marked copy to
	*/
	static void
	mark_child_deleted(xmlNode old_child, xmlNode new_parent)
	{
	// Re-create the child element so we can check ACLs
	xmlNode *candidate = pcmk__xml_copy(new_parent, old_child);

	// Clear flags on new child and its children
	reset_xml_node_flags(candidate);

	// Check whether ACLs allow the deletion
	pcmk__apply_acl(xmlDocGetRootElement(candidate->doc));

	// Remove the child again (which will track it in document's deleted_objs)
	free_xml_with_position(candidate,
	pcmk__xml_position(old_child, pcmk__xf_skip));

	if (pcmk__xml_match(new_parent, old_child, true) == NULL) {
	pcmk__set_xml_flags((xml_node_private_t *) (old_child->_private),
	pcmk__xf_skip);
	}
	}

	static void
	mark_child_moved(xmlNode old_child, xmlNode new_parent, xmlNode *new_child,
	int p_old, int p_new)
	{
	xml_node_private_t *nodepriv = new_child->_private;

	crm_trace("Child element %s with id='%s' moved from position %d to %d under %s",
	new_child->name, pcmk__s(pcmk__xe_id(new_child), "<no id>"),
	p_old, p_new, new_parent->name);
	pcmk__mark_xml_node_dirty(new_parent);
	pcmk__set_xml_flags(nodepriv, pcmk__xf_moved);

	if (p_old > p_new) {
	nodepriv = old_child->_private;
	} else {
	nodepriv = new_child->_private;
	}
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	}

	// Given original and new XML, mark new XML portions that have changed
	static void
	mark_xml_changes(xmlNode old_xml, xmlNode new_xml, bool check_top)
	{
	xmlNode *cIter = NULL;
	xml_node_private_t *nodepriv = NULL;

	CRM_CHECK(new_xml != NULL, return);
	if (old_xml == NULL) {
	pcmk__mark_xml_created(new_xml);
	pcmk__apply_creation_acl(new_xml, check_top);
	return;
	}

	nodepriv = new_xml->_private;
	CRM_CHECK(nodepriv != NULL, return);

	if(nodepriv->flags & pcmk__xf_processed) {
	/* Avoid re-comparing nodes */
	return;
	}
	pcmk__set_xml_flags(nodepriv, pcmk__xf_processed);

	xml_diff_attrs(old_xml, new_xml);

	// Check for differences in the original children
	for (cIter = pcmk__xml_first_child(old_xml); cIter != NULL; ) {
	xmlNode *old_child = cIter;
	xmlNode *new_child = pcmk__xml_match(new_xml, cIter, true);

	cIter = pcmk__xml_next(cIter);
	if(new_child) {
	mark_xml_changes(old_child, new_child, TRUE);

	} else {
	mark_child_deleted(old_child, new_xml);
	}
	}

	// Check for moved or created children
	for (cIter = pcmk__xml_first_child(new_xml); cIter != NULL; ) {
	xmlNode *new_child = cIter;
	xmlNode *old_child = pcmk__xml_match(old_xml, cIter, true);

	cIter = pcmk__xml_next(cIter);
	if(old_child == NULL) {
	// This is a newly created child
	nodepriv = new_child->_private;
	pcmk__set_xml_flags(nodepriv, pcmk__xf_skip);
	mark_xml_changes(old_child, new_child, TRUE);

	} else {
	/* Check for movement, we already checked for differences */
	int p_new = pcmk__xml_position(new_child, pcmk__xf_skip);
	int p_old = pcmk__xml_position(old_child, pcmk__xf_skip);

	if(p_old != p_new) {
	mark_child_moved(old_child, new_xml, new_child, p_old, p_new);
	}
	}
	}
	}

	void
	xml_calculate_significant_changes(xmlNode old_xml, xmlNode new_xml)
	{
	pcmk__set_xml_doc_flag(new_xml, pcmk__xf_lazy);
	xml_calculate_changes(old_xml, new_xml);
	}

	// Called functions may set the \p pcmk__xf_skip flag on parts of \p old_xml
	void
	xml_calculate_changes(xmlNode old_xml, xmlNode new_xml)
	{
	CRM_CHECK((old_xml != NULL) && (new_xml != NULL)
	&& pcmk__xe_is(old_xml, (const char *) new_xml->name)
	&& pcmk__str_eq(pcmk__xe_id(old_xml), pcmk__xe_id(new_xml),
	pcmk__str_none),
	return);

	if(xml_tracking_changes(new_xml) == FALSE) {
	xml_track_changes(new_xml, NULL, NULL, FALSE);
	}

	mark_xml_changes(old_xml, new_xml, FALSE);
	}

	/*!
	* \internal
	* \brief Find a comment with matching content in specified XML
	*
	* \param[in] root XML to search
	* \param[in] search_comment Comment whose content should be searched for
	* \param[in] exact If true, comment must also be at same position
	*/
	xmlNode *
	pcmk__xc_match(const xmlNode root, const xmlNode search_comment, bool exact)
	{
	xmlNode *a_child = NULL;
	int search_offset = pcmk__xml_position(search_comment, pcmk__xf_skip);

	CRM_CHECK(search_comment->type == XML_COMMENT_NODE, return NULL);

	for (a_child = pcmk__xml_first_child(root); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	if (exact) {
	int offset = pcmk__xml_position(a_child, pcmk__xf_skip);
	xml_node_private_t *nodepriv = a_child->_private;

	if (offset < search_offset) {
	continue;

	} else if (offset > search_offset) {
	return NULL;
	}

	if (pcmk_is_set(nodepriv->flags, pcmk__xf_skip)) {
	continue;
	}
	}

	if (a_child->type == XML_COMMENT_NODE
	&& pcmk__str_eq((const char )a_child->content, (const char )search_comment->content, pcmk__str_casei)) {
	return a_child;

	} else if (exact) {
	return NULL;
	}
	}

	return NULL;
	}

	/*!
	* \internal
	* \brief Make one XML comment match another (in content)
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* comment child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML comment node
	* \param[in] update Make comment content match this (must not be NULL)
	*
	* \note At least one of \parent and \target must be non-NULL
	*/
	void
	pcmk__xc_update(xmlNode parent, xmlNode target, xmlNode *update)
	{
	CRM_CHECK(update != NULL, return);
	CRM_CHECK(update->type == XML_COMMENT_NODE, return);

	if (target == NULL) {
	target = pcmk__xc_match(parent, update, false);
	}

	if (target == NULL) {
	pcmk__xml_copy(parent, update);

	} else if (!pcmk__str_eq((const char )target->content, (const char )update->content, pcmk__str_casei)) {
	xmlFree(target->content);
	target->content = xmlStrdup(update->content);
	}
	}

	/*!
	* \internal
	* \brief Make one XML tree match another (in children and attributes)
	*
	* \param[in,out] parent If \p target is NULL and this is not, add or update
	* child of this XML node that matches \p update
	* \param[in,out] target If not NULL, update this XML
	* \param[in] update Make the desired XML match this (must not be NULL)
	* \param[in] as_diff If false, expand "++" when making attributes match
	*
	* \note At least one of \p parent and \p target must be non-NULL
	*/
	void
	pcmk__xml_update(xmlNode parent, xmlNode target, xmlNode *update,
	bool as_diff)
	{
	xmlNode *a_child = NULL;
	const char *object_name = NULL,
	*object_href = NULL,
	*object_href_val = NULL;

	#if XML_PARSER_DEBUG
	crm_log_xml_trace(update, "update:");
	crm_log_xml_trace(target, "target:");
	#endif

	CRM_CHECK(update != NULL, return);

	if (update->type == XML_COMMENT_NODE) {
	pcmk__xc_update(parent, target, update);
	return;
	}

	object_name = (const char *) update->name;
	object_href_val = pcmk__xe_id(update);
	if (object_href_val != NULL) {
	object_href = PCMK_XA_ID;
	} else {
	object_href_val = crm_element_value(update, PCMK_XA_ID_REF);
	object_href = (object_href_val == NULL)? NULL : PCMK_XA_ID_REF;
	}

	CRM_CHECK(object_name != NULL, return);
	CRM_CHECK(target != NULL \|\| parent != NULL, return);

	if (target == NULL) {
	target = pcmk__xe_match(parent, object_name,
	object_href, object_href_val);
	}

	if (target == NULL) {
	target = create_xml_node(parent, object_name);
	CRM_CHECK(target != NULL, return);
	#if XML_PARSER_DEBUG
	crm_trace("Added <%s%s%s%s%s/>", pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");

	} else {
	crm_trace("Found node <%s%s%s%s%s/> to update",
	pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	}

	CRM_CHECK(pcmk__xe_is(target, (const char *) update->name), return);

	if (as_diff == FALSE) {
	/* So that expand_plus_plus() gets called */
	copy_in_properties(target, update);

	} else {
	/* No need for expand_plus_plus(), just raw speed */
	for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
	a = a->next) {
	const char *p_value = pcmk__xml_attr_value(a);

	/* Remove it first so the ordering of the update is preserved */
	xmlUnsetProp(target, a->name);
	xmlSetProp(target, a->name, (pcmkXmlStr) p_value);
	}
	}

	for (a_child = pcmk__xml_first_child(update); a_child != NULL;
	a_child = pcmk__xml_next(a_child)) {
	#if XML_PARSER_DEBUG
	crm_trace("Updating child <%s%s%s%s%s/>",
	pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	pcmk__xml_update(target, NULL, a_child, as_diff);
	}

	#if XML_PARSER_DEBUG
	crm_trace("Finished with <%s%s%s%s%s/>", pcmk__s(object_name, "<null>"),
	object_href ? " " : "",
	object_href ? object_href : "",
	object_href ? "=" : "",
	object_href ? object_href_val : "");
	#endif
	}

	gboolean
	update_xml_child(xmlNode * child, xmlNode * to_update)
	{
	gboolean can_update = TRUE;
	xmlNode *child_of_child = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(to_update != NULL, return FALSE);

	if (!pcmk__xe_is(to_update, (const char *) child->name)) {
	can_update = FALSE;

	} else if (!pcmk__str_eq(pcmk__xe_id(to_update), pcmk__xe_id(child),
	pcmk__str_none)) {
	can_update = FALSE;

	} else if (can_update) {
	#if XML_PARSER_DEBUG
	crm_log_xml_trace(child, "Update match found...");
	#endif
	pcmk__xml_update(NULL, child, to_update, false);
	}

	for (child_of_child = pcmk__xml_first_child(child); child_of_child != NULL;
	child_of_child = pcmk__xml_next(child_of_child)) {
	/* only update the first one */
	if (can_update) {
	break;
	}
	can_update = update_xml_child(child_of_child, to_update);
	}

	return can_update;
	}

	int
	find_xml_children(xmlNode ** children, xmlNode * root,
	const char tag, const char field, const char *value, gboolean search_matches)
	{
	int match_found = 0;

	CRM_CHECK(root != NULL, return FALSE);
	CRM_CHECK(children != NULL, return FALSE);

	if ((tag != NULL) && !pcmk__xe_is(root, tag)) {

	} else if (value != NULL && !pcmk__str_eq(value, crm_element_value(root, field), pcmk__str_casei)) {

	} else {
	if (*children == NULL) {
	*children = create_xml_node(NULL, __func__);
	}
	pcmk__xml_copy(*children, root);
	match_found = 1;
	}

	if (search_matches \|\| match_found == 0) {
	xmlNode *child = NULL;

	for (child = pcmk__xml_first_child(root); child != NULL;
	child = pcmk__xml_next(child)) {
	match_found += find_xml_children(children, child, tag, field, value, search_matches);
	}
	}

	return match_found;
	}

	gboolean
	replace_xml_child(xmlNode * parent, xmlNode * child, xmlNode * update, gboolean delete_only)
	{
	gboolean can_delete = FALSE;
	xmlNode *child_of_child = NULL;

	const char *up_id = NULL;
	const char *child_id = NULL;
	const char *right_val = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(update != NULL, return FALSE);

	up_id = pcmk__xe_id(update);
	child_id = pcmk__xe_id(child);

	if (up_id == NULL \|\| (child_id && strcmp(child_id, up_id) == 0)) {
	can_delete = TRUE;
	}
	if (!pcmk__xe_is(update, (const char *) child->name)) {
	can_delete = FALSE;
	}
	if (can_delete && delete_only) {
	for (xmlAttrPtr a = pcmk__xe_first_attr(update); a != NULL;
	a = a->next) {
	const char p_name = (const char ) a->name;
	const char *p_value = pcmk__xml_attr_value(a);

	right_val = crm_element_value(child, p_name);
	if (!pcmk__str_eq(p_value, right_val, pcmk__str_casei)) {
	can_delete = FALSE;
	}
	}
	}

	if (can_delete && parent != NULL) {
	crm_log_xml_trace(child, "Delete match found...");
	if (delete_only \|\| update == NULL) {
	free_xml(child);

	} else {
	xmlNode *old = child;
	xmlNode *new = xmlCopyNode(update, 1);

	CRM_ASSERT(new != NULL);

	// May be unnecessary but avoids slight changes to some test outputs
	reset_xml_node_flags(new);

	old = xmlReplaceNode(old, new);

	if (xml_tracking_changes(new)) {
	// Replaced sections may have included relevant ACLs
	pcmk__apply_acl(new);
	}
	xml_calculate_changes(old, new);
	xmlFreeNode(old);
	}
	return TRUE;

	} else if (can_delete) {
	crm_log_xml_debug(child, "Cannot delete the search root");
	can_delete = FALSE;
	}

	child_of_child = pcmk__xml_first_child(child);
	while (child_of_child) {
	xmlNode *next = pcmk__xml_next(child_of_child);

	can_delete = replace_xml_child(child, child_of_child, update, delete_only);

	/* only delete the first one */
	if (can_delete) {
	child_of_child = NULL;
	} else {
	child_of_child = next;
	}
	}

	return can_delete;
	}

	xmlNode *
	sorted_xml(xmlNode input, xmlNode parent, gboolean recursive)
	{
	xmlNode *child = NULL;
	GSList *nvpairs = NULL;
	xmlNode *result = NULL;

	CRM_CHECK(input != NULL, return NULL);

	result = create_xml_node(parent, (const char *) input->name);
	nvpairs = pcmk_xml_attrs2nvpairs(input);
	nvpairs = pcmk_sort_nvpairs(nvpairs);
	pcmk_nvpairs2xml_attrs(nvpairs, result);
	pcmk_free_nvpairs(nvpairs);

	for (child = pcmk__xml_first_child(input); child != NULL;
	child = pcmk__xml_next(child)) {

	if (recursive) {
	sorted_xml(child, result, recursive);
	} else {
	pcmk__xml_copy(result, child);
	}
	}

	return result;
	}

	xmlNode *
	first_named_child(const xmlNode parent, const char name)
	{
	xmlNode *match = NULL;

	for (match = pcmk__xe_first_child(parent); match != NULL;
	match = pcmk__xe_next(match)) {
	/*
	* name == NULL gives first child regardless of name; this is
	* semantically incorrect in this function, but may be necessary
	* due to prior use of xml_child_iter_filter
	*/
	if ((name == NULL) \|\| pcmk__xe_is(match, name)) {
	return match;
	}
	}
	return NULL;
	}

	/*!
	* \brief Get next instance of same XML tag
	*
	* \param[in] sibling XML tag to start from
	*
	* \return Next sibling XML tag with same name
	*/
	xmlNode *
	crm_next_same_xml(const xmlNode *sibling)
	{
	xmlNode *match = pcmk__xe_next(sibling);

	while (match != NULL) {
	if (pcmk__xe_is(match, (const char *) sibling->name)) {
	return match;
	}
	match = pcmk__xe_next(match);
	}
	return NULL;
	}

	void
	crm_xml_init(void)
	{
	static bool init = true;

	if(init) {
	init = false;
	/* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
	* pcmk__realloc()s and it can take upwards of 18 seconds (yes, seconds)
	* to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
	* less than 1 second.
	*/
	xmlSetBufferAllocationScheme(XML_BUFFER_ALLOC_DOUBLEIT);

	/* Populate and free the _private field when nodes are created and destroyed */
	xmlDeregisterNodeDefault(free_private_data);
	xmlRegisterNodeDefault(new_private_data);

	crm_schema_init();
	}
	}

	void
	crm_xml_cleanup(void)
	{
	crm_schema_cleanup();
	xmlCleanupParser();
	}

	#define XPATH_MAX 512

	xmlNode *
	expand_idref(xmlNode * input, xmlNode * top)
	{
	char *xpath = NULL;
	const char *ref = NULL;
	xmlNode *result = NULL;

	if (input == NULL) {
	return NULL;
	}

	ref = crm_element_value(input, PCMK_XA_ID_REF);
	if (ref == NULL) {
	return input;
	}

	if (top == NULL) {
	top = input;
	}

	xpath = crm_strdup_printf("//%s[@" PCMK_XA_ID "='%s']", input->name, ref);
	result = get_xpath_object(xpath, top, LOG_DEBUG);
	if (result == NULL) { // Not possible with schema validation enabled
	pcmk__config_err("Ignoring invalid %s configuration: "
	PCMK_XA_ID_REF " '%s' does not reference "
	"a valid object " CRM_XS " xpath=%s",
	input->name, ref, xpath);
	}
	free(xpath);
	return result;
	}

	char *
	pcmk__xml_artefact_root(enum pcmk__xml_artefact_ns ns)
	{
	static const char *base = NULL;
	char *ret = NULL;

	if (base == NULL) {
	base = pcmk__env_option(PCMK__ENV_SCHEMA_DIRECTORY);
	}
	if (pcmk__str_empty(base)) {
	base = CRM_SCHEMA_DIRECTORY;
	}

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_legacy_xslt:
	ret = strdup(base);
	break;
	case pcmk__xml_artefact_ns_base_rng:
	case pcmk__xml_artefact_ns_base_xslt:
	ret = crm_strdup_printf("%s/base", base);
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}
	return ret;
	}

	static char *
	find_artefact(enum pcmk__xml_artefact_ns ns, const char path, const char filespec)
	{
	char *ret = NULL;

	switch (ns) {
	case pcmk__xml_artefact_ns_legacy_rng:
	case pcmk__xml_artefact_ns_base_rng:
	if (pcmk__ends_with(filespec, ".rng")) {
	ret = crm_strdup_printf("%s/%s", path, filespec);
	} else {
	ret = crm_strdup_printf("%s/%s.rng", path, filespec);
	}
	break;
	case pcmk__xml_artefact_ns_legacy_xslt:
	case pcmk__xml_artefact_ns_base_xslt:
	if (pcmk__ends_with(filespec, ".xsl")) {
	ret = crm_strdup_printf("%s/%s", path, filespec);
	} else {
	ret = crm_strdup_printf("%s/%s.xsl", path, filespec);
	}
	break;
	default:
	crm_err("XML artefact family specified as %u not recognized", ns);
	}

	return ret;
	}

	char *
	pcmk__xml_artefact_path(enum pcmk__xml_artefact_ns ns, const char *filespec)
	{
	struct stat sb;
	char *base = pcmk__xml_artefact_root(ns);
	char *ret = NULL;

	ret = find_artefact(ns, base, filespec);
	free(base);

	if (stat(ret, &sb) != 0 \|\| !S_ISREG(sb.st_mode)) {
	const char *remote_schema_dir = pcmk__remote_schema_dir();
	ret = find_artefact(ns, remote_schema_dir, filespec);
	}

	return ret;
	}

	void
	pcmk__xe_set_propv(xmlNodePtr node, va_list pairs)
	{
	while (true) {
	const char name, value;

	name = va_arg(pairs, const char *);
	if (name == NULL) {
	return;
	}

	value = va_arg(pairs, const char *);
	if (value != NULL) {
	crm_xml_add(node, name, value);
	}
	}
	}

	void
	pcmk__xe_set_props(xmlNodePtr node, ...)
	{
	va_list pairs;
	va_start(pairs, node);
	pcmk__xe_set_propv(node, pairs);
	va_end(pairs);
	}

	int
	pcmk__xe_foreach_child(xmlNode xml, const char child_element_name,
	int (handler)(xmlNode xml, void *userdata),
	void *userdata)
	{
	xmlNode *children = (xml? xml->children : NULL);

	CRM_ASSERT(handler != NULL);

	for (xmlNode *node = children; node != NULL; node = node->next) {
	if ((node->type == XML_ELEMENT_NODE)
	&& ((child_element_name == NULL)
	\|\| pcmk__xe_is(node, child_element_name))) {
	int rc = handler(node, userdata);

	if (rc != pcmk_rc_ok) {
	return rc;
	}
	}
	}

	return pcmk_rc_ok;
	}

	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	#include <crm/common/xml_compat.h>

	xmlNode *
	find_entity(xmlNode parent, const char node_name, const char *id)
	{
	return pcmk__xe_match(parent, node_name,
	((id == NULL)? id : PCMK_XA_ID), id);
	}

	void
	crm_destroy_xml(gpointer data)
	{
	free_xml(data);
	}

	xmlDoc *
	getDocPtr(xmlNode *node)
	{
	xmlDoc *doc = NULL;

	CRM_CHECK(node != NULL, return NULL);

	doc = node->doc;
	if (doc == NULL) {
	doc = xmlNewDoc((pcmkXmlStr) "1.0");
	xmlDocSetRootElement(doc, node);
	}
	return doc;
	}

	xmlNode *
	add_node_copy(xmlNode parent, xmlNode src_node)
	{
	xmlNode *child = NULL;

	CRM_CHECK((parent != NULL) && (src_node != NULL), return NULL);

	child = xmlDocCopyNode(src_node, parent->doc, 1);
	if (child == NULL) {
	return NULL;
	}
	xmlAddChild(parent, child);
	pcmk__mark_xml_created(child);
	return child;
	}

	int
	add_node_nocopy(xmlNode parent, const char name, xmlNode *child)
	{
	add_node_copy(parent, child);
	free_xml(child);
	return 1;
	}

	gboolean
	xml_has_children(const xmlNode * xml_root)
	{
	if (xml_root != NULL && xml_root->children != NULL) {
	return TRUE;
	}
	return FALSE;
	}

	char *
	crm_xml_escape(const char *text)
	{
	size_t length = 0;
	char *copy = NULL;

	if (text == NULL) {
	return NULL;
	}

	length = strlen(text);
	copy = strdup(text);
	CRM_ASSERT(copy != NULL);
	for (size_t index = 0; index <= length; index++) {
	if(copy[index] & 0x80 && copy[index+1] & 0x80){
	index++;
	continue;
	}
	switch (copy[index]) {
	case 0:
	// Sanity only; loop should stop at the last non-null byte
	break;
	case '<':
	copy = replace_text(copy, &index, &length, "<");
	break;
	case '>':
	copy = replace_text(copy, &index, &length, ">");
	break;
	case '"':
	copy = replace_text(copy, &index, &length, """);
	break;
	case '\'':
	copy = replace_text(copy, &index, &length, "'");
	break;
	case '&':
	copy = replace_text(copy, &index, &length, "&");
	break;
	case '\t':
	/* Might as well just expand to a few spaces... */
	copy = replace_text(copy, &index, &length, " ");
	break;
	case '\n':
	copy = replace_text(copy, &index, &length, "\\n");
	break;
	case '\r':
	copy = replace_text(copy, &index, &length, "\\r");
	break;
	default:
	/* Check for and replace non-printing characters with their octal equivalent */
	if(copy[index] < ' ' \|\| copy[index] > '~') {
	char *replace = crm_strdup_printf("\\%.3o", copy[index]);

	copy = replace_text(copy, &index, &length, replace);
	free(replace);
	}
	}
	}
	return copy;
	}

	xmlNode *
	copy_xml(xmlNode *src)
	{
	xmlDoc *doc = xmlNewDoc((pcmkXmlStr) "1.0");
	xmlNode *copy = xmlDocCopyNode(src, doc, 1);

	CRM_ASSERT(copy != NULL);
	xmlDocSetRootElement(doc, copy);
	return copy;
	}

	xmlNode *
	filename2xml(const char *filename)
	{
	return pcmk__xml_parse_file(filename);
	}

	xmlNode *
	stdin2xml(void)
	{
	return pcmk__xml_parse_file(NULL);
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c
	index e702feb321..b7a9c8ce2f 100644
	--- a/lib/fencing/st_client.c
	+++ b/lib/fencing/st_client.c
	@@ -1,2720 +1,2720 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdlib.h>
	#include <stdio.h>
	#include <stdbool.h>
	#include <string.h>
	#include <ctype.h>
	#include <inttypes.h>
	#include <sys/types.h>
	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>

	#include "fencing_private.h"

	CRM_TRACE_INIT_DATA(stonith);

	// Used as stonith_t:st_private
	typedef struct stonith_private_s {
	char *token;
	crm_ipc_t *ipc;
	mainloop_io_t *source;
	GHashTable *stonith_op_callback_table;
	GList *notify_list;
	int notify_refcnt;
	bool notify_deletes;

	void (op_callback) (stonith_t st, stonith_callback_data_t * data);

	} stonith_private_t;

	// Used as stonith_event_t:opaque
	struct event_private {
	pcmk__action_result_t result;
	};

	typedef struct stonith_notify_client_s {
	const char *event;
	const char obj_id; / implement one day */
	const char obj_type; / implement one day */
	void (notify) (stonith_t st, stonith_event_t * e);
	bool delete;

	} stonith_notify_client_t;

	typedef struct stonith_callback_client_s {
	void (callback) (stonith_t st, stonith_callback_data_t * data);
	const char *id;
	void *user_data;
	gboolean only_success;
	gboolean allow_timeout_updates;
	struct timer_rec_s *timer;

	} stonith_callback_client_t;

	struct notify_blob_s {
	stonith_t *stonith;
	xmlNode *xml;
	};

	struct timer_rec_s {
	int call_id;
	int timeout;
	guint ref;
	stonith_t *stonith;
	};

	typedef int (stonith_op_t) (const char , int, const char , xmlNode ,
	xmlNode , xmlNode , xmlNode , xmlNode );

	bool stonith_dispatch(stonith_t * st);
	xmlNode stonith_create_op(int call_id, const char token, const char op, xmlNode data,
	int call_options);
	static int stonith_send_command(stonith_t stonith, const char op,
	xmlNode data, xmlNode *output_data,
	int call_options, int timeout);

	static void stonith_connection_destroy(gpointer user_data);
	static void stonith_send_notification(gpointer data, gpointer user_data);
	static int stonith_api_del_notification(stonith_t *stonith,
	const char *event);
	/*!
	* \brief Get agent namespace by name
	*
	* \param[in] namespace_s Name of namespace as string
	*
	* \return Namespace as enum value
	*/
	enum stonith_namespace
	stonith_text2namespace(const char *namespace_s)
	{
	if (pcmk__str_eq(namespace_s, "any", pcmk__str_null_matches)) {
	return st_namespace_any;

	} else if (!strcmp(namespace_s, "redhat")
	\|\| !strcmp(namespace_s, "stonith-ng")) {
	return st_namespace_rhcs;

	} else if (!strcmp(namespace_s, "internal")) {
	return st_namespace_internal;

	} else if (!strcmp(namespace_s, "heartbeat")) {
	return st_namespace_lha;
	}
	return st_namespace_invalid;
	}

	/*!
	* \brief Get agent namespace name
	*
	* \param[in] namespace Namespace as enum value
	*
	* \return Namespace name as string
	*/
	const char *
	stonith_namespace2text(enum stonith_namespace st_namespace)
	{
	switch (st_namespace) {
	case st_namespace_any: return "any";
	case st_namespace_rhcs: return "stonith-ng";
	case st_namespace_internal: return "internal";
	case st_namespace_lha: return "heartbeat";
	default: break;
	}
	return "unsupported";
	}

	/*!
	* \brief Determine namespace of a fence agent
	*
	* \param[in] agent Fence agent type
	* \param[in] namespace_s Name of agent namespace as string, if known
	*
	* \return Namespace of specified agent, as enum value
	*/
	enum stonith_namespace
	stonith_get_namespace(const char agent, const char namespace_s)
	{
	if (pcmk__str_eq(namespace_s, "internal", pcmk__str_none)) {
	return st_namespace_internal;
	}

	if (stonith__agent_is_rhcs(agent)) {
	return st_namespace_rhcs;
	}

	#if HAVE_STONITH_STONITH_H
	if (stonith__agent_is_lha(agent)) {
	return st_namespace_lha;
	}
	#endif

	crm_err("Unknown fence agent: %s", agent);
	return st_namespace_invalid;
	}

	gboolean
	stonith__watchdog_fencing_enabled_for_node_api(stonith_t st, const char node)
	{
	gboolean rv = FALSE;
	stonith_t *stonith_api = st?st:stonith_api_new();
	char *list = NULL;

	if(stonith_api) {
	if (stonith_api->state == stonith_disconnected) {
	int rc = stonith_api->cmds->connect(stonith_api, "stonith-api", NULL);

	if (rc != pcmk_ok) {
	crm_err("Failed connecting to Stonith-API for watchdog-fencing-query.");
	}
	}

	if (stonith_api->state != stonith_disconnected) {
	/* caveat!!!
	* this might fail when when stonithd is just updating the device-list
	* probably something we should fix as well for other api-calls */
	int rc = stonith_api->cmds->list(stonith_api, st_opt_sync_call, STONITH_WATCHDOG_ID, &list, 0);
	if ((rc != pcmk_ok) \|\| (list == NULL)) {
	/* due to the race described above it can happen that
	* we drop in here - so as not to make remote nodes
	* panic on that answer
	*/
	if (rc == -ENODEV) {
	crm_notice("Cluster does not have watchdog fencing device");
	} else {
	crm_warn("Could not check for watchdog fencing device: %s",
	pcmk_strerror(rc));
	}
	} else if (list[0] == '\0') {
	rv = TRUE;
	} else {
	GList *targets = stonith__parse_targets(list);
	rv = pcmk__str_in_list(node, targets, pcmk__str_casei);
	g_list_free_full(targets, free);
	}
	free(list);
	if (!st) {
	/* if we're provided the api we still might have done the
	* connection - but let's assume the caller won't bother
	*/
	stonith_api->cmds->disconnect(stonith_api);
	}
	}

	if (!st) {
	stonith_api_delete(stonith_api);
	}
	} else {
	crm_err("Stonith-API for watchdog-fencing-query couldn't be created.");
	}
	crm_trace("Pacemaker assumes node %s %sto do watchdog-fencing.",
	node, rv?"":"not ");
	return rv;
	}

	gboolean
	stonith__watchdog_fencing_enabled_for_node(const char *node)
	{
	return stonith__watchdog_fencing_enabled_for_node_api(NULL, node);
	}

	/* when cycling through the list we don't want to delete items
	so just mark them and when we know nobody is using the list
	loop over it to remove the marked items
	*/
	static void
	foreach_notify_entry (stonith_private_t *private,
	GFunc func,
	gpointer user_data)
	{
	private->notify_refcnt++;
	g_list_foreach(private->notify_list, func, user_data);
	private->notify_refcnt--;
	if ((private->notify_refcnt == 0) &&
	private->notify_deletes) {
	GList *list_item = private->notify_list;

	private->notify_deletes = FALSE;
	while (list_item != NULL)
	{
	stonith_notify_client_t *list_client = list_item->data;
	GList *next = g_list_next(list_item);

	if (list_client->delete) {
	free(list_client);
	private->notify_list =
	g_list_delete_link(private->notify_list, list_item);
	}
	list_item = next;
	}
	}
	}

	static void
	stonith_connection_destroy(gpointer user_data)
	{
	stonith_t *stonith = user_data;
	stonith_private_t *native = NULL;
	struct notify_blob_s blob;

	crm_trace("Sending destroyed notification");
	blob.stonith = stonith;
	blob.xml = create_xml_node(NULL, PCMK__XE_NOTIFY);

	native = stonith->st_private;
	native->ipc = NULL;
	native->source = NULL;

	free(native->token); native->token = NULL;
	stonith->state = stonith_disconnected;
	crm_xml_add(blob.xml, PCMK__XA_T, PCMK__VALUE_ST_NOTIFY);
	crm_xml_add(blob.xml, PCMK__XA_SUBT, PCMK__VALUE_ST_NOTIFY_DISCONNECT);

	foreach_notify_entry(native, stonith_send_notification, &blob);
	free_xml(blob.xml);
	}

	xmlNode *
	create_device_registration_xml(const char *id, enum stonith_namespace namespace,
	const char *agent,
	const stonith_key_value_t *params,
	const char *rsc_provides)
	{
	xmlNode *data = create_xml_node(NULL, PCMK__XE_ST_DEVICE_ID);
	xmlNode *args = create_xml_node(data, PCMK__XE_ATTRIBUTES);

	#if HAVE_STONITH_STONITH_H
	if (namespace == st_namespace_any) {
	namespace = stonith_get_namespace(agent, NULL);
	}
	if (namespace == st_namespace_lha) {
	hash2field((gpointer) "plugin", (gpointer) agent, args);
	agent = "fence_legacy";
	}
	#endif

	crm_xml_add(data, PCMK_XA_ID, id);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK_XA_AGENT, agent);
	if ((namespace != st_namespace_any) && (namespace != st_namespace_invalid)) {
	crm_xml_add(data, PCMK__XA_NAMESPACE,
	stonith_namespace2text(namespace));
	}
	if (rsc_provides) {
	crm_xml_add(data, PCMK__XA_RSC_PROVIDES, rsc_provides);
	}

	for (; params; params = params->next) {
	hash2field((gpointer) params->key, (gpointer) params->value, args);
	}

	return data;
	}

	static int
	stonith_api_register_device(stonith_t *st, int call_options,
	const char id, const char namespace_s,
	const char *agent,
	const stonith_key_value_t *params)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_device_registration_xml(id,
	stonith_text2namespace(namespace_s),
	agent, params, NULL);

	rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_remove_device(stonith_t * st, int call_options, const char *name)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK_XA_ID, name);
	rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_remove_level_full(stonith_t *st, int options,
	const char node, const char pattern,
	const char attr, const char value, int level)
	{
	int rc = 0;
	xmlNode *data = NULL;

	CRM_CHECK(node \|\| pattern \|\| (attr && value), return -EINVAL);

	data = create_xml_node(NULL, PCMK_XE_FENCING_LEVEL);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);

	if (node) {
	crm_xml_add(data, PCMK_XA_TARGET, node);

	} else if (pattern) {
	crm_xml_add(data, PCMK_XA_TARGET_PATTERN, pattern);

	} else {
	crm_xml_add(data, PCMK_XA_TARGET_ATTRIBUTE, attr);
	crm_xml_add(data, PCMK_XA_TARGET_VALUE, value);
	}

	crm_xml_add_int(data, PCMK_XA_INDEX, level);
	rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_remove_level(stonith_t * st, int options, const char *node, int level)
	{
	return stonith_api_remove_level_full(st, options, node,
	NULL, NULL, NULL, level);
	}

	/*!
	* \internal
	* \brief Create XML for fence topology level registration request
	*
	* \param[in] node If not NULL, target level by this node name
	* \param[in] pattern If not NULL, target by node name using this regex
	* \param[in] attr If not NULL, target by this node attribute
	* \param[in] value If not NULL, target by this node attribute value
	* \param[in] level Index number of level to register
	* \param[in] device_list List of devices in level
	*
	* \return Newly allocated XML tree on success, NULL otherwise
	*
	* \note The caller should set only one of node, pattern or attr/value.
	*/
	xmlNode *
	create_level_registration_xml(const char node, const char pattern,
	const char attr, const char value,
	int level, const stonith_key_value_t *device_list)
	{
	GString *list = NULL;
	xmlNode *data;

	CRM_CHECK(node \|\| pattern \|\| (attr && value), return NULL);

	data = create_xml_node(NULL, PCMK_XE_FENCING_LEVEL);
	CRM_CHECK(data, return NULL);

	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add_int(data, PCMK_XA_ID, level);
	crm_xml_add_int(data, PCMK_XA_INDEX, level);

	if (node) {
	crm_xml_add(data, PCMK_XA_TARGET, node);

	} else if (pattern) {
	crm_xml_add(data, PCMK_XA_TARGET_PATTERN, pattern);

	} else {
	crm_xml_add(data, PCMK_XA_TARGET_ATTRIBUTE, attr);
	crm_xml_add(data, PCMK_XA_TARGET_VALUE, value);
	}

	for (; device_list; device_list = device_list->next) {
	pcmk__add_separated_word(&list, 1024, device_list->value, ",");
	}

	if (list != NULL) {
	crm_xml_add(data, PCMK_XA_DEVICES, (const char *) list->str);
	g_string_free(list, TRUE);
	}
	return data;
	}

	static int
	stonith_api_register_level_full(stonith_t st, int options, const char node,
	const char pattern, const char attr,
	const char *value, int level,
	const stonith_key_value_t *device_list)
	{
	int rc = 0;
	xmlNode *data = create_level_registration_xml(node, pattern, attr, value,
	level, device_list);
	CRM_CHECK(data != NULL, return -EINVAL);

	rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_register_level(stonith_t * st, int options, const char *node, int level,
	const stonith_key_value_t * device_list)
	{
	return stonith_api_register_level_full(st, options, node, NULL, NULL, NULL,
	level, device_list);
	}

	static int
	stonith_api_device_list(stonith_t *stonith, int call_options,
	const char namespace_s, stonith_key_value_t *devices,
	int timeout)
	{
	int count = 0;
	enum stonith_namespace ns = stonith_text2namespace(namespace_s);

	if (devices == NULL) {
	crm_err("Parameter error: stonith_api_device_list");
	return -EFAULT;
	}

	#if HAVE_STONITH_STONITH_H
	// Include Linux-HA agents if requested
	if ((ns == st_namespace_any) \|\| (ns == st_namespace_lha)) {
	count += stonith__list_lha_agents(devices);
	}
	#endif

	// Include Red Hat agents if requested
	if ((ns == st_namespace_any) \|\| (ns == st_namespace_rhcs)) {
	count += stonith__list_rhcs_agents(devices);
	}

	return count;
	}

	// See stonith_api_operations_t:metadata() documentation
	static int
	stonith_api_device_metadata(stonith_t *stonith, int call_options,
	const char agent, const char namespace_s,
	char **output, int timeout_sec)
	{
	/* By executing meta-data directly, we can get it from stonith_admin when
	* the cluster is not running, which is important for higher-level tools.
	*/

	enum stonith_namespace ns = stonith_get_namespace(agent, namespace_s);

	if (timeout_sec <= 0) {
	timeout_sec = PCMK_DEFAULT_METADATA_TIMEOUT_MS;
	}

	crm_trace("Looking up metadata for %s agent %s",
	stonith_namespace2text(ns), agent);

	switch (ns) {
	case st_namespace_rhcs:
	return stonith__rhcs_metadata(agent, timeout_sec, output);

	#if HAVE_STONITH_STONITH_H
	case st_namespace_lha:
	return stonith__lha_metadata(agent, timeout_sec, output);
	#endif

	default:
	crm_err("Can't get fence agent '%s' meta-data: No such agent",
	agent);
	break;
	}
	return -ENODEV;
	}

	static int
	stonith_api_query(stonith_t * stonith, int call_options, const char *target,
	stonith_key_value_t ** devices, int timeout)
	{
	int rc = 0, lpc = 0, max = 0;

	xmlNode *data = NULL;
	xmlNode *output = NULL;
	xmlXPathObjectPtr xpathObj = NULL;

	CRM_CHECK(devices != NULL, return -EINVAL);

	data = create_xml_node(NULL, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_ST_TARGET, target);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ACTION, PCMK_ACTION_OFF);
	rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout);

	if (rc < 0) {
	return rc;
	}

	xpathObj = xpath_search(output, "//@agent");
	if (xpathObj) {
	max = numXpathResults(xpathObj);

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match != NULL) {
	xmlChar *match_path = xmlGetNodePath(match);

	crm_info("%s[%d] = %s", "//@agent", lpc, match_path);
	free(match_path);
	devices = stonith_key_value_add(devices, NULL,
	crm_element_value(match,
	PCMK_XA_ID));
	}
	}

	freeXpathObject(xpathObj);
	}

	free_xml(output);
	free_xml(data);
	return max;
	}

	/*!
	* \internal
	* \brief Make a STONITH_OP_EXEC request
	*
	* \param[in,out] stonith Fencer connection
	* \param[in] call_options Bitmask of \c stonith_call_options
	* \param[in] id Fence device ID that request is for
	* \param[in] action Agent action to request (list, status, monitor)
	* \param[in] target Name of target node for requested action
	* \param[in] timeout_sec Error if not completed within this many seconds
	* \param[out] output Where to set agent output
	*/
	static int
	stonith_api_call(stonith_t stonith, int call_options, const char id,
	const char action, const char target, int timeout_sec,
	xmlNode **output)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, PCMK__XE_ST_DEVICE_ID);
	crm_xml_add(data, PCMK__XA_ST_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ID, id);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ACTION, action);
	crm_xml_add(data, PCMK__XA_ST_TARGET, target);

	rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output,
	call_options, timeout_sec);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_list(stonith_t * stonith, int call_options, const char id, char *list_info,
	int timeout)
	{
	int rc;
	xmlNode *output = NULL;

	rc = stonith_api_call(stonith, call_options, id, PCMK_ACTION_LIST, NULL,
	timeout, &output);

	if (output && list_info) {
	const char *list_str;

	list_str = crm_element_value(output, PCMK__XA_ST_OUTPUT);

	if (list_str) {
	*list_info = strdup(list_str);
	}
	}

	if (output) {
	free_xml(output);
	}

	return rc;
	}

	static int
	stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout)
	{
	return stonith_api_call(stonith, call_options, id, PCMK_ACTION_MONITOR,
	NULL, timeout, NULL);
	}

	static int
	stonith_api_status(stonith_t * stonith, int call_options, const char id, const char port,
	int timeout)
	{
	return stonith_api_call(stonith, call_options, id, PCMK_ACTION_STATUS, port,
	timeout, NULL);
	}

	static int
	stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char *node,
	const char *action, int timeout, int tolerance, int delay)
	{
	int rc = 0;
	xmlNode *data = NULL;

	data = create_xml_node(NULL, __func__);
	crm_xml_add(data, PCMK__XA_ST_TARGET, node);
	crm_xml_add(data, PCMK__XA_ST_DEVICE_ACTION, action);
	crm_xml_add_int(data, PCMK__XA_ST_TIMEOUT, timeout);
	crm_xml_add_int(data, PCMK__XA_ST_TOLERANCE, tolerance);
	crm_xml_add_int(data, PCMK__XA_ST_DELAY, delay);

	rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout);
	free_xml(data);

	return rc;
	}

	static int
	stonith_api_fence(stonith_t * stonith, int call_options, const char node, const char action,
	int timeout, int tolerance)
	{
	return stonith_api_fence_with_delay(stonith, call_options, node, action,
	timeout, tolerance, 0);
	}

	static int
	stonith_api_confirm(stonith_t * stonith, int call_options, const char *target)
	{
	stonith__set_call_options(call_options, target, st_opt_manual_ack);
	return stonith_api_fence(stonith, call_options, target, PCMK_ACTION_OFF, 0,
	0);
	}

	static int
	stonith_api_history(stonith_t * stonith, int call_options, const char *node,
	stonith_history_t ** history, int timeout)
	{
	int rc = 0;
	xmlNode *data = NULL;
	xmlNode *output = NULL;
	stonith_history_t *last = NULL;

	*history = NULL;

	if (node) {
	data = create_xml_node(NULL, __func__);
	crm_xml_add(data, PCMK__XA_ST_TARGET, node);
	}

	stonith__set_call_options(call_options, node, st_opt_sync_call);
	rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output,
	call_options, timeout);
	free_xml(data);

	if (rc == 0) {
	xmlNode *op = NULL;
	xmlNode *reply = get_xpath_object("//" PCMK__XE_ST_HISTORY, output,
	LOG_NEVER);

	for (op = pcmk__xml_first_child(reply); op != NULL;
	op = pcmk__xml_next(op)) {
	stonith_history_t *kvp;
	long long completed;
	long long completed_nsec = 0L;

	kvp = calloc(1, sizeof(stonith_history_t));
	kvp->target = crm_element_value_copy(op, PCMK__XA_ST_TARGET);
	kvp->action = crm_element_value_copy(op, PCMK__XA_ST_DEVICE_ACTION);
	kvp->origin = crm_element_value_copy(op, PCMK__XA_ST_ORIGIN);
	kvp->delegate = crm_element_value_copy(op, PCMK__XA_ST_DELEGATE);
	kvp->client = crm_element_value_copy(op, PCMK__XA_ST_CLIENTNAME);
	crm_element_value_ll(op, PCMK__XA_ST_DATE, &completed);
	kvp->completed = (time_t) completed;
	crm_element_value_ll(op, PCMK__XA_ST_DATE_NSEC, &completed_nsec);
	kvp->completed_nsec = completed_nsec;
	crm_element_value_int(op, PCMK__XA_ST_STATE, &kvp->state);
	kvp->exit_reason = crm_element_value_copy(op, PCMK_XA_EXIT_REASON);

	if (last) {
	last->next = kvp;
	} else {
	*history = kvp;
	}
	last = kvp;
	}
	}

	free_xml(output);

	return rc;
	}

	void stonith_history_free(stonith_history_t *history)
	{
	stonith_history_t hp, hp_old;

	for (hp = history; hp; hp_old = hp, hp = hp->next, free(hp_old)) {
	free(hp->target);
	free(hp->action);
	free(hp->origin);
	free(hp->delegate);
	free(hp->client);
	free(hp->exit_reason);
	}
	}

	static gint
	stonithlib_GCompareFunc(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const stonith_notify_client_t *a_client = a;
	const stonith_notify_client_t *b_client = b;

	if (a_client->delete \|\| b_client->delete) {
	/* make entries marked for deletion not findable */
	return -1;
	}
	CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
	rc = strcmp(a_client->event, b_client->event);
	if (rc == 0) {
	if (a_client->notify == NULL \|\| b_client->notify == NULL) {
	return 0;

	} else if (a_client->notify == b_client->notify) {
	return 0;

	} else if (((long)a_client->notify) < ((long)b_client->notify)) {
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return -1;
	}
	crm_err("callbacks for %s are not equal: %p vs. %p",
	a_client->event, a_client->notify, b_client->notify);
	return 1;
	}
	return rc;
	}

	xmlNode *
	stonith_create_op(int call_id, const char token, const char op, xmlNode * data, int call_options)
	{
	xmlNode *op_msg = create_xml_node(NULL, PCMK__XE_STONITH_COMMAND);

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(op_msg, PCMK__XA_ST_OP, op);
	crm_xml_add_int(op_msg, PCMK__XA_ST_CALLID, call_id);
	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(op_msg, PCMK__XA_ST_CALLOPT, call_options);

	if (data != NULL) {
	add_message_xml(op_msg, PCMK__XA_ST_CALLDATA, data);
	}

	return op_msg;
	}

	static void
	stonith_destroy_op_callback(gpointer data)
	{
	stonith_callback_client_t *blob = data;

	if (blob->timer && blob->timer->ref > 0) {
	g_source_remove(blob->timer->ref);
	}
	free(blob->timer);
	free(blob);
	}

	static int
	stonith_api_signoff(stonith_t * stonith)
	{
	stonith_private_t *native = stonith->st_private;

	crm_debug("Disconnecting from the fencer");

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;

	} else if (native->ipc) {
	/* Not attached to mainloop */
	crm_ipc_t *ipc = native->ipc;

	native->ipc = NULL;
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	}

	free(native->token); native->token = NULL;
	stonith->state = stonith_disconnected;
	return pcmk_ok;
	}

	static int
	stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks)
	{
	stonith_private_t *private = stonith->st_private;

	if (all_callbacks) {
	private->op_callback = NULL;
	g_hash_table_destroy(private->stonith_op_callback_table);
	private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);

	} else if (call_id == 0) {
	private->op_callback = NULL;

	} else {
	pcmk__intkey_table_remove(private->stonith_op_callback_table, call_id);
	}
	return pcmk_ok;
	}

	/*!
	* \internal
	* \brief Invoke a (single) specified fence action callback
	*
	* \param[in,out] st Fencer API connection
	* \param[in] call_id If positive, call ID of completed fence action,
	* otherwise legacy return code for early failure
	* \param[in,out] result Full result for action
	* \param[in,out] userdata User data to pass to callback
	* \param[in] callback Fence action callback to invoke
	*/
	static void
	invoke_fence_action_callback(stonith_t *st, int call_id,
	pcmk__action_result_t *result,
	void *userdata,
	void (callback) (stonith_t st,
	stonith_callback_data_t *data))
	{
	stonith_callback_data_t data = { 0, };

	data.call_id = call_id;
	data.rc = pcmk_rc2legacy(stonith__result2rc(result));
	data.userdata = userdata;
	data.opaque = (void *) result;

	callback(st, &data);
	}

	/*!
	* \internal
	* \brief Invoke any callbacks registered for a specified fence action result
	*
	* Given a fence action result from the fencer, invoke any callback registered
	* for that action, as well as any global callback registered.
	*
	* \param[in,out] stonith Fencer API connection
	* \param[in] msg If non-NULL, fencer reply
	* \param[in] call_id If \p msg is NULL, call ID of action that timed out
	*/
	static void
	invoke_registered_callbacks(stonith_t stonith, const xmlNode msg, int call_id)
	{
	stonith_private_t *private = NULL;
	stonith_callback_client_t *cb_info = NULL;
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	CRM_CHECK(stonith != NULL, return);
	CRM_CHECK(stonith->st_private != NULL, return);

	private = stonith->st_private;

	if (msg == NULL) {
	// Fencer didn't reply in time
	pcmk__set_result(&result, CRM_EX_ERROR, PCMK_EXEC_TIMEOUT,
	"Fencer accepted request but did not reply in time");
	CRM_LOG_ASSERT(call_id > 0);

	} else {
	// We have the fencer reply
	if ((crm_element_value_int(msg, PCMK__XA_ST_CALLID, &call_id) != 0)
	\|\| (call_id <= 0)) {
	crm_log_xml_warn(msg, "Bad fencer reply");
	}
	stonith__xe_get_result(msg, &result);
	}

	if (call_id > 0) {
	cb_info = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
	call_id);
	}

	if ((cb_info != NULL) && (cb_info->callback != NULL)
	&& (pcmk__result_ok(&result) \|\| !(cb_info->only_success))) {
	crm_trace("Invoking callback %s for call %d",
	pcmk__s(cb_info->id, "without ID"), call_id);
	invoke_fence_action_callback(stonith, call_id, &result,
	cb_info->user_data, cb_info->callback);

	} else if ((private->op_callback == NULL) && !pcmk__result_ok(&result)) {
	crm_warn("Fencing action without registered callback failed: %d (%s%s%s)",
	result.exit_status,
	pcmk_exec_status_str(result.execution_status),
	((result.exit_reason == NULL)? "" : ": "),
	((result.exit_reason == NULL)? "" : result.exit_reason));
	crm_log_xml_debug(msg, "Failed fence update");
	}

	if (private->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	invoke_fence_action_callback(stonith, call_id, &result, NULL,
	private->op_callback);
	}

	if (cb_info != NULL) {
	stonith_api_del_callback(stonith, call_id, FALSE);
	}
	pcmk__reset_result(&result);
	}

	static gboolean
	stonith_async_timeout_handler(gpointer data)
	{
	struct timer_rec_s *timer = data;

	crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout);
	invoke_registered_callbacks(timer->stonith, NULL, timer->call_id);

	/* Always return TRUE, never remove the handler
	* We do that in stonith_del_callback()
	*/
	return TRUE;
	}

	static void
	set_callback_timeout(stonith_callback_client_t * callback, stonith_t * stonith, int call_id,
	int timeout)
	{
	struct timer_rec_s *async_timer = callback->timer;

	if (timeout <= 0) {
	return;
	}

	if (!async_timer) {
	async_timer = calloc(1, sizeof(struct timer_rec_s));
	callback->timer = async_timer;
	}

	async_timer->stonith = stonith;
	async_timer->call_id = call_id;
	/* Allow a fair bit of grace to allow the server to tell us of a timeout
	* This is only a fallback
	*/
	async_timer->timeout = (timeout + 60) * 1000;
	if (async_timer->ref) {
	g_source_remove(async_timer->ref);
	}
	async_timer->ref =
	g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer);
	}

	static void
	update_callback_timeout(int call_id, int timeout, stonith_t * st)
	{
	stonith_callback_client_t *callback = NULL;
	stonith_private_t *private = st->st_private;

	callback = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
	call_id);
	if (!callback \|\| !callback->allow_timeout_updates) {
	return;
	}

	set_callback_timeout(callback, st, call_id, timeout);
	}

	static int
	stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
	{
	const char *type = NULL;
	struct notify_blob_s blob;

	stonith_t *st = userdata;
	stonith_private_t *private = NULL;

	CRM_ASSERT(st != NULL);
	private = st->st_private;

	blob.stonith = st;
	- blob.xml = string2xml(buffer);
	+ blob.xml = pcmk__xml_parse_string(buffer);
	if (blob.xml == NULL) {
	crm_warn("Received malformed message from fencer: %s", buffer);
	return 0;
	}

	/* do callbacks */
	type = crm_element_value(blob.xml, PCMK__XA_T);
	crm_trace("Activating %s callbacks...", type);

	if (pcmk__str_eq(type, PCMK__VALUE_STONITH_NG, pcmk__str_none)) {
	invoke_registered_callbacks(st, blob.xml, 0);

	} else if (pcmk__str_eq(type, PCMK__VALUE_ST_NOTIFY, pcmk__str_none)) {
	foreach_notify_entry(private, stonith_send_notification, &blob);

	} else if (pcmk__str_eq(type, PCMK__VALUE_ST_ASYNC_TIMEOUT_VALUE,
	pcmk__str_none)) {
	int call_id = 0;
	int timeout = 0;

	crm_element_value_int(blob.xml, PCMK__XA_ST_TIMEOUT, &timeout);
	crm_element_value_int(blob.xml, PCMK__XA_ST_CALLID, &call_id);

	update_callback_timeout(call_id, timeout, st);
	} else {
	crm_err("Unknown message type: %s", type);
	crm_log_xml_warn(blob.xml, "BadReply");
	}

	free_xml(blob.xml);
	return 1;
	}

	static int
	stonith_api_signon(stonith_t * stonith, const char name, int stonith_fd)
	{
	int rc = pcmk_ok;
	stonith_private_t *native = NULL;
	const char *display_name = name? name : "client";

	struct ipc_client_callbacks st_callbacks = {
	.dispatch = stonith_dispatch_internal,
	.destroy = stonith_connection_destroy
	};

	CRM_CHECK(stonith != NULL, return -EINVAL);

	native = stonith->st_private;
	CRM_ASSERT(native != NULL);

	crm_debug("Attempting fencer connection by %s with%s mainloop",
	display_name, (stonith_fd? "out" : ""));

	stonith->state = stonith_connected_command;
	if (stonith_fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new("stonith-ng", 0);
	if (native->ipc != NULL) {
	rc = pcmk__connect_generic_ipc(native->ipc);
	if (rc == pcmk_rc_ok) {
	rc = pcmk__ipc_fd(native->ipc, stonith_fd);
	if (rc != pcmk_rc_ok) {
	crm_debug("Couldn't get file descriptor for IPC: %s",
	pcmk_rc_str(rc));
	}
	}
	if (rc != pcmk_rc_ok) {
	crm_ipc_close(native->ipc);
	crm_ipc_destroy(native->ipc);
	native->ipc = NULL;
	}
	}

	} else {
	/* With mainloop */
	native->source =
	mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	rc = -ENOTCONN;
	} else {
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, PCMK__XE_STONITH_COMMAND);

	crm_xml_add(hello, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
	crm_xml_add(hello, PCMK__XA_ST_OP, CRM_OP_REGISTER);
	crm_xml_add(hello, PCMK__XA_ST_CLIENTNAME, name);
	rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply);

	if (rc < 0) {
	crm_debug("Couldn't register with the fencer: %s "
	CRM_XS " rc=%d", pcmk_strerror(rc), rc);
	rc = -ECOMM;

	} else if (reply == NULL) {
	crm_debug("Couldn't register with the fencer: no reply");
	rc = -EPROTO;

	} else {
	const char *msg_type = crm_element_value(reply, PCMK__XA_ST_OP);

	native->token = crm_element_value_copy(reply, PCMK__XA_ST_CLIENTID);
	if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_none)) {
	crm_debug("Couldn't register with the fencer: invalid reply type '%s'",
	(msg_type? msg_type : "(missing)"));
	crm_log_xml_debug(reply, "Invalid fencer reply");
	rc = -EPROTO;

	} else if (native->token == NULL) {
	crm_debug("Couldn't register with the fencer: no token in reply");
	crm_log_xml_debug(reply, "Invalid fencer reply");
	rc = -EPROTO;

	} else {
	crm_debug("Connection to fencer by %s succeeded (registration token: %s)",
	display_name, native->token);
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);
	}

	if (rc != pcmk_ok) {
	crm_debug("Connection attempt to fencer by %s failed: %s "
	CRM_XS " rc=%d", display_name, pcmk_strerror(rc), rc);
	stonith->cmds->disconnect(stonith);
	}
	return rc;
	}

	static int
	stonith_set_notification(stonith_t * stonith, const char *callback, int enabled)
	{
	int rc = pcmk_ok;
	xmlNode *notify_msg = create_xml_node(NULL, __func__);
	stonith_private_t *native = stonith->st_private;

	if (stonith->state != stonith_disconnected) {

	crm_xml_add(notify_msg, PCMK__XA_ST_OP, STONITH_OP_NOTIFY);
	if (enabled) {
	crm_xml_add(notify_msg, PCMK__XA_ST_NOTIFY_ACTIVATE, callback);
	} else {
	crm_xml_add(notify_msg, PCMK__XA_ST_NOTIFY_DEACTIVATE, callback);
	}

	rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, -1, NULL);
	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc);
	rc = -ECOMM;
	} else {
	rc = pcmk_ok;
	}
	}

	free_xml(notify_msg);
	return rc;
	}

	static int
	stonith_api_add_notification(stonith_t * stonith, const char *event,
	void (callback) (stonith_t stonith, stonith_event_t * e))
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = NULL;

	private = stonith->st_private;
	crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list));

	new_client = calloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = callback;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	if (list_item != NULL) {
	crm_warn("Callback already present");
	free(new_client);
	return -ENOTUNIQ;

	} else {
	private->notify_list = g_list_append(private->notify_list, new_client);

	stonith_set_notification(stonith, event, 1);

	crm_trace("Callback added (%d)", g_list_length(private->notify_list));
	}
	return pcmk_ok;
	}

	static void
	del_notify_entry(gpointer data, gpointer user_data)
	{
	stonith_notify_client_t *entry = data;
	stonith_t * stonith = user_data;

	if (!entry->delete) {
	crm_debug("Removing callback for %s events", entry->event);
	stonith_api_del_notification(stonith, entry->event);
	}
	}

	static int
	stonith_api_del_notification(stonith_t * stonith, const char *event)
	{
	GList *list_item = NULL;
	stonith_notify_client_t *new_client = NULL;
	stonith_private_t *private = stonith->st_private;

	if (event == NULL) {
	foreach_notify_entry(private, del_notify_entry, stonith);
	crm_trace("Removed callback");

	return pcmk_ok;
	}

	crm_debug("Removing callback for %s events", event);

	new_client = calloc(1, sizeof(stonith_notify_client_t));
	new_client->event = event;
	new_client->notify = NULL;

	list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);

	stonith_set_notification(stonith, event, 0);

	if (list_item != NULL) {
	stonith_notify_client_t *list_client = list_item->data;

	if (private->notify_refcnt) {
	list_client->delete = TRUE;
	private->notify_deletes = TRUE;
	} else {
	private->notify_list = g_list_remove(private->notify_list, list_client);
	free(list_client);
	}

	crm_trace("Removed callback");

	} else {
	crm_trace("Callback not present");
	}
	free(new_client);
	return pcmk_ok;
	}

	static int
	stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int options,
	void user_data, const char callback_name,
	void (callback) (stonith_t st, stonith_callback_data_t * data))
	{
	stonith_callback_client_t *blob = NULL;
	stonith_private_t *private = NULL;

	CRM_CHECK(stonith != NULL, return -EINVAL);
	CRM_CHECK(stonith->st_private != NULL, return -EINVAL);
	private = stonith->st_private;

	if (call_id == 0) { // Add global callback
	private->op_callback = callback;

	} else if (call_id < 0) { // Call failed immediately, so call callback now
	if (!(options & st_opt_report_only_success)) {
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	crm_trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id));
	pcmk__set_result(&result, CRM_EX_ERROR,
	stonith__legacy2status(call_id), NULL);
	invoke_fence_action_callback(stonith, call_id, &result,
	user_data, callback);
	} else {
	crm_warn("Fencer call failed: %s", pcmk_strerror(call_id));
	}
	return FALSE;
	}

	blob = calloc(1, sizeof(stonith_callback_client_t));
	blob->id = callback_name;
	blob->only_success = (options & st_opt_report_only_success) ? TRUE : FALSE;
	blob->user_data = user_data;
	blob->callback = callback;
	blob->allow_timeout_updates = (options & st_opt_timeout_updates) ? TRUE : FALSE;

	if (timeout > 0) {
	set_callback_timeout(blob, stonith, call_id, timeout);
	}

	pcmk__intkey_table_insert(private->stonith_op_callback_table, call_id,
	blob);
	crm_trace("Added callback to %s for call %d", callback_name, call_id);

	return TRUE;
	}

	static void
	stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
	{
	int call = GPOINTER_TO_INT(key);
	stonith_callback_client_t *blob = value;

	crm_debug("Call %d (%s): pending", call, pcmk__s(blob->id, "no ID"));
	}

	void
	stonith_dump_pending_callbacks(stonith_t * stonith)
	{
	stonith_private_t *private = stonith->st_private;

	if (private->stonith_op_callback_table == NULL) {
	return;
	}
	return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL);
	}

	/*!
	* \internal
	* \brief Get the data section of a fencer notification
	*
	* \param[in] msg Notification XML
	* \param[in] ntype Notification type
	*/
	static xmlNode *
	get_event_data_xml(xmlNode msg, const char ntype)
	{
	char *data_addr = crm_strdup_printf("//%s", ntype);
	xmlNode *data = get_xpath_object(data_addr, msg, LOG_DEBUG);

	free(data_addr);
	return data;
	}

	/*
	<notify t="st_notify" subt="st_device_register" st_op="st_device_register" st_rc="0" >
	<st_calldata >
	<stonith_command t="stonith-ng" st_async_id="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_op="st_device_register" st_callid="2" st_callopt="4096" st_timeout="0" st_clientid="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_clientname="cts-fence-helper" >
	<st_calldata >
	<st_device_id id="test-id" origin="create_device_registration_xml" agent="fence_virsh" namespace="stonith-ng" >
	<attributes ipaddr="localhost" pcmk-portmal="some-host=pcmk-1 pcmk-3=3,4" login="root" identity_file="/root/.ssh/id_dsa" />
	</st_device_id>
	</st_calldata>
	</stonith_command>
	</st_calldata>
	</notify>

	<notify t="st_notify" subt="st_notify_fence" st_op="st_notify_fence" st_rc="0" >
	<st_calldata >
	<st_notify_fence st_rc="0" st_target="some-host" st_op="st_fence" st_delegate="test-id" st_origin="61dd7759-e229-4be7-b1f8-ef49dd14d9f0" />
	</st_calldata>
	</notify>
	*/
	static stonith_event_t *
	xml_to_event(xmlNode *msg)
	{
	stonith_event_t *event = calloc(1, sizeof(stonith_event_t));
	struct event_private *event_private = NULL;

	CRM_ASSERT(event != NULL);

	event->opaque = calloc(1, sizeof(struct event_private));
	CRM_ASSERT(event->opaque != NULL);
	event_private = (struct event_private *) event->opaque;

	crm_log_xml_trace(msg, "stonith_notify");

	// All notification types have the operation result and notification subtype
	stonith__xe_get_result(msg, &event_private->result);
	event->operation = crm_element_value_copy(msg, PCMK__XA_ST_OP);

	// @COMPAT The API originally provided the result as a legacy return code
	event->result = pcmk_rc2legacy(stonith__result2rc(&event_private->result));

	// Some notification subtypes have additional information

	if (pcmk__str_eq(event->operation, PCMK__VALUE_ST_NOTIFY_FENCE,
	pcmk__str_none)) {
	xmlNode *data = get_event_data_xml(msg, event->operation);

	if (data == NULL) {
	crm_err("No data for %s event", event->operation);
	crm_log_xml_notice(msg, "BadEvent");
	} else {
	event->origin = crm_element_value_copy(data, PCMK__XA_ST_ORIGIN);
	event->action = crm_element_value_copy(data,
	PCMK__XA_ST_DEVICE_ACTION);
	event->target = crm_element_value_copy(data, PCMK__XA_ST_TARGET);
	event->executioner = crm_element_value_copy(data,
	PCMK__XA_ST_DELEGATE);
	event->id = crm_element_value_copy(data, PCMK__XA_ST_REMOTE_OP);
	event->client_origin =
	crm_element_value_copy(data, PCMK__XA_ST_CLIENTNAME);
	event->device = crm_element_value_copy(data, PCMK__XA_ST_DEVICE_ID);
	}

	} else if (pcmk__str_any_of(event->operation,
	STONITH_OP_DEVICE_ADD, STONITH_OP_DEVICE_DEL,
	STONITH_OP_LEVEL_ADD, STONITH_OP_LEVEL_DEL,
	NULL)) {
	xmlNode *data = get_event_data_xml(msg, event->operation);

	if (data == NULL) {
	crm_err("No data for %s event", event->operation);
	crm_log_xml_notice(msg, "BadEvent");
	} else {
	event->device = crm_element_value_copy(data, PCMK__XA_ST_DEVICE_ID);
	}
	}

	return event;
	}

	static void
	event_free(stonith_event_t * event)
	{
	struct event_private *event_private = event->opaque;

	free(event->id);
	free(event->type);
	free(event->message);
	free(event->operation);
	free(event->origin);
	free(event->action);
	free(event->target);
	free(event->executioner);
	free(event->device);
	free(event->client_origin);
	pcmk__reset_result(&event_private->result);
	free(event->opaque);
	free(event);
	}

	static void
	stonith_send_notification(gpointer data, gpointer user_data)
	{
	struct notify_blob_s *blob = user_data;
	stonith_notify_client_t *entry = data;
	stonith_event_t *st_event = NULL;
	const char *event = NULL;

	if (blob->xml == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(blob->xml, PCMK__XA_SUBT);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->delete) {
	crm_trace("Skipping callback - marked for deletion");
	return;

	} else if (entry->notify == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (!pcmk__str_eq(entry->event, event, pcmk__str_none)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	st_event = xml_to_event(blob->xml);

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->notify(blob->stonith, st_event);
	crm_trace("Callback invoked...");

	event_free(st_event);
	}

	/*!
	* \internal
	* \brief Create and send an API request
	*
	* \param[in,out] stonith Stonith connection
	* \param[in] op API operation to request
	* \param[in] data Data to attach to request
	* \param[out] output_data If not NULL, will be set to reply if synchronous
	* \param[in] call_options Bitmask of stonith_call_options to use
	* \param[in] timeout Error if not completed within this many seconds
	*
	* \return pcmk_ok (for synchronous requests) or positive call ID
	* (for asynchronous requests) on success, -errno otherwise
	*/
	static int
	stonith_send_command(stonith_t * stonith, const char op, xmlNode data, xmlNode ** output_data,
	int call_options, int timeout)
	{
	int rc = 0;
	int reply_id = -1;

	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;
	stonith_private_t *native = NULL;

	CRM_ASSERT(stonith && stonith->st_private && op);
	native = stonith->st_private;

	if (output_data != NULL) {
	*output_data = NULL;
	}

	if ((stonith->state == stonith_disconnected) \|\| (native->token == NULL)) {
	return -ENOTCONN;
	}

	/* Increment the call ID, which must be positive to avoid conflicting with
	* error codes. This shouldn't be a problem unless the client mucked with
	* it or the counter wrapped around.
	*/
	stonith->call_id++;
	if (stonith->call_id < 1) {
	stonith->call_id = 1;
	}

	op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options);
	if (op_msg == NULL) {
	return -EINVAL;
	}

	crm_xml_add_int(op_msg, PCMK__XA_ST_TIMEOUT, timeout);
	crm_trace("Sending %s message to fencer with timeout %ds", op, timeout);

	if (data) {
	const char *delay_s = crm_element_value(data, PCMK__XA_ST_DELAY);

	if (delay_s) {
	crm_xml_add(op_msg, PCMK__XA_ST_DELAY, delay_s);
	}
	}

	{
	enum crm_ipc_flags ipc_flags = crm_ipc_flags_none;

	if (call_options & st_opt_sync_call) {
	pcmk__set_ipc_flags(ipc_flags, "stonith command",
	crm_ipc_client_response);
	}
	rc = crm_ipc_send(native->ipc, op_msg, ipc_flags,
	1000 * (timeout + 60), &op_reply);
	}
	free_xml(op_msg);

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc);
	rc = -ECOMM;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (!(call_options & st_opt_sync_call)) {
	crm_trace("Async call %d, returning", stonith->call_id);
	free_xml(op_reply);
	return stonith->call_id;
	}

	crm_element_value_int(op_reply, PCMK__XA_ST_CALLID, &reply_id);

	if (reply_id == stonith->call_id) {
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	crm_trace("Synchronous reply %d received", reply_id);

	stonith__xe_get_result(op_reply, &result);
	rc = pcmk_rc2legacy(stonith__result2rc(&result));
	pcmk__reset_result(&result);

	if ((call_options & st_opt_discard_reply) \|\| output_data == NULL) {
	crm_trace("Discarding reply");

	} else {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	} else if (reply_id <= 0) {
	crm_err("Received bad reply: No id set");
	crm_log_xml_err(op_reply, "Bad reply");
	free_xml(op_reply);
	rc = -ENOMSG;

	} else {
	crm_err("Received bad reply: %d (wanted %d)", reply_id, stonith->call_id);
	crm_log_xml_err(op_reply, "Old reply");
	free_xml(op_reply);
	rc = -ENOMSG;
	}

	done:
	if (!crm_ipc_connected(native->ipc)) {
	crm_err("Fencer disconnected");
	free(native->token); native->token = NULL;
	stonith->state = stonith_disconnected;
	}

	free_xml(op_reply);
	return rc;
	}

	/* Not used with mainloop */
	bool
	stonith_dispatch(stonith_t * st)
	{
	gboolean stay_connected = TRUE;
	stonith_private_t *private = NULL;

	CRM_ASSERT(st != NULL);
	private = st->st_private;

	while (crm_ipc_ready(private->ipc)) {

	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);

	stonith_dispatch_internal(msg, strlen(msg), st);
	}

	if (!crm_ipc_connected(private->ipc)) {
	crm_err("Connection closed");
	stay_connected = FALSE;
	}
	}

	return stay_connected;
	}

	static int
	stonith_api_free(stonith_t * stonith)
	{
	int rc = pcmk_ok;

	crm_trace("Destroying %p", stonith);

	if (stonith->state != stonith_disconnected) {
	crm_trace("Unregistering notifications and disconnecting %p first",
	stonith);
	stonith->cmds->remove_notification(stonith, NULL);
	rc = stonith->cmds->disconnect(stonith);
	}

	if (stonith->state == stonith_disconnected) {
	stonith_private_t *private = stonith->st_private;

	crm_trace("Removing %d callbacks", g_hash_table_size(private->stonith_op_callback_table));
	g_hash_table_destroy(private->stonith_op_callback_table);

	crm_trace("Destroying %d notification clients", g_list_length(private->notify_list));
	g_list_free_full(private->notify_list, free);

	free(stonith->st_private);
	free(stonith->cmds);
	free(stonith);

	} else {
	crm_err("Not free'ing active connection: %s (%d)", pcmk_strerror(rc), rc);
	}

	return rc;
	}

	void
	stonith_api_delete(stonith_t * stonith)
	{
	crm_trace("Destroying %p", stonith);
	if(stonith) {
	stonith->cmds->free(stonith);
	}
	}

	static int
	stonith_api_validate(stonith_t st, int call_options, const char rsc_id,
	const char namespace_s, const char agent,
	const stonith_key_value_t *params, int timeout_sec,
	char output, char error_output)
	{
	/* Validation should be done directly via the agent, so we can get it from
	* stonith_admin when the cluster is not running, which is important for
	* higher-level tools.
	*/

	int rc = pcmk_ok;

	/* Use a dummy node name in case the agent requires a target. We assume the
	* actual target doesn't matter for validation purposes (if in practice,
	* that is incorrect, we will need to allow the caller to pass the target).
	*/
	const char *target = "node1";
	const char *host_arg = NULL;

	GHashTable *params_table = pcmk__strkey_table(free, free);

	// Convert parameter list to a hash table
	for (; params; params = params->next) {
	if (pcmk__str_eq(params->key, PCMK_STONITH_HOST_ARGUMENT,
	pcmk__str_none)) {
	host_arg = params->value;
	}
	if (!pcmk_stonith_param(params->key)) {
	pcmk__insert_dup(params_table, params->key, params->value);
	}
	}

	#if SUPPORT_CIBSECRETS
	rc = pcmk__substitute_secrets(rsc_id, params_table);
	if (rc != pcmk_rc_ok) {
	crm_warn("Could not replace secret parameters for validation of %s: %s",
	agent, pcmk_rc_str(rc));
	// rc is standard return value, don't return it in this function
	}
	#endif

	if (output) {
	*output = NULL;
	}
	if (error_output) {
	*error_output = NULL;
	}

	if (timeout_sec <= 0) {
	timeout_sec = PCMK_DEFAULT_METADATA_TIMEOUT_MS; // Questionable
	}

	switch (stonith_get_namespace(agent, namespace_s)) {
	case st_namespace_rhcs:
	rc = stonith__rhcs_validate(st, call_options, target, agent,
	params_table, host_arg, timeout_sec,
	output, error_output);
	break;

	#if HAVE_STONITH_STONITH_H
	case st_namespace_lha:
	rc = stonith__lha_validate(st, call_options, target, agent,
	params_table, timeout_sec, output,
	error_output);
	break;
	#endif

	case st_namespace_invalid:
	errno = ENOENT;
	rc = -errno;

	if (error_output) {
	*error_output = crm_strdup_printf("Agent %s not found", agent);
	} else {
	crm_err("Agent %s not found", agent);
	}

	break;

	default:
	errno = EOPNOTSUPP;
	rc = -errno;

	if (error_output) {
	*error_output = crm_strdup_printf("Agent %s does not support validation",
	agent);
	} else {
	crm_err("Agent %s does not support validation", agent);
	}

	break;
	}

	g_hash_table_destroy(params_table);
	return rc;
	}

	stonith_t *
	stonith_api_new(void)
	{
	stonith_t *new_stonith = NULL;
	stonith_private_t *private = NULL;

	new_stonith = calloc(1, sizeof(stonith_t));
	if (new_stonith == NULL) {
	return NULL;
	}

	private = calloc(1, sizeof(stonith_private_t));
	if (private == NULL) {
	free(new_stonith);
	return NULL;
	}
	new_stonith->st_private = private;

	private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);
	private->notify_list = NULL;
	private->notify_refcnt = 0;
	private->notify_deletes = FALSE;

	new_stonith->call_id = 1;
	new_stonith->state = stonith_disconnected;

	new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t));
	if (new_stonith->cmds == NULL) {
	free(new_stonith->st_private);
	free(new_stonith);
	return NULL;
	}

	/* INDENT-OFF */
	new_stonith->cmds->free = stonith_api_free;
	new_stonith->cmds->connect = stonith_api_signon;
	new_stonith->cmds->disconnect = stonith_api_signoff;

	new_stonith->cmds->list = stonith_api_list;
	new_stonith->cmds->monitor = stonith_api_monitor;
	new_stonith->cmds->status = stonith_api_status;
	new_stonith->cmds->fence = stonith_api_fence;
	new_stonith->cmds->fence_with_delay = stonith_api_fence_with_delay;
	new_stonith->cmds->confirm = stonith_api_confirm;
	new_stonith->cmds->history = stonith_api_history;

	new_stonith->cmds->list_agents = stonith_api_device_list;
	new_stonith->cmds->metadata = stonith_api_device_metadata;

	new_stonith->cmds->query = stonith_api_query;
	new_stonith->cmds->remove_device = stonith_api_remove_device;
	new_stonith->cmds->register_device = stonith_api_register_device;

	new_stonith->cmds->remove_level = stonith_api_remove_level;
	new_stonith->cmds->remove_level_full = stonith_api_remove_level_full;
	new_stonith->cmds->register_level = stonith_api_register_level;
	new_stonith->cmds->register_level_full = stonith_api_register_level_full;

	new_stonith->cmds->remove_callback = stonith_api_del_callback;
	new_stonith->cmds->register_callback = stonith_api_add_callback;
	new_stonith->cmds->remove_notification = stonith_api_del_notification;
	new_stonith->cmds->register_notification = stonith_api_add_notification;

	new_stonith->cmds->validate = stonith_api_validate;
	/* INDENT-ON */

	return new_stonith;
	}

	/*!
	* \brief Make a blocking connection attempt to the fencer
	*
	* \param[in,out] st Fencer API object
	* \param[in] name Client name to use with fencer
	* \param[in] max_attempts Return error if this many attempts fail
	*
	* \return pcmk_ok on success, result of last attempt otherwise
	*/
	int
	stonith_api_connect_retry(stonith_t st, const char name, int max_attempts)
	{
	int rc = -EINVAL; // if max_attempts is not positive

	for (int attempt = 1; attempt <= max_attempts; attempt++) {
	rc = st->cmds->connect(st, name, NULL);
	if (rc == pcmk_ok) {
	return pcmk_ok;
	} else if (attempt < max_attempts) {
	crm_notice("Fencer connection attempt %d of %d failed (retrying in 2s): %s "
	CRM_XS " rc=%d",
	attempt, max_attempts, pcmk_strerror(rc), rc);
	sleep(2);
	}
	}
	crm_notice("Could not connect to fencer: %s " CRM_XS " rc=%d",
	pcmk_strerror(rc), rc);
	return rc;
	}

	stonith_key_value_t *
	stonith_key_value_add(stonith_key_value_t * head, const char key, const char value)
	{
	stonith_key_value_t p, end;

	p = calloc(1, sizeof(stonith_key_value_t));
	pcmk__str_update(&p->key, key);
	pcmk__str_update(&p->value, value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values)
	{
	stonith_key_value_t *p;

	while (head) {
	p = head->next;
	if (keys) {
	free(head->key);
	}
	if (values) {
	free(head->value);
	}
	free(head);
	head = p;
	}
	}

	#define api_log_open() openlog("stonith-api", LOG_CONS \| LOG_NDELAY \| LOG_PID, LOG_DAEMON)
	#define api_log(level, fmt, args...) syslog(level, "%s: "fmt, __func__, args)

	int
	stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off)
	{
	int rc = pcmk_ok;
	stonith_t *st = stonith_api_new();
	const char *action = off? PCMK_ACTION_OFF : PCMK_ACTION_REBOOT;

	api_log_open();
	if (st == NULL) {
	api_log(LOG_ERR, "API initialization failed, could not kick (%s) node %u/%s",
	action, nodeid, uname);
	return -EPROTO;
	}

	rc = st->cmds->connect(st, "stonith-api", NULL);
	if (rc != pcmk_ok) {
	api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)",
	action, nodeid, uname, pcmk_strerror(rc), rc);
	} else {
	char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);
	int opts = 0;

	stonith__set_call_options(opts, name,
	st_opt_sync_call\|st_opt_allow_suicide);
	if ((uname == NULL) && (nodeid > 0)) {
	stonith__set_call_options(opts, name, st_opt_cs_nodeid);
	}
	rc = st->cmds->fence(st, opts, name, action, timeout, 0);
	free(name);

	if (rc != pcmk_ok) {
	api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)",
	action, nodeid, uname, pcmk_strerror(rc), rc);
	} else {
	api_log(LOG_NOTICE, "Node %u/%s kicked: %s", nodeid, uname, action);
	}
	}

	stonith_api_delete(st);
	return rc;
	}

	time_t
	stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress)
	{
	int rc = pcmk_ok;
	time_t when = 0;
	stonith_t *st = stonith_api_new();
	stonith_history_t history = NULL, hp = NULL;

	if (st == NULL) {
	api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: "
	"API initialization failed", nodeid, uname);
	return when;
	}

	rc = st->cmds->connect(st, "stonith-api", NULL);
	if (rc != pcmk_ok) {
	api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc);
	} else {
	int entries = 0;
	int progress = 0;
	int completed = 0;
	int opts = 0;
	char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);

	stonith__set_call_options(opts, name, st_opt_sync_call);
	if ((uname == NULL) && (nodeid > 0)) {
	stonith__set_call_options(opts, name, st_opt_cs_nodeid);
	}
	rc = st->cmds->history(st, opts, name, &history, 120);
	free(name);

	for (hp = history; hp; hp = hp->next) {
	entries++;
	if (in_progress) {
	progress++;
	if (hp->state != st_done && hp->state != st_failed) {
	when = time(NULL);
	}

	} else if (hp->state == st_done) {
	completed++;
	if (hp->completed > when) {
	when = hp->completed;
	}
	}
	}

	stonith_history_free(history);

	if(rc == pcmk_ok) {
	api_log(LOG_INFO, "Found %d entries for %u/%s: %d in progress, %d completed", entries, nodeid, uname, progress, completed);
	} else {
	api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: %s (%d)", nodeid, uname, pcmk_strerror(rc), rc);
	}
	}

	stonith_api_delete(st);

	if(when) {
	api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when);
	}
	return when;
	}

	bool
	stonith_agent_exists(const char *agent, int timeout)
	{
	stonith_t *st = NULL;
	stonith_key_value_t *devices = NULL;
	stonith_key_value_t *dIter = NULL;
	bool rc = FALSE;

	if (agent == NULL) {
	return rc;
	}

	st = stonith_api_new();
	if (st == NULL) {
	crm_err("Could not list fence agents: API memory allocation failed");
	return FALSE;
	}
	st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout == 0 ? 120 : timeout);

	for (dIter = devices; dIter != NULL; dIter = dIter->next) {
	if (pcmk__str_eq(dIter->value, agent, pcmk__str_none)) {
	rc = TRUE;
	break;
	}
	}

	stonith_key_value_freeall(devices, 1, 1);
	stonith_api_delete(st);
	return rc;
	}

	const char *
	stonith_action_str(const char *action)
	{
	if (action == NULL) {
	return "fencing";
	} else if (strcmp(action, PCMK_ACTION_ON) == 0) {
	return "unfencing";
	} else if (strcmp(action, PCMK_ACTION_OFF) == 0) {
	return "turning off";
	} else {
	return action;
	}
	}

	/*!
	* \internal
	* \brief Parse a target name from one line of a target list string
	*
	* \param[in] line One line of a target list string
	* \param[in] len String length of line
	* \param[in,out] output List to add newly allocated target name to
	*/
	static void
	parse_list_line(const char line, int len, GList *output)
	{
	size_t i = 0;
	size_t entry_start = 0;

	/* Skip complaints about additional parameters device doesn't understand
	*
	* @TODO Document or eliminate the implied restriction of target names
	*/
	if (strstr(line, "invalid") \|\| strstr(line, "variable")) {
	crm_debug("Skipping list output line: %s", line);
	return;
	}

	// Process line content, character by character
	for (i = 0; i <= len; i++) {

	if (isspace(line[i]) \|\| (line[i] == ',') \|\| (line[i] == ';')
	\|\| (line[i] == '\0')) {
	// We've found a separator (i.e. the end of an entry)

	int rc = 0;
	char *entry = NULL;

	if (i == entry_start) {
	// Skip leading and sequential separators
	entry_start = i + 1;
	continue;
	}

	entry = calloc(i - entry_start + 1, sizeof(char));
	CRM_ASSERT(entry != NULL);

	/* Read entry, stopping at first separator
	*
	* @TODO Document or eliminate these character restrictions
	*/
	rc = sscanf(line + entry_start, "%[a-zA-Z0-9_-.]", entry);
	if (rc != 1) {
	crm_warn("Could not parse list output entry: %s "
	CRM_XS " entry_start=%d position=%d",
	line + entry_start, entry_start, i);
	free(entry);

	} else if (pcmk__strcase_any_of(entry, PCMK_ACTION_ON,
	PCMK_ACTION_OFF, NULL)) {
	/* Some agents print the target status in the list output,
	* though none are known now (the separate list-status command
	* is used for this, but it can also print "UNKNOWN"). To handle
	* this possibility, skip such entries.
	*
	* @TODO Document or eliminate the implied restriction of target
	* names.
	*/
	free(entry);

	} else {
	// We have a valid entry
	output = g_list_append(output, entry);
	}
	entry_start = i + 1;
	}
	}
	}

	/*!
	* \internal
	* \brief Parse a list of targets from a string
	*
	* \param[in] list_output Target list as a string
	*
	* \return List of target names
	* \note The target list string format is flexible, to allow for user-specified
	* lists such pcmk_host_list and the output of an agent's list action
	* (whether direct or via the API, which escapes newlines). There may be
	* multiple lines, separated by either a newline or an escaped newline
	* (backslash n). Each line may have one or more target names, separated
	* by any combination of whitespace, commas, and semi-colons. Lines
	* containing "invalid" or "variable" will be ignored entirely. Target
	* names "on" or "off" (case-insensitive) will be ignored. Target names
	* may contain only alphanumeric characters, underbars (_), dashes (-),
	* and dots (.) (if any other character occurs in the name, it and all
	* subsequent characters in the name will be ignored).
	* \note The caller is responsible for freeing the result with
	* g_list_free_full(result, free).
	*/
	GList *
	stonith__parse_targets(const char *target_spec)
	{
	GList *targets = NULL;

	if (target_spec != NULL) {
	size_t out_len = strlen(target_spec);
	size_t line_start = 0; // Starting index of line being processed

	for (size_t i = 0; i <= out_len; ++i) {
	if ((target_spec[i] == '\n') \|\| (target_spec[i] == '\0')
	\|\| ((target_spec[i] == '\\') && (target_spec[i + 1] == 'n'))) {
	// We've reached the end of one line of output

	int len = i - line_start;

	if (len > 0) {
	char *line = strndup(target_spec + line_start, len);

	line[len] = '\0'; // Because it might be a newline
	parse_list_line(line, len, &targets);
	free(line);
	}
	if (target_spec[i] == '\\') {
	++i; // backslash-n takes up two positions
	}
	line_start = i + 1;
	}
	}
	}
	return targets;
	}

	/*!
	* \internal
	* \brief Check whether a fencing failure was followed by an equivalent success
	*
	* \param[in] event Fencing failure
	* \param[in] top_history Complete fencing history (must be sorted by
	* stonith__sort_history() beforehand)
	*
	* \return The name of the node that executed the fencing if a later successful
	* event exists, or NULL if no such event exists
	*/
	const char *
	stonith__later_succeeded(const stonith_history_t *event,
	const stonith_history_t *top_history)
	{
	const char *other = NULL;

	for (const stonith_history_t *prev_hp = top_history;
	prev_hp != NULL; prev_hp = prev_hp->next) {
	if (prev_hp == event) {
	break;
	}
	if ((prev_hp->state == st_done) &&
	pcmk__str_eq(event->target, prev_hp->target, pcmk__str_casei) &&
	pcmk__str_eq(event->action, prev_hp->action, pcmk__str_none) &&
	((event->completed < prev_hp->completed) \|\|
	((event->completed == prev_hp->completed) && (event->completed_nsec < prev_hp->completed_nsec)))) {

	if ((event->delegate == NULL)
	\|\| pcmk__str_eq(event->delegate, prev_hp->delegate,
	pcmk__str_casei)) {
	// Prefer equivalent fencing by same executioner
	return prev_hp->delegate;

	} else if (other == NULL) {
	// Otherwise remember first successful executioner
	other = (prev_hp->delegate == NULL)? "some node" : prev_hp->delegate;
	}
	}
	}
	return other;
	}

	/*!
	* \internal
	* \brief Sort fencing history, pending first then by most recently completed
	*
	* \param[in,out] history List of stonith actions
	*
	* \return New head of sorted \p history
	*/
	stonith_history_t *
	stonith__sort_history(stonith_history_t *history)
	{
	stonith_history_t new = NULL, pending = NULL, hp, np, *tmp;

	for (hp = history; hp; ) {
	tmp = hp->next;
	if ((hp->state == st_done) \|\| (hp->state == st_failed)) {
	/* sort into new */
	if ((!new) \|\| (hp->completed > new->completed) \|\|
	((hp->completed == new->completed) && (hp->completed_nsec > new->completed_nsec))) {
	hp->next = new;
	new = hp;
	} else {
	np = new;
	do {
	if ((!np->next) \|\| (hp->completed > np->next->completed) \|\|
	((hp->completed == np->next->completed) && (hp->completed_nsec > np->next->completed_nsec))) {
	hp->next = np->next;
	np->next = hp;
	break;
	}
	np = np->next;
	} while (1);
	}
	} else {
	/* put into pending */
	hp->next = pending;
	pending = hp;
	}
	hp = tmp;
	}

	/* pending actions don't have a completed-stamp so make them go front */
	if (pending) {
	stonith_history_t *last_pending = pending;

	while (last_pending->next) {
	last_pending = last_pending->next;
	}

	last_pending->next = new;
	new = pending;
	}
	return new;
	}

	/*!
	* \brief Return string equivalent of an operation state value
	*
	* \param[in] state Fencing operation state value
	*
	* \return Human-friendly string equivalent of state
	*/
	const char *
	stonith_op_state_str(enum op_state state)
	{
	switch (state) {
	case st_query: return "querying";
	case st_exec: return "executing";
	case st_done: return "completed";
	case st_duplicate: return "duplicate";
	case st_failed: return "failed";
	}
	return "unknown";
	}

	stonith_history_t *
	stonith__first_matching_event(stonith_history_t *history,
	bool (matching_fn)(stonith_history_t , void *),
	void *user_data)
	{
	for (stonith_history_t *hp = history; hp; hp = hp->next) {
	if (matching_fn(hp, user_data)) {
	return hp;
	}
	}

	return NULL;
	}

	bool
	stonith__event_state_pending(stonith_history_t history, void user_data)
	{
	return history->state != st_failed && history->state != st_done;
	}

	bool
	stonith__event_state_eq(stonith_history_t history, void user_data)
	{
	return history->state == GPOINTER_TO_INT(user_data);
	}

	bool
	stonith__event_state_neq(stonith_history_t history, void user_data)
	{
	return history->state != GPOINTER_TO_INT(user_data);
	}

	void
	stonith__device_parameter_flags(uint32_t device_flags, const char device_name,
	xmlNode *metadata)
	{
	xmlXPathObjectPtr xpath = NULL;
	int max = 0;
	int lpc = 0;

	CRM_CHECK((device_flags != NULL) && (metadata != NULL), return);

	xpath = xpath_search(metadata, "//" PCMK_XE_PARAMETER);
	max = numXpathResults(xpath);

	if (max <= 0) {
	freeXpathObject(xpath);
	return;
	}

	for (lpc = 0; lpc < max; lpc++) {
	const char *parameter = NULL;
	xmlNode *match = getXpathResult(xpath, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if (match == NULL) {
	continue;
	}

	parameter = crm_element_value(match, PCMK_XA_NAME);

	if (pcmk__str_eq(parameter, "plug", pcmk__str_casei)) {
	stonith__set_device_flags(*device_flags, device_name,
	st_device_supports_parameter_plug);

	} else if (pcmk__str_eq(parameter, "port", pcmk__str_casei)) {
	stonith__set_device_flags(*device_flags, device_name,
	st_device_supports_parameter_port);
	}
	}

	freeXpathObject(xpath);
	}

	/*!
	* \internal
	* \brief Retrieve fence agent meta-data asynchronously
	*
	* \param[in] agent Agent to execute
	* \param[in] timeout_sec Error if not complete within this time
	* \param[in] callback Function to call with result (this will always be
	* called, whether by this function directly or
	* later via the main loop, and on success the
	* metadata will be in its result argument's
	* action_stdout)
	* \param[in,out] user_data User data to pass to callback
	*
	* \return Standard Pacemaker return code
	* \note The caller must use a main loop. This function is not a
	* stonith_api_operations_t method because it does not need a stonith_t
	* object and does not go through the fencer, but executes the agent
	* directly.
	*/
	int
	stonith__metadata_async(const char *agent, int timeout_sec,
	void (*callback)(int pid,
	const pcmk__action_result_t *result,
	void *user_data),
	void *user_data)
	{
	switch (stonith_get_namespace(agent, NULL)) {
	case st_namespace_rhcs:
	{
	stonith_action_t *action = NULL;
	int rc = pcmk_ok;

	action = stonith__action_create(agent, PCMK_ACTION_METADATA,
	NULL, 0, timeout_sec, NULL,
	NULL, NULL);

	rc = stonith__execute_async(action, user_data, callback, NULL);
	if (rc != pcmk_ok) {
	callback(0, stonith__action_result(action), user_data);
	stonith__destroy_action(action);
	}
	return pcmk_legacy2rc(rc);
	}

	#if HAVE_STONITH_STONITH_H
	case st_namespace_lha:
	// LHA metadata is simply synthesized, so simulate async
	{
	pcmk__action_result_t result = {
	.exit_status = CRM_EX_OK,
	.execution_status = PCMK_EXEC_DONE,
	.exit_reason = NULL,
	.action_stdout = NULL,
	.action_stderr = NULL,
	};

	stonith__lha_metadata(agent, timeout_sec,
	&result.action_stdout);
	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	return pcmk_rc_ok;
	}
	#endif

	default:
	{
	pcmk__action_result_t result = {
	.exit_status = CRM_EX_NOSUCH,
	.execution_status = PCMK_EXEC_ERROR_HARD,
	.exit_reason = crm_strdup_printf("No such agent '%s'",
	agent),
	.action_stdout = NULL,
	.action_stderr = NULL,
	};

	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	return ENOENT;
	}
	}
	}

	/*!
	* \internal
	* \brief Return the exit status from an async action callback
	*
	* \param[in] data Callback data
	*
	* \return Exit status from callback data
	*/
	int
	stonith__exit_status(const stonith_callback_data_t *data)
	{
	if ((data == NULL) \|\| (data->opaque == NULL)) {
	return CRM_EX_ERROR;
	}
	return ((pcmk__action_result_t *) data->opaque)->exit_status;
	}

	/*!
	* \internal
	* \brief Return the execution status from an async action callback
	*
	* \param[in] data Callback data
	*
	* \return Execution status from callback data
	*/
	int
	stonith__execution_status(const stonith_callback_data_t *data)
	{
	if ((data == NULL) \|\| (data->opaque == NULL)) {
	return PCMK_EXEC_UNKNOWN;
	}
	return ((pcmk__action_result_t *) data->opaque)->execution_status;
	}

	/*!
	* \internal
	* \brief Return the exit reason from an async action callback
	*
	* \param[in] data Callback data
	*
	* \return Exit reason from callback data
	*/
	const char *
	stonith__exit_reason(const stonith_callback_data_t *data)
	{
	if ((data == NULL) \|\| (data->opaque == NULL)) {
	return NULL;
	}
	return ((pcmk__action_result_t *) data->opaque)->exit_reason;
	}

	/*!
	* \internal
	* \brief Return the exit status from an event notification
	*
	* \param[in] event Event
	*
	* \return Exit status from event
	*/
	int
	stonith__event_exit_status(const stonith_event_t *event)
	{
	if ((event == NULL) \|\| (event->opaque == NULL)) {
	return CRM_EX_ERROR;
	} else {
	struct event_private *event_private = event->opaque;

	return event_private->result.exit_status;
	}
	}

	/*!
	* \internal
	* \brief Return the execution status from an event notification
	*
	* \param[in] event Event
	*
	* \return Execution status from event
	*/
	int
	stonith__event_execution_status(const stonith_event_t *event)
	{
	if ((event == NULL) \|\| (event->opaque == NULL)) {
	return PCMK_EXEC_UNKNOWN;
	} else {
	struct event_private *event_private = event->opaque;

	return event_private->result.execution_status;
	}
	}

	/*!
	* \internal
	* \brief Return the exit reason from an event notification
	*
	* \param[in] event Event
	*
	* \return Exit reason from event
	*/
	const char *
	stonith__event_exit_reason(const stonith_event_t *event)
	{
	if ((event == NULL) \|\| (event->opaque == NULL)) {
	return NULL;
	} else {
	struct event_private *event_private = event->opaque;

	return event_private->result.exit_reason;
	}
	}

	/*!
	* \internal
	* \brief Return a human-friendly description of a fencing event
	*
	* \param[in] event Event to describe
	*
	* \return Newly allocated string with description of \p event
	* \note The caller is responsible for freeing the return value.
	* This function asserts on memory errors and never returns NULL.
	*/
	char *
	stonith__event_description(const stonith_event_t *event)
	{
	// Use somewhat readable defaults
	const char *origin = pcmk__s(event->client_origin, "a client");
	const char *origin_node = pcmk__s(event->origin, "a node");
	const char *executioner = pcmk__s(event->executioner, "the cluster");
	const char *device = pcmk__s(event->device, "unknown");
	const char *action = pcmk__s(event->action, event->operation);
	const char *target = pcmk__s(event->target, "no node");
	const char *reason = stonith__event_exit_reason(event);
	const char *status;

	if (action == NULL) {
	action = "(unknown)";
	}

	if (stonith__event_execution_status(event) != PCMK_EXEC_DONE) {
	status = pcmk_exec_status_str(stonith__event_execution_status(event));
	} else if (stonith__event_exit_status(event) != CRM_EX_OK) {
	status = pcmk_exec_status_str(PCMK_EXEC_ERROR);
	} else {
	status = crm_exit_str(CRM_EX_OK);
	}

	if (pcmk__str_eq(event->operation, PCMK__VALUE_ST_NOTIFY_HISTORY,
	pcmk__str_none)) {
	return crm_strdup_printf("Fencing history may have changed");

	} else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_ADD,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing device (%s) was added", device);

	} else if (pcmk__str_eq(event->operation, STONITH_OP_DEVICE_DEL,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing device (%s) was removed", device);

	} else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_ADD,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing topology level (%s) was added",
	device);

	} else if (pcmk__str_eq(event->operation, STONITH_OP_LEVEL_DEL,
	pcmk__str_none)) {
	return crm_strdup_printf("A fencing topology level (%s) was removed",
	device);
	}

	// event->operation should be PCMK__VALUE_ST_NOTIFY_FENCE at this point

	return crm_strdup_printf("Operation %s of %s by %s for %s@%s: %s%s%s%s (ref=%s)",
	action, target, executioner, origin, origin_node,
	status,
	((reason == NULL)? "" : " ("), pcmk__s(reason, ""),
	((reason == NULL)? "" : ")"),
	pcmk__s(event->id, "(none)"));
	}


	// Deprecated functions kept only for backward API compatibility
	// LCOV_EXCL_START

	const char get_stonith_provider(const char agent, const char *provider);

	const char *
	get_stonith_provider(const char agent, const char provider)
	{
	return stonith_namespace2text(stonith_get_namespace(agent, provider));
	}

	// LCOV_EXCL_STOP
	// End deprecated API
	diff --git a/lib/fencing/st_rhcs.c b/lib/fencing/st_rhcs.c
	index 2ba507e446..8d2d6f4871 100644
	--- a/lib/fencing/st_rhcs.c
	+++ b/lib/fencing/st_rhcs.c
	@@ -1,324 +1,324 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <string.h>
	#include <sys/stat.h>
	#include <glib.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>

	#include "fencing_private.h"

	#define RH_STONITH_PREFIX "fence_"

	/*!
	* \internal
	* \brief Add available RHCS-compatible agents to a list
	*
	* \param[in,out] List to add to
	*
	* \return Number of agents added
	*/
	int
	stonith__list_rhcs_agents(stonith_key_value_t **devices)
	{
	// Essentially: ls -1 @sbin_dir@/fence_*

	int count = 0, i;
	struct dirent **namelist;
	const int file_num = scandir(PCMK__FENCE_BINDIR, &namelist, 0, alphasort);

	#if _POSIX_C_SOURCE < 200809L && !(defined(O_SEARCH) \|\| defined(O_PATH))
	char buffer[FILENAME_MAX + 1];
	#elif defined(O_SEARCH)
	const int dirfd = open(PCMK__FENCE_BINDIR, O_SEARCH);
	#else
	const int dirfd = open(PCMK__FENCE_BINDIR, O_PATH);
	#endif

	for (i = 0; i < file_num; i++) {
	struct stat prop;

	if (pcmk__starts_with(namelist[i]->d_name, RH_STONITH_PREFIX)) {
	#if _POSIX_C_SOURCE < 200809L && !(defined(O_SEARCH) \|\| defined(O_PATH))
	snprintf(buffer, sizeof(buffer), "%s/%s", PCMK__FENCE_BINDIR,
	namelist[i]->d_name);
	if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
	#else
	if (dirfd == -1) {
	if (i == 0) {
	crm_notice("Problem with listing %s directory"
	CRM_XS "errno=%d", RH_STONITH_PREFIX, errno);
	}
	free(namelist[i]);
	continue;
	}
	/* note: we can possibly prevent following symlinks here,
	which may be a good idea, but fall on the nose when
	these agents are moved elsewhere & linked back */
	if (fstatat(dirfd, namelist[i]->d_name, &prop, 0) == 0
	&& S_ISREG(prop.st_mode)) {
	#endif
	devices = stonith_key_value_add(devices, NULL,
	namelist[i]->d_name);
	count++;
	}
	}
	free(namelist[i]);
	}
	if (file_num > 0) {
	free(namelist);
	}
	#if _POSIX_C_SOURCE >= 200809L \|\| defined(O_SEARCH) \|\| defined(O_PATH)
	if (dirfd >= 0) {
	close(dirfd);
	}
	#endif
	return count;
	}

	static void
	stonith_rhcs_parameter_not_required(xmlNode metadata, const char parameter)
	{
	char *xpath = NULL;
	xmlXPathObject *xpathObj = NULL;

	CRM_CHECK(metadata != NULL, return);
	CRM_CHECK(parameter != NULL, return);

	xpath = crm_strdup_printf("//" PCMK_XE_PARAMETER "[@" PCMK_XA_NAME "='%s']",
	parameter);
	/* Fudge metadata so that the parameter isn't required in config
	* Pacemaker handles and adds it */
	xpathObj = xpath_search(metadata, xpath);
	if (numXpathResults(xpathObj) > 0) {
	xmlNode *tmp = getXpathResult(xpathObj, 0);

	crm_xml_add(tmp, "required", "0");
	}
	freeXpathObject(xpathObj);
	free(xpath);
	}

	/*!
	* \brief Execute RHCS-compatible agent's metadata action
	*
	* \param[in] agent Agent to execute
	* \param[in] timeout_sec Action timeout
	* \param[out] metadata Where to store output xmlNode (or NULL to ignore)
	*/
	static int
	stonith__rhcs_get_metadata(const char *agent, int timeout_sec,
	xmlNode **metadata)
	{
	xmlNode *xml = NULL;
	xmlNode *actions = NULL;
	xmlXPathObject *xpathObj = NULL;
	stonith_action_t *action = stonith__action_create(agent,
	PCMK_ACTION_METADATA,
	NULL, 0, timeout_sec,
	NULL, NULL, NULL);
	int rc = stonith__execute(action);
	pcmk__action_result_t *result = stonith__action_result(action);

	if (result == NULL) {
	if (rc < 0) {
	crm_warn("Could not execute metadata action for %s: %s "
	CRM_XS " rc=%d", agent, pcmk_strerror(rc), rc);
	}
	stonith__destroy_action(action);
	return rc;
	}

	if (result->execution_status != PCMK_EXEC_DONE) {
	crm_warn("Could not execute metadata action for %s: %s",
	agent, pcmk_exec_status_str(result->execution_status));
	rc = pcmk_rc2legacy(stonith__result2rc(result));
	stonith__destroy_action(action);
	return rc;
	}

	if (!pcmk__result_ok(result)) {
	crm_warn("Metadata action for %s returned error code %d",
	agent, result->exit_status);
	rc = pcmk_rc2legacy(stonith__result2rc(result));
	stonith__destroy_action(action);
	return rc;
	}

	if (result->action_stdout == NULL) {
	crm_warn("Metadata action for %s returned no data", agent);
	stonith__destroy_action(action);
	return -ENODATA;
	}

	- xml = string2xml(result->action_stdout);
	+ xml = pcmk__xml_parse_string(result->action_stdout);
	stonith__destroy_action(action);

	if (xml == NULL) {
	crm_warn("Metadata for %s is invalid", agent);
	return -pcmk_err_schema_validation;
	}

	xpathObj = xpath_search(xml, "//" PCMK_XE_ACTIONS);
	if (numXpathResults(xpathObj) > 0) {
	actions = getXpathResult(xpathObj, 0);
	}
	freeXpathObject(xpathObj);

	// Add start and stop (implemented by pacemaker, not agent) to meta-data
	xpathObj = xpath_search(xml,
	"//" PCMK_XE_ACTION
	"[@" PCMK_XA_NAME "='" PCMK_ACTION_STOP "']");
	if (numXpathResults(xpathObj) <= 0) {
	xmlNode *tmp = NULL;
	const char *timeout_str = NULL;

	timeout_str = pcmk__readable_interval(PCMK_DEFAULT_ACTION_TIMEOUT_MS);

	tmp = create_xml_node(actions, PCMK_XE_ACTION);
	crm_xml_add(tmp, PCMK_XA_NAME, PCMK_ACTION_STOP);
	crm_xml_add(tmp, PCMK_META_TIMEOUT, timeout_str);

	tmp = create_xml_node(actions, PCMK_XE_ACTION);
	crm_xml_add(tmp, PCMK_XA_NAME, PCMK_ACTION_START);
	crm_xml_add(tmp, PCMK_META_TIMEOUT, timeout_str);
	}
	freeXpathObject(xpathObj);

	// Fudge metadata so parameters are not required in config (pacemaker adds them)
	stonith_rhcs_parameter_not_required(xml, "action");
	stonith_rhcs_parameter_not_required(xml, "plug");
	stonith_rhcs_parameter_not_required(xml, "port");

	if (metadata) {
	*metadata = xml;

	} else {
	free_xml(xml);
	}

	return pcmk_ok;
	}

	/*!
	* \brief Retrieve metadata for RHCS-compatible fence agent
	*
	* \param[in] agent Agent to execute
	* \param[in] timeout_sec Action timeout
	* \param[out] output Where to store action output (or NULL to ignore)
	*/
	int
	stonith__rhcs_metadata(const char agent, int timeout_sec, char *output)
	{
	char *buffer = NULL;
	xmlNode *xml = NULL;

	int rc = stonith__rhcs_get_metadata(agent, timeout_sec, &xml);

	if (rc != pcmk_ok) {
	free_xml(xml);
	return rc;
	}

	buffer = dump_xml_formatted_with_text(xml);
	free_xml(xml);
	if (buffer == NULL) {
	return -pcmk_err_schema_validation;
	}
	if (output) {
	*output = buffer;
	} else {
	free(buffer);
	}
	return pcmk_ok;
	}

	bool
	stonith__agent_is_rhcs(const char *agent)
	{
	struct stat prop;
	char *buffer = crm_strdup_printf(PCMK__FENCE_BINDIR "/%s", agent);
	int rc = stat(buffer, &prop);

	free(buffer);
	return (rc >= 0) && S_ISREG(prop.st_mode);
	}

	int
	stonith__rhcs_validate(stonith_t st, int call_options, const char target,
	const char agent, GHashTable params,
	const char * host_arg, int timeout,
	char output, char error_output)
	{
	int rc = pcmk_ok;
	int remaining_timeout = timeout;
	xmlNode *metadata = NULL;
	stonith_action_t *action = NULL;
	pcmk__action_result_t *result = NULL;

	if (host_arg == NULL) {
	time_t start_time = time(NULL);

	rc = stonith__rhcs_get_metadata(agent, remaining_timeout, &metadata);

	if (rc == pcmk_ok) {
	uint32_t device_flags = 0;

	stonith__device_parameter_flags(&device_flags, agent, metadata);
	if (pcmk_is_set(device_flags, st_device_supports_parameter_port)) {
	host_arg = "port";

	} else if (pcmk_is_set(device_flags,
	st_device_supports_parameter_plug)) {
	host_arg = "plug";
	}
	}

	free_xml(metadata);

	remaining_timeout -= time(NULL) - start_time;

	if (rc == -ETIME \|\| remaining_timeout <= 0 ) {
	return -ETIME;
	}

	} else if (pcmk__str_eq(host_arg, PCMK_VALUE_NONE, pcmk__str_casei)) {
	host_arg = NULL;
	}

	action = stonith__action_create(agent, PCMK_ACTION_VALIDATE_ALL, target, 0,
	remaining_timeout, params, NULL, host_arg);

	rc = stonith__execute(action);
	result = stonith__action_result(action);

	if (result != NULL) {
	rc = pcmk_rc2legacy(stonith__result2rc(result));

	// Take ownership of output so stonith__destroy_action() doesn't free it
	if (output != NULL) {
	*output = result->action_stdout;
	result->action_stdout = NULL;
	}
	if (error_output != NULL) {
	*error_output = result->action_stderr;
	result->action_stderr = NULL;
	}
	}
	stonith__destroy_action(action);
	return rc;
	}
	diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
	index 2fdb2a45e2..00c4436b31 100644
	--- a/lib/lrmd/lrmd_client.c
	+++ b/lib/lrmd/lrmd_client.c
	@@ -1,2580 +1,2580 @@
	/*
	* Copyright 2012-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdint.h> // uint32_t, uint64_t
	#include <stdarg.h>
	#include <string.h>
	#include <ctype.h>
	#include <errno.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <glib.h>
	#include <dirent.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/lrmd_internal.h>
	#include <crm/services.h>
	#include <crm/services_internal.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipc_internal.h>
	#include <crm/common/remote_internal.h>
	#include <crm/common/xml.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h> // stonith__*

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# include <gnutls/gnutls.h>
	#endif

	#include <sys/socket.h>
	#include <netinet/in.h>
	#include <netinet/ip.h>
	#include <arpa/inet.h>
	#include <netdb.h>

	#define MAX_TLS_RECV_WAIT 10000

	CRM_TRACE_INIT_DATA(lrmd);

	static int lrmd_api_disconnect(lrmd_t * lrmd);
	static int lrmd_api_is_connected(lrmd_t * lrmd);

	/* IPC proxy functions */
	int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
	static void lrmd_internal_proxy_dispatch(lrmd_t lrmd, xmlNode msg);
	void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg));

	#ifdef HAVE_GNUTLS_GNUTLS_H
	# define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000 /* 5 seconds */
	gnutls_psk_client_credentials_t psk_cred_s;
	static void lrmd_tls_disconnect(lrmd_t * lrmd);
	static int global_remote_msg_id = 0;
	static void lrmd_tls_connection_destroy(gpointer userdata);
	#endif

	typedef struct lrmd_private_s {
	uint64_t type;
	char *token;
	mainloop_io_t *source;

	/* IPC parameters */
	crm_ipc_t *ipc;

	pcmk__remote_t *remote;

	/* Extra TLS parameters */
	char *remote_nodename;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	char *server;
	int port;
	gnutls_psk_client_credentials_t psk_cred_c;

	/* while the async connection is occurring, this is the id
	* of the connection timeout timer. */
	int async_timer;
	int sock;
	/* since tls requires a round trip across the network for a
	* request/reply, there are times where we just want to be able
	* to send a request from the client and not wait around (or even care
	* about) what the reply is. */
	int expected_late_replies;
	GList *pending_notify;
	crm_trigger_t *process_notify;
	#endif

	lrmd_event_callback callback;

	/* Internal IPC proxy msg passing for remote guests */
	void (proxy_callback)(lrmd_t lrmd, void userdata, xmlNode msg);
	void *proxy_callback_userdata;
	char *peer_version;
	} lrmd_private_t;

	static lrmd_list_t *
	lrmd_list_add(lrmd_list_t * head, const char *value)
	{
	lrmd_list_t p, end;

	p = calloc(1, sizeof(lrmd_list_t));
	p->val = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_list_freeall(lrmd_list_t * head)
	{
	lrmd_list_t *p;

	while (head) {
	char val = (char )head->val;

	p = head->next;
	free(val);
	free(head);
	head = p;
	}
	}

	lrmd_key_value_t *
	lrmd_key_value_add(lrmd_key_value_t * head, const char key, const char value)
	{
	lrmd_key_value_t p, end;

	p = calloc(1, sizeof(lrmd_key_value_t));
	p->key = strdup(key);
	p->value = strdup(value);

	end = head;
	while (end && end->next) {
	end = end->next;
	}

	if (end) {
	end->next = p;
	} else {
	head = p;
	}

	return head;
	}

	void
	lrmd_key_value_freeall(lrmd_key_value_t * head)
	{
	lrmd_key_value_t *p;

	while (head) {
	p = head->next;
	free(head->key);
	free(head->value);
	free(head);
	head = p;
	}
	}

	/*!
	* \brief Create a new lrmd_event_data_t object
	*
	* \param[in] rsc_id ID of resource involved in event
	* \param[in] task Action name
	* \param[in] interval_ms Action interval
	*
	* \return Newly allocated and initialized lrmd_event_data_t
	* \note This functions asserts on memory errors, so the return value is
	* guaranteed to be non-NULL. The caller is responsible for freeing the
	* result with lrmd_free_event().
	*/
	lrmd_event_data_t *
	lrmd_new_event(const char rsc_id, const char task, guint interval_ms)
	{
	lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));

	CRM_ASSERT(event != NULL);
	pcmk__str_update((char **) &event->rsc_id, rsc_id);
	pcmk__str_update((char **) &event->op_type, task);
	event->interval_ms = interval_ms;
	return event;
	}

	lrmd_event_data_t *
	lrmd_copy_event(lrmd_event_data_t * event)
	{
	lrmd_event_data_t *copy = NULL;

	copy = calloc(1, sizeof(lrmd_event_data_t));

	copy->type = event->type;
	pcmk__str_update((char **) &copy->rsc_id, event->rsc_id);
	pcmk__str_update((char **) &copy->op_type, event->op_type);
	pcmk__str_update((char **) &copy->user_data, event->user_data);
	copy->call_id = event->call_id;
	copy->timeout = event->timeout;
	copy->interval_ms = event->interval_ms;
	copy->start_delay = event->start_delay;
	copy->rsc_deleted = event->rsc_deleted;
	copy->rc = event->rc;
	copy->op_status = event->op_status;
	pcmk__str_update((char **) &copy->output, event->output);
	copy->t_run = event->t_run;
	copy->t_rcchange = event->t_rcchange;
	copy->exec_time = event->exec_time;
	copy->queue_time = event->queue_time;
	copy->connection_rc = event->connection_rc;
	copy->params = pcmk__str_table_dup(event->params);
	pcmk__str_update((char **) &copy->remote_nodename, event->remote_nodename);
	pcmk__str_update((char **) &copy->exit_reason, event->exit_reason);

	return copy;
	}

	/*!
	* \brief Free an executor event
	*
	* \param[in,out] Executor event object to free
	*/
	void
	lrmd_free_event(lrmd_event_data_t *event)
	{
	if (event == NULL) {
	return;
	}
	// @TODO Why are these const char *?
	free((void *) event->rsc_id);
	free((void *) event->op_type);
	free((void *) event->user_data);
	free((void *) event->remote_nodename);
	lrmd__reset_result(event);
	if (event->params != NULL) {
	g_hash_table_destroy(event->params);
	}
	free(event);
	}

	static void
	lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
	{
	const char *type;
	const char *proxy_session = crm_element_value(msg,
	PCMK__XA_LRMD_IPC_SESSION);
	lrmd_private_t *native = lrmd->lrmd_private;
	lrmd_event_data_t event = { 0, };

	if (proxy_session != NULL) {
	/* this is proxy business */
	lrmd_internal_proxy_dispatch(lrmd, msg);
	return;
	} else if (!native->callback) {
	/* no callback set */
	crm_trace("notify event received but client has not set callback");
	return;
	}

	event.remote_nodename = native->remote_nodename;
	type = crm_element_value(msg, PCMK__XA_LRMD_OP);
	crm_element_value_int(msg, PCMK__XA_LRMD_CALLID, &event.call_id);
	event.rsc_id = crm_element_value(msg, PCMK__XA_LRMD_RSC_ID);

	if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
	event.type = lrmd_event_register;
	} else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
	event.type = lrmd_event_unregister;
	} else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
	time_t epoch = 0;

	crm_element_value_int(msg, PCMK__XA_LRMD_TIMEOUT, &event.timeout);
	crm_element_value_ms(msg, PCMK__XA_LRMD_RSC_INTERVAL,
	&event.interval_ms);
	crm_element_value_int(msg, PCMK__XA_LRMD_RSC_START_DELAY,
	&event.start_delay);
	crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_RC, (int *) &event.rc);
	crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_OP_STATUS,
	&event.op_status);
	crm_element_value_int(msg, PCMK__XA_LRMD_RSC_DELETED,
	&event.rsc_deleted);

	crm_element_value_epoch(msg, PCMK__XA_LRMD_RUN_TIME, &epoch);
	event.t_run = (unsigned int) epoch;

	crm_element_value_epoch(msg, PCMK__XA_LRMD_RCCHANGE_TIME, &epoch);
	event.t_rcchange = (unsigned int) epoch;

	crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_TIME,
	(int *) &event.exec_time);
	crm_element_value_int(msg, PCMK__XA_LRMD_QUEUE_TIME,
	(int *) &event.queue_time);

	event.op_type = crm_element_value(msg, PCMK__XA_LRMD_RSC_ACTION);
	event.user_data = crm_element_value(msg,
	PCMK__XA_LRMD_RSC_USERDATA_STR);
	event.type = lrmd_event_exec_complete;

	/* output and exit_reason may be freed by a callback */
	event.output = crm_element_value_copy(msg, PCMK__XA_LRMD_RSC_OUTPUT);
	lrmd__set_result(&event, event.rc, event.op_status,
	crm_element_value(msg, PCMK__XA_LRMD_RSC_EXIT_REASON));

	event.params = xml2list(msg);
	} else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
	event.type = lrmd_event_new_client;
	} else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
	event.type = lrmd_event_poke;
	} else {
	return;
	}

	crm_trace("op %s notify event received", type);
	native->callback(&event);

	if (event.params) {
	g_hash_table_destroy(event.params);
	}
	lrmd__reset_result(&event);
	}

	// \return Always 0, to indicate that IPC mainloop source should be kept
	static int
	lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->callback != NULL) {
	- xmlNode *msg = string2xml(buffer);
	+ xmlNode *msg = pcmk__xml_parse_string(buffer);

	lrmd_dispatch_internal(lrmd, msg);
	free_xml(msg);
	}
	return 0;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_free_xml(gpointer userdata)
	{
	free_xml((xmlNode *) userdata);
	}

	static bool
	remote_executor_connected(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	return (native->remote->tls_session != NULL);
	}

	/*!
	* \internal
	* \brief TLS dispatch function (for both trigger and file descriptor sources)
	*
	* \param[in,out] userdata API connection
	*
	* \return Always return a nonnegative value, which as a file descriptor
	* dispatch function means keep the mainloop source, and as a
	* trigger dispatch function, 0 means remove the trigger from the
	* mainloop while 1 means keep it (and job completed)
	*/
	static int
	lrmd_tls_dispatch(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *xml = NULL;
	int rc = pcmk_rc_ok;

	if (!remote_executor_connected(lrmd)) {
	crm_trace("TLS dispatch triggered after disconnect");
	return 0;
	}

	crm_trace("TLS dispatch triggered");

	/* First check if there are any pending notifies to process that came
	* while we were waiting for replies earlier. */
	if (native->pending_notify) {
	GList *iter = NULL;

	crm_trace("Processing pending notifies");
	for (iter = native->pending_notify; iter; iter = iter->next) {
	lrmd_dispatch_internal(lrmd, iter->data);
	}
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	/* Next read the current buffer and see if there are any messages to handle. */
	switch (pcmk__remote_ready(native->remote, 0)) {
	case pcmk_rc_ok:
	rc = pcmk__read_remote_message(native->remote, -1);
	xml = pcmk__remote_message_xml(native->remote);
	break;
	case ETIME:
	// Nothing to read, check if a full message is already in buffer
	xml = pcmk__remote_message_xml(native->remote);
	break;
	default:
	rc = ENOTCONN;
	break;
	}
	while (xml) {
	const char *msg_type = crm_element_value(xml,
	PCMK__XA_LRMD_REMOTE_MSG_TYPE);
	if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
	lrmd_dispatch_internal(lrmd, xml);
	} else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
	if (native->expected_late_replies > 0) {
	native->expected_late_replies--;
	} else {
	int reply_id = 0;
	crm_element_value_int(xml, PCMK__XA_LRMD_CALLID, &reply_id);
	/* if this happens, we want to know about it */
	crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
	}
	}
	free_xml(xml);
	xml = pcmk__remote_message_xml(native->remote);
	}

	if (rc == ENOTCONN) {
	crm_info("Lost %s executor connection while reading data",
	(native->remote_nodename? native->remote_nodename : "local"));
	lrmd_tls_disconnect(lrmd);
	return 0;
	}
	return 1;
	}
	#endif

	/* Not used with mainloop */
	int
	lrmd_poll(lrmd_t * lrmd, int timeout)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	return crm_ipc_ready(native->ipc);

	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	if (native->pending_notify) {
	return 1;
	} else {
	int rc = pcmk__remote_ready(native->remote, 0);

	switch (rc) {
	case pcmk_rc_ok:
	return 1;
	case ETIME:
	return 0;
	default:
	return pcmk_rc2legacy(rc);
	}
	}
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	return -EPROTONOSUPPORT;
	}
	}

	/* Not used with mainloop */
	bool
	lrmd_dispatch(lrmd_t * lrmd)
	{
	lrmd_private_t *private = NULL;

	CRM_ASSERT(lrmd != NULL);

	private = lrmd->lrmd_private;
	switch (private->type) {
	case pcmk__client_ipc:
	while (crm_ipc_ready(private->ipc)) {
	if (crm_ipc_read(private->ipc) > 0) {
	const char *msg = crm_ipc_buffer(private->ipc);

	lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
	}
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	lrmd_tls_dispatch(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	private->type);
	}

	if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("Connection closed");
	return FALSE;
	}

	return TRUE;
	}

	static xmlNode *
	lrmd_create_op(const char token, const char op, xmlNode *data, int timeout,
	enum lrmd_call_options options)
	{
	xmlNode *op_msg = create_xml_node(NULL, PCMK__XE_LRMD_COMMAND);

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, PCMK__XA_T, PCMK__VALUE_LRMD);
	crm_xml_add(op_msg, PCMK__XA_LRMD_OP, op);
	crm_xml_add_int(op_msg, PCMK__XA_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(op_msg, PCMK__XA_LRMD_CALLOPT, options);

	if (data != NULL) {
	add_message_xml(op_msg, PCMK__XE_LRMD_CALLDATA, data);
	}

	crm_trace("Created executor %s command with call options %.8lx (%d)",
	op, (long)options, options);
	return op_msg;
	}

	static void
	lrmd_ipc_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	crm_info("Disconnected from local executor");
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	crm_info("Disconnected from remote executor on %s",
	native->remote_nodename);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type %d (bug?)",
	native->type);
	}

	/* Prevent these from being cleaned up in lrmd_api_disconnect() */
	native->ipc = NULL;
	native->source = NULL;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_disconnect;
	event.remote_nodename = native->remote_nodename;
	native->callback(&event);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_connection_destroy(gpointer userdata)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;

	crm_info("TLS connection destroyed");

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	}
	if (native->psk_cred_c) {
	gnutls_psk_free_client_credentials(native->psk_cred_c);
	}
	if (native->sock) {
	close(native->sock);
	}
	if (native->process_notify) {
	mainloop_destroy_trigger(native->process_notify);
	native->process_notify = NULL;
	}
	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}

	free(native->remote->buffer);
	free(native->remote->start_state);
	native->remote->buffer = NULL;
	native->remote->start_state = NULL;
	native->source = 0;
	native->sock = 0;
	native->psk_cred_c = NULL;
	native->remote->tls_session = NULL;
	native->sock = 0;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.remote_nodename = native->remote_nodename;
	event.type = lrmd_event_disconnect;
	native->callback(&event);
	}
	return;
	}

	// \return Standard Pacemaker return code
	int
	lrmd__remote_send_xml(pcmk__remote_t session, xmlNode msg, uint32_t id,
	const char *msg_type)
	{
	crm_xml_add_int(msg, PCMK__XA_LRMD_REMOTE_MSG_ID, id);
	crm_xml_add(msg, PCMK__XA_LRMD_REMOTE_MSG_TYPE, msg_type);
	return pcmk__remote_send_xml(session, msg);
	}

	// \return Standard Pacemaker return code
	static int
	read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
	xmlNode **reply)
	{
	lrmd_private_t *native = lrmd->lrmd_private;
	time_t start = time(NULL);
	const char *msg_type = NULL;
	int reply_id = 0;
	int remaining_timeout = 0;
	int rc = pcmk_rc_ok;

	/* A timeout of 0 here makes no sense. We have to wait a period of time
	* for the response to come back. If -1 or 0, default to 10 seconds. */
	if (total_timeout <= 0 \|\| total_timeout > MAX_TLS_RECV_WAIT) {
	total_timeout = MAX_TLS_RECV_WAIT;
	}

	for (reply = NULL; reply == NULL; ) {

	*reply = pcmk__remote_message_xml(native->remote);
	if (*reply == NULL) {
	/* read some more off the tls buffer if we still have time left. */
	if (remaining_timeout) {
	remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
	} else {
	remaining_timeout = total_timeout;
	}
	if (remaining_timeout <= 0) {
	return ETIME;
	}

	rc = pcmk__read_remote_message(native->remote, remaining_timeout);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	*reply = pcmk__remote_message_xml(native->remote);
	if (*reply == NULL) {
	return ENOMSG;
	}
	}

	crm_element_value_int(*reply, PCMK__XA_LRMD_REMOTE_MSG_ID, &reply_id);
	msg_type = crm_element_value(*reply, PCMK__XA_LRMD_REMOTE_MSG_TYPE);

	if (!msg_type) {
	crm_err("Empty msg type received while waiting for reply");
	free_xml(*reply);
	*reply = NULL;
	} else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
	/* got a notify while waiting for reply, trigger the notify to be processed later */
	crm_info("queueing notify");
	native->pending_notify = g_list_append(native->pending_notify, *reply);
	if (native->process_notify) {
	crm_info("notify trigger set.");
	mainloop_set_trigger(native->process_notify);
	}
	*reply = NULL;
	} else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
	/* msg isn't a reply, make some noise */
	crm_err("Expected a reply, got %s", msg_type);
	free_xml(*reply);
	*reply = NULL;
	} else if (reply_id != expected_reply_id) {
	if (native->expected_late_replies > 0) {
	native->expected_late_replies--;
	} else {
	crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
	}
	free_xml(*reply);
	*reply = NULL;
	}
	}

	if (native->remote->buffer && native->process_notify) {
	mainloop_set_trigger(native->process_notify);
	}

	return rc;
	}

	// \return Standard Pacemaker return code
	static int
	send_remote_message(lrmd_t lrmd, xmlNode msg)
	{
	int rc = pcmk_rc_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	global_remote_msg_id++;
	if (global_remote_msg_id <= 0) {
	global_remote_msg_id = 1;
	}

	rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
	"request");
	if (rc != pcmk_rc_ok) {
	crm_err("Disconnecting because TLS message could not be sent to "
	"Pacemaker Remote: %s", pcmk_rc_str(rc));
	lrmd_tls_disconnect(lrmd);
	}
	return rc;
	}

	static int
	lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
	{
	int rc = 0;
	xmlNode *xml = NULL;

	if (!remote_executor_connected(lrmd)) {
	return -ENOTCONN;
	}

	rc = send_remote_message(lrmd, msg);
	if (rc != pcmk_rc_ok) {
	return pcmk_rc2legacy(rc);
	}

	rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
	if (rc != pcmk_rc_ok) {
	crm_err("Disconnecting remote after request %d reply not received: %s "
	CRM_XS " rc=%d timeout=%dms",
	global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
	lrmd_tls_disconnect(lrmd);
	}

	if (reply) {
	*reply = xml;
	} else {
	free_xml(xml);
	}

	return pcmk_rc2legacy(rc);
	}
	#endif

	static int
	lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	return rc;
	}

	static int
	lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = send_remote_message(lrmd, msg);
	if (rc == pcmk_rc_ok) {
	/* we don't want to wait around for the reply, but
	* since the request/reply protocol needs to behave the same
	* as libqb, a reply will eventually come later anyway. */
	native->expected_late_replies++;
	}
	rc = pcmk_rc2legacy(rc);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	return rc;
	}

	static int
	lrmd_api_is_connected(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	return crm_ipc_connected(native->ipc);
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	return remote_executor_connected(lrmd);
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	return 0;
	}
	}

	/*!
	* \internal
	* \brief Send a prepared API command to the executor
	*
	* \param[in,out] lrmd Existing connection to the executor
	* \param[in] op Name of API command to send
	* \param[in] data Command data XML to add to the sent command
	* \param[out] output_data If expecting a reply, it will be stored here
	* \param[in] timeout Timeout in milliseconds (if 0, defaults to
	* a sensible value per the type of connection,
	* standard vs. pacemaker remote);
	* also propagated to the command XML
	* \param[in] call_options Call options to pass to server when sending
	* \param[in] expect_reply If TRUE, wait for a reply from the server;
	* must be TRUE for IPC (as opposed to TLS) clients
	*
	* \return pcmk_ok on success, -errno on error
	*/
	static int
	lrmd_send_command(lrmd_t lrmd, const char op, xmlNode *data,
	xmlNode **output_data, int timeout,
	enum lrmd_call_options options, gboolean expect_reply)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *op_msg = NULL;
	xmlNode *op_reply = NULL;

	if (!lrmd_api_is_connected(lrmd)) {
	return -ENOTCONN;
	}

	if (op == NULL) {
	crm_err("No operation specified");
	return -EINVAL;
	}

	CRM_CHECK(native->token != NULL,;
	);
	crm_trace("Sending %s op to executor", op);

	op_msg = lrmd_create_op(native->token, op, data, timeout, options);

	if (op_msg == NULL) {
	return -EINVAL;
	}

	if (expect_reply) {
	rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
	} else {
	rc = lrmd_send_xml_no_reply(lrmd, op_msg);
	goto done;
	}

	if (rc < 0) {
	crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
	goto done;

	} else if(op_reply == NULL) {
	rc = -ENOMSG;
	goto done;
	}

	rc = pcmk_ok;
	crm_trace("%s op reply received", op);
	if (crm_element_value_int(op_reply, PCMK__XA_LRMD_RC, &rc) != 0) {
	rc = -ENOMSG;
	goto done;
	}

	crm_log_xml_trace(op_reply, "Reply");

	if (output_data) {
	*output_data = op_reply;
	op_reply = NULL; /* Prevent subsequent free */
	}

	done:
	if (lrmd_api_is_connected(lrmd) == FALSE) {
	crm_err("Executor disconnected");
	}

	free_xml(op_msg);
	free_xml(op_reply);
	return rc;
	}

	static int
	lrmd_api_poke_connection(lrmd_t * lrmd)
	{
	int rc;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *data = create_xml_node(NULL, PCMK__XE_LRMD_RSC);

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
	(native->type == pcmk__client_ipc));
	free_xml(data);

	return rc < 0 ? rc : pcmk_ok;
	}

	// \return Standard Pacemaker return code
	int
	lrmd__validate_remote_settings(lrmd_t lrmd, GHashTable hash)
	{
	int rc = pcmk_rc_ok;
	const char *value;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *data = create_xml_node(NULL, PCMK__XA_LRMD_OP);

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);

	value = g_hash_table_lookup(hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
	if ((value) &&
	(stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
	crm_xml_add(data, PCMK__XA_LRMD_WATCHDOG, value);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
	(native->type == pcmk__client_ipc));
	free_xml(data);
	return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
	}

	static int
	lrmd_handshake(lrmd_t * lrmd, const char *name)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;
	xmlNode *reply = NULL;
	xmlNode *hello = create_xml_node(NULL, PCMK__XE_LRMD_COMMAND);

	crm_xml_add(hello, PCMK__XA_T, PCMK__VALUE_LRMD);
	crm_xml_add(hello, PCMK__XA_LRMD_OP, CRM_OP_REGISTER);
	crm_xml_add(hello, PCMK__XA_LRMD_CLIENTNAME, name);
	crm_xml_add(hello, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);

	/* advertise that we are a proxy provider */
	if (native->proxy_callback) {
	pcmk__xe_set_bool_attr(hello, PCMK__XA_LRMD_IS_IPC_PROVIDER, true);
	}

	rc = lrmd_send_xml(lrmd, hello, -1, &reply);

	if (rc < 0) {
	crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
	rc = -ECOMM;
	} else if (reply == NULL) {
	crm_err("Did not receive registration reply");
	rc = -EPROTO;
	} else {
	const char *version = crm_element_value(reply,
	PCMK__XA_LRMD_PROTOCOL_VERSION);
	const char *msg_type = crm_element_value(reply, PCMK__XA_LRMD_OP);
	const char *tmp_ticket = crm_element_value(reply,
	PCMK__XA_LRMD_CLIENTID);
	const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE);
	long long uptime = -1;

	crm_element_value_int(reply, PCMK__XA_LRMD_RC, &rc);

	/* The remote executor may add its uptime to the XML reply, which is
	* useful in handling transient attributes when the connection to the
	* remote node unexpectedly drops. If no parameter is given, just
	* default to -1.
	*/
	crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
	native->remote->uptime = uptime;

	if (start_state) {
	native->remote->start_state = strdup(start_state);
	}

	if (rc == -EPROTO) {
	crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
	LRMD_PROTOCOL_VERSION, version);
	crm_log_xml_err(reply, "Protocol Error");

	} else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
	crm_err("Invalid registration message: %s", msg_type);
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else if (tmp_ticket == NULL) {
	crm_err("No registration token provided");
	crm_log_xml_err(reply, "Bad reply");
	rc = -EPROTO;
	} else {
	crm_trace("Obtained registration token: %s", tmp_ticket);
	native->token = strdup(tmp_ticket);
	native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
	rc = pcmk_ok;
	}
	}

	free_xml(reply);
	free_xml(hello);

	if (rc != pcmk_ok) {
	lrmd_api_disconnect(lrmd);
	}
	return rc;
	}

	static int
	lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	struct ipc_client_callbacks lrmd_callbacks = {
	.dispatch = lrmd_ipc_dispatch,
	.destroy = lrmd_ipc_connection_destroy
	};

	crm_info("Connecting to executor");

	if (fd) {
	/* No mainloop */
	native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
	if (native->ipc != NULL) {
	rc = pcmk__connect_generic_ipc(native->ipc);
	if (rc == pcmk_rc_ok) {
	rc = pcmk__ipc_fd(native->ipc, fd);
	}
	if (rc != pcmk_rc_ok) {
	crm_err("Connection to executor failed: %s", pcmk_rc_str(rc));
	rc = -ENOTCONN;
	}
	}
	} else {
	native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
	native->ipc = mainloop_get_ipc_client(native->source);
	}

	if (native->ipc == NULL) {
	crm_debug("Could not connect to the executor API");
	rc = -ENOTCONN;
	}

	return rc;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	copy_gnutls_datum(gnutls_datum_t dest, gnutls_datum_t source)
	{
	CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));

	dest->data = gnutls_malloc(source->size);
	CRM_ASSERT(dest->data);

	memcpy(dest->data, source->data, source->size);
	dest->size = source->size;
	}

	static void
	clear_gnutls_datum(gnutls_datum_t *datum)
	{
	gnutls_free(datum->data);
	datum->data = NULL;
	datum->size = 0;
	}

	#define KEY_READ_LEN 256 // Chunk size for reading key from file

	// \return Standard Pacemaker return code
	static int
	read_gnutls_key(const char location, gnutls_datum_t key)
	{
	FILE *stream = NULL;
	size_t buf_len = KEY_READ_LEN;

	if ((location == NULL) \|\| (key == NULL)) {
	return EINVAL;
	}

	stream = fopen(location, "r");
	if (stream == NULL) {
	return errno;
	}

	key->data = gnutls_malloc(buf_len);
	key->size = 0;
	while (!feof(stream)) {
	int next = fgetc(stream);

	if (next == EOF) {
	if (!feof(stream)) {
	crm_warn("Pacemaker Remote key read was partially successful "
	"(copy in memory may be corrupted)");
	}
	break;
	}
	if (key->size == buf_len) {
	buf_len = key->size + KEY_READ_LEN;
	key->data = gnutls_realloc(key->data, buf_len);
	CRM_ASSERT(key->data);
	}
	key->data[key->size++] = (unsigned char) next;
	}
	fclose(stream);

	if (key->size == 0) {
	clear_gnutls_datum(key);
	return ENOKEY;
	}
	return pcmk_rc_ok;
	}

	// Cache the most recently used Pacemaker Remote authentication key

	struct key_cache_s {
	time_t updated; // When cached key was read (valid for 1 minute)
	const char *location; // Where cached key was read from
	gnutls_datum_t key; // Cached key
	};

	static bool
	key_is_cached(struct key_cache_s *key_cache)
	{
	return key_cache->updated != 0;
	}

	static bool
	key_cache_expired(struct key_cache_s *key_cache)
	{
	return (time(NULL) - key_cache->updated) >= 60;
	}

	static void
	clear_key_cache(struct key_cache_s *key_cache)
	{
	clear_gnutls_datum(&(key_cache->key));
	if ((key_cache->updated != 0) \|\| (key_cache->location != NULL)) {
	key_cache->updated = 0;
	key_cache->location = NULL;
	crm_debug("Cleared Pacemaker Remote key cache");
	}
	}

	static void
	get_cached_key(struct key_cache_s key_cache, gnutls_datum_t key)
	{
	copy_gnutls_datum(key, &(key_cache->key));
	crm_debug("Using cached Pacemaker Remote key from %s",
	pcmk__s(key_cache->location, "unknown location"));
	}

	static void
	cache_key(struct key_cache_s key_cache, gnutls_datum_t key,
	const char *location)
	{
	key_cache->updated = time(NULL);
	key_cache->location = location;
	copy_gnutls_datum(&(key_cache->key), key);
	crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
	pcmk__s(location, "unknown location"));
	}

	/*!
	* \internal
	* \brief Get Pacemaker Remote authentication key from file or cache
	*
	* \param[in] location Path to key file to try (this memory must
	* persist across all calls of this function)
	* \param[out] key Key from location or cache
	*
	* \return Standard Pacemaker return code
	*/
	static int
	get_remote_key(const char location, gnutls_datum_t key)
	{
	static struct key_cache_s key_cache = { 0, };
	int rc = pcmk_rc_ok;

	if ((location == NULL) \|\| (key == NULL)) {
	return EINVAL;
	}

	if (key_is_cached(&key_cache)) {
	if (key_cache_expired(&key_cache)) {
	clear_key_cache(&key_cache);
	} else {
	get_cached_key(&key_cache, key);
	return pcmk_rc_ok;
	}
	}

	rc = read_gnutls_key(location, key);
	if (rc != pcmk_rc_ok) {
	return rc;
	}
	cache_key(&key_cache, key, location);
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Initialize the Pacemaker Remote authentication key
	*
	* Try loading the Pacemaker Remote authentication key from cache if available,
	* otherwise from these locations, in order of preference: the value of the
	* PCMK_authkey_location environment variable, if set; the Pacemaker default key
	* file location; or (for historical reasons) /etc/corosync/authkey.
	*
	* \param[out] key Where to store key
	*
	* \return Standard Pacemaker return code
	*/
	int
	lrmd__init_remote_key(gnutls_datum_t *key)
	{
	static const char *env_location = NULL;
	static bool need_env = true;

	int env_rc = pcmk_rc_ok;
	int default_rc = pcmk_rc_ok;
	int alt_rc = pcmk_rc_ok;

	bool env_is_default = false;
	bool env_is_fallback = false;

	if (need_env) {
	env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
	need_env = false;
	}

	// Try location in environment variable, if set
	if (env_location != NULL) {
	env_rc = get_remote_key(env_location, key);
	if (env_rc == pcmk_rc_ok) {
	return pcmk_rc_ok;
	}

	env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
	env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);

	/* @TODO It would be more secure to fail, rather than fall back to the
	* default, if an explicitly set key location is not readable, and it
	* would be better to never use the Corosync location as a fallback.
	* However, that would break any deployments currently working with the
	* fallbacks.
	*/
	}

	// Try default location, if environment wasn't explicitly set to it
	if (env_is_default) {
	default_rc = env_rc;
	} else {
	default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
	}

	// Try fallback location, if environment wasn't set to it and default failed
	if (env_is_fallback) {
	alt_rc = env_rc;
	} else if (default_rc != pcmk_rc_ok) {
	alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
	}

	// We have all results, so log and return

	if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
	&& (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed

	crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
	env_location,
	env_is_default? "" : "or default location ",
	env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
	!env_is_default && !env_is_fallback? " " : "",
	env_is_fallback? "" : "or fallback location ",
	env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
	pcmk_rc_str(env_rc));
	return ENOKEY;
	}

	if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
	crm_warn("Could not read Pacemaker Remote key from %s "
	"(using %s location %s instead): %s",
	env_location,
	(default_rc == pcmk_rc_ok)? "default" : "fallback",
	(default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
	pcmk_rc_str(env_rc));
	return pcmk_rc_ok;
	}

	if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
	// Environment unset, defaults failed
	crm_warn("Could not read Pacemaker Remote key from default location %s"
	" (or fallback location %s): %s",
	DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
	pcmk_rc_str(default_rc));
	return ENOKEY;
	}

	return pcmk_rc_ok; // Environment variable unset, a default worked
	}

	static void
	lrmd_gnutls_global_init(void)
	{
	static int gnutls_init = 0;

	if (!gnutls_init) {
	crm_gnutls_global_init();
	}
	gnutls_init = 1;
	}
	#endif

	static void
	report_async_connection_result(lrmd_t * lrmd, int rc)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->callback) {
	lrmd_event_data_t event = { 0, };
	event.type = lrmd_event_connect;
	event.remote_nodename = native->remote_nodename;
	event.connection_rc = rc;
	native->callback(&event);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static inline int
	lrmd__tls_client_handshake(pcmk__remote_t *remote)
	{
	return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
	}

	/*!
	* \internal
	* \brief Add trigger and file descriptor mainloop sources for TLS
	*
	* \param[in,out] lrmd API connection with established TLS session
	* \param[in] do_handshake Whether to perform executor handshake
	*
	* \return Standard Pacemaker return code
	*/
	static int
	add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
	{
	lrmd_private_t *native = lrmd->lrmd_private;
	int rc = pcmk_rc_ok;

	char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
	native->server, native->port);

	struct mainloop_fd_callbacks tls_fd_callbacks = {
	.dispatch = lrmd_tls_dispatch,
	.destroy = lrmd_tls_connection_destroy,
	};

	native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
	lrmd_tls_dispatch, lrmd);
	native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
	&tls_fd_callbacks);

	/* Async connections lose the client name provided by the API caller, so we
	* have to use our generated name here to perform the executor handshake.
	*
	* @TODO Keep track of the caller-provided name. Perhaps we should be using
	* that name in this function instead of generating one anyway.
	*/
	if (do_handshake) {
	rc = lrmd_handshake(lrmd, name);
	rc = pcmk_legacy2rc(rc);
	}
	free(name);
	return rc;
	}

	static void
	lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
	{
	lrmd_t *lrmd = userdata;
	lrmd_private_t *native = lrmd->lrmd_private;
	gnutls_datum_t psk_key = { NULL, 0 };

	native->async_timer = 0;

	if (rc != pcmk_rc_ok) {
	lrmd_tls_connection_destroy(lrmd);
	crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
	return;
	}

	/* The TCP connection was successful, so establish the TLS connection.
	* @TODO make this async to avoid blocking code in client
	*/

	native->sock = sock;

	rc = lrmd__init_remote_key(&psk_key);
	if (rc != pcmk_rc_ok) {
	crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	lrmd_tls_connection_destroy(lrmd);
	report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
	return;
	}

	gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
	gnutls_free(psk_key.data);

	native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
	GNUTLS_CRD_PSK,
	native->psk_cred_c);
	if (native->remote->tls_session == NULL) {
	lrmd_tls_connection_destroy(lrmd);
	report_async_connection_result(lrmd, -EPROTO);
	return;
	}

	if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
	crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
	native->server, native->port);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = NULL;
	lrmd_tls_connection_destroy(lrmd);
	report_async_connection_result(lrmd, -EKEYREJECTED);
	return;
	}

	crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
	native->server, native->port);
	rc = add_tls_to_mainloop(lrmd, true);
	report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
	}

	static int
	lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /ms / )
	{
	int rc;
	int timer_id = 0;
	lrmd_private_t *native = lrmd->lrmd_private;

	lrmd_gnutls_global_init();
	native->sock = -1;
	rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
	&(native->sock), lrmd, lrmd_tcp_connect_cb);
	if (rc != pcmk_rc_ok) {
	crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	return pcmk_rc2legacy(rc);
	}
	native->async_timer = timer_id;
	return pcmk_ok;
	}

	static int
	lrmd_tls_connect(lrmd_t * lrmd, int *fd)
	{
	int rc;

	lrmd_private_t *native = lrmd->lrmd_private;
	gnutls_datum_t psk_key = { NULL, 0 };

	lrmd_gnutls_global_init();

	native->sock = -1;
	rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
	&(native->sock), NULL, NULL);
	if (rc != pcmk_rc_ok) {
	crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
	CRM_XS " rc=%d",
	native->server, native->port, pcmk_rc_str(rc), rc);
	lrmd_tls_connection_destroy(lrmd);
	return -ENOTCONN;
	}

	rc = lrmd__init_remote_key(&psk_key);
	if (rc != pcmk_rc_ok) {
	lrmd_tls_connection_destroy(lrmd);
	return pcmk_rc2legacy(rc);
	}

	gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
	gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
	gnutls_free(psk_key.data);

	native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
	GNUTLS_CRD_PSK,
	native->psk_cred_c);
	if (native->remote->tls_session == NULL) {
	lrmd_tls_connection_destroy(lrmd);
	return -EPROTO;
	}

	if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
	crm_err("Session creation for %s:%d failed", native->server, native->port);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = NULL;
	lrmd_tls_connection_destroy(lrmd);
	return -EKEYREJECTED;
	}

	crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
	native->port);

	if (fd) {
	*fd = native->sock;
	} else {
	add_tls_to_mainloop(lrmd, false);
	}
	return pcmk_ok;
	}
	#endif

	static int
	lrmd_api_connect(lrmd_t * lrmd, const char name, int fd)
	{
	int rc = -ENOTCONN;
	lrmd_private_t *native = lrmd->lrmd_private;

	switch (native->type) {
	case pcmk__client_ipc:
	rc = lrmd_ipc_connect(lrmd, fd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = lrmd_tls_connect(lrmd, fd);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	if (rc == pcmk_ok) {
	rc = lrmd_handshake(lrmd, name);
	}

	return rc;
	}

	static int
	lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
	{
	int rc = pcmk_ok;
	lrmd_private_t *native = lrmd->lrmd_private;

	CRM_CHECK(native && native->callback, return -EINVAL);

	switch (native->type) {
	case pcmk__client_ipc:
	/* fake async connection with ipc. it should be fast
	* enough that we gain very little from async */
	rc = lrmd_api_connect(lrmd, name, NULL);
	if (!rc) {
	report_async_connection_result(lrmd, rc);
	}
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	rc = lrmd_tls_connect_async(lrmd, timeout);
	if (rc) {
	/* connection failed, report rc now */
	report_async_connection_result(lrmd, rc);
	}
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	return rc;
	}

	static void
	lrmd_ipc_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;
	native->ipc = NULL;

	} else if (native->ipc) {
	/* Not attached to mainloop */
	crm_ipc_t *ipc = native->ipc;

	native->ipc = NULL;
	crm_ipc_close(ipc);
	crm_ipc_destroy(ipc);
	}
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	static void
	lrmd_tls_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->remote->tls_session) {
	gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
	gnutls_deinit(*native->remote->tls_session);
	gnutls_free(native->remote->tls_session);
	native->remote->tls_session = 0;
	}

	if (native->async_timer) {
	g_source_remove(native->async_timer);
	native->async_timer = 0;
	}

	if (native->source != NULL) {
	/* Attached to mainloop */
	mainloop_del_ipc_client(native->source);
	native->source = NULL;

	} else if (native->sock) {
	close(native->sock);
	native->sock = 0;
	}

	if (native->pending_notify) {
	g_list_free_full(native->pending_notify, lrmd_free_xml);
	native->pending_notify = NULL;
	}
	}
	#endif

	static int
	lrmd_api_disconnect(lrmd_t * lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;
	int rc = pcmk_ok;

	switch (native->type) {
	case pcmk__client_ipc:
	crm_debug("Disconnecting from local executor");
	lrmd_ipc_disconnect(lrmd);
	break;
	#ifdef HAVE_GNUTLS_GNUTLS_H
	case pcmk__client_tls:
	crm_debug("Disconnecting from remote executor on %s",
	native->remote_nodename);
	lrmd_tls_disconnect(lrmd);
	break;
	#endif
	default:
	crm_err("Unsupported executor connection type (bug?): %d",
	native->type);
	rc = -EPROTONOSUPPORT;
	}

	free(native->token);
	native->token = NULL;

	free(native->peer_version);
	native->peer_version = NULL;
	return rc;
	}

	static int
	lrmd_api_register_rsc(lrmd_t * lrmd,
	const char *rsc_id,
	const char *class,
	const char provider, const char type, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = NULL;

	if (!class \|\| !type \|\| !rsc_id) {
	return -EINVAL;
	}
	if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
	&& (provider == NULL)) {
	return -EINVAL;
	}

	data = create_xml_node(NULL, PCMK__XE_LRMD_RSC);

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, PCMK__XA_LRMD_CLASS, class);
	crm_xml_add(data, PCMK__XA_LRMD_PROVIDER, provider);
	crm_xml_add(data, PCMK__XA_LRMD_TYPE, type);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	static int
	lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, PCMK__XE_LRMD_RSC);

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
	free_xml(data);

	return rc;
	}

	lrmd_rsc_info_t *
	lrmd_new_rsc_info(const char rsc_id, const char standard,
	const char provider, const char type)
	{
	lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));

	CRM_ASSERT(rsc_info);
	pcmk__str_update(&rsc_info->id, rsc_id);
	pcmk__str_update(&rsc_info->standard, standard);
	pcmk__str_update(&rsc_info->provider, provider);
	pcmk__str_update(&rsc_info->type, type);
	return rsc_info;
	}

	lrmd_rsc_info_t *
	lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
	rsc_info->provider, rsc_info->type);
	}

	void
	lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
	{
	if (!rsc_info) {
	return;
	}
	free(rsc_info->id);
	free(rsc_info->type);
	free(rsc_info->standard);
	free(rsc_info->provider);
	free(rsc_info);
	}

	static lrmd_rsc_info_t *
	lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
	{
	lrmd_rsc_info_t *rsc_info = NULL;
	xmlNode *data = create_xml_node(NULL, PCMK__XE_LRMD_RSC);
	xmlNode *output = NULL;
	const char *class = NULL;
	const char *provider = NULL;
	const char *type = NULL;

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
	lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
	free_xml(data);

	if (!output) {
	return NULL;
	}

	class = crm_element_value(output, PCMK__XA_LRMD_CLASS);
	provider = crm_element_value(output, PCMK__XA_LRMD_PROVIDER);
	type = crm_element_value(output, PCMK__XA_LRMD_TYPE);

	if (!class \|\| !type) {
	free_xml(output);
	return NULL;
	} else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
	&& !provider) {
	free_xml(output);
	return NULL;
	}

	rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
	free_xml(output);
	return rsc_info;
	}

	void
	lrmd_free_op_info(lrmd_op_info_t *op_info)
	{
	if (op_info) {
	free(op_info->rsc_id);
	free(op_info->action);
	free(op_info->interval_ms_s);
	free(op_info->timeout_ms_s);
	free(op_info);
	}
	}

	static int
	lrmd_api_get_recurring_ops(lrmd_t lrmd, const char rsc_id, int timeout_ms,
	enum lrmd_call_options options, GList **output)
	{
	xmlNode *data = NULL;
	xmlNode *output_xml = NULL;
	int rc = pcmk_ok;

	if (output == NULL) {
	return -EINVAL;
	}
	*output = NULL;

	// Send request
	if (rsc_id) {
	data = create_xml_node(NULL, PCMK__XE_LRMD_RSC);
	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
	}
	rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
	timeout_ms, options, TRUE);
	if (data) {
	free_xml(data);
	}

	// Process reply
	if ((rc != pcmk_ok) \|\| (output_xml == NULL)) {
	return rc;
	}
	for (xmlNode *rsc_xml = first_named_child(output_xml, PCMK__XE_LRMD_RSC);
	(rsc_xml != NULL) && (rc == pcmk_ok);
	rsc_xml = crm_next_same_xml(rsc_xml)) {

	rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
	if (rsc_id == NULL) {
	crm_err("Could not parse recurring operation information from executor");
	continue;
	}
	for (xmlNode *op_xml = first_named_child(rsc_xml, PCMK__XE_LRMD_RSC_OP);
	op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {

	lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));

	if (op_info == NULL) {
	rc = -ENOMEM;
	break;
	}
	op_info->rsc_id = strdup(rsc_id);
	op_info->action = crm_element_value_copy(op_xml,
	PCMK__XA_LRMD_RSC_ACTION);
	op_info->interval_ms_s =
	crm_element_value_copy(op_xml, PCMK__XA_LRMD_RSC_INTERVAL);
	op_info->timeout_ms_s =
	crm_element_value_copy(op_xml, PCMK__XA_LRMD_TIMEOUT);
	output = g_list_prepend(output, op_info);
	}
	}
	free_xml(output_xml);
	return rc;
	}


	static void
	lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	native->callback = callback;
	}

	void
	lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void userdata, void (callback)(lrmd_t lrmd, void userdata, xmlNode *msg))
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	native->proxy_callback = callback;
	native->proxy_callback_userdata = userdata;
	}

	void
	lrmd_internal_proxy_dispatch(lrmd_t lrmd, xmlNode msg)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->proxy_callback) {
	crm_log_xml_trace(msg, "PROXY_INBOUND");
	native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
	}
	}

	int
	lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
	{
	if (lrmd == NULL) {
	return -ENOTCONN;
	}
	crm_xml_add(msg, PCMK__XA_LRMD_OP, CRM_OP_IPC_FWD);

	crm_log_xml_trace(msg, "PROXY_OUTBOUND");
	return lrmd_send_xml_no_reply(lrmd, msg);
	}

	static int
	stonith_get_metadata(const char provider, const char type, char **output)
	{
	int rc = pcmk_ok;
	stonith_t *stonith_api = stonith_api_new();

	if (stonith_api == NULL) {
	crm_err("Could not get fence agent meta-data: API memory allocation failed");
	return -ENOMEM;
	}

	rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
	provider, output, 0);
	if ((rc == pcmk_ok) && (*output == NULL)) {
	rc = -EIO;
	}
	stonith_api->cmds->free(stonith_api);
	return rc;
	}

	static int
	lrmd_api_get_metadata(lrmd_t lrmd, const char standard, const char *provider,
	const char type, char *output,
	enum lrmd_call_options options)
	{
	return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
	output, options, NULL);
	}

	static int
	lrmd_api_get_metadata_params(lrmd_t lrmd, const char standard,
	const char provider, const char type,
	char **output, enum lrmd_call_options options,
	lrmd_key_value_t *params)
	{
	svc_action_t *action = NULL;
	GHashTable *params_table = NULL;

	if (!standard \|\| !type) {
	lrmd_key_value_freeall(params);
	return -EINVAL;
	}

	if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	lrmd_key_value_freeall(params);
	return stonith_get_metadata(provider, type, output);
	}

	params_table = pcmk__strkey_table(free, free);
	for (const lrmd_key_value_t *param = params; param; param = param->next) {
	pcmk__insert_dup(params_table, param->key, param->value);
	}
	action = services__create_resource_action(type, standard, provider, type,
	PCMK_ACTION_META_DATA, 0,
	PCMK_DEFAULT_METADATA_TIMEOUT_MS,
	params_table, 0);
	lrmd_key_value_freeall(params);

	if (action == NULL) {
	return -ENOMEM;
	}
	if (action->rc != PCMK_OCF_UNKNOWN) {
	services_action_free(action);
	return -EINVAL;
	}

	if (!services_action_sync(action)) {
	crm_err("Failed to retrieve meta-data for %s:%s:%s",
	standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	if (!action->stdout_data) {
	crm_err("Failed to receive meta-data for %s:%s:%s",
	standard, provider, type);
	services_action_free(action);
	return -EIO;
	}

	*output = strdup(action->stdout_data);
	services_action_free(action);

	return pcmk_ok;
	}

	static int
	lrmd_api_exec(lrmd_t lrmd, const char rsc_id, const char *action,
	const char *userdata, guint interval_ms,
	int timeout, /* ms */
	int start_delay, /* ms */
	enum lrmd_call_options options, lrmd_key_value_t * params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, PCMK__XE_LRMD_RSC);
	xmlNode *args = create_xml_node(data, PCMK__XE_ATTRIBUTES);
	lrmd_key_value_t *tmp = NULL;

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_USERDATA_STR, userdata);
	crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
	crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
	crm_xml_add_int(data, PCMK__XA_LRMD_RSC_START_DELAY, start_delay);

	for (tmp = params; tmp; tmp = tmp->next) {
	hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	/* timeout is in ms */
	static int
	lrmd_api_exec_alert(lrmd_t lrmd, const char alert_id, const char *alert_path,
	int timeout, lrmd_key_value_t *params)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, PCMK__XE_LRMD_ALERT);
	xmlNode *args = create_xml_node(data, PCMK__XE_ATTRIBUTES);
	lrmd_key_value_t *tmp = NULL;

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_LRMD_ALERT_ID, alert_id);
	crm_xml_add(data, PCMK__XA_LRMD_ALERT_PATH, alert_path);
	crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);

	for (tmp = params; tmp; tmp = tmp->next) {
	hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
	}

	rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
	lrmd_opt_notify_orig_only, TRUE);
	free_xml(data);

	lrmd_key_value_freeall(params);
	return rc;
	}

	static int
	lrmd_api_cancel(lrmd_t lrmd, const char rsc_id, const char *action,
	guint interval_ms)
	{
	int rc = pcmk_ok;
	xmlNode *data = create_xml_node(NULL, PCMK__XE_LRMD_RSC);

	crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
	crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
	crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
	rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
	free_xml(data);
	return rc;
	}

	static int
	list_stonith_agents(lrmd_list_t ** resources)
	{
	int rc = 0;
	stonith_t *stonith_api = stonith_api_new();
	stonith_key_value_t *stonith_resources = NULL;
	stonith_key_value_t *dIter = NULL;

	if (stonith_api == NULL) {
	crm_err("Could not list fence agents: API memory allocation failed");
	return -ENOMEM;
	}
	stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
	&stonith_resources, 0);
	stonith_api->cmds->free(stonith_api);

	for (dIter = stonith_resources; dIter; dIter = dIter->next) {
	rc++;
	if (resources) {
	resources = lrmd_list_add(resources, dIter->value);
	}
	}

	stonith_key_value_freeall(stonith_resources, 1, 0);
	return rc;
	}

	static int
	lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
	const char *provider)
	{
	int rc = 0;
	int stonith_count = 0; // Initially, whether to include stonith devices

	if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	stonith_count = 1;

	} else {
	GList *gIter = NULL;
	GList *agents = resources_list_agents(class, provider);

	for (gIter = agents; gIter != NULL; gIter = gIter->next) {
	resources = lrmd_list_add(resources, (const char *)gIter->data);
	rc++;
	}
	g_list_free_full(agents, free);

	if (!class) {
	stonith_count = 1;
	}
	}

	if (stonith_count) {
	// Now, if stonith devices are included, how many there are
	stonith_count = list_stonith_agents(resources);
	if (stonith_count > 0) {
	rc += stonith_count;
	}
	}
	if (rc == 0) {
	crm_notice("No agents found for class %s", class);
	rc = -EPROTONOSUPPORT;
	}
	return rc;
	}

	static bool
	does_provider_have_agent(const char agent, const char provider, const char *class)
	{
	bool found = false;
	GList *agents = NULL;
	GList *gIter2 = NULL;

	agents = resources_list_agents(class, provider);
	for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
	if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
	found = true;
	}
	}
	g_list_free_full(agents, free);
	return found;
	}

	static int
	lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char agent, lrmd_list_t * providers)
	{
	int rc = pcmk_ok;
	char *provider = NULL;
	GList *ocf_providers = NULL;
	GList *gIter = NULL;

	ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);

	for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
	provider = gIter->data;
	if (!agent \|\| does_provider_have_agent(agent, provider,
	PCMK_RESOURCE_CLASS_OCF)) {
	providers = lrmd_list_add(providers, (const char *)gIter->data);
	rc++;
	}
	}

	g_list_free_full(ocf_providers, free);
	return rc;
	}

	static int
	lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
	{
	int rc = 0;
	GList *standards = NULL;
	GList *gIter = NULL;

	standards = resources_list_standards();

	for (gIter = standards; gIter != NULL; gIter = gIter->next) {
	supported = lrmd_list_add(supported, (const char *)gIter->data);
	rc++;
	}

	if (list_stonith_agents(NULL) > 0) {
	supported = lrmd_list_add(supported, PCMK_RESOURCE_CLASS_STONITH);
	rc++;
	}

	g_list_free_full(standards, free);
	return rc;
	}

	/*!
	* \internal
	* \brief Create an executor API object
	*
	* \param[out] api Will be set to newly created API object (it is the
	* caller's responsibility to free this value with
	* lrmd_api_delete() if this function succeeds)
	* \param[in] nodename If the object will be used for a remote connection,
	* the node name to use in cluster for remote executor
	* \param[in] server If the object will be used for a remote connection,
	* the resolvable host name to connect to
	* \param[in] port If the object will be used for a remote connection,
	* port number on \p server to connect to
	*
	* \return Standard Pacemaker return code
	* \note If the caller leaves one of \p nodename or \p server NULL, the other's
	* value will be used for both. If the caller leaves both NULL, an API
	* object will be created for a local executor connection.
	*/
	int
	lrmd__new(lrmd_t *api, const char nodename, const char *server, int port)
	{
	lrmd_private_t *pvt = NULL;

	if (api == NULL) {
	return EINVAL;
	}
	*api = NULL;

	// Allocate all memory needed

	*api = calloc(1, sizeof(lrmd_t));
	if (*api == NULL) {
	return ENOMEM;
	}

	pvt = calloc(1, sizeof(lrmd_private_t));
	if (pvt == NULL) {
	lrmd_api_delete(*api);
	*api = NULL;
	return ENOMEM;
	}
	(*api)->lrmd_private = pvt;

	// @TODO Do we need to do this for local connections?
	pvt->remote = calloc(1, sizeof(pcmk__remote_t));

	(*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));

	if ((pvt->remote == NULL) \|\| ((*api)->cmds == NULL)) {
	lrmd_api_delete(*api);
	*api = NULL;
	return ENOMEM;
	}

	// Set methods
	(*api)->cmds->connect = lrmd_api_connect;
	(*api)->cmds->connect_async = lrmd_api_connect_async;
	(*api)->cmds->is_connected = lrmd_api_is_connected;
	(*api)->cmds->poke_connection = lrmd_api_poke_connection;
	(*api)->cmds->disconnect = lrmd_api_disconnect;
	(*api)->cmds->register_rsc = lrmd_api_register_rsc;
	(*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
	(*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
	(*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
	(*api)->cmds->set_callback = lrmd_api_set_callback;
	(*api)->cmds->get_metadata = lrmd_api_get_metadata;
	(*api)->cmds->exec = lrmd_api_exec;
	(*api)->cmds->cancel = lrmd_api_cancel;
	(*api)->cmds->list_agents = lrmd_api_list_agents;
	(*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
	(*api)->cmds->list_standards = lrmd_api_list_standards;
	(*api)->cmds->exec_alert = lrmd_api_exec_alert;
	(*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;

	if ((nodename == NULL) && (server == NULL)) {
	pvt->type = pcmk__client_ipc;
	} else {
	#ifdef HAVE_GNUTLS_GNUTLS_H
	if (nodename == NULL) {
	nodename = server;
	} else if (server == NULL) {
	server = nodename;
	}
	pvt->type = pcmk__client_tls;
	pvt->remote_nodename = strdup(nodename);
	pvt->server = strdup(server);
	if ((pvt->remote_nodename == NULL) \|\| (pvt->server == NULL)) {
	lrmd_api_delete(*api);
	*api = NULL;
	return ENOMEM;
	}
	pvt->port = port;
	if (pvt->port == 0) {
	pvt->port = crm_default_remote_port();
	}
	#else
	crm_err("Cannot communicate with Pacemaker Remote "
	"because GnuTLS is not enabled for this build");
	lrmd_api_delete(*api);
	*api = NULL;
	return EOPNOTSUPP;
	#endif
	}
	return pcmk_rc_ok;
	}

	lrmd_t *
	lrmd_api_new(void)
	{
	lrmd_t *api = NULL;

	CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
	return api;
	}

	lrmd_t *
	lrmd_remote_api_new(const char nodename, const char server, int port)
	{
	lrmd_t *api = NULL;

	CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
	return api;
	}

	void
	lrmd_api_delete(lrmd_t * lrmd)
	{
	if (lrmd == NULL) {
	return;
	}
	if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
	if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
	lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
	}
	free(lrmd->cmds);
	}
	if (lrmd->lrmd_private != NULL) {
	lrmd_private_t *native = lrmd->lrmd_private;

	#ifdef HAVE_GNUTLS_GNUTLS_H
	free(native->server);
	#endif
	free(native->remote_nodename);
	free(native->remote);
	free(native->token);
	free(native->peer_version);
	free(lrmd->lrmd_private);
	}
	free(lrmd);
	}

	struct metadata_cb {
	void (callback)(int pid, const pcmk__action_result_t result,
	void *user_data);
	void *user_data;
	};

	/*!
	* \internal
	* \brief Process asynchronous metadata completion
	*
	* \param[in,out] action Metadata action that completed
	*/
	static void
	metadata_complete(svc_action_t *action)
	{
	struct metadata_cb metadata_cb = (struct metadata_cb ) action->cb_data;
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	pcmk__set_result(&result, action->rc, action->status,
	services__exit_reason(action));
	pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);

	metadata_cb->callback(0, &result, metadata_cb->user_data);
	result.action_stdout = NULL; // Prevent free, because action owns it
	result.action_stderr = NULL; // Prevent free, because action owns it
	pcmk__reset_result(&result);
	free(metadata_cb);
	}

	/*!
	* \internal
	* \brief Retrieve agent metadata asynchronously
	*
	* \param[in] rsc Resource agent specification
	* \param[in] callback Function to call with result (this will always be
	* called, whether by this function directly or later
	* via the main loop, and on success the metadata will
	* be in its result argument's action_stdout)
	* \param[in,out] user_data User data to pass to callback
	*
	* \return Standard Pacemaker return code
	* \note This function is not a lrmd_api_operations_t method because it does not
	* need an lrmd_t object and does not go through the executor, but
	* executes the agent directly.
	*/
	int
	lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
	void (*callback)(int pid,
	const pcmk__action_result_t *result,
	void *user_data),
	void *user_data)
	{
	svc_action_t *action = NULL;
	struct metadata_cb *metadata_cb = NULL;
	pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;

	CRM_CHECK(callback != NULL, return EINVAL);

	if ((rsc == NULL) \|\| (rsc->standard == NULL) \|\| (rsc->type == NULL)) {
	pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
	PCMK_EXEC_ERROR_FATAL,
	"Invalid resource specification");
	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	return EINVAL;
	}

	if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
	return stonith__metadata_async(rsc->type,
	PCMK_DEFAULT_METADATA_TIMEOUT_MS / 1000,
	callback, user_data);
	}

	action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
	rsc->standard, rsc->provider,
	rsc->type,
	PCMK_ACTION_META_DATA, 0,
	PCMK_DEFAULT_METADATA_TIMEOUT_MS,
	NULL, 0);
	if (action == NULL) {
	pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Out of memory");
	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	return ENOMEM;
	}
	if (action->rc != PCMK_OCF_UNKNOWN) {
	pcmk__set_result(&result, action->rc, action->status,
	services__exit_reason(action));
	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	services_action_free(action);
	return EINVAL;
	}

	action->cb_data = calloc(1, sizeof(struct metadata_cb));
	if (action->cb_data == NULL) {
	services_action_free(action);
	pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
	"Out of memory");
	callback(0, &result, user_data);
	pcmk__reset_result(&result);
	return ENOMEM;
	}

	metadata_cb = (struct metadata_cb *) action->cb_data;
	metadata_cb->callback = callback;
	metadata_cb->user_data = user_data;
	if (!services_action_async(action, metadata_complete)) {
	services_action_free(action);
	return pcmk_rc_error; // @TODO Derive from action->rc and ->status
	}

	// The services library has taken responsibility for action
	return pcmk_rc_ok;
	}

	/*!
	* \internal
	* \brief Set the result of an executor event
	*
	* \param[in,out] event Executor event to set
	* \param[in] rc OCF exit status of event
	* \param[in] op_status Executor status of event
	* \param[in] exit_reason Human-friendly description of event
	*/
	void
	lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
	const char *exit_reason)
	{
	if (event == NULL) {
	return;
	}

	event->rc = rc;
	event->op_status = op_status;
	pcmk__str_update((char **) &event->exit_reason, exit_reason);
	}

	/*!
	* \internal
	* \brief Clear an executor event's exit reason, output, and error output
	*
	* \param[in,out] event Executor event to reset
	*/
	void
	lrmd__reset_result(lrmd_event_data_t *event)
	{
	if (event == NULL) {
	return;
	}

	free((void *) event->exit_reason);
	event->exit_reason = NULL;

	free((void *) event->output);
	event->output = NULL;
	}

	/*!
	* \internal
	* \brief Get the uptime of a remote resource connection
	*
	* When the cluster connects to a remote resource, part of that resource's
	* handshake includes the uptime of the remote resource's connection. This
	* uptime is stored in the lrmd_t object.
	*
	* \return The connection's uptime, or -1 if unknown
	*/
	time_t
	lrmd__uptime(lrmd_t *lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->remote == NULL) {
	return -1;
	} else {
	return native->remote->uptime;
	}
	}

	const char *
	lrmd__node_start_state(lrmd_t *lrmd)
	{
	lrmd_private_t *native = lrmd->lrmd_private;

	if (native->remote == NULL) {
	return NULL;
	} else {
	return native->remote->start_state;
	}
	}
	diff --git a/lib/lrmd/proxy_common.c b/lib/lrmd/proxy_common.c
	index 3f4a90d16e..11eaa92695 100644
	--- a/lib/lrmd/proxy_common.c
	+++ b/lib/lrmd/proxy_common.c
	@@ -1,314 +1,314 @@
	/*
	* Copyright 2015-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>
	#include <unistd.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/services.h>
	#include <crm/common/mainloop.h>

	#include <crm/pengine/status.h>
	#include <crm/cib.h>
	#include <crm/lrmd.h>
	#include <crm/lrmd_internal.h>

	int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
	GHashTable *proxy_table = NULL;

	static void
	remote_proxy_notify_destroy(lrmd_t lrmd, const char session_id)
	{
	/* sending to the remote node that an ipc connection has been destroyed */
	xmlNode *msg = create_xml_node(NULL, PCMK__XE_LRMD_IPC_PROXY);
	crm_xml_add(msg, PCMK__XA_LRMD_IPC_OP, LRMD_IPC_OP_DESTROY);
	crm_xml_add(msg, PCMK__XA_LRMD_IPC_SESSION, session_id);
	lrmd_internal_proxy_send(lrmd, msg);
	free_xml(msg);
	}

	/*!
	* \internal
	* \brief Acknowledge a remote proxy shutdown request
	*
	* \param[in,out] lrmd Connection to proxy
	*/
	void
	remote_proxy_ack_shutdown(lrmd_t *lrmd)
	{
	xmlNode *msg = create_xml_node(NULL, PCMK__XE_LRMD_IPC_PROXY);
	crm_xml_add(msg, PCMK__XA_LRMD_IPC_OP, LRMD_IPC_OP_SHUTDOWN_ACK);
	lrmd_internal_proxy_send(lrmd, msg);
	free_xml(msg);
	}

	/*!
	* \internal
	* \brief Reject a remote proxy shutdown request
	*
	* \param[in,out] lrmd Connection to proxy
	*/
	void
	remote_proxy_nack_shutdown(lrmd_t *lrmd)
	{
	xmlNode *msg = create_xml_node(NULL, PCMK__XE_LRMD_IPC_PROXY);
	crm_xml_add(msg, PCMK__XA_LRMD_IPC_OP, LRMD_IPC_OP_SHUTDOWN_NACK);
	lrmd_internal_proxy_send(lrmd, msg);
	free_xml(msg);
	}

	void
	remote_proxy_relay_event(remote_proxy_t proxy, xmlNode msg)
	{
	/* sending to the remote node an event msg. */
	xmlNode *event = create_xml_node(NULL, PCMK__XE_LRMD_IPC_PROXY);
	crm_xml_add(event, PCMK__XA_LRMD_IPC_OP, LRMD_IPC_OP_EVENT);
	crm_xml_add(event, PCMK__XA_LRMD_IPC_SESSION, proxy->session_id);
	add_message_xml(event, PCMK__XE_LRMD_IPC_MSG, msg);
	crm_log_xml_explicit(event, "EventForProxy");
	lrmd_internal_proxy_send(proxy->lrm, event);
	free_xml(event);
	}

	void
	remote_proxy_relay_response(remote_proxy_t proxy, xmlNode msg, int msg_id)
	{
	/* sending to the remote node a response msg. */
	xmlNode *response = create_xml_node(NULL, PCMK__XE_LRMD_IPC_PROXY);
	crm_xml_add(response, PCMK__XA_LRMD_IPC_OP, LRMD_IPC_OP_RESPONSE);
	crm_xml_add(response, PCMK__XA_LRMD_IPC_SESSION, proxy->session_id);
	crm_xml_add_int(response, PCMK__XA_LRMD_IPC_MSG_ID, msg_id);
	add_message_xml(response, PCMK__XE_LRMD_IPC_MSG, msg);
	lrmd_internal_proxy_send(proxy->lrm, response);
	free_xml(response);
	}

	static void
	remote_proxy_end_session(remote_proxy_t *proxy)
	{
	if (proxy == NULL) {
	return;
	}
	crm_trace("ending session ID %s", proxy->session_id);

	if (proxy->source) {
	mainloop_del_ipc_client(proxy->source);
	}
	}

	void
	remote_proxy_free(gpointer data)
	{
	remote_proxy_t *proxy = data;

	crm_trace("freed proxy session ID %s", proxy->session_id);
	free(proxy->node_name);
	free(proxy->session_id);
	free(proxy);
	}

	int
	remote_proxy_dispatch(const char *buffer, ssize_t length, gpointer userdata)
	{
	// Async responses from cib and friends to clients via pacemaker-remoted
	xmlNode *xml = NULL;
	uint32_t flags = 0;
	remote_proxy_t *proxy = userdata;

	- xml = string2xml(buffer);
	+ xml = pcmk__xml_parse_string(buffer);
	if (xml == NULL) {
	crm_warn("Received a NULL msg from IPC service.");
	return 1;
	}

	flags = crm_ipc_buffer_flags(proxy->ipc);
	if (flags & crm_ipc_proxied_relay_response) {
	crm_trace("Passing response back to %.8s on %s: %.200s - request id: %d", proxy->session_id, proxy->node_name, buffer, proxy->last_request_id);
	remote_proxy_relay_response(proxy, xml, proxy->last_request_id);
	proxy->last_request_id = 0;

	} else {
	crm_trace("Passing event back to %.8s on %s: %.200s", proxy->session_id, proxy->node_name, buffer);
	remote_proxy_relay_event(proxy, xml);
	}
	free_xml(xml);
	return 1;
	}


	void
	remote_proxy_disconnected(gpointer userdata)
	{
	remote_proxy_t *proxy = userdata;

	crm_trace("destroying %p", proxy);

	proxy->source = NULL;
	proxy->ipc = NULL;

	if(proxy->lrm) {
	remote_proxy_notify_destroy(proxy->lrm, proxy->session_id);
	proxy->lrm = NULL;
	}

	g_hash_table_remove(proxy_table, proxy->session_id);
	}

	remote_proxy_t *
	remote_proxy_new(lrmd_t lrmd, struct ipc_client_callbacks proxy_callbacks,
	const char node_name, const char session_id, const char *channel)
	{
	remote_proxy_t *proxy = NULL;

	if(channel == NULL) {
	crm_err("No channel specified to proxy");
	remote_proxy_notify_destroy(lrmd, session_id);
	return NULL;
	}

	proxy = calloc(1, sizeof(remote_proxy_t));

	proxy->node_name = strdup(node_name);
	proxy->session_id = strdup(session_id);
	proxy->lrm = lrmd;

	if (!strcmp(pcmk__message_name(crm_system_name), CRM_SYSTEM_CRMD)
	&& !strcmp(pcmk__message_name(channel), CRM_SYSTEM_CRMD)) {
	// The controller doesn't need to connect to itself
	proxy->is_local = TRUE;

	} else {
	proxy->source = mainloop_add_ipc_client(channel, G_PRIORITY_LOW, 0, proxy, proxy_callbacks);
	proxy->ipc = mainloop_get_ipc_client(proxy->source);
	if (proxy->source == NULL) {
	remote_proxy_free(proxy);
	remote_proxy_notify_destroy(lrmd, session_id);
	return NULL;
	}
	}

	crm_trace("new remote proxy client established to %s on %s, session id %s",
	channel, node_name, session_id);
	g_hash_table_insert(proxy_table, proxy->session_id, proxy);

	return proxy;
	}

	void
	remote_proxy_cb(lrmd_t lrmd, const char node_name, xmlNode *msg)
	{
	const char *op = crm_element_value(msg, PCMK__XA_LRMD_IPC_OP);
	const char *session = crm_element_value(msg, PCMK__XA_LRMD_IPC_SESSION);
	remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
	int msg_id = 0;

	/* sessions are raw ipc connections to IPC,
	* all we do is proxy requests/responses exactly
	* like they are given to us at the ipc level. */

	CRM_CHECK(op != NULL, return);
	CRM_CHECK(session != NULL, return);

	crm_element_value_int(msg, PCMK__XA_LRMD_IPC_MSG_ID, &msg_id);
	/* This is msg from remote ipc client going to real ipc server */

	if (pcmk__str_eq(op, LRMD_IPC_OP_DESTROY, pcmk__str_casei)) {
	remote_proxy_end_session(proxy);

	} else if (pcmk__str_eq(op, LRMD_IPC_OP_REQUEST, pcmk__str_casei)) {
	int flags = 0;
	xmlNode *request = get_message_xml(msg, PCMK__XE_LRMD_IPC_MSG);
	const char *name = crm_element_value(msg, PCMK__XA_LRMD_IPC_CLIENT);

	CRM_CHECK(request != NULL, return);

	if (proxy == NULL) {
	/* proxy connection no longer exists */
	remote_proxy_notify_destroy(lrmd, session);
	return;
	}

	// Controller requests MUST be handled by the controller, not us
	CRM_CHECK(proxy->is_local == FALSE,
	remote_proxy_end_session(proxy); return);

	if (!crm_ipc_connected(proxy->ipc)) {
	remote_proxy_end_session(proxy);
	return;
	}
	proxy->last_request_id = 0;
	crm_element_value_int(msg, PCMK__XA_LRMD_IPC_MSG_FLAGS, &flags);
	crm_xml_add(request, PCMK_XE_ACL_ROLE, "pacemaker-remote");

	CRM_ASSERT(node_name);
	pcmk__update_acl_user(request, PCMK__XA_LRMD_IPC_USER, node_name);

	if (pcmk_is_set(flags, crm_ipc_proxied)) {
	const char *type = crm_element_value(request, PCMK__XA_T);
	int rc = 0;

	if (pcmk__str_eq(type, PCMK__VALUE_ATTRD, pcmk__str_none)
	&& (crm_element_value(request, PCMK__XA_ATTR_HOST) == NULL)
	&& pcmk__str_any_of(crm_element_value(request, PCMK_XA_TASK),
	PCMK__ATTRD_CMD_UPDATE,
	PCMK__ATTRD_CMD_UPDATE_BOTH,
	PCMK__ATTRD_CMD_UPDATE_DELAY, NULL)) {
	pcmk__xe_add_node(request, proxy->node_name, 0);
	}

	rc = crm_ipc_send(proxy->ipc, request, flags, 5000, NULL);

	if(rc < 0) {
	xmlNode *op_reply = create_xml_node(NULL, PCMK__XE_NACK);

	crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);

	/* Send a n'ack so the caller doesn't block */
	crm_xml_add(op_reply, PCMK_XA_FUNCTION, __func__);
	crm_xml_add_int(op_reply, PCMK__XA_LINE, __LINE__);
	crm_xml_add_int(op_reply, PCMK_XA_RC, rc);
	remote_proxy_relay_response(proxy, op_reply, msg_id);
	free_xml(op_reply);

	} else {
	crm_trace("Relayed %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
	proxy->last_request_id = msg_id;
	}

	} else {
	int rc = pcmk_ok;
	xmlNode *op_reply = NULL;
	// @COMPAT pacemaker_remoted <= 1.1.10

	crm_trace("Relaying %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);

	rc = crm_ipc_send(proxy->ipc, request, flags, 10000, &op_reply);
	if(rc < 0) {
	crm_err("Could not relay %s request %d from %s to %s for %s: %s (%d)",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name, pcmk_strerror(rc), rc);
	} else {
	crm_trace("Relayed %s request %d from %s to %s for %s",
	op, msg_id, proxy->node_name, crm_ipc_name(proxy->ipc), name);
	}

	if(op_reply) {
	remote_proxy_relay_response(proxy, op_reply, msg_id);
	free_xml(op_reply);
	}
	}
	} else {
	crm_err("Unknown proxy operation: %s", op);
	}
	}
	diff --git a/lib/pacemaker/pcmk_verify.c b/lib/pacemaker/pcmk_verify.c
	index a69f2ac74e..368ef9ed65 100644
	--- a/lib/pacemaker/pcmk_verify.c
	+++ b/lib/pacemaker/pcmk_verify.c
	@@ -1,151 +1,151 @@
	/*
	* Copyright 2023-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU Lesser General Public License
	* version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <crm/cib/internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>
	#include <crm/common/scheduler.h>
	#include <pacemaker-internal.h>
	#include <pacemaker.h>

	#include <stdint.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>

	#include "libpacemaker_private.h"

	int
	pcmk__parse_cib(pcmk__output_t out, const char cib_source, xmlNodePtr *cib_object)
	{
	// @COMPAT Take an enum for cib_source instead of trying to figure it out?
	const char *first = cib_source;

	if (cib_source == NULL) {
	crm_info("Reading XML from: live cluster");
	return cib__signon_query(out, NULL, cib_object);
	}

	while (isspace(*first)) {
	first++;
	}

	if (*first == '<') {
	- *cib_object = string2xml(cib_source);
	+ *cib_object = pcmk__xml_parse_string(cib_source);
	} else {
	*cib_object = pcmk__xml_parse_file(cib_source);
	}

	return (*cib_object == NULL)? ENODATA : pcmk_rc_ok;
	}

	int
	pcmk__verify(pcmk_scheduler_t scheduler, pcmk__output_t out, xmlNode *cib_object)
	{
	int rc = pcmk_rc_ok;
	xmlNode *status = NULL;
	xmlNode *cib_object_copy = NULL;

	if (!pcmk__xe_is(cib_object, PCMK_XE_CIB)) {
	rc = EBADMSG;
	out->err(out, "This tool can only check complete configurations (i.e. those starting with <cib>).");
	goto verify_done;
	}

	status = pcmk_find_cib_element(cib_object, PCMK_XE_STATUS);
	if (status == NULL) {
	create_xml_node(cib_object, PCMK_XE_STATUS);
	}

	if (pcmk__validate_xml(cib_object, NULL, (xmlRelaxNGValidityErrorFunc) out->err, out) == FALSE) {
	crm_config_error = TRUE;
	rc = pcmk_rc_schema_validation;
	goto verify_done;

	} else if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
	crm_config_error = TRUE;
	out->err(out, "The cluster will NOT be able to use this configuration.\n"
	"Please manually update the configuration to conform to the %s syntax.",
	xml_latest_schema());
	rc = pcmk_rc_schema_validation;
	goto verify_done;
	}

	/* Process the configuration to set crm_config_error/crm_config_warning.
	*
	* @TODO Some parts of the configuration are unpacked only when needed (for
	* example, action configuration), so we aren't necessarily checking those.
	*/
	if (cib_object != NULL) {
	unsigned long long flags = pcmk_sched_no_counts\|pcmk_sched_no_compat;

	if (status == NULL) {
	// No status available, so do minimal checks
	flags \|= pcmk_sched_validate_only;
	}
	cib_object_copy = pcmk__xml_copy(NULL, cib_object);

	/* The scheduler takes ownership of the XML object and potentially
	* frees it later. We want the caller of pcmk__verify to retain
	* ownership of the passed-in XML object, hence we pass in a copy
	* to the scheduler.
	*/
	pcmk__schedule_actions(cib_object_copy, flags, scheduler);
	}

	verify_done:
	if (crm_config_error) {
	rc = pcmk_rc_schema_validation;
	pcmk__config_err("CIB did not pass schema validation");
	} else if (crm_config_warning) {
	rc = pcmk_rc_schema_validation;
	}
	return rc;
	}

	int
	pcmk_verify(xmlNodePtr xml, const char cib_source)
	{
	pcmk_scheduler_t *scheduler = NULL;
	pcmk__output_t *out = NULL;
	int rc = pcmk_rc_ok;

	xmlNode *cib_object = NULL;

	rc = pcmk__xml_output_new(&out, xml);
	if (rc != pcmk_rc_ok) {
	return rc;
	}

	pe__register_messages(out);
	pcmk__register_lib_messages(out);

	rc = pcmk__parse_cib(out, cib_source, &cib_object);
	if (rc != pcmk_rc_ok) {
	out->err(out, "Couldn't parse input");
	goto done;
	}

	scheduler = pe_new_working_set();
	if (scheduler == NULL) {
	rc = errno;
	out->err(out, "Couldn't allocate scheduler data: %s", pcmk_rc_str(rc));
	goto done;
	}

	scheduler->priv = out;
	rc = pcmk__verify(scheduler, out, cib_object);

	done:
	pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
	free_xml(cib_object);
	pe_free_working_set(scheduler);
	return rc;
	}
	diff --git a/lib/pengine/tests/rules/pe_cron_range_satisfied_test.c b/lib/pengine/tests/rules/pe_cron_range_satisfied_test.c
	index e3bb43c20c..fa77e79e9a 100644
	--- a/lib/pengine/tests/rules/pe_cron_range_satisfied_test.c
	+++ b/lib/pengine/tests/rules/pe_cron_range_satisfied_test.c
	@@ -1,196 +1,196 @@
	/*
	* Copyright 2020-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/common/unittest_internal.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/rules_internal.h>

	static void
	run_one_test(const char t, const char x, int expected) {
	crm_time_t *tm = crm_time_new(t);
	- xmlNodePtr xml = string2xml(x);
	+ xmlNodePtr xml = pcmk__xml_parse_string(x);

	assert_int_equal(pe_cron_range_satisfied(tm, xml), expected);

	crm_time_free(tm);
	free_xml(xml);
	}

	static void
	no_time_given(void **state) {
	assert_int_equal(pe_cron_range_satisfied(NULL, NULL), pcmk_rc_op_unsatisfied);
	}

	static void
	any_time_satisfies_empty_spec(void **state) {
	crm_time_t *tm = crm_time_new(NULL);

	assert_int_equal(pe_cron_range_satisfied(tm, NULL), pcmk_rc_ok);

	crm_time_free(tm);
	}

	static void
	time_satisfies_year_spec(void **state) {
	run_one_test("2020-01-01",
	"<date_spec " PCMK_XA_ID "='spec' years='2020'/>",
	pcmk_rc_ok);
	}

	static void
	time_after_year_spec(void **state) {
	run_one_test("2020-01-01",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2019'/>",
	pcmk_rc_after_range);
	}

	static void
	time_satisfies_year_range(void **state) {
	run_one_test("2020-01-01",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2010-2030'/>",
	pcmk_rc_ok);
	}

	static void
	time_before_year_range(void **state) {
	run_one_test("2000-01-01",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2010-2030'/>",
	pcmk_rc_before_range);
	}

	static void
	time_after_year_range(void **state) {
	run_one_test("2020-01-01",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2010-2015'/>",
	pcmk_rc_after_range);
	}

	static void
	range_without_start_year_passes(void **state) {
	run_one_test("2010-01-01",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='-2020'/>",
	pcmk_rc_ok);
	}

	static void
	range_without_end_year_passes(void **state) {
	run_one_test("2010-01-01",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2000-'/>",
	pcmk_rc_ok);
	run_one_test("2000-10-01",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2000-'/>",
	pcmk_rc_ok);
	}

	static void
	yeardays_satisfies(void **state) {
	run_one_test("2020-01-30",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARDAYS "='30'/>",
	pcmk_rc_ok);
	}

	static void
	time_after_yeardays_spec(void **state) {
	run_one_test("2020-02-15",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARDAYS "='40'/>",
	pcmk_rc_after_range);
	}

	static void
	yeardays_feb_29_satisfies(void **state) {
	run_one_test("2016-02-29",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARDAYS "='60'/>",
	pcmk_rc_ok);
	}

	static void
	exact_ymd_satisfies(void **state) {
	run_one_test("2001-12-31",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2001' "
	PCMK_XA_MONTHS "='12' "
	PCMK_XA_MONTHDAYS "='31'/>",
	pcmk_rc_ok);
	}

	static void
	range_in_month_satisfies(void **state) {
	run_one_test("2001-06-10",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2001' "
	PCMK_XA_MONTHS "='6' "
	PCMK_XA_MONTHDAYS "='1-10'/>",
	pcmk_rc_ok);
	}

	static void
	exact_ymd_after_range(void **state) {
	run_one_test("2001-12-31",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2001' "
	PCMK_XA_MONTHS "='12' "
	PCMK_XA_MONTHDAYS "='30'/>",
	pcmk_rc_after_range);
	}

	static void
	time_after_monthdays_range(void **state) {
	run_one_test("2001-06-10",
	"<" PCMK_XE_DATE_SPEC " "
	PCMK_XA_ID "='spec' "
	PCMK_XA_YEARS "='2001' "
	PCMK_XA_MONTHS "='6' "
	PCMK_XA_MONTHDAYS "='11-15'/>",
	pcmk_rc_before_range);
	}

	PCMK__UNIT_TEST(NULL, NULL,
	cmocka_unit_test(no_time_given),
	cmocka_unit_test(any_time_satisfies_empty_spec),
	cmocka_unit_test(time_satisfies_year_spec),
	cmocka_unit_test(time_after_year_spec),
	cmocka_unit_test(time_satisfies_year_range),
	cmocka_unit_test(time_before_year_range),
	cmocka_unit_test(time_after_year_range),
	cmocka_unit_test(range_without_start_year_passes),
	cmocka_unit_test(range_without_end_year_passes),
	cmocka_unit_test(yeardays_satisfies),
	cmocka_unit_test(time_after_yeardays_spec),
	cmocka_unit_test(yeardays_feb_29_satisfies),
	cmocka_unit_test(exact_ymd_satisfies),
	cmocka_unit_test(range_in_month_satisfies),
	cmocka_unit_test(exact_ymd_after_range),
	cmocka_unit_test(time_after_monthdays_range))
	diff --git a/tools/cibadmin.c b/tools/cibadmin.c
	index 3f3d45b0f1..57f4f8de37 100644
	--- a/tools/cibadmin.c
	+++ b/tools/cibadmin.c
	@@ -1,950 +1,950 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>
	#include <stdio.h>
	#include <crm/crm.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/ipc.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h>

	#include <pacemaker-internal.h>

	#define SUMMARY "query and edit the Pacemaker configuration"

	#define INDENT " "

	enum cibadmin_section_type {
	cibadmin_section_all = 0,
	cibadmin_section_scope,
	cibadmin_section_xpath,
	};

	static int request_id = 0;

	static cib_t *the_cib = NULL;
	static GMainLoop *mainloop = NULL;
	static crm_exit_t exit_code = CRM_EX_OK;

	static struct {
	const char *cib_action;
	int cmd_options;
	enum cibadmin_section_type section_type;
	char *cib_section;
	char *validate_with;
	gint message_timeout_sec;
	enum pcmk__acl_render_how acl_render_mode;
	gchar *cib_user;
	gchar *dest_node;
	gchar *input_file;
	gchar *input_xml;
	gboolean input_stdin;
	bool delete_all;
	gboolean allow_create;
	gboolean force;
	gboolean get_node_path;
	gboolean local;
	gboolean no_children;
	gboolean sync_call;

	/* @COMPAT: For "-!" version option. Not advertised nor marked as
	* deprecated, but accepted.
	*/
	gboolean extended_version;

	//! \deprecated
	gboolean no_bcast;
	} options;

	int do_init(void);
	static int do_work(xmlNode input, xmlNode *output);
	void cibadmin_op_callback(xmlNode msg, int call_id, int rc, xmlNode output,
	void *user_data);

	static void
	print_xml_output(xmlNode * xml)
	{
	if (!xml) {
	return;
	} else if (xml->type != XML_ELEMENT_NODE) {
	return;
	}

	if (pcmk_is_set(options.cmd_options, cib_xpath_address)) {
	const char *id = crm_element_value(xml, PCMK_XA_ID);

	if (pcmk__xe_is(xml, PCMK__XE_XPATH_QUERY)) {
	xmlNode *child = NULL;

	for (child = xml->children; child; child = child->next) {
	print_xml_output(child);
	}

	} else if (id) {
	printf("%s\n", id);
	}

	} else {
	char *buffer = dump_xml_formatted(xml);
	fprintf(stdout, "%s", buffer);
	free(buffer);
	}
	}

	// Upgrade requested but already at latest schema
	static void
	report_schema_unchanged(void)
	{
	const char *err = pcmk_rc_str(pcmk_rc_schema_unchanged);

	crm_info("Upgrade unnecessary: %s\n", err);
	printf("Upgrade unnecessary: %s\n", err);
	exit_code = CRM_EX_OK;
	}

	/*!
	* \internal
	* \brief Check whether the current CIB action is dangerous
	* \return true if \p options.cib_action is dangerous, or false otherwise
	*/
	static inline bool
	cib_action_is_dangerous(void)
	{
	return options.no_bcast \|\| options.delete_all
	\|\| pcmk__str_any_of(options.cib_action,
	PCMK__CIB_REQUEST_UPGRADE,
	PCMK__CIB_REQUEST_ERASE,
	NULL);
	}

	/*!
	* \internal
	* \brief Determine whether the given CIB scope is valid for \p cibadmin
	*
	* \param[in] scope Scope to validate
	*
	* \return true if \p scope is valid, or false otherwise
	* \note An invalid scope applies the operation to the entire CIB.
	*/
	static inline bool
	scope_is_valid(const char *scope)
	{
	return pcmk__str_any_of(scope,
	PCMK_XE_CONFIGURATION,
	PCMK_XE_NODES,
	PCMK_XE_RESOURCES,
	PCMK_XE_CONSTRAINTS,
	PCMK_XE_CRM_CONFIG,
	PCMK_XE_RSC_DEFAULTS,
	PCMK_XE_OP_DEFAULTS,
	PCMK_XE_ACLS,
	PCMK_XE_FENCING_TOPOLOGY,
	PCMK_XE_TAGS,
	PCMK_XE_ALERTS,
	PCMK_XE_STATUS,
	NULL);
	}

	static gboolean
	command_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	options.delete_all = false;

	if (pcmk__str_any_of(option_name, "-u", "--upgrade", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_UPGRADE;

	} else if (pcmk__str_any_of(option_name, "-Q", "--query", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_QUERY;

	} else if (pcmk__str_any_of(option_name, "-E", "--erase", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_ERASE;

	} else if (pcmk__str_any_of(option_name, "-B", "--bump", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_BUMP;

	} else if (pcmk__str_any_of(option_name, "-C", "--create", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_CREATE;

	} else if (pcmk__str_any_of(option_name, "-M", "--modify", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_MODIFY;

	} else if (pcmk__str_any_of(option_name, "-P", "--patch", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_APPLY_PATCH;

	} else if (pcmk__str_any_of(option_name, "-R", "--replace", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_REPLACE;

	} else if (pcmk__str_any_of(option_name, "-D", "--delete", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_DELETE;

	} else if (pcmk__str_any_of(option_name, "-d", "--delete-all", NULL)) {
	options.cib_action = PCMK__CIB_REQUEST_DELETE;
	options.delete_all = true;

	} else if (pcmk__str_any_of(option_name, "-a", "--empty", NULL)) {
	options.cib_action = "empty";
	pcmk__str_update(&options.validate_with, optarg);

	} else if (pcmk__str_any_of(option_name, "-5", "--md5-sum", NULL)) {
	options.cib_action = "md5-sum";

	} else if (pcmk__str_any_of(option_name, "-6", "--md5-sum-versioned",
	NULL)) {
	options.cib_action = "md5-sum-versioned";

	} else {
	// Should be impossible
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	show_access_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	if (pcmk__str_eq(optarg, "auto", pcmk__str_null_matches)) {
	options.acl_render_mode = pcmk__acl_render_default;

	} else if (g_strcmp0(optarg, "namespace") == 0) {
	options.acl_render_mode = pcmk__acl_render_namespace;

	} else if (g_strcmp0(optarg, "text") == 0) {
	options.acl_render_mode = pcmk__acl_render_text;

	} else if (g_strcmp0(optarg, "color") == 0) {
	options.acl_render_mode = pcmk__acl_render_color;

	} else {
	g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
	"Invalid value '%s' for option '%s'",
	optarg, option_name);
	return FALSE;
	}
	return TRUE;
	}

	static gboolean
	section_cb(const gchar option_name, const gchar optarg, gpointer data,
	GError **error)
	{
	if (pcmk__str_any_of(option_name, "-o", "--scope", NULL)) {
	options.section_type = cibadmin_section_scope;

	} else if (pcmk__str_any_of(option_name, "-A", "--xpath", NULL)) {
	options.section_type = cibadmin_section_xpath;

	} else {
	// Should be impossible
	return FALSE;
	}

	pcmk__str_update(&options.cib_section, optarg);
	return TRUE;
	}

	static GOptionEntry command_entries[] = {
	{ "upgrade", 'u', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Upgrade the configuration to the latest syntax", NULL },

	{ "query", 'Q', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Query the contents of the CIB", NULL },

	{ "erase", 'E', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Erase the contents of the whole CIB", NULL },

	{ "bump", 'B', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Increase the CIB's epoch value by 1", NULL },

	{ "create", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Create an object in the CIB (will fail if object already exists)",
	NULL },

	{ "modify", 'M', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Find object somewhere in CIB's XML tree and update it (fails if object "
	"does not exist unless -c is also specified)",
	NULL },

	{ "patch", 'P', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Supply an update in the form of an XML diff (see crm_diff(8))", NULL },

	{ "replace", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Recursively replace an object in the CIB", NULL },

	{ "delete", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Delete first object matching supplied criteria (for example, "
	"<" PCMK_XE_OP " " PCMK_XA_ID "=\"rsc1_op1\" "
	PCMK_XA_NAME "=\"monitor\"/>).\n"
	INDENT "The XML element name and all attributes must match in order for "
	"the element to be deleted.",
	NULL },

	{ "delete-all", 'd', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	command_cb,
	"When used with --xpath, remove all matching objects in the "
	"configuration instead of just the first one",
	NULL },

	{ "empty", 'a', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	command_cb,
	"Output an empty CIB. Accepts an optional schema name argument to use as "
	"the " PCMK_XA_VALIDATE_WITH " value.\n"
	INDENT "If no schema is given, the latest will be used.",
	"[schema]" },

	{ "md5-sum", '5', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, command_cb,
	"Calculate the on-disk CIB digest", NULL },

	{ "md5-sum-versioned", '6', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
	command_cb, "Calculate an on-the-wire versioned CIB digest", NULL },

	{ NULL }
	};

	static GOptionEntry data_entries[] = {
	/* @COMPAT: These arguments should be last-wins. We can have an enum option
	* that stores the input type, along with a single string option that stores
	* the XML string for --xml-text, filename for --xml-file, or NULL for
	* --xml-pipe.
	*/
	{ "xml-text", 'X', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING,
	&options.input_xml, "Retrieve XML from the supplied string", "value" },

	{ "xml-file", 'x', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME,
	&options.input_file, "Retrieve XML from the named file", "value" },

	{ "xml-pipe", 'p', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.input_stdin, "Retrieve XML from stdin", NULL },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "force", 'f', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.force,
	"Force the action to be performed", NULL },

	{ "timeout", 't', G_OPTION_FLAG_NONE, G_OPTION_ARG_INT,
	&options.message_timeout_sec,
	"Time (in seconds) to wait before declaring the operation failed",
	"value" },

	{ "user", 'U', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.cib_user,
	"Run the command with permissions of the named user (valid only for the "
	"root and " CRM_DAEMON_USER " accounts)", "value" },

	{ "sync-call", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.sync_call, "Wait for call to complete before returning", NULL },

	{ "local", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &options.local,
	"Command takes effect locally (should be used only for queries)", NULL },

	{ "scope", 'o', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, section_cb,
	"Limit scope of operation to specific section of CIB\n"
	INDENT "Valid values: " PCMK_XE_CONFIGURATION ", " PCMK_XE_NODES
	", " PCMK_XE_RESOURCES ", " PCMK_XE_CONSTRAINTS
	", " PCMK_XE_CRM_CONFIG ", " PCMK_XE_RSC_DEFAULTS ",\n"
	INDENT " " PCMK_XE_OP_DEFAULTS ", " PCMK_XE_ACLS
	", " PCMK_XE_FENCING_TOPOLOGY ", " PCMK_XE_TAGS ", " PCMK_XE_ALERTS
	", " PCMK_XE_STATUS "\n"
	INDENT "If both --scope/-o and --xpath/-a are specified, the last one to "
	"appear takes effect",
	"value" },

	{ "xpath", 'A', G_OPTION_FLAG_NONE, G_OPTION_ARG_CALLBACK, section_cb,
	"A valid XPath to use instead of --scope/-o\n"
	INDENT "If both --scope/-o and --xpath/-a are specified, the last one to "
	"appear takes effect",
	"value" },

	{ "node-path", 'e', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.get_node_path,
	"When performing XPath queries, return paths of any matches found\n"
	INDENT "(for example, "
	"\"/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION
	"/" PCMK_XE_RESOURCES "/" PCMK_XE_CLONE
	"[@" PCMK_XA_ID "='dummy-clone']"
	"/" PCMK_XE_PRIMITIVE "[@" PCMK_XA_ID "='dummy']\")",
	NULL },

	{ "show-access", 'S', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
	show_access_cb,
	"Whether to use syntax highlighting for ACLs (with -Q/--query and "
	"-U/--user)\n"
	INDENT "Allowed values: 'color' (default for terminal), 'text' (plain text, "
	"default for non-terminal),\n"
	INDENT " 'namespace', or 'auto' (use default value)\n"
	INDENT "Default value: 'auto'",
	"[value]" },

	{ "allow-create", 'c', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.allow_create,
	"(Advanced) Allow target of --modify/-M to be created if it does not "
	"exist",
	NULL },

	{ "no-children", 'n', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE,
	&options.no_children,
	"(Advanced) When querying an object, do not include its children in the "
	"result",
	NULL },

	{ "node", 'N', G_OPTION_FLAG_NONE, G_OPTION_ARG_STRING, &options.dest_node,
	"(Advanced) Send command to the specified host", "value" },

	// @COMPAT: Deprecated
	{ "no-bcast", 'b', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE,
	&options.no_bcast, "deprecated", NULL },

	// @COMPAT: Deprecated
	{ "host", 'h', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_STRING,
	&options.dest_node, "deprecated", NULL },

	{ NULL }
	};

	static GOptionContext *
	build_arg_context(pcmk__common_args_t *args)
	{
	const char *desc = NULL;
	GOptionContext *context = NULL;

	GOptionEntry extra_prog_entries[] = {
	// @COMPAT: Deprecated
	{ "extended-version", '!', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE,
	&options.extended_version, "deprecated", NULL },

	{ NULL }
	};

	desc = "Examples:\n\n"
	"Query the configuration from the local node:\n\n"
	"\t# cibadmin --query --local\n\n"
	"Query just the cluster options configuration:\n\n"
	"\t# cibadmin --query --scope " PCMK_XE_CRM_CONFIG "\n\n"
	"Query all '" PCMK_META_TARGET_ROLE "' settings:\n\n"
	"\t# cibadmin --query --xpath "
	"\"//" PCMK_XE_NVPAIR
	"[@" PCMK_XA_NAME "='" PCMK_META_TARGET_ROLE"']\"\n\n"
	"Remove all '" PCMK_META_IS_MANAGED "' settings:\n\n"
	"\t# cibadmin --delete-all --xpath "
	"\"//" PCMK_XE_NVPAIR
	"[@" PCMK_XA_NAME "='" PCMK_META_IS_MANAGED "']\"\n\n"
	"Remove the resource named 'old':\n\n"
	"\t# cibadmin --delete --xml-text "
	"'<" PCMK_XE_PRIMITIVE " " PCMK_XA_ID "=\"old\"/>'\n\n"
	"Remove all resources from the configuration:\n\n"
	"\t# cibadmin --replace --scope " PCMK_XE_RESOURCES
	" --xml-text '<" PCMK_XE_RESOURCES "/>'\n\n"
	"Replace complete configuration with contents of "
	"$HOME/pacemaker.xml:\n\n"
	"\t# cibadmin --replace --xml-file $HOME/pacemaker.xml\n\n"
	"Replace " PCMK_XE_CONSTRAINTS " section of configuration with "
	"contents of $HOME/constraints.xml:\n\n"
	"\t# cibadmin --replace --scope " PCMK_XE_CONSTRAINTS
	" --xml-file $HOME/constraints.xml\n\n"
	"Increase configuration version to prevent old configurations from "
	"being loaded accidentally:\n\n"
	"\t# cibadmin --modify --xml-text "
	"'<" PCMK_XE_CIB " " PCMK_XA_ADMIN_EPOCH
	"=\"" PCMK_XA_ADMIN_EPOCH "++\"/>'\n\n"
	"Edit the configuration with your favorite $EDITOR:\n\n"
	"\t# cibadmin --query > $HOME/local.xml\n\n"
	"\t# $EDITOR $HOME/local.xml\n\n"
	"\t# cibadmin --replace --xml-file $HOME/local.xml\n\n"
	"Assuming terminal, render configuration in color (green for "
	"writable, blue for readable, red for\n"
	"denied) to visualize permissions for user tony:\n\n"
	"\t# cibadmin --show-access=color --query --user tony \| less -r\n\n"
	"SEE ALSO:\n"
	" crm(8), pcs(8), crm_shadow(8), crm_diff(8)\n";

	context = pcmk__build_arg_context(args, NULL, NULL, "<command>");
	g_option_context_set_description(context, desc);

	pcmk__add_main_args(context, extra_prog_entries);

	pcmk__add_arg_group(context, "commands", "Commands:", "Show command help",
	command_entries);
	pcmk__add_arg_group(context, "data", "Data:", "Show data help",
	data_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	int rc = pcmk_rc_ok;
	const char *source = NULL;
	xmlNode *output = NULL;
	xmlNode *input = NULL;
	gchar *acl_cred = NULL;

	GError *error = NULL;

	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "ANSUXhotx");
	GOptionContext *context = build_arg_context(args);

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	if (g_strv_length(processed_args) > 1) {
	gchar *help = g_option_context_get_help(context, TRUE, NULL);
	GString *extra = g_string_sized_new(128);

	for (int lpc = 1; processed_args[lpc] != NULL; lpc++) {
	if (extra->len > 0) {
	g_string_append_c(extra, ' ');
	}
	g_string_append(extra, processed_args[lpc]);
	}

	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"non-option ARGV-elements: %s\n\n%s", extra->str, help);
	g_free(help);
	g_string_free(extra, TRUE);
	goto done;
	}

	if (args->version \|\| options.extended_version) {
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	/* FIXME: When cibadmin is converted to use formatted output, this can
	* be replaced by out->version with the appropriate boolean flag.
	*
	* options.extended_version is deprecated and will be removed in a
	* future release.
	*/
	pcmk__cli_help(options.extended_version? '!' : 'v');
	}

	/* At LOG_ERR, stderr for CIB calls is rather verbose. Several lines like
	*
	* (func@file:line) error: CIB <op> failures <XML>
	*
	* In cibadmin we explicitly output the XML portion without the prefixes. So
	* we default to LOG_CRIT.
	*/
	pcmk__cli_init_logging("cibadmin", 0);
	set_crm_log_level(LOG_CRIT);

	if (args->verbosity > 0) {
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_verbose);

	for (int i = 0; i < args->verbosity; i++) {
	crm_bump_log_level(argc, argv);
	}
	}

	if (options.cib_action == NULL) {
	// @COMPAT: Create a default command if other tools have one
	gchar *help = g_option_context_get_help(context, TRUE, NULL);

	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Must specify a command option\n\n%s", help);
	g_free(help);
	goto done;
	}

	if (strcmp(options.cib_action, "empty") == 0) {
	// Output an empty CIB
	char *buf = NULL;

	output = createEmptyCib(1);
	crm_xml_add(output, PCMK_XA_VALIDATE_WITH, options.validate_with);
	buf = dump_xml_formatted(output);
	fprintf(stdout, "%s", buf);
	free(buf);
	goto done;
	}

	if (cib_action_is_dangerous() && !options.force) {
	exit_code = CRM_EX_UNSAFE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command is considered dangerous. To prevent "
	"accidental destruction of the cluster, the --force flag "
	"is required in order to proceed.");
	goto done;
	}

	if (options.message_timeout_sec < 1) {
	// Set default timeout
	options.message_timeout_sec = 30;
	}

	if (options.section_type == cibadmin_section_xpath) {
	// Enable getting section by XPath
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_xpath);

	} else if (options.section_type == cibadmin_section_scope) {
	if (!scope_is_valid(options.cib_section)) {
	// @COMPAT: Consider requiring --force to proceed
	fprintf(stderr,
	"Invalid value '%s' for '--scope'. Operation will apply "
	"to the entire CIB.\n", options.cib_section);
	}
	}

	if (options.allow_create) {
	// Allow target of --modify/-M to be created if it does not exist
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_can_create);
	}

	if (options.delete_all) {
	// With cibadmin_section_xpath, remove all matching objects
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_multiple);
	}

	if (options.get_node_path) {
	/* Enable getting node path of XPath query matches.
	* Meaningful only if options.section_type == cibadmin_section_xpath.
	*/
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_xpath_address);
	}

	if (options.local) {
	// Configure command to take effect only locally
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_scope_local);
	}

	// @COMPAT: Deprecated option
	if (options.no_bcast) {
	// Configure command to take effect only locally and not to broadcast
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_inhibit_bcast\|cib_scope_local);
	}

	if (options.no_children) {
	// When querying an object, don't include its children in the result
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_no_children);
	}

	if (options.sync_call
	\|\| (options.acl_render_mode != pcmk__acl_render_none)) {
	/* Wait for call to complete before returning.
	*
	* The ACL render modes work only with sync calls due to differences in
	* output handling between sync/async. It shouldn't matter to the user
	* whether the call is synchronous; for a CIB query, we have to wait for
	* the result in order to display it in any case.
	*/
	cib__set_call_options(options.cmd_options, crm_system_name,
	cib_sync_call);
	}

	if (options.input_file != NULL) {
	input = pcmk__xml_parse_file(options.input_file);
	source = options.input_file;

	} else if (options.input_xml != NULL) {
	- input = string2xml(options.input_xml);
	+ input = pcmk__xml_parse_string(options.input_xml);
	source = "input string";

	} else if (options.input_stdin) {
	input = pcmk__xml_parse_file(NULL);
	source = "STDIN";

	} else if (options.acl_render_mode != pcmk__acl_render_none) {
	char *username = pcmk__uid2username(geteuid());
	bool required = pcmk_acl_required(username);

	free(username);

	if (required) {
	if (options.force) {
	fprintf(stderr, "The supplied command can provide skewed"
	" result since it is run under user that also"
	" gets guarded per ACLs on their own right."
	" Continuing since --force flag was"
	" provided.\n");

	} else {
	exit_code = CRM_EX_UNSAFE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command can provide skewed result "
	"since it is run under user that also gets guarded "
	"per ACLs in their own right. To accept the risk "
	"of such a possible distortion (without even "
	"knowing it at this time), use the --force flag.");
	goto done;
	}
	}

	if (options.cib_user == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"The supplied command requires -U user specified.");
	goto done;
	}

	/* We already stopped/warned ACL-controlled users about consequences.
	*
	* Note: acl_cred takes ownership of options.cib_user here.
	* options.cib_user is set to NULL so that the CIB is obtained as the
	* user running the cibadmin command. The CIB must be obtained as a user
	* with full permissions in order to show the CIB correctly annotated
	* for the options.cib_user's permissions.
	*/
	acl_cred = options.cib_user;
	options.cib_user = NULL;
	}

	if (input != NULL) {
	crm_log_xml_debug(input, "[admin input]");

	} else if (source != NULL) {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Couldn't parse input from %s.", source);
	goto done;
	}

	if (pcmk__str_eq(options.cib_action, "md5-sum", pcmk__str_casei)) {
	char *digest = NULL;

	if (input == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Please supply XML to process with -X, -x, or -p");
	goto done;
	}

	digest = calculate_on_disk_digest(input);
	fprintf(stderr, "Digest: ");
	fprintf(stdout, "%s\n", pcmk__s(digest, "<null>"));
	free(digest);
	goto done;

	} else if (strcmp(options.cib_action, "md5-sum-versioned") == 0) {
	char *digest = NULL;
	const char *version = NULL;

	if (input == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Please supply XML to process with -X, -x, or -p");
	goto done;
	}

	version = crm_element_value(input, PCMK_XA_CRM_FEATURE_SET);
	digest = calculate_xml_versioned_digest(input, FALSE, TRUE, version);
	fprintf(stderr, "Versioned (%s) digest: ", version);
	fprintf(stdout, "%s\n", pcmk__s(digest, "<null>"));
	free(digest);
	goto done;
	}

	rc = do_init();
	if (rc != pcmk_ok) {
	rc = pcmk_legacy2rc(rc);
	exit_code = pcmk_rc2exitc(rc);

	crm_err("Init failed, could not perform requested operations: %s",
	pcmk_rc_str(rc));
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Init failed, could not perform requested operations: %s",
	pcmk_rc_str(rc));
	goto done;
	}

	rc = do_work(input, &output);
	if (rc > 0) {
	/* wait for the reply by creating a mainloop and running it until
	* the callbacks are invoked...
	*/
	request_id = rc;

	the_cib->cmds->register_callback(the_cib, request_id,
	options.message_timeout_sec, FALSE,
	NULL, "cibadmin_op_callback",
	cibadmin_op_callback);

	mainloop = g_main_loop_new(NULL, FALSE);

	crm_trace("%s waiting for reply from the local CIB", crm_system_name);

	crm_info("Starting mainloop");
	g_main_loop_run(mainloop);

	} else if ((rc == -pcmk_err_schema_unchanged)
	&& (strcmp(options.cib_action,
	PCMK__CIB_REQUEST_UPGRADE) == 0)) {
	report_schema_unchanged();

	} else if (rc < 0) {
	rc = pcmk_legacy2rc(rc);
	crm_err("Call failed: %s", pcmk_rc_str(rc));
	fprintf(stderr, "Call failed: %s\n", pcmk_rc_str(rc));

	if (rc == pcmk_rc_schema_validation) {
	if (strcmp(options.cib_action, PCMK__CIB_REQUEST_UPGRADE) == 0) {
	xmlNode *obj = NULL;
	int version = 0;

	if (the_cib->cmds->query(the_cib, NULL, &obj,
	options.cmd_options) == pcmk_ok) {
	update_validation(&obj, &version, 0, TRUE, FALSE);
	}
	free_xml(obj);

	} else if (output) {
	validate_xml_verbose(output);
	}
	}
	exit_code = pcmk_rc2exitc(rc);
	}

	if ((output != NULL)
	&& (options.acl_render_mode != pcmk__acl_render_none)) {

	xmlDoc *acl_evaled_doc;
	rc = pcmk__acl_annotate_permissions(acl_cred, output->doc, &acl_evaled_doc);
	if (rc == pcmk_rc_ok) {
	xmlChar *rendered = NULL;

	rc = pcmk__acl_evaled_render(acl_evaled_doc,
	options.acl_render_mode, &rendered);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not render evaluated access: %s",
	pcmk_rc_str(rc));
	goto done;
	}
	printf("%s\n", (char *) rendered);
	free(rendered);

	} else {
	exit_code = CRM_EX_CONFIG;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Could not evaluate access per request (%s, error: %s)",
	acl_cred, pcmk_rc_str(rc));
	goto done;
	}

	} else if (output != NULL) {
	print_xml_output(output);
	}

	crm_trace("%s exiting normally", crm_system_name);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	g_free(options.cib_user);
	g_free(options.dest_node);
	g_free(options.input_file);
	g_free(options.input_xml);
	free(options.cib_section);
	free(options.validate_with);

	g_free(acl_cred);
	free_xml(input);
	free_xml(output);

	rc = cib__clean_up_connection(&the_cib);
	if (exit_code == CRM_EX_OK) {
	exit_code = pcmk_rc2exitc(rc);
	}

	pcmk__output_and_clear_error(&error, NULL);
	crm_exit(exit_code);
	}

	static int
	do_work(xmlNode input, xmlNode *output)
	{
	/* construct the request */
	the_cib->call_timeout = options.message_timeout_sec;
	if ((strcmp(options.cib_action, PCMK__CIB_REQUEST_REPLACE) == 0)
	&& pcmk__xe_is(input, PCMK_XE_CIB)) {
	xmlNode *status = pcmk_find_cib_element(input, PCMK_XE_STATUS);

	if (status == NULL) {
	create_xml_node(input, PCMK_XE_STATUS);
	}
	}

	crm_trace("Passing \"%s\" to variant_op...", options.cib_action);
	return cib_internal_op(the_cib, options.cib_action, options.dest_node,
	options.cib_section, input, output,
	options.cmd_options, options.cib_user);
	}

	int
	do_init(void)
	{
	int rc = pcmk_ok;

	the_cib = cib_new();
	rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
	if (rc != pcmk_ok) {
	crm_err("Could not connect to the CIB: %s", pcmk_strerror(rc));
	fprintf(stderr, "Could not connect to the CIB: %s\n",
	pcmk_strerror(rc));
	}

	return rc;
	}

	void
	cibadmin_op_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	rc = pcmk_legacy2rc(rc);
	exit_code = pcmk_rc2exitc(rc);

	if (rc == pcmk_rc_schema_unchanged) {
	report_schema_unchanged();

	} else if (rc != pcmk_rc_ok) {
	crm_warn("Call %s failed: %s " CRM_XS " rc=%d",
	options.cib_action, pcmk_rc_str(rc), rc);
	fprintf(stderr, "Call %s failed: %s\n",
	options.cib_action, pcmk_rc_str(rc));
	print_xml_output(output);

	} else if ((strcmp(options.cib_action, PCMK__CIB_REQUEST_QUERY) == 0)
	&& (output == NULL)) {
	crm_err("Query returned no output");
	crm_log_xml_err(msg, "no output");

	} else if (output == NULL) {
	crm_info("Call passed");

	} else {
	crm_info("Call passed");
	print_xml_output(output);
	}

	if (call_id == request_id) {
	g_main_loop_quit(mainloop);

	} else {
	crm_info("Message was not the response we were looking for (%d vs. %d)",
	call_id, request_id);
	}
	}
	diff --git a/tools/crm_diff.c b/tools/crm_diff.c
	index f0d4e6153a..60cece9ac5 100644
	--- a/tools/crm_diff.c
	+++ b/tools/crm_diff.c
	@@ -1,379 +1,379 @@
	/*
	* Copyright 2005-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <sys/param.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/common/cmdline_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/ipc.h>
	#include <crm/cib.h>

	#define SUMMARY "Compare two Pacemaker configurations (in XML format) to produce a custom diff-like output, " \
	"or apply such an output as a patch"

	struct {
	gboolean apply;
	gboolean as_cib;
	gboolean no_version;
	gboolean raw_1;
	gboolean raw_2;
	gboolean use_stdin;
	char *xml_file_1;
	char *xml_file_2;
	} options;

	gboolean new_string_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean original_string_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);
	gboolean patch_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	static GOptionEntry original_xml_entries[] = {
	{ "original", 'o', 0, G_OPTION_ARG_STRING, &options.xml_file_1,
	"XML is contained in the named file",
	"FILE" },
	{ "original-string", 'O', 0, G_OPTION_ARG_CALLBACK, original_string_cb,
	"XML is contained in the supplied string",
	"STRING" },

	{ NULL }
	};

	static GOptionEntry operation_entries[] = {
	{ "new", 'n', 0, G_OPTION_ARG_STRING, &options.xml_file_2,
	"Compare the original XML to the contents of the named file",
	"FILE" },
	{ "new-string", 'N', 0, G_OPTION_ARG_CALLBACK, new_string_cb,
	"Compare the original XML with the contents of the supplied string",
	"STRING" },
	{ "patch", 'p', 0, G_OPTION_ARG_CALLBACK, patch_cb,
	"Patch the original XML with the contents of the named file",
	"FILE" },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "cib", 'c', 0, G_OPTION_ARG_NONE, &options.as_cib,
	"Compare/patch the inputs as a CIB (includes versions details)",
	NULL },
	{ "stdin", 's', 0, G_OPTION_ARG_NONE, &options.use_stdin,
	"",
	NULL },
	{ "no-version", 'u', 0, G_OPTION_ARG_NONE, &options.no_version,
	"Generate the difference without versions details",
	NULL },

	{ NULL }
	};

	gboolean
	new_string_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.raw_2 = TRUE;
	pcmk__str_update(&options.xml_file_2, optarg);
	return TRUE;
	}

	gboolean
	original_string_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.raw_1 = TRUE;
	pcmk__str_update(&options.xml_file_1, optarg);
	return TRUE;
	}

	gboolean
	patch_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	options.apply = TRUE;
	pcmk__str_update(&options.xml_file_2, optarg);
	return TRUE;
	}

	static void
	print_patch(xmlNode *patch)
	{
	char *buffer = dump_xml_formatted(patch);

	printf("%s", buffer);
	free(buffer);
	fflush(stdout);
	}

	// \return Standard Pacemaker return code
	static int
	apply_patch(xmlNode input, xmlNode patch, gboolean as_cib)
	{
	xmlNode *output = pcmk__xml_copy(NULL, input);
	int rc = xml_apply_patchset(output, patch, as_cib);

	rc = pcmk_legacy2rc(rc);
	if (rc != pcmk_rc_ok) {
	fprintf(stderr, "Could not apply patch: %s\n", pcmk_rc_str(rc));
	free_xml(output);
	return rc;
	}

	if (output != NULL) {
	const char *version;
	char *buffer;

	print_patch(output);

	version = crm_element_value(output, PCMK_XA_CRM_FEATURE_SET);
	buffer = calculate_xml_versioned_digest(output, FALSE, TRUE, version);
	crm_trace("Digest: %s", pcmk__s(buffer, "<null>\n"));
	free(buffer);
	free_xml(output);
	}
	return pcmk_rc_ok;
	}

	static void
	log_patch_cib_versions(xmlNode *patch)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	const char *fmt = NULL;
	const char *digest = NULL;

	xml_patch_versions(patch, add, del);
	fmt = crm_element_value(patch, PCMK_XA_FORMAT);
	digest = crm_element_value(patch, PCMK__XA_DIGEST);

	if (add[2] != del[2] \|\| add[1] != del[1] \|\| add[0] != del[0]) {
	crm_info("Patch: --- %d.%d.%d %s", del[0], del[1], del[2], fmt);
	crm_info("Patch: +++ %d.%d.%d %s", add[0], add[1], add[2], digest);
	}
	}

	static void
	strip_patch_cib_version(xmlNode patch, const char *vfields, size_t nvfields)
	{
	int format = 1;

	crm_element_value_int(patch, PCMK_XA_FORMAT, &format);
	if (format == 2) {
	xmlNode *version_xml = find_xml_node(patch, PCMK_XE_VERSION, FALSE);

	if (version_xml) {
	free_xml(version_xml);
	}

	} else {
	int i = 0;

	const char *tags[] = {
	PCMK__XE_DIFF_REMOVED,
	PCMK__XE_DIFF_ADDED,
	};

	for (i = 0; i < PCMK__NELEM(tags); i++) {
	xmlNode *tmp = NULL;
	int lpc;

	tmp = find_xml_node(patch, tags[i], FALSE);
	if (tmp) {
	for (lpc = 0; lpc < nvfields; lpc++) {
	xml_remove_prop(tmp, vfields[lpc]);
	}

	tmp = find_xml_node(tmp, PCMK_XE_CIB, FALSE);
	if (tmp) {
	for (lpc = 0; lpc < nvfields; lpc++) {
	xml_remove_prop(tmp, vfields[lpc]);
	}
	}
	}
	}
	}
	}

	// \return Standard Pacemaker return code
	static int
	generate_patch(xmlNode object_1, xmlNode object_2, const char *xml_file_2,
	gboolean as_cib, gboolean no_version)
	{
	const char *vfields[] = {
	PCMK_XA_ADMIN_EPOCH,
	PCMK_XA_EPOCH,
	PCMK_XA_NUM_UPDATES,
	};

	xmlNode *output = NULL;

	/* If we're ignoring the version, make the version information
	* identical, so it isn't detected as a change. */
	if (no_version) {
	int lpc;

	for (lpc = 0; lpc < PCMK__NELEM(vfields); lpc++) {
	crm_copy_xml_element(object_1, object_2, vfields[lpc]);
	}
	}

	xml_track_changes(object_2, NULL, object_2, FALSE);
	if(as_cib) {
	xml_calculate_significant_changes(object_1, object_2);
	} else {
	xml_calculate_changes(object_1, object_2);
	}
	crm_log_xml_debug(object_2, (xml_file_2? xml_file_2: "target"));

	output = xml_create_patchset(0, object_1, object_2, NULL, FALSE);

	pcmk__log_xml_changes(LOG_INFO, object_2);
	xml_accept_changes(object_2);

	if (output == NULL) {
	return pcmk_rc_ok; // No changes
	}

	patchset_process_digest(output, object_1, object_2, as_cib);

	if (as_cib) {
	log_patch_cib_versions(output);

	} else if (no_version) {
	strip_patch_cib_version(output, vfields, PCMK__NELEM(vfields));
	}

	pcmk__log_xml_patchset(LOG_NOTICE, output);
	print_patch(output);
	free_xml(output);

	/* pcmk_rc_error means there's a non-empty diff.
	* @COMPAT Choose a more descriptive return code, like one that maps to
	* CRM_EX_DIGEST?
	*/
	return pcmk_rc_error;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t *args) {
	GOptionContext *context = NULL;

	const char *description = "Examples:\n\n"
	"Obtain the two different configuration files by running cibadmin on the two cluster setups to compare:\n\n"
	"\t# cibadmin --query > cib-old.xml\n\n"
	"\t# cibadmin --query > cib-new.xml\n\n"
	"Calculate and save the difference between the two files:\n\n"
	"\t# crm_diff --original cib-old.xml --new cib-new.xml > patch.xml\n\n"
	"Apply the patch to the original file:\n\n"
	"\t# crm_diff --original cib-old.xml --patch patch.xml > updated.xml\n\n"
	"Apply the patch to the running cluster:\n\n"
	"\t# cibadmin --patch -x patch.xml\n";

	context = pcmk__build_arg_context(args, NULL, NULL, NULL);
	g_option_context_set_description(context, description);

	pcmk__add_arg_group(context, "xml", "Original XML:",
	"Show original XML options", original_xml_entries);
	pcmk__add_arg_group(context, "operation", "Operation:",
	"Show operation options", operation_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	xmlNode *object_1 = NULL;
	xmlNode *object_2 = NULL;

	crm_exit_t exit_code = CRM_EX_OK;
	GError *error = NULL;

	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "nopNO");
	GOptionContext *context = build_arg_context(args);

	int rc = pcmk_rc_ok;

	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_diff", args->verbosity);

	if (args->version) {
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);
	/* FIXME: When crm_diff is converted to use formatted output, this can go. */
	pcmk__cli_help('v');
	}

	if (options.apply && options.no_version) {
	fprintf(stderr, "warning: -u/--no-version ignored with -p/--patch\n");
	} else if (options.as_cib && options.no_version) {
	fprintf(stderr, "error: -u/--no-version incompatible with -c/--cib\n");
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	if (options.raw_1) {
	- object_1 = string2xml(options.xml_file_1);
	+ object_1 = pcmk__xml_parse_string(options.xml_file_1);

	} else if (options.use_stdin) {
	fprintf(stderr, "Input first XML fragment:");
	object_1 = pcmk__xml_parse_file(NULL);

	} else if (options.xml_file_1 != NULL) {
	object_1 = pcmk__xml_parse_file(options.xml_file_1);
	}

	if (options.raw_2) {
	- object_2 = string2xml(options.xml_file_2);
	+ object_2 = pcmk__xml_parse_string(options.xml_file_2);

	} else if (options.use_stdin) {
	fprintf(stderr, "Input second XML fragment:");
	object_1 = pcmk__xml_parse_file(NULL);

	} else if (options.xml_file_2 != NULL) {
	object_2 = pcmk__xml_parse_file(options.xml_file_2);
	}

	if (object_1 == NULL) {
	fprintf(stderr, "Could not parse the first XML fragment\n");
	exit_code = CRM_EX_DATAERR;
	goto done;
	}
	if (object_2 == NULL) {
	fprintf(stderr, "Could not parse the second XML fragment\n");
	exit_code = CRM_EX_DATAERR;
	goto done;
	}

	if (options.apply) {
	rc = apply_patch(object_1, object_2, options.as_cib);
	} else {
	rc = generate_patch(object_1, object_2, options.xml_file_2, options.as_cib, options.no_version);
	}
	exit_code = pcmk_rc2exitc(rc);

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);
	free(options.xml_file_1);
	free(options.xml_file_2);
	free_xml(object_1);
	free_xml(object_2);

	pcmk__output_and_clear_error(&error, NULL);
	crm_exit(exit_code);
	}
	diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
	index c90a8a73f4..28962340b2 100644
	--- a/tools/crm_resource_print.c
	+++ b/tools/crm_resource_print.c
	@@ -1,835 +1,835 @@
	/*
	* Copyright 2004-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <stdint.h>

	#include <crm_resource.h>
	#include <crm/common/lists_internal.h>
	#include <crm/common/output.h>
	#include <crm/common/results.h>

	#define cons_string(x) x?x:"NA"
	static int
	print_constraint(xmlNode xml_obj, void userdata)
	{
	pcmk_scheduler_t scheduler = (pcmk_scheduler_t ) userdata;
	pcmk__output_t *out = scheduler->priv;
	xmlNode *lifetime = NULL;
	const char *id = crm_element_value(xml_obj, PCMK_XA_ID);

	if (id == NULL) {
	return pcmk_rc_ok;
	}

	// @COMPAT PCMK__XE_LIFETIME is deprecated
	lifetime = first_named_child(xml_obj, PCMK__XE_LIFETIME);
	if (pe_evaluate_rules(lifetime, NULL, scheduler->now, NULL) == FALSE) {
	return pcmk_rc_ok;
	}

	if (!pcmk__xe_is(xml_obj, PCMK_XE_RSC_COLOCATION)) {
	return pcmk_rc_ok;
	}

	out->info(out, "Constraint %s %s %s %s %s %s %s",
	xml_obj->name,
	cons_string(crm_element_value(xml_obj, PCMK_XA_ID)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_RSC)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_WITH_RSC)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_SCORE)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_RSC_ROLE)),
	cons_string(crm_element_value(xml_obj, PCMK_XA_WITH_RSC_ROLE)));

	return pcmk_rc_ok;
	}

	void
	cli_resource_print_cts_constraints(pcmk_scheduler_t *scheduler)
	{
	pcmk__xe_foreach_child(pcmk_find_cib_element(scheduler->input,
	PCMK_XE_CONSTRAINTS),
	NULL, print_constraint, scheduler);
	}

	void
	cli_resource_print_cts(pcmk_resource_t rsc, pcmk__output_t out)
	{
	const char *host = NULL;
	bool needs_quorum = TRUE;
	const char *rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
	const char *rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
	const char *rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
	pcmk_node_t *node = pcmk__current_node(rsc);

	if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
	needs_quorum = FALSE;
	} else {
	// @TODO check requires in resource meta-data and rsc_defaults
	}

	if (node != NULL) {
	host = node->details->uname;
	}

	out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld %#.16llx",
	rsc->xml->name, rsc->id,
	rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
	rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
	rsc->flags);

	g_list_foreach(rsc->children, (GFunc) cli_resource_print_cts, out);
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print_operations(const char rsc_id, const char host_uname,
	bool active, pcmk_scheduler_t *scheduler)
	{
	pcmk__output_t *out = scheduler->priv;
	int rc = pcmk_rc_no_output;
	GList *ops = find_operations(rsc_id, host_uname, active, scheduler);

	if (!ops) {
	return rc;
	}

	out->begin_list(out, NULL, NULL, "Resource Operations");
	rc = pcmk_rc_ok;

	for (GList *lpc = ops; lpc != NULL; lpc = lpc->next) {
	xmlNode xml_op = (xmlNode ) lpc->data;
	out->message(out, "node-and-op", scheduler, xml_op);
	}

	out->end_list(out);
	return rc;
	}

	// \return Standard Pacemaker return code
	int
	cli_resource_print(pcmk_resource_t rsc, pcmk_scheduler_t scheduler,
	bool expanded)
	{
	pcmk__output_t *out = scheduler->priv;
	uint32_t show_opts = pcmk_show_pending;
	GList *all = NULL;

	all = g_list_prepend(all, (gpointer) "*");

	out->begin_list(out, NULL, NULL, "Resource Config");
	out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc, all, all);
	out->message(out, "resource-config", rsc, !expanded);
	out->end_list(out);

	g_list_free(all);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Attributes");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_default(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	if (status == PCMK_EXEC_DONE) {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* returned <exit-code> (<exit-description>[: <exit-reason>])
	*/
	out->info(out, "Operation %s%s%s (%s%s%s:%s) returned %d (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, (int) rc, services_ocf_exitcode_str((int) rc),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	} else {
	/* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
	* could not be executed (<execution-status>[: <exit-reason>])
	*/
	out->err(out,
	"Operation %s%s%s (%s%s%s:%s) could not be executed (%s%s%s)",
	action,
	((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
	class,
	((provider == NULL)? "" : ":"),
	((provider == NULL)? "" : provider),
	type, pcmk_exec_status_str(status),
	((exit_reason == NULL)? "" : ": "),
	((exit_reason == NULL)? "" : exit_reason));
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("agent-status", "int", "const char ", "const char ", "const char *",
	"const char ", "const char ", "crm_exit_t", "const char *")
	static int
	agent_status_xml(pcmk__output_t *out, va_list args) {
	int status = va_arg(args, int);
	const char action G_GNUC_UNUSED = va_arg(args, const char );
	const char name G_GNUC_UNUSED = va_arg(args, const char );
	const char class G_GNUC_UNUSED = va_arg(args, const char );
	const char provider G_GNUC_UNUSED = va_arg(args, const char );
	const char type G_GNUC_UNUSED = va_arg(args, const char );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	const char exit_reason = va_arg(args, const char );

	char *exit_s = pcmk__itoa(rc);
	const char *message = services_ocf_exitcode_str((int) rc);
	char *status_s = pcmk__itoa(status);
	const char *execution_message = pcmk_exec_status_str(status);

	pcmk__output_create_xml_node(out, PCMK_XE_AGENT_STATUS,
	PCMK_XA_CODE, exit_s,
	PCMK_XA_MESSAGE, message,
	PCMK_XA_EXECUTION_CODE, status_s,
	PCMK_XA_EXECUTION_MESSAGE, execution_message,
	PCMK_XA_REASON, exit_reason,
	NULL);

	free(exit_s);
	free(status_s);

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t ", "const char ",
	"const char *")
	static int
	attribute_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );
	const char value = va_arg(args, const char );

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	return pcmk_rc_ok;
	} else {
	out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
	}
	return pcmk_rc_ok;
	}
	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_default(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	if (rsc_name == NULL) {
	out->list_item(out, NULL, "Overriding the cluster configuration with '%s' = '%s'",
	name, value);
	} else {
	out->list_item(out, NULL, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
	rsc_name, name, value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("override", "const char ", "const char ", "const char *")
	static int
	override_xml(pcmk__output_t *out, va_list args) {
	const char rsc_name = va_arg(args, const char );
	const char name = va_arg(args, const char );
	const char value = va_arg(args, const char );

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_OVERRIDE,
	PCMK_XA_NAME, name,
	PCMK_XA_VALUE, value,
	NULL);

	if (rsc_name != NULL) {
	crm_xml_add(node, PCMK_XA_RSC, rsc_name);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_default(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	out->begin_list(out, NULL, NULL, "Properties");
	out->list_item(out, attr, "%s", value);
	out->end_list(out);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t ", "const char ")
	static int
	property_list_text(pcmk__output_t *out, va_list args) {
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	const char attr = va_arg(args, const char );

	const char *value = crm_element_value(rsc->xml, attr);

	if (value != NULL) {
	pcmk__formatted_printf(out, "%s\n", value);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_default(pcmk__output_t *out, va_list args) {
	int verbose = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, PCMK_XE_OVERRIDES);

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	/* hide output for validate-all if not in verbose */
	if ((verbose == 0)
	&& pcmk__str_eq(action, PCMK_ACTION_VALIDATE_ALL, pcmk__str_casei)) {
	return pcmk_rc_ok;
	}

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	- doc = string2xml(stdout_data);
	+ doc = pcmk__xml_parse_string(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, PCMK_XE_COMMAND, stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char ", "const char ",
	"const char ", "const char ", "const char ", "GHashTable ",
	"crm_exit_t", "int", "const char ", "const char ", "const char *")
	static int
	resource_agent_action_xml(pcmk__output_t *out, va_list args) {
	int verbose G_GNUC_UNUSED = va_arg(args, int);

	const char class = va_arg(args, const char );
	const char provider = va_arg(args, const char );
	const char type = va_arg(args, const char );
	const char rsc_name = va_arg(args, const char );
	const char action = va_arg(args, const char );
	GHashTable overrides = va_arg(args, GHashTable );
	crm_exit_t rc = va_arg(args, crm_exit_t);
	int status = va_arg(args, int);
	const char exit_reason = va_arg(args, const char );
	const char stdout_data = va_arg(args, const char );
	const char stderr_data = va_arg(args, const char );

	xmlNodePtr node = NULL;

	node = pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE_AGENT_ACTION,
	PCMK_XA_ACTION, action,
	PCMK_XA_CLASS, class,
	PCMK_XA_TYPE, type,
	NULL);

	if (rsc_name) {
	crm_xml_add(node, PCMK_XA_RSC, rsc_name);
	}

	crm_xml_add(node, PCMK_XA_PROVIDER, provider);

	if (overrides) {
	GHashTableIter iter;
	const char *name = NULL;
	const char *value = NULL;

	out->begin_list(out, NULL, NULL, PCMK_XE_OVERRIDES);

	g_hash_table_iter_init(&iter, overrides);
	while (g_hash_table_iter_next(&iter, (gpointer ) &name, (gpointer ) &value)) {
	out->message(out, "override", rsc_name, name, value);
	}

	out->end_list(out);
	}

	out->message(out, "agent-status", status, action, rsc_name, class, provider,
	type, rc, exit_reason);

	if (stdout_data \|\| stderr_data) {
	xmlNodePtr doc = NULL;

	if (stdout_data != NULL) {
	- doc = string2xml(stdout_data);
	+ doc = pcmk__xml_parse_string(stdout_data);
	}
	if (doc != NULL) {
	out->output_xml(out, PCMK_XE_COMMAND, stdout_data);
	xmlFreeNode(doc);
	} else {
	out->subprocess_output(out, rc, stdout_data, stderr_data);
	}
	}

	pcmk__output_xml_pop_parent(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_default(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	if (checks->flags == 0) {
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Checks");

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
	parent->id);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
	parent->id, checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	out->list_item(out, "check",
	"'%s' cannot run on unhealthy nodes due to "
	PCMK_OPT_NODE_HEALTH_STRATEGY "='%s'",
	parent->id,
	pcmk__cluster_option(checks->rsc->cluster->config_hash,
	PCMK_OPT_NODE_HEALTH_STRATEGY));
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
	static int
	resource_check_list_xml(pcmk__output_t *out, va_list args) {
	resource_checks_t checks = va_arg(args, resource_checks_t );

	const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);

	xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_CHECK,
	PCMK_XA_ID, parent->id,
	NULL);

	if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_REMAIN_STOPPED, true);
	}

	if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_PROMOTABLE, false);
	}

	if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_UNMANAGED, true);
	}

	if (pcmk_is_set(checks->flags, rsc_locked)) {
	crm_xml_add(node, PCMK_XA_LOCKED_TO_HYPHEN, checks->lock_node);
	}

	if (pcmk_is_set(checks->flags, rsc_node_health)) {
	pcmk__xe_set_bool_attr(node, PCMK_XA_UNHEALTHY, true);
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const char ")
	static int
	resource_search_list_default(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const char requested_name = va_arg(args, const char );

	bool printed = false;
	int rc = pcmk_rc_no_output;

	if (!out->is_quiet(out) && nodes == NULL) {
	out->err(out, "resource %s is NOT running", requested_name);
	return rc;
	}

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;

	if (!printed) {
	out->begin_list(out, NULL, NULL, "Nodes");
	printed = true;
	rc = pcmk_rc_ok;
	}

	if (out->is_quiet(out)) {
	out->list_item(out, "node", "%s", ni->node_name);
	} else {
	const char *role_text = "";

	if (ni->promoted) {
	#ifdef PCMK__COMPAT_2_0
	role_text = " " PCMK__ROLE_PROMOTED_LEGACY;
	#else
	role_text = " " PCMK__ROLE_PROMOTED;
	#endif
	}
	out->list_item(out, "node", "resource %s is running on: %s%s",
	requested_name, ni->node_name, role_text);
	}
	}

	if (printed) {
	out->end_list(out);
	}

	return rc;
	}

	PCMK__OUTPUT_ARGS("resource-search-list", "GList ", "const char ")
	static int
	resource_search_list_xml(pcmk__output_t *out, va_list args)
	{
	GList nodes = va_arg(args, GList );
	const char requested_name = va_arg(args, const char );

	pcmk__output_xml_create_parent(out, PCMK_XE_NODES,
	PCMK_XA_RESOURCE, requested_name,
	NULL);

	for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
	node_info_t ni = (node_info_t ) lpc->data;
	xmlNodePtr sub_node = pcmk__output_create_xml_text_node(out,
	PCMK_XE_NODE,
	ni->node_name);

	if (ni->promoted) {
	crm_xml_add(sub_node, PCMK_XA_STATE, "promoted");
	}
	}

	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_default(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	out->begin_list(out, NULL, NULL, "Resource Reasons");

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	rsc->fns->location(rsc, &hosts, TRUE);

	if (hosts == NULL) {
	out->list_item(out, "reason", "Resource %s is not running", rsc->id);
	} else {
	out->list_item(out, "reason", "Resource %s is running", rsc->id);
	}

	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	hosts = NULL;
	}

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	} else {
	out->list_item(out, "reason", "Resource %s is not running on host %s",
	rsc->id, host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is running on host %s",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
	rsc->id, host_uname);
	cli_resource_check(out, rsc, node);
	}

	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	out->list_item(out, "reason", "Resource %s is %srunning",
	rsc->id, (hosts? "" : "not "));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	out->end_list(out);
	return pcmk_rc_ok;
	}

	PCMK__OUTPUT_ARGS("resource-reasons-list", "GList ", "pcmk_resource_t ",
	"pcmk_node_t *")
	static int
	resource_reasons_list_xml(pcmk__output_t *out, va_list args)
	{
	GList resources = va_arg(args, GList );
	pcmk_resource_t rsc = va_arg(args, pcmk_resource_t );
	pcmk_node_t node = va_arg(args, pcmk_node_t );

	const char *host_uname = (node == NULL)? NULL : node->details->uname;

	xmlNodePtr xml_node = pcmk__output_xml_create_parent(out, PCMK_XE_REASON,
	NULL);

	if ((rsc == NULL) && (host_uname == NULL)) {
	GList *lpc = NULL;
	GList *hosts = NULL;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCES, NULL);

	for (lpc = resources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;
	const char *running = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	running = pcmk__btoa(hosts != NULL);

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_RUNNING, running,
	NULL);

	cli_resource_check(out, rsc, NULL);
	pcmk__output_xml_pop_parent(out);
	g_list_free(hosts);
	hosts = NULL;
	}

	pcmk__output_xml_pop_parent(out);

	} else if ((rsc != NULL) && (host_uname != NULL)) {
	if (resource_is_running_on(rsc, host_uname)) {
	crm_xml_add(xml_node, PCMK_XA_RUNNING_ON, host_uname);
	}

	cli_resource_check(out, rsc, node);

	} else if ((rsc == NULL) && (host_uname != NULL)) {
	const char* host_uname = node->details->uname;
	GList *allResources = node->details->allocated_rsc;
	GList *activeResources = node->details->running_rsc;
	GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
	GList *lpc = NULL;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCES, NULL);

	for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_RUNNING, PCMK_VALUE_TRUE,
	PCMK_XA_HOST, host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
	pcmk_resource_t rsc = (pcmk_resource_t ) lpc->data;

	pcmk__output_xml_create_parent(out, PCMK_XE_RESOURCE,
	PCMK_XA_ID, rsc->id,
	PCMK_XA_RUNNING, PCMK_VALUE_FALSE,
	PCMK_XA_HOST, host_uname,
	NULL);

	cli_resource_check(out, rsc, node);
	pcmk__output_xml_pop_parent(out);
	}

	pcmk__output_xml_pop_parent(out);
	g_list_free(allResources);
	g_list_free(activeResources);
	g_list_free(unactiveResources);

	} else if ((rsc != NULL) && (host_uname == NULL)) {
	GList *hosts = NULL;

	rsc->fns->location(rsc, &hosts, TRUE);
	crm_xml_add(xml_node, PCMK_XA_RUNNING, pcmk__btoa(hosts != NULL));
	cli_resource_check(out, rsc, NULL);
	g_list_free(hosts);
	}

	return pcmk_rc_ok;
	}

	static void
	add_resource_name(pcmk_resource_t rsc, pcmk__output_t out)
	{
	if (rsc->children == NULL) {
	/* Sometimes PCMK_XE_RESOURCE might act as a PCMK_XA_NAME instead of an
	* XML element name, depending on whether the command line causes
	* --xml-simple-list to be forced.
	*/
	out->list_item(out, PCMK_XE_RESOURCE, "%s", rsc->id);
	} else {
	g_list_foreach(rsc->children, (GFunc) add_resource_name, out);
	}
	}

	PCMK__OUTPUT_ARGS("resource-names-list", "GList *")
	static int
	resource_names(pcmk__output_t *out, va_list args) {
	GList resources = va_arg(args, GList );

	if (resources == NULL) {
	out->err(out, "NO resources configured\n");
	return pcmk_rc_no_output;
	}

	out->begin_list(out, NULL, NULL, "Resource Names");
	g_list_foreach(resources, (GFunc) add_resource_name, out);
	out->end_list(out);
	return pcmk_rc_ok;
	}

	static pcmk__message_entry_t fmt_functions[] = {
	{ "agent-status", "default", agent_status_default },
	{ "agent-status", "xml", agent_status_xml },
	{ "attribute-list", "default", attribute_list_default },
	{ "attribute-list", "text", attribute_list_text },
	{ "override", "default", override_default },
	{ "override", "xml", override_xml },
	{ "property-list", "default", property_list_default },
	{ "property-list", "text", property_list_text },
	{ "resource-agent-action", "default", resource_agent_action_default },
	{ "resource-agent-action", "xml", resource_agent_action_xml },
	{ "resource-check-list", "default", resource_check_list_default },
	{ "resource-check-list", "xml", resource_check_list_xml },
	{ "resource-search-list", "default", resource_search_list_default },
	{ "resource-search-list", "xml", resource_search_list_xml },
	{ "resource-reasons-list", "default", resource_reasons_list_default },
	{ "resource-reasons-list", "xml", resource_reasons_list_xml },
	{ "resource-names-list", "default", resource_names },

	{ NULL, NULL, NULL }
	};

	void
	crm_resource_register_messages(pcmk__output_t *out) {
	pcmk__register_messages(out, fmt_functions);
	}
	diff --git a/tools/crm_rule.c b/tools/crm_rule.c
	index 9844902eb1..6bc70cb056 100644
	--- a/tools/crm_rule.c
	+++ b/tools/crm_rule.c
	@@ -1,228 +1,228 @@
	/*
	* Copyright 2019-2024 the Pacemaker project contributors
	*
	* The version control history for this file may have further details.
	*
	* This source code is licensed under the GNU General Public License version 2
	* or later (GPLv2+) WITHOUT ANY WARRANTY.
	*/

	#include <crm_internal.h>

	#include <crm/common/cmdline_internal.h>
	#include <crm/common/output_internal.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/rules_internal.h>
	#include <crm/pengine/status.h>
	#include <pacemaker-internal.h>

	#include <sys/stat.h>

	#define SUMMARY "evaluate rules from the Pacemaker configuration"

	static pcmk__supported_format_t formats[] = {
	PCMK__SUPPORTED_FORMAT_NONE,
	PCMK__SUPPORTED_FORMAT_TEXT,
	PCMK__SUPPORTED_FORMAT_XML,
	{ NULL, NULL, NULL }
	};

	enum crm_rule_mode {
	crm_rule_mode_none,
	crm_rule_mode_check
	};

	struct {
	char *date;
	char *input_xml;
	enum crm_rule_mode mode;
	gchar **rules;
	} options = {
	.mode = crm_rule_mode_none
	};

	static gboolean mode_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error);

	static GOptionEntry mode_entries[] = {
	{ "check", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, mode_cb,
	"Check whether a rule is in effect",
	NULL },

	{ NULL }
	};

	static GOptionEntry data_entries[] = {
	{ "xml-text", 'X', 0, G_OPTION_ARG_STRING, &options.input_xml,
	"Use argument for XML (or stdin if '-')",
	NULL },

	{ NULL }
	};

	static GOptionEntry addl_entries[] = {
	{ "date", 'd', 0, G_OPTION_ARG_STRING, &options.date,
	"Whether the rule is in effect on a given date",
	NULL },
	{ "rule", 'r', 0, G_OPTION_ARG_STRING_ARRAY, &options.rules,
	"The ID of the rule to check (may be specified multiple times)",
	NULL },

	{ NULL }
	};

	static gboolean
	mode_cb(const gchar option_name, const gchar optarg, gpointer data, GError **error) {
	if (strcmp(option_name, "c")) {
	options.mode = crm_rule_mode_check;
	}

	return TRUE;
	}

	static GOptionContext *
	build_arg_context(pcmk__common_args_t args, GOptionGroup *group) {
	GOptionContext *context = NULL;

	context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);

	pcmk__add_arg_group(context, "modes", "Modes (mutually exclusive):",
	"Show modes of operation", mode_entries);
	pcmk__add_arg_group(context, "data", "Data:",
	"Show data options", data_entries);
	pcmk__add_arg_group(context, "additional", "Additional Options:",
	"Show additional options", addl_entries);
	return context;
	}

	int
	main(int argc, char **argv)
	{
	crm_time_t *rule_date = NULL;
	xmlNode *input = NULL;

	int rc = pcmk_rc_ok;
	crm_exit_t exit_code = CRM_EX_OK;

	GError *error = NULL;

	pcmk__output_t *out = NULL;

	GOptionGroup *output_group = NULL;
	pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
	GOptionContext *context = build_arg_context(args, &output_group);
	gchar **processed_args = pcmk__cmdline_preproc(argv, "drX");

	pcmk__register_formats(output_group, formats);
	if (!g_option_context_parse_strv(context, &processed_args, &error)) {
	exit_code = CRM_EX_USAGE;
	goto done;
	}

	pcmk__cli_init_logging("crm_rule", args->verbosity);

	rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
	if (rc != pcmk_rc_ok) {
	exit_code = CRM_EX_ERROR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating output format %s: %s",
	args->output_ty, pcmk_rc_str(rc));
	goto done;
	}

	pcmk__register_lib_messages(out);

	if (args->version) {
	out->version(out, false);
	goto done;
	}

	/* Check command line arguments before opening a connection to
	* the CIB manager or doing anything else important.
	*/
	switch(options.mode) {
	case crm_rule_mode_check:
	if (options.rules == NULL) {
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "--check requires use of --rule=");
	goto done;
	}

	break;

	default:
	exit_code = CRM_EX_USAGE;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "No mode operation given");
	goto done;
	break;
	}

	/* Set up some defaults. */
	rule_date = crm_time_new(options.date);
	if (rule_date == NULL) {
	if (options.date != NULL) {
	exit_code = CRM_EX_DATAERR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Invalid date specified: '%s'", options.date);

	} else {
	// Should never happen
	exit_code = CRM_EX_OSERR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"No --date given and can't determine current date");
	}
	goto done;
	}

	// Parse the input XML specified by the command-line options, if any
	if (pcmk__str_eq(options.input_xml, "-", pcmk__str_none)) {
	input = pcmk__xml_parse_file(NULL);

	if (input == NULL) {
	exit_code = CRM_EX_DATAERR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Couldn't parse input from STDIN");
	goto done;
	}
	} else if (options.input_xml != NULL) {
	- input = string2xml(options.input_xml);
	+ input = pcmk__xml_parse_string(options.input_xml);

	if (input == NULL) {
	exit_code = CRM_EX_DATAERR;
	g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
	"Couldn't parse input string: %s", options.input_xml);
	goto done;
	}
	}

	/* Now do whichever operation mode was asked for. There's only one at the
	* moment so this looks a little silly, but I expect there will be more
	* modes in the future.
	*/
	switch(options.mode) {
	case crm_rule_mode_check:
	rc = pcmk__check_rules(out, input, rule_date,
	(const char **) options.rules);
	exit_code = pcmk_rc2exitc(rc);
	break;

	default:
	break;
	}

	done:
	g_strfreev(processed_args);
	pcmk__free_arg_context(context);

	crm_time_free(rule_date);
	free_xml(input);

	pcmk__output_and_clear_error(&error, out);

	if (out != NULL) {
	out->finish(out, exit_code, true, NULL);
	pcmk__output_free(out);
	}

	pcmk__unregister_formats();
	return crm_exit(exit_code);
	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Jul 8, 3:07 PM (3 h, 40 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 2002132
Default Alt Text: (829 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions