diff --git a/daemons/attrd/attrd_ipc.c b/daemons/attrd/attrd_ipc.c
index bb5ff77d76..286bb612d4 100644
--- a/daemons/attrd/attrd_ipc.c
+++ b/daemons/attrd/attrd_ipc.c
@@ -1,633 +1,633 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <errno.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <sys/types.h>
 
 #include <crm/cluster.h>
 #include <crm/cluster/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/acl_internal.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/logging.h>
 #include <crm/common/results.h>
 #include <crm/common/strings_internal.h>
 #include <crm/common/util.h>
 
 #include "pacemaker-attrd.h"
 
 static qb_ipcs_service_t *ipcs = NULL;
 
 /*!
  * \internal
  * \brief Build the XML reply to a client query
  *
  * \param[in] attr Name of requested attribute
  * \param[in] host Name of requested host (or NULL for all hosts)
  *
  * \return New XML reply
  * \note Caller is responsible for freeing the resulting XML
  */
 static xmlNode *build_query_reply(const char *attr, const char *host)
 {
     xmlNode *reply = create_xml_node(NULL, __func__);
     attribute_t *a;
 
     if (reply == NULL) {
         return NULL;
     }
     crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_ATTRD);
     crm_xml_add(reply, PCMK__XA_SUBT, PCMK__ATTRD_CMD_QUERY);
     crm_xml_add(reply, PCMK__XA_ATTR_VERSION, ATTRD_PROTOCOL_VERSION);
 
     /* If desired attribute exists, add its value(s) to the reply */
     a = g_hash_table_lookup(attributes, attr);
     if (a) {
         attribute_value_t *v;
         xmlNode *host_value;
 
         crm_xml_add(reply, PCMK__XA_ATTR_NAME, attr);
 
         /* Allow caller to use "localhost" to refer to local node */
         if (pcmk__str_eq(host, "localhost", pcmk__str_casei)) {
             host = attrd_cluster->uname;
             crm_trace("Mapped localhost to %s", host);
         }
 
         /* If a specific node was requested, add its value */
         if (host) {
             v = g_hash_table_lookup(a->values, host);
             host_value = create_xml_node(reply, PCMK_XE_NODE);
             if (host_value == NULL) {
                 free_xml(reply);
                 return NULL;
             }
             pcmk__xe_add_node(host_value, host, 0);
             crm_xml_add(host_value, PCMK__XA_ATTR_VALUE,
                         (v? v->current : NULL));
 
         /* Otherwise, add all nodes' values */
         } else {
             GHashTableIter iter;
 
             g_hash_table_iter_init(&iter, a->values);
             while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &v)) {
                 host_value = create_xml_node(reply, PCMK_XE_NODE);
                 if (host_value == NULL) {
                     free_xml(reply);
                     return NULL;
                 }
                 pcmk__xe_add_node(host_value, v->nodename, 0);
                 crm_xml_add(host_value, PCMK__XA_ATTR_VALUE, v->current);
             }
         }
     }
     return reply;
 }
 
 xmlNode *
 attrd_client_clear_failure(pcmk__request_t *request)
 {
     xmlNode *xml = request->xml;
     const char *rsc, *op, *interval_spec;
 
     if (minimum_protocol_version >= 2) {
         /* Propagate to all peers (including ourselves).
          * This ends up at attrd_peer_message().
          */
         attrd_send_message(NULL, xml, false);
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
         return NULL;
     }
 
     rsc = crm_element_value(xml, PCMK__XA_ATTR_RESOURCE);
     op = crm_element_value(xml, PCMK__XA_ATTR_OPERATION);
     interval_spec = crm_element_value(xml, PCMK__XA_ATTR_INTERVAL);
 
     /* Map this to an update */
     crm_xml_add(xml, PCMK__XA_TASK, PCMK__ATTRD_CMD_UPDATE);
 
     /* Add regular expression matching desired attributes */
 
     if (rsc) {
         char *pattern;
 
         if (op == NULL) {
             pattern = crm_strdup_printf(ATTRD_RE_CLEAR_ONE, rsc);
 
         } else {
             guint interval_ms = 0U;
 
             pcmk_parse_interval_spec(interval_spec, &interval_ms);
             pattern = crm_strdup_printf(ATTRD_RE_CLEAR_OP,
                                         rsc, op, interval_ms);
         }
 
         crm_xml_add(xml, PCMK__XA_ATTR_PATTERN, pattern);
         free(pattern);
 
     } else {
         crm_xml_add(xml, PCMK__XA_ATTR_PATTERN, ATTRD_RE_CLEAR_ALL);
     }
 
     /* Make sure attribute and value are not set, so we delete via regex */
     xml_remove_prop(xml, PCMK__XA_ATTR_NAME);
     xml_remove_prop(xml, PCMK__XA_ATTR_VALUE);
 
     return attrd_client_update(request);
 }
 
 xmlNode *
 attrd_client_peer_remove(pcmk__request_t *request)
 {
     xmlNode *xml = request->xml;
 
     // Host and ID are not used in combination, rather host has precedence
     const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
     char *host_alloc = NULL;
 
     attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags);
 
     if (host == NULL) {
         int nodeid = 0;
 
         crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID, &nodeid);
         if (nodeid > 0) {
             crm_node_t *node = NULL;
             char *host_alloc = NULL;
 
             node = pcmk__search_node_caches(nodeid, NULL,
                                             pcmk__node_search_cluster);
             if (node && node->uname) {
                 // Use cached name if available
                 host = node->uname;
             } else {
                 // Otherwise ask cluster layer
                 host_alloc = get_node_name(nodeid);
                 host = host_alloc;
             }
             pcmk__xe_add_node(xml, host, 0);
         }
     }
 
     if (host) {
         crm_info("Client %s is requesting all values for %s be removed",
                  pcmk__client_name(request->ipc_client), host);
         attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
         free(host_alloc);
     } else {
         crm_info("Ignoring request by client %s to remove all peer values without specifying peer",
                  pcmk__client_name(request->ipc_client));
     }
 
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     return NULL;
 }
 
 xmlNode *
 attrd_client_query(pcmk__request_t *request)
 {
     xmlNode *query = request->xml;
     xmlNode *reply = NULL;
     const char *attr = NULL;
 
     crm_debug("Query arrived from %s", pcmk__client_name(request->ipc_client));
 
     /* Request must specify attribute name to query */
     attr = crm_element_value(query, PCMK__XA_ATTR_NAME);
     if (attr == NULL) {
         pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
                             "Ignoring malformed query from %s (no attribute name given)",
                             pcmk__client_name(request->ipc_client));
         return NULL;
     }
 
     /* Build the XML reply */
     reply = build_query_reply(attr, crm_element_value(query,
                                                       PCMK__XA_ATTR_NODE_NAME));
     if (reply == NULL) {
         pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
                             "Could not respond to query from %s: could not create XML reply",
                             pcmk__client_name(request->ipc_client));
         return NULL;
     } else {
         pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     }
 
     request->ipc_client->request_id = 0;
     return reply;
 }
 
 xmlNode *
 attrd_client_refresh(pcmk__request_t *request)
 {
     crm_info("Updating all attributes");
 
     attrd_send_ack(request->ipc_client, request->ipc_id, request->ipc_flags);
     attrd_write_attributes(attrd_write_all|attrd_write_no_delay);
 
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     return NULL;
 }
 
 static void
 handle_missing_host(xmlNode *xml)
 {
     const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
 
     if (host == NULL) {
         crm_trace("Inferring host");
         pcmk__xe_add_node(xml, attrd_cluster->uname, attrd_cluster->nodeid);
     }
 }
 
 /* Convert a single IPC message with a regex into one with multiple children, one
  * for each regex match.
  */
 static int
 expand_regexes(xmlNode *xml, const char *attr, const char *value, const char *regex)
 {
     if (attr == NULL && regex) {
         bool matched = false;
         GHashTableIter aIter;
         regex_t r_patt;
 
         crm_debug("Setting %s to %s", regex, value);
         if (regcomp(&r_patt, regex, REG_EXTENDED|REG_NOSUB)) {
             return EINVAL;
         }
 
         g_hash_table_iter_init(&aIter, attributes);
         while (g_hash_table_iter_next(&aIter, (gpointer *) & attr, NULL)) {
             int status = regexec(&r_patt, attr, 0, NULL, 0);
 
             if (status == 0) {
                 xmlNode *child = create_xml_node(xml, PCMK_XE_OP);
 
                 crm_trace("Matched %s with %s", attr, regex);
                 matched = true;
 
                 /* Copy all the attributes from the parent over, but remove the
                  * regex and replace it with the name.
                  */
                 attrd_copy_xml_attributes(xml, child);
                 xml_remove_prop(child, PCMK__XA_ATTR_PATTERN);
                 crm_xml_add(child, PCMK__XA_ATTR_NAME, attr);
             }
         }
 
         regfree(&r_patt);
 
         /* Return a code if we never matched anything.  This should not be treated
          * as an error.  It indicates there was a regex, and it was a valid regex,
          * but simply did not match anything and the caller should not continue
          * doing any regex-related processing.
          */
         if (!matched) {
             return pcmk_rc_op_unsatisfied;
         }
 
     } else if (attr == NULL) {
         return pcmk_rc_bad_nvpair;
     }
 
     return pcmk_rc_ok;
 }
 
 static int
 handle_regexes(pcmk__request_t *request)
 {
     xmlNode *xml = request->xml;
     int rc = pcmk_rc_ok;
 
     const char *attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
     const char *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
     const char *regex = crm_element_value(xml, PCMK__XA_ATTR_PATTERN);
 
     rc = expand_regexes(xml, attr, value, regex);
 
     if (rc == EINVAL) {
         pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
                             "Bad regex '%s' for update from client %s", regex,
                             pcmk__client_name(request->ipc_client));
 
     } else if (rc == pcmk_rc_bad_nvpair) {
         crm_err("Update request did not specify attribute or regular expression");
         pcmk__format_result(&request->result, CRM_EX_ERROR, PCMK_EXEC_ERROR,
                             "Client %s update request did not specify attribute or regular expression",
                             pcmk__client_name(request->ipc_client));
     }
 
     return rc;
 }
 
 static int
 handle_value_expansion(const char **value, xmlNode *xml, const char *op,
                        const char *attr)
 {
     attribute_t *a = g_hash_table_lookup(attributes, attr);
 
     if (a == NULL && pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_DELAY, pcmk__str_none)) {
         return EINVAL;
     }
 
     if (*value && attrd_value_needs_expansion(*value)) {
         int int_value;
         attribute_value_t *v = NULL;
 
         if (a) {
             const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
             v = g_hash_table_lookup(a->values, host);
         }
 
         int_value = attrd_expand_value(*value, (v? v->current : NULL));
 
         crm_info("Expanded %s=%s to %d", attr, *value, int_value);
         crm_xml_add_int(xml, PCMK__XA_ATTR_VALUE, int_value);
 
         /* Replacing the value frees the previous memory, so re-query it */
         *value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
     }
 
     return pcmk_rc_ok;
 }
 
 static void
 send_update_msg_to_cluster(pcmk__request_t *request, xmlNode *xml)
 {
     if (pcmk__str_eq(attrd_request_sync_point(xml), PCMK__VALUE_CLUSTER, pcmk__str_none)) {
         /* The client is waiting on the cluster-wide sync point.  In this case,
          * the response ACK is not sent until this attrd broadcasts the update
          * and receives its own confirmation back from all peers.
          */
         attrd_expect_confirmations(request, attrd_cluster_sync_point_update);
         attrd_send_message(NULL, xml, true); /* ends up at attrd_peer_message() */
 
     } else {
         /* The client is either waiting on the local sync point or was not
          * waiting on any sync point at all.  For the local sync point, the
          * response ACK is sent in attrd_peer_update.  For clients not
          * waiting on any sync point, the response ACK is sent in
          * handle_update_request immediately before this function was called.
          */
         attrd_send_message(NULL, xml, false); /* ends up at attrd_peer_message() */
     }
 }
 
 static int
 send_child_update(xmlNode *child, void *data)
 {
     pcmk__request_t *request = (pcmk__request_t *) data;
 
     /* Calling pcmk__set_result is handled by one of these calls to
      * attrd_client_update, so no need to do it again here.
      */
     request->xml = child;
     attrd_client_update(request);
     return pcmk_rc_ok;
 }
 
 xmlNode *
 attrd_client_update(pcmk__request_t *request)
 {
     xmlNode *xml = NULL;
     const char *attr, *value, *regex;
 
     CRM_CHECK((request != NULL) && (request->xml != NULL), return NULL);
 
     xml = request->xml;
 
     /* If the message has children, that means it is a message from a newer
      * client that supports sending multiple operations at a time.  There are
      * two ways we can handle that.
      */
     if (xml->children != NULL) {
         if (ATTRD_SUPPORTS_MULTI_MESSAGE(minimum_protocol_version)) {
             /* First, if all peers support a certain protocol version, we can
              * just broadcast the big message and they'll handle it.  However,
              * we also need to apply all the transformations in this function
              * to the children since they don't happen anywhere else.
              */
             for (xmlNode *child = first_named_child(xml, PCMK_XE_OP);
                  child != NULL; child = crm_next_same_xml(child)) {
 
                 attr = crm_element_value(child, PCMK__XA_ATTR_NAME);
                 value = crm_element_value(child, PCMK__XA_ATTR_VALUE);
 
                 handle_missing_host(child);
 
                 if (handle_value_expansion(&value, child, request->op, attr) == EINVAL) {
                     pcmk__format_result(&request->result, CRM_EX_NOSUCH, PCMK_EXEC_ERROR,
                                         "Attribute %s does not exist", attr);
                     return NULL;
                 }
             }
 
             send_update_msg_to_cluster(request, xml);
             pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
 
         } else {
             /* Save the original xml node pointer so it can be restored after iterating
              * over all the children.
              */
             xmlNode *orig_xml = request->xml;
 
             /* Second, if they do not support that protocol version, split it
              * up into individual messages and call attrd_client_update on
              * each one.
              */
             pcmk__xe_foreach_child(xml, PCMK_XE_OP, send_child_update, request);
             request->xml = orig_xml;
         }
 
         return NULL;
     }
 
     attr = crm_element_value(xml, PCMK__XA_ATTR_NAME);
     value = crm_element_value(xml, PCMK__XA_ATTR_VALUE);
     regex = crm_element_value(xml, PCMK__XA_ATTR_PATTERN);
 
     if (handle_regexes(request) != pcmk_rc_ok) {
         /* Error handling was already dealt with in handle_regexes, so just return. */
         return NULL;
     } else if (regex) {
         /* Recursively call attrd_client_update on the new message with regexes
          * expanded.  If supported by the attribute daemon, this means that all
          * matches can also be handled atomically.
          */
         return attrd_client_update(request);
     }
 
     handle_missing_host(xml);
 
     if (handle_value_expansion(&value, xml, request->op, attr) == EINVAL) {
         pcmk__format_result(&request->result, CRM_EX_NOSUCH, PCMK_EXEC_ERROR,
                             "Attribute %s does not exist", attr);
         return NULL;
     }
 
     crm_debug("Broadcasting %s[%s]=%s%s", attr, crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME),
               value, (attrd_election_won()? " (writer)" : ""));
 
     send_update_msg_to_cluster(request, xml);
     pcmk__set_result(&request->result, CRM_EX_OK, PCMK_EXEC_DONE, NULL);
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Accept a new client IPC connection
  *
  * \param[in,out] c    New connection
  * \param[in]     uid  Client user id
  * \param[in]     gid  Client group id
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 static int32_t
 attrd_ipc_accept(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
 {
     crm_trace("New client connection %p", c);
     if (attrd_shutting_down(false)) {
         crm_info("Ignoring new connection from pid %d during shutdown",
                  pcmk__client_pid(c));
-        return -EPERM;
+        return -ECONNREFUSED;
     }
 
     if (pcmk__new_client(c, uid, gid) == NULL) {
         return -EIO;
     }
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Destroy a client IPC connection
  *
  * \param[in] c  Connection to destroy
  *
  * \return FALSE (i.e. do not re-run this callback)
  */
 static int32_t
 attrd_ipc_closed(qb_ipcs_connection_t *c)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     if (client == NULL) {
         crm_trace("Ignoring request to clean up unknown connection %p", c);
     } else {
         crm_trace("Cleaning up closed client connection %p", c);
 
         /* Remove the client from the sync point waitlist if it's present. */
         attrd_remove_client_from_waitlist(client);
 
         /* And no longer wait for confirmations from any peers. */
         attrd_do_not_wait_for_client(client);
 
         pcmk__free_client(client);
     }
 
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Destroy a client IPC connection
  *
  * \param[in,out] c  Connection to destroy
  *
  * \note We handle a destroyed connection the same as a closed one,
  *       but we need a separate handler because the return type is different.
  */
 static void
 attrd_ipc_destroy(qb_ipcs_connection_t *c)
 {
     crm_trace("Destroying client connection %p", c);
     attrd_ipc_closed(c);
 }
 
 static int32_t
 attrd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     pcmk__client_t *client = pcmk__find_client(c);
     xmlNode *xml = NULL;
 
     // Sanity-check, and parse XML from IPC data
     CRM_CHECK((c != NULL) && (client != NULL), return 0);
     if (data == NULL) {
         crm_debug("No IPC data from PID %d", pcmk__client_pid(c));
         return 0;
     }
 
     xml = pcmk__client_data2xml(client, data, &id, &flags);
 
     if (xml == NULL) {
         crm_debug("Unrecognizable IPC data from PID %d", pcmk__client_pid(c));
         pcmk__ipc_send_ack(client, id, flags, PCMK__XE_ACK, NULL,
                            CRM_EX_PROTOCOL);
         return 0;
 
     } else {
         pcmk__request_t request = {
             .ipc_client     = client,
             .ipc_id         = id,
             .ipc_flags      = flags,
             .peer           = NULL,
             .xml            = xml,
             .call_options   = 0,
             .result         = PCMK__UNKNOWN_RESULT,
         };
 
         CRM_ASSERT(client->user != NULL);
         pcmk__update_acl_user(xml, PCMK__XA_ATTR_USER, client->user);
 
         request.op = crm_element_value_copy(request.xml, PCMK__XA_TASK);
         CRM_CHECK(request.op != NULL, return 0);
 
         attrd_handle_request(&request);
         pcmk__reset_request(&request);
     }
 
     free_xml(xml);
     return 0;
 }
 
 static struct qb_ipcs_service_handlers ipc_callbacks = {
     .connection_accept = attrd_ipc_accept,
     .connection_created = NULL,
     .msg_process = attrd_ipc_dispatch,
     .connection_closed = attrd_ipc_closed,
     .connection_destroyed = attrd_ipc_destroy
 };
 
 void
 attrd_ipc_fini(void)
 {
     if (ipcs != NULL) {
         pcmk__drop_all_clients(ipcs);
         qb_ipcs_destroy(ipcs);
         ipcs = NULL;
     }
 }
 
 /*!
  * \internal
  * \brief Set up attrd IPC communication
  */
 void
 attrd_init_ipc(void)
 {
     pcmk__serve_attrd_ipc(&ipcs, &ipc_callbacks);
 }
diff --git a/daemons/based/based_callbacks.c b/daemons/based/based_callbacks.c
index cba0c653ef..bdd1cf6899 100644
--- a/daemons/based/based_callbacks.c
+++ b/daemons/based/based_callbacks.c
@@ -1,1728 +1,1728 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>   // PRIu64
 
 #include <glib.h>
 #include <libxml/tree.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/cluster/internal.h>
 
 #include <crm/common/xml.h>
 #include <crm/common/remote_internal.h>
 
 #include <pacemaker-based.h>
 
 #define EXIT_ESCALATION_MS 10000
 
 static unsigned long cib_local_bcast_num = 0;
 
 typedef struct cib_local_notify_s {
     xmlNode *notify_src;
     char *client_id;
     gboolean from_peer;
     gboolean sync_reply;
 } cib_local_notify_t;
 
 int next_client_id = 0;
 
 gboolean legacy_mode = FALSE;
 
 qb_ipcs_service_t *ipcs_ro = NULL;
 qb_ipcs_service_t *ipcs_rw = NULL;
 qb_ipcs_service_t *ipcs_shm = NULL;
 
 static int cib_process_command(xmlNode *request,
                                const cib__operation_t *operation,
                                cib__op_fn_t op_function, xmlNode **reply,
                                xmlNode **cib_diff, bool privileged);
 
 static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data,
                                     size_t size, gboolean privileged);
 
 gboolean
 cib_legacy_mode(void)
 {
     return legacy_mode;
 }
 
 static int32_t
 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
 {
     if (cib_shutdown_flag) {
         crm_info("Ignoring new IPC client [%d] during shutdown",
                  pcmk__client_pid(c));
-        return -EPERM;
+        return -ECONNREFUSED;
     }
 
     if (pcmk__new_client(c, uid, gid) == NULL) {
         return -EIO;
     }
     return 0;
 }
 
 static int32_t
 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     crm_trace("%p message from %s", c, client->id);
     return cib_common_callback(c, data, size, TRUE);
 }
 
 static int32_t
 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     crm_trace("%p message from %s", c, client->id);
     return cib_common_callback(c, data, size, FALSE);
 }
 
 /* Error code means? */
 static int32_t
 cib_ipc_closed(qb_ipcs_connection_t * c)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     if (client == NULL) {
         return 0;
     }
     crm_trace("Connection %p", c);
     pcmk__free_client(client);
     return 0;
 }
 
 static void
 cib_ipc_destroy(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p", c);
     cib_ipc_closed(c);
     if (cib_shutdown_flag) {
         cib_shutdown(0);
     }
 }
 
 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
     .connection_accept = cib_ipc_accept,
     .connection_created = NULL,
     .msg_process = cib_ipc_dispatch_ro,
     .connection_closed = cib_ipc_closed,
     .connection_destroyed = cib_ipc_destroy
 };
 
 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
     .connection_accept = cib_ipc_accept,
     .connection_created = NULL,
     .msg_process = cib_ipc_dispatch_rw,
     .connection_closed = cib_ipc_closed,
     .connection_destroyed = cib_ipc_destroy
 };
 
 /*!
  * \internal
  * \brief Create reply XML for a CIB request
  *
  * \param[in] op            CIB operation type
  * \param[in] call_id       CIB call ID
  * \param[in] client_id     CIB client ID
  * \param[in] call_options  Group of <tt>enum cib_call_options</tt> flags
  * \param[in] rc            Request return code
  * \param[in] call_data     Request output data
  *
  * \return Reply XML
  *
  * \note The caller is responsible for freeing the return value using
  *       \p free_xml().
  */
 static xmlNode *
 create_cib_reply(const char *op, const char *call_id, const char *client_id,
                  int call_options, int rc, xmlNode *call_data)
 {
     xmlNode *reply = create_xml_node(NULL, "cib-reply");
 
     CRM_ASSERT(reply != NULL);
 
     crm_xml_add(reply, PCMK__XA_T, T_CIB);
     crm_xml_add(reply, F_CIB_OPERATION, op);
     crm_xml_add(reply, F_CIB_CALLID, call_id);
     crm_xml_add(reply, F_CIB_CLIENTID, client_id);
     crm_xml_add_int(reply, F_CIB_CALLOPTS, call_options);
     crm_xml_add_int(reply, F_CIB_RC, rc);
 
     if (call_data != NULL) {
         crm_trace("Attaching reply output");
         add_message_xml(reply, F_CIB_CALLDATA, call_data);
     }
 
     crm_log_xml_explicit(reply, "cib:reply");
     return reply;
 }
 
 static void
 do_local_notify(const xmlNode *notify_src, const char *client_id,
                 bool sync_reply, bool from_peer)
 {
     int rid = 0;
     int call_id = 0;
     pcmk__client_t *client_obj = NULL;
 
     CRM_ASSERT(notify_src && client_id);
 
     crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);
 
     client_obj = pcmk__find_client_by_id(client_id);
     if (client_obj == NULL) {
         crm_debug("Could not send response %d: client %s not found",
                   call_id, client_id);
         return;
     }
 
     if (sync_reply) {
         if (client_obj->ipcs) {
             CRM_LOG_ASSERT(client_obj->request_id);
 
             rid = client_obj->request_id;
             client_obj->request_id = 0;
 
             crm_trace("Sending response %d to client %s%s",
                       rid, pcmk__client_name(client_obj),
                       (from_peer? " (originator of delegated request)" : ""));
         } else {
             crm_trace("Sending response (call %d) to client %s%s",
                       call_id, pcmk__client_name(client_obj),
                       (from_peer? " (originator of delegated request)" : ""));
         }
 
     } else {
         crm_trace("Sending event %d to client %s%s",
                   call_id, pcmk__client_name(client_obj),
                   (from_peer? " (originator of delegated request)" : ""));
     }
 
     switch (PCMK__CLIENT_TYPE(client_obj)) {
         case pcmk__client_ipc:
             {
                 int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
                                             (sync_reply? crm_ipc_flags_none
                                              : crm_ipc_server_event));
 
                 if (rc != pcmk_rc_ok) {
                     crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
                              (sync_reply? "Synchronous" : "Asynchronous"),
                              pcmk__client_name(client_obj), pcmk_rc_str(rc),
                              rc);
                 }
             }
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case pcmk__client_tls:
 #endif
         case pcmk__client_tcp:
             pcmk__remote_send_xml(client_obj->remote, notify_src);
             break;
         default:
             crm_err("Unknown transport for client %s "
                     CRM_XS " flags=%#016" PRIx64,
                     pcmk__client_name(client_obj), client_obj->flags);
     }
 }
 
 void
 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
                            pcmk__client_t *cib_client, gboolean privileged)
 {
     const char *op = crm_element_value(op_request, F_CIB_OPERATION);
     int call_options = cib_none;
 
     crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
 
     /* Requests with cib_transaction set should not be sent to based directly
      * (outside of a commit-transaction request)
      */
     if (pcmk_is_set(call_options, cib_transaction)) {
         return;
     }
 
     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
         if (flags & crm_ipc_client_response) {
             xmlNode *ack = create_xml_node(NULL, __func__);
 
             crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
             crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
             pcmk__ipc_send_xml(cib_client, id, ack, flags);
             cib_client->request_id = 0;
             free_xml(ack);
         }
         return;
 
     } else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
         /* Update the notify filters for this client */
         int on_off = 0;
         crm_exit_t status = CRM_EX_OK;
         uint64_t bit = UINT64_C(0);
         const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);
 
         crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
 
         crm_debug("Setting %s callbacks %s for client %s",
                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 
         if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
             bit = cib_notify_post;
 
         } else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
             bit = cib_notify_pre;
 
         } else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
             bit = cib_notify_confirm;
 
         } else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
             bit = cib_notify_diff;
 
         } else {
             status = CRM_EX_INVALID_PARAM;
         }
 
         if (bit != 0) {
             if (on_off) {
                 pcmk__set_client_flags(cib_client, bit);
             } else {
                 pcmk__clear_client_flags(cib_client, bit);
             }
         }
 
         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_ACK, NULL, status);
         return;
     }
 
     cib_process_request(op_request, privileged, cib_client);
 }
 
 int32_t
 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     int call_options = 0;
     pcmk__client_t *cib_client = pcmk__find_client(c);
     xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
 
     if (op_request) {
         crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
     }
 
     if (op_request == NULL) {
         crm_trace("Invalid message from %p", c);
         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_NACK, NULL,
                            CRM_EX_PROTOCOL);
         return 0;
 
     } else if(cib_client == NULL) {
         crm_trace("Invalid client %p", c);
         return 0;
     }
 
     if (pcmk_is_set(call_options, cib_sync_call)) {
         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
         cib_client->request_id = id;    /* Reply only to the last one */
     }
 
     if (cib_client->name == NULL) {
         const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);
 
         if (value == NULL) {
             cib_client->name = pcmk__itoa(cib_client->pid);
         } else {
             cib_client->name = strdup(value);
             if (crm_is_daemon_name(value)) {
                 pcmk__set_client_flags(cib_client, cib_is_daemon);
             }
         }
     }
 
     /* Allow cluster daemons more leeway before being evicted */
     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
         const char *qmax = cib_config_lookup(PCMK_OPT_CLUSTER_IPC_LIMIT);
 
         if (pcmk__set_client_queue_max(cib_client, qmax)) {
             crm_trace("IPC threshold for client %s[%u] is now %u",
                       pcmk__client_name(cib_client), cib_client->pid,
                       cib_client->queue_max);
         }
     }
 
     crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
     crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
 
     CRM_LOG_ASSERT(cib_client->user != NULL);
     pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);
 
     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
     free_xml(op_request);
 
     return 0;
 }
 
 static uint64_t ping_seq = 0;
 static char *ping_digest = NULL;
 static bool ping_modified_since = FALSE;
 
 static gboolean
 cib_digester_cb(gpointer data)
 {
     if (based_is_primary) {
         char buffer[32];
         xmlNode *ping = create_xml_node(NULL, PCMK__XE_PING);
 
         ping_seq++;
         free(ping_digest);
         ping_digest = NULL;
         ping_modified_since = FALSE;
         snprintf(buffer, 32, "%" PRIu64, ping_seq);
         crm_trace("Requesting peer digests (%s)", buffer);
 
         crm_xml_add(ping, PCMK__XA_T, T_CIB);
         crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
         crm_xml_add(ping, F_CIB_PING_ID, buffer);
 
         crm_xml_add(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
         send_cluster_message(NULL, crm_msg_cib, ping, TRUE);
 
         free_xml(ping);
     }
     return FALSE;
 }
 
 static void
 process_ping_reply(xmlNode *reply) 
 {
     uint64_t seq = 0;
     const char *host = crm_element_value(reply, PCMK__XA_SRC);
 
     xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
     const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
     const char *digest = crm_element_value(pong, PCMK__XA_DIGEST);
 
     if (seq_s == NULL) {
         crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
         return;
 
     } else {
         long long seq_ll;
 
         if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
             return;
         }
         seq = (uint64_t) seq_ll;
     }
 
     if(digest == NULL) {
         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 
     } else if(seq != ping_seq) {
         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 
     } else if(ping_modified_since) {
         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 
     } else {
         const char *version = crm_element_value(pong, PCMK_XA_CRM_FEATURE_SET);
 
         if(ping_digest == NULL) {
             crm_trace("Calculating new digest");
             ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
         }
 
         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
             xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);
             const char *admin_epoch_s = NULL;
             const char *epoch_s = NULL;
             const char *num_updates_s = NULL;
 
             if (remote_cib != NULL) {
                 admin_epoch_s = crm_element_value(remote_cib,
                                                   PCMK_XA_ADMIN_EPOCH);
                 epoch_s = crm_element_value(remote_cib, PCMK_XA_EPOCH);
                 num_updates_s = crm_element_value(remote_cib,
                                                   PCMK_XA_NUM_UPDATES);
             }
 
             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
                        crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH),
                        crm_element_value(the_cib, PCMK_XA_EPOCH),
                        crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
                        ping_digest, host,
                        pcmk__s(admin_epoch_s, "_"),
                        pcmk__s(epoch_s, "_"),
                        pcmk__s(num_updates_s, "_"),
                        digest, remote_cib);
 
             if(remote_cib && remote_cib->children) {
                 // Additional debug
                 xml_calculate_changes(the_cib, remote_cib);
                 pcmk__log_xml_changes(LOG_INFO, remote_cib);
                 crm_trace("End of differences");
             }
 
             free_xml(remote_cib);
             sync_our_cib(reply, FALSE);
         }
     }
 }
 
 static void
 local_notify_destroy_callback(gpointer data)
 {
     cib_local_notify_t *notify = data;
 
     free_xml(notify->notify_src);
     free(notify->client_id);
     free(notify);
 }
 
 static void
 check_local_notify(int bcast_id)
 {
     const cib_local_notify_t *notify = NULL;
 
     if (!local_notify_queue) {
         return;
     }
 
     notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);
 
     if (notify) {
         do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
                         notify->from_peer);
         pcmk__intkey_table_remove(local_notify_queue, bcast_id);
     }
 }
 
 static void
 queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
                    gboolean from_peer)
 {
     cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));
 
     notify->notify_src = notify_src;
     notify->client_id = strdup(client_id);
     notify->sync_reply = sync_reply;
     notify->from_peer = from_peer;
 
     if (!local_notify_queue) {
         local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
     }
     pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
     // cppcheck doesn't know notify will get freed when hash table is destroyed
     // cppcheck-suppress memleak
 }
 
 static void
 parse_local_options_v1(const pcmk__client_t *cib_client,
                        const cib__operation_t *operation, int call_options,
                        const char *host, const char *op, gboolean *local_notify,
                        gboolean *needs_reply, gboolean *process,
                        gboolean *needs_forward)
 {
     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
         && !pcmk_is_set(call_options, cib_inhibit_bcast)) {
         /* we need to send an update anyway */
         *needs_reply = TRUE;
     } else {
         *needs_reply = FALSE;
     }
 
     if (host == NULL && (call_options & cib_scope_local)) {
         crm_trace("Processing locally scoped %s op from client %s",
                   op, pcmk__client_name(cib_client));
         *local_notify = TRUE;
 
     } else if ((host == NULL) && based_is_primary) {
         crm_trace("Processing %s op locally from client %s as primary",
                   op, pcmk__client_name(cib_client));
         *local_notify = TRUE;
 
     } else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
         crm_trace("Processing locally addressed %s op from client %s",
                   op, pcmk__client_name(cib_client));
         *local_notify = TRUE;
 
     } else if (stand_alone) {
         *needs_forward = FALSE;
         *local_notify = TRUE;
         *process = TRUE;
 
     } else {
         crm_trace("%s op from %s needs to be forwarded to client %s",
                   op, pcmk__client_name(cib_client),
                   pcmk__s(host, "the primary instance"));
         *needs_forward = TRUE;
         *process = FALSE;
     }
 }
 
 static void
 parse_local_options_v2(const pcmk__client_t *cib_client,
                        const cib__operation_t *operation, int call_options,
                        const char *host, const char *op, gboolean *local_notify,
                        gboolean *needs_reply, gboolean *process,
                        gboolean *needs_forward)
 {
     // Process locally and notify local client
     *process = TRUE;
     *needs_reply = FALSE;
     *local_notify = TRUE;
     *needs_forward = FALSE;
 
     if (pcmk_is_set(operation->flags, cib__op_attr_local)) {
         /* Always process locally if cib__op_attr_local is set.
          *
          * @COMPAT: Currently host is ignored. At a compatibility break, throw
          * an error (from cib_process_request() or earlier) if host is not NULL or
          * OUR_NODENAME.
          */
         crm_trace("Processing always-local %s op from client %s",
                   op, pcmk__client_name(cib_client));
 
         if (!pcmk__str_eq(host, OUR_NODENAME,
                           pcmk__str_casei|pcmk__str_null_matches)) {
 
             crm_warn("Operation '%s' is always local but its target host is "
                      "set to '%s'",
                      op, host);
         }
         return;
     }
 
     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
         || !pcmk__str_eq(host, OUR_NODENAME,
                          pcmk__str_casei|pcmk__str_null_matches)) {
 
         // Forward modifying and non-local requests via cluster
         *process = FALSE;
         *needs_reply = FALSE;
         *local_notify = FALSE;
         *needs_forward = TRUE;
 
         crm_trace("%s op from %s needs to be forwarded to %s",
                   op, pcmk__client_name(cib_client),
                   pcmk__s(host, "all nodes"));
         return;
     }
 
     if (stand_alone) {
         crm_trace("Processing %s op from client %s (stand-alone)",
                   op, pcmk__client_name(cib_client));
 
     } else {
         crm_trace("Processing %saddressed %s op from client %s",
                   ((host != NULL)? "locally " : "un"),
                   op, pcmk__client_name(cib_client));
     }
 }
 
 static void
 parse_local_options(const pcmk__client_t *cib_client,
                     const cib__operation_t *operation, int call_options,
                     const char *host, const char *op, gboolean *local_notify,
                     gboolean *needs_reply, gboolean *process,
                     gboolean *needs_forward)
 {
     if(cib_legacy_mode()) {
         parse_local_options_v1(cib_client, operation, call_options, host,
                                op, local_notify, needs_reply, process,
                                needs_forward);
     } else {
         parse_local_options_v2(cib_client, operation, call_options, host,
                                op, local_notify, needs_reply, process,
                                needs_forward);
     }
 }
 
 static gboolean
 parse_peer_options_v1(const cib__operation_t *operation, xmlNode *request,
                       gboolean *local_notify, gboolean *needs_reply,
                       gboolean *process)
 {
     const char *op = NULL;
     const char *host = NULL;
     const char *delegated = NULL;
     const char *originator = crm_element_value(request, PCMK__XA_SRC);
     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 
     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 
     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
         *needs_reply = FALSE;
         if (is_reply) {
             *local_notify = TRUE;
             crm_trace("Processing global/peer update from %s"
                       " that originated from us", originator);
         } else {
             crm_trace("Processing global/peer update from %s", originator);
         }
         return TRUE;
     }
 
     op = crm_element_value(request, F_CIB_OPERATION);
     crm_trace("Processing legacy %s request sent by %s", op, originator);
 
     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
         /* Always process these */
         *local_notify = FALSE;
         if (reply_to == NULL || is_reply) {
             *process = TRUE;
         }
         if (is_reply) {
             *needs_reply = FALSE;
         }
         return *process;
     }
 
     if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
         process_ping_reply(request);
         return FALSE;
     }
 
     if (is_reply) {
         crm_trace("Forward reply sent from %s to local clients", originator);
         *process = FALSE;
         *needs_reply = FALSE;
         *local_notify = TRUE;
         return TRUE;
     }
 
     host = crm_element_value(request, F_CIB_HOST);
     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
         crm_trace("Processing %s request sent to us from %s", op, originator);
         return TRUE;
 
     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
         crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
         *needs_reply = TRUE;
         return TRUE;
 
     } else if ((host == NULL) && based_is_primary) {
         crm_trace("Processing %s request sent to primary instance from %s",
                   op, originator);
         return TRUE;
     }
 
     delegated = crm_element_value(request, F_CIB_DELEGATED);
     if (delegated != NULL) {
         crm_trace("Ignoring message for primary instance");
 
     } else if (host != NULL) {
         /* this is for a specific instance and we're not it */
         crm_trace("Ignoring msg for instance on %s", host);
 
     } else if ((reply_to == NULL) && !based_is_primary) {
         // This is for the primary instance, and we're not it
         crm_trace("Ignoring reply for primary instance");
 
     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
         if (reply_to != NULL) {
             crm_debug("Processing %s from %s", op, originator);
             *needs_reply = FALSE;
 
         } else {
             crm_debug("Processing %s reply from %s", op, originator);
         }
         return TRUE;
 
     } else {
         crm_err("Nothing for us to do?");
         crm_log_xml_err(request, "Peer[inbound]");
     }
 
     return FALSE;
 }
 
 static gboolean
 parse_peer_options_v2(const cib__operation_t *operation, xmlNode *request,
                       gboolean *local_notify, gboolean *needs_reply,
                       gboolean *process)
 {
     const char *host = NULL;
     const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
     const char *op = crm_element_value(request, F_CIB_OPERATION);
     const char *originator = crm_element_value(request, PCMK__XA_SRC);
     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 
     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 
     if (originator == NULL) { // Shouldn't be possible
         originator = "peer";
     }
 
     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
         /* sync_our_cib() sets F_CIB_ISREPLY */
         if (reply_to) {
             delegated = reply_to;
         }
         goto skip_is_reply;
 
     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
                             pcmk__str_none)) {
         // Nothing to do
 
     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
         process_ping_reply(request);
         return FALSE;
 
     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
         /* Only the DC (node with the oldest software) should process
          * this operation if F_CIB_SCHEMA_MAX is unset
          *
          * If the DC is happy it will then send out another
          * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
          * upgrade.
          *
          * Except this time F_CIB_SCHEMA_MAX will be set which puts a
          * limit on how far newer nodes will go
          */
         const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
         const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);
 
         crm_trace("Parsing upgrade %s for %s with max=%s and upgrade_rc=%s",
                   (is_reply? "reply" : "request"),
                   (based_is_primary? "primary" : "secondary"),
                   pcmk__s(max, "none"), pcmk__s(upgrade_rc, "none"));
 
         if (upgrade_rc != NULL) {
             // Our upgrade request was rejected by DC, notify clients of result
             crm_xml_add(request, F_CIB_RC, upgrade_rc);
 
         } else if ((max == NULL) && based_is_primary) {
             /* We are the DC, check if this upgrade is allowed */
             goto skip_is_reply;
 
         } else if(max) {
             /* Ok, go ahead and upgrade to 'max' */
             goto skip_is_reply;
 
         } else {
             // Ignore broadcast client requests when we're not primary
             return FALSE;
         }
 
     } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
         crm_info("Detected legacy %s global update from %s", op, originator);
         send_sync_request(NULL);
         legacy_mode = TRUE;
         return FALSE;
 
     } else if (is_reply
                && pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
         return FALSE;
 
     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
         *local_notify = FALSE;
         if (reply_to == NULL) {
             *process = TRUE;
         } else { // Not possible?
             crm_debug("Ignoring shutdown request from %s because reply_to=%s",
                       originator, reply_to);
         }
         return *process;
     }
 
     if (is_reply) {
         crm_trace("Will notify local clients for %s reply from %s",
                   op, originator);
         *process = FALSE;
         *needs_reply = FALSE;
         *local_notify = TRUE;
         return TRUE;
     }
 
   skip_is_reply:
     *process = TRUE;
     *needs_reply = FALSE;
 
     *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei);
 
     host = crm_element_value(request, F_CIB_HOST);
     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
         crm_trace("Processing %s request sent to us from %s", op, originator);
         *needs_reply = TRUE;
         return TRUE;
 
     } else if (host != NULL) {
         crm_trace("Ignoring %s request intended for CIB manager on %s",
                   op, host);
         return FALSE;
 
     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
         *needs_reply = TRUE;
     }
 
     crm_trace("Processing %s request broadcast by %s call %s on %s "
               "(local clients will%s be notified)", op,
               pcmk__s(crm_element_value(request, F_CIB_CLIENTNAME), "client"),
               pcmk__s(crm_element_value(request, F_CIB_CALLID), "without ID"),
               originator, (*local_notify? "" : "not"));
     return TRUE;
 }
 
 static gboolean
 parse_peer_options(const cib__operation_t *operation, xmlNode *request,
                    gboolean *local_notify, gboolean *needs_reply,
                    gboolean *process)
 {
     /* TODO: What happens when an update comes in after node A
      * requests the CIB from node B, but before it gets the reply (and
      * sends out the replace operation)
      */
     if(cib_legacy_mode()) {
         return parse_peer_options_v1(operation, request, local_notify,
                                      needs_reply, process);
     } else {
         return parse_peer_options_v2(operation, request, local_notify,
                                      needs_reply, process);
     }
 }
 
 /*!
  * \internal
  * \brief Forward a CIB request to the appropriate target host(s)
  *
  * \param[in] request  CIB request to forward
  */
 static void
 forward_request(xmlNode *request)
 {
     const char *op = crm_element_value(request, F_CIB_OPERATION);
     const char *section = crm_element_value(request, F_CIB_SECTION);
     const char *host = crm_element_value(request, F_CIB_HOST);
     const char *originator = crm_element_value(request, PCMK__XA_SRC);
     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
     const char *call_id = crm_element_value(request, F_CIB_CALLID);
     crm_node_t *peer = NULL;
 
     int log_level = LOG_INFO;
 
     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
         log_level = LOG_DEBUG;
     }
 
     do_crm_log(log_level,
                "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
                pcmk__s(op, "invalid"),
                pcmk__s(section, "all"),
                pcmk__s(host, (cib_legacy_mode()? "primary" : "all")),
                pcmk__s(originator, "local"),
                pcmk__s(client_name, "unspecified"),
                pcmk__s(call_id, "unspecified"));
 
     crm_xml_add(request, F_CIB_DELEGATED, OUR_NODENAME);
 
     if (host != NULL) {
         peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster);
     }
     send_cluster_message(peer, crm_msg_cib, request, FALSE);
 
     // Return the request to its original state
     xml_remove_prop(request, F_CIB_DELEGATED);
 }
 
 static gboolean
 send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
 {
     CRM_ASSERT(msg != NULL);
 
     if (broadcast) {
         /* @COMPAT: Legacy code
          *
          * This successful call modified the CIB, and the change needs to be
          * broadcast (sent via cluster to all nodes).
          */
         int diff_add_updates = 0;
         int diff_add_epoch = 0;
         int diff_add_admin_epoch = 0;
 
         int diff_del_updates = 0;
         int diff_del_epoch = 0;
         int diff_del_admin_epoch = 0;
 
         const char *digest = NULL;
         int format = 1;
 
         CRM_LOG_ASSERT(result_diff != NULL);
         digest = crm_element_value(result_diff, PCMK__XA_DIGEST);
         crm_element_value_int(result_diff, PCMK_XA_FORMAT, &format);
 
         cib_diff_version_details(result_diff,
                                  &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
                                  &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
         crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
 
         crm_xml_add(msg, F_CIB_ISREPLY, originator);
         pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
         crm_xml_add(msg, F_CIB_OPERATION, PCMK__CIB_REQUEST_APPLY_PATCH);
         crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);
 
         if (format == 1) {
             CRM_ASSERT(digest != NULL);
         }
 
         add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
         crm_log_xml_explicit(msg, "copy");
         return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
 
     } else if (originator != NULL) {
         /* send reply via HA to originating node */
         crm_trace("Sending request result to %s only", originator);
         crm_xml_add(msg, F_CIB_ISREPLY, originator);
         return send_cluster_message(pcmk__get_node(0, originator, NULL,
                                                    pcmk__node_search_cluster),
                                     crm_msg_cib, msg, FALSE);
     }
 
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Handle an IPC or CPG message containing a request
  *
  * \param[in,out] request        Request XML
  * \param[in] privileged         Whether privileged commands may be run
  *                               (see cib_server_ops[] definition)
  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
  *
  * \return Legacy Pacemaker return code
  */
 int
 cib_process_request(xmlNode *request, gboolean privileged,
                     const pcmk__client_t *cib_client)
 {
     // @TODO: Break into multiple smaller functions
     int call_options = 0;
 
     gboolean process = TRUE;        // Whether to process request locally now
     gboolean is_update = TRUE;      // Whether request would modify CIB
     gboolean needs_reply = TRUE;    // Whether to build a reply
     gboolean local_notify = FALSE;  // Whether to notify (local) requester
     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
 
     xmlNode *op_reply = NULL;
     xmlNode *result_diff = NULL;
 
     int rc = pcmk_ok;
     const char *op = crm_element_value(request, F_CIB_OPERATION);
     const char *originator = crm_element_value(request, PCMK__XA_SRC);
     const char *host = crm_element_value(request, F_CIB_HOST);
     const char *target = NULL;
     const char *call_id = crm_element_value(request, F_CIB_CALLID);
     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 
     const cib__operation_t *operation = NULL;
     cib__op_fn_t op_function = NULL;
 
     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
 
     if ((host != NULL) && (*host == '\0')) {
         host = NULL;
     }
 
     // @TODO: Improve trace messages. Target is accurate only for legacy mode.
     if (host) {
         target = host;
 
     } else if (call_options & cib_scope_local) {
         target = "local host";
 
     } else {
         target = "primary";
     }
 
     if (cib_client == NULL) {
         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
                   op, client_name, call_id, originator, target, reply_to);
     } else {
         crm_xml_add(request, PCMK__XA_SRC, OUR_NODENAME);
         crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
     }
 
     rc = cib__get_operation(op, &operation);
     rc = pcmk_rc2legacy(rc);
     if (rc != pcmk_ok) {
         /* TODO: construct error reply? */
         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
         return rc;
     }
 
     op_function = based_get_op_function(operation);
     if (op_function == NULL) {
         crm_err("Operation %s not supported by CIB manager", op);
         return -EOPNOTSUPP;
     }
 
     if (cib_client != NULL) {
         parse_local_options(cib_client, operation, call_options, host, op,
                             &local_notify, &needs_reply, &process,
                             &needs_forward);
 
     } else if (!parse_peer_options(operation, request, &local_notify,
                                    &needs_reply, &process)) {
         return rc;
     }
 
     if (pcmk_is_set(call_options, cib_transaction)) {
         /* All requests in a transaction are processed locally against a working
          * CIB copy, and we don't notify for individual requests because the
          * entire transaction is atomic.
          *
          * We still call the option parser functions above, for the sake of log
          * messages and checking whether we're the target for peer requests.
          */
         process = TRUE;
         needs_reply = FALSE;
         local_notify = FALSE;
         needs_forward = FALSE;
     }
 
     is_update = pcmk_is_set(operation->flags, cib__op_attr_modifies);
 
     if (pcmk_is_set(call_options, cib_discard_reply)) {
         /* If the request will modify the CIB, and we are in legacy mode, we
          * need to build a reply so we can broadcast a diff, even if the
          * requester doesn't want one.
          */
         needs_reply = is_update && cib_legacy_mode();
         local_notify = FALSE;
         crm_trace("Client is not interested in the reply");
     }
 
     if (needs_forward) {
         forward_request(request);
         return rc;
     }
 
     if (cib_status != pcmk_ok) {
         rc = cib_status;
         crm_err("Operation ignored, cluster configuration is invalid."
                 " Please repair and restart: %s", pcmk_strerror(cib_status));
 
         op_reply = create_cib_reply(op, call_id, client_id, call_options, rc,
                                     the_cib);
 
     } else if (process) {
         time_t finished = 0;
         time_t now = time(NULL);
         int level = LOG_INFO;
         const char *section = crm_element_value(request, F_CIB_SECTION);
         const char *admin_epoch_s = NULL;
         const char *epoch_s = NULL;
         const char *num_updates_s = NULL;
 
         rc = cib_process_command(request, operation, op_function, &op_reply,
                                  &result_diff, privileged);
 
         if (!is_update) {
             level = LOG_TRACE;
 
         } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
             switch (rc) {
                 case pcmk_ok:
                     level = LOG_INFO;
                     break;
                 case -pcmk_err_old_data:
                 case -pcmk_err_diff_resync:
                 case -pcmk_err_diff_failed:
                     level = LOG_TRACE;
                     break;
                 default:
                     level = LOG_ERR;
             }
 
         } else if (rc != pcmk_ok) {
             level = LOG_WARNING;
         }
 
         if (the_cib != NULL) {
             admin_epoch_s = crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH);
             epoch_s = crm_element_value(the_cib, PCMK_XA_EPOCH);
             num_updates_s = crm_element_value(the_cib, PCMK_XA_NUM_UPDATES);
         }
 
         do_crm_log(level,
                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
                    originator ? originator : "local", client_name, call_id,
                    pcmk__s(admin_epoch_s, "0"),
                    pcmk__s(epoch_s, "0"),
                    pcmk__s(num_updates_s, "0"));
 
         finished = time(NULL);
         if ((finished - now) > 3) {
             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
             crm_write_blackbox(0, NULL);
         }
 
         if (op_reply == NULL && (needs_reply || local_notify)) {
             crm_err("Unexpected NULL reply to message");
             crm_log_xml_err(request, "null reply");
             needs_reply = FALSE;
             local_notify = FALSE;
         }
     }
 
     if (is_update && !cib_legacy_mode()) {
         crm_trace("Completed pre-sync update from %s/%s/%s%s",
                   originator ? originator : "local", client_name, call_id,
                   local_notify?" with local notification":"");
 
     } else if (!needs_reply || stand_alone) {
         // This was a non-originating secondary update
         crm_trace("Completed update as secondary");
 
     } else if (cib_legacy_mode() &&
                rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
         gboolean broadcast = FALSE;
 
         cib_local_bcast_num++;
         crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
         broadcast = send_peer_reply(request, result_diff, originator, TRUE);
 
         if (broadcast && client_id && local_notify && op_reply) {
 
             /* If we have been asked to sync the reply,
              * and a bcast msg has gone out, we queue the local notify
              * until we know the bcast message has been received */
             local_notify = FALSE;
             crm_trace("Queuing local %ssync notification for %s",
                       (call_options & cib_sync_call) ? "" : "a-", client_id);
 
             queue_local_notify(op_reply, client_id,
                                pcmk_is_set(call_options, cib_sync_call),
                                (cib_client == NULL));
             op_reply = NULL;    /* the reply is queued, so don't free here */
         }
 
     } else if ((cib_client == NULL)
                && !pcmk_is_set(call_options, cib_discard_reply)) {
 
         if (is_update == FALSE || result_diff == NULL) {
             crm_trace("Request not broadcast: R/O call");
 
         } else if (call_options & cib_inhibit_bcast) {
             crm_trace("Request not broadcast: inhibited");
 
         } else if (rc != pcmk_ok) {
             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
 
         } else {
             crm_trace("Directing reply to %s", originator);
         }
 
         send_peer_reply(op_reply, result_diff, originator, FALSE);
     }
 
     if (local_notify && client_id) {
         crm_trace("Performing local %ssync notification for %s",
                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
                   client_id);
         if (process == FALSE) {
             do_local_notify(request, client_id,
                             pcmk_is_set(call_options, cib_sync_call),
                             (cib_client == NULL));
         } else {
             do_local_notify(op_reply, client_id,
                             pcmk_is_set(call_options, cib_sync_call),
                             (cib_client == NULL));
         }
     }
 
     free_xml(op_reply);
     free_xml(result_diff);
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Get a CIB operation's input from the request XML
  *
  * \param[in]  request  CIB request XML
  * \param[in]  type     CIB operation type
  * \param[out] section  Where to store CIB section name
  *
  * \return Input XML for CIB operation
  *
  * \note If not \c NULL, the return value is a non-const pointer to part of
  *       \p request. The caller should not free it directly.
  */
 static xmlNode *
 prepare_input(const xmlNode *request, enum cib__op_type type,
               const char **section)
 {
     xmlNode *input = NULL;
 
     *section = NULL;
 
     switch (type) {
         case cib__op_apply_patch:
             if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
                 input = get_message_xml(request, F_CIB_UPDATE_DIFF);
             } else {
                 input = get_message_xml(request, F_CIB_CALLDATA);
             }
             break;
 
         default:
             input = get_message_xml(request, F_CIB_CALLDATA);
             *section = crm_element_value(request, F_CIB_SECTION);
             break;
     }
 
     // Grab the specified section
     if ((*section != NULL) && pcmk__xe_is(input, PCMK_XE_CIB)) {
         input = pcmk_find_cib_element(input, *section);
     }
 
     return input;
 }
 
 // v1 and v2 patch formats
 #define XPATH_CONFIG_CHANGE         \
     "//" PCMK_XE_CRM_CONFIG " | "   \
     "//" PCMK_XE_CHANGE             \
     "[contains(@" PCMK_XA_PATH ",'/" PCMK_XE_CRM_CONFIG "/')]"
 
 static bool
 contains_config_change(xmlNode *diff)
 {
     bool changed = false;
 
     if (diff) {
         xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE);
 
         if (numXpathResults(xpathObj) > 0) {
             changed = true;
         }
         freeXpathObject(xpathObj);
     }
     return changed;
 }
 
 static int
 cib_process_command(xmlNode *request, const cib__operation_t *operation,
                     cib__op_fn_t op_function, xmlNode **reply,
                     xmlNode **cib_diff, bool privileged)
 {
     xmlNode *input = NULL;
     xmlNode *output = NULL;
     xmlNode *result_cib = NULL;
 
     int call_options = 0;
 
     const char *op = NULL;
     const char *section = NULL;
     const char *call_id = crm_element_value(request, F_CIB_CALLID);
     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 
     int rc = pcmk_ok;
 
     bool config_changed = false;
     bool manage_counters = true;
 
     static mainloop_timer_t *digest_timer = NULL;
 
     CRM_ASSERT(cib_status == pcmk_ok);
 
     if(digest_timer == NULL) {
         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
     }
 
     *reply = NULL;
     *cib_diff = NULL;
 
     /* Start processing the request... */
     op = crm_element_value(request, F_CIB_OPERATION);
     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
 
     if (!privileged && pcmk_is_set(operation->flags, cib__op_attr_privileged)) {
         rc = -EACCES;
         crm_trace("Failed due to lack of privileges: %s", pcmk_strerror(rc));
         goto done;
     }
 
     input = prepare_input(request, operation->type, &section);
 
     if (!pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
         rc = cib_perform_op(NULL, op, call_options, op_function, true, section,
                             request, input, false, &config_changed, &the_cib,
                             &result_cib, NULL, &output);
 
         CRM_CHECK(result_cib == NULL, free_xml(result_cib));
         goto done;
     }
 
     /* @COMPAT: Handle a valid write action (legacy)
      *
      * @TODO: Re-evaluate whether this is all truly legacy. The cib_force_diff
      * portion is. However, F_CIB_GLOBAL_UPDATE may be set by a sync operation
      * even in non-legacy mode, and manage_counters tells xml_create_patchset()
      * whether to update version/epoch info.
      */
     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
         manage_counters = false;
         cib__set_call_options(call_options, "call", cib_force_diff);
         crm_trace("Global update detected");
 
         CRM_LOG_ASSERT(pcmk__str_any_of(op,
                                         PCMK__CIB_REQUEST_APPLY_PATCH,
                                         PCMK__CIB_REQUEST_REPLACE,
                                         NULL));
     }
 
     ping_modified_since = TRUE;
     if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
         crm_trace("Skipping update: inhibit broadcast");
         manage_counters = false;
     }
 
     // result_cib must not be modified after cib_perform_op() returns
     rc = cib_perform_op(NULL, op, call_options, op_function, false, section,
                         request, input, manage_counters, &config_changed,
                         &the_cib, &result_cib, cib_diff, &output);
 
     // @COMPAT: Legacy code
     if (!manage_counters) {
         int format = 1;
 
         // If the diff is NULL at this point, it's because nothing changed
         if (*cib_diff != NULL) {
             crm_element_value_int(*cib_diff, PCMK_XA_FORMAT, &format);
         }
 
         if (format == 1) {
             config_changed = cib__config_changed_v1(NULL, NULL, cib_diff);
         }
     }
 
     /* Always write to disk for successful ops with the flag set. This also
      * negates the need to detect ordering changes.
      */
     if ((rc == pcmk_ok)
         && pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
 
         config_changed = true;
     }
 
     if ((rc == pcmk_ok)
         && !pcmk_any_flags_set(call_options, cib_dryrun|cib_transaction)) {
 
         if (result_cib != the_cib) {
             if (pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
                 config_changed = true;
             }
 
             crm_trace("Activating %s->%s%s",
                       crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
                       crm_element_value(result_cib, PCMK_XA_NUM_UPDATES),
                       (config_changed? " changed" : ""));
 
             rc = activateCibXml(result_cib, config_changed, op);
             if (rc != pcmk_ok) {
                 crm_err("Failed to activate new CIB: %s", pcmk_strerror(rc));
             }
         }
 
         if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) {
             cib_read_config(config_hash, result_cib);
         }
 
         /* @COMPAT Nodes older than feature set 3.19.0 don't support
          * transactions. In a mixed-version cluster with nodes <3.19.0, we must
          * sync the updated CIB, so that the older nodes receive the changes.
          * Any node that has already applied the transaction will ignore the
          * synced CIB.
          *
          * To ensure the updated CIB is synced from only one node, we sync it
          * from the originator.
          */
         if ((operation->type == cib__op_commit_transact)
             && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei)
             && compare_version(crm_element_value(the_cib,
                                                  PCMK_XA_CRM_FEATURE_SET),
                                "3.19.0") < 0) {
 
             sync_our_cib(request, TRUE);
         }
 
         mainloop_timer_stop(digest_timer);
         mainloop_timer_start(digest_timer);
 
     } else if (rc == -pcmk_err_schema_validation) {
         CRM_ASSERT(result_cib != the_cib);
 
         if (output != NULL) {
             crm_log_xml_info(output, "cib:output");
             free_xml(output);
         }
 
         output = result_cib;
 
     } else {
         crm_trace("Not activating %d %d %s", rc,
                   pcmk_is_set(call_options, cib_dryrun),
                   crm_element_value(result_cib, PCMK_XA_NUM_UPDATES));
 
         if (result_cib != the_cib) {
             free_xml(result_cib);
         }
     }
 
     if (!pcmk_any_flags_set(call_options,
                             cib_dryrun|cib_inhibit_notify|cib_transaction)) {
         crm_trace("Sending notifications %d",
                   pcmk_is_set(call_options, cib_dryrun));
         cib_diff_notify(op, rc, call_id, client_id, client_name, originator,
                         input, *cib_diff);
     }
 
     pcmk__log_xml_patchset(LOG_TRACE, *cib_diff);
 
   done:
     if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) {
         *reply = create_cib_reply(op, call_id, client_id, call_options, rc,
                                   output);
     }
 
     if (output != the_cib) {
         free_xml(output);
     }
     crm_trace("done");
     return rc;
 }
 
 void
 cib_peer_callback(xmlNode * msg, void *private_data)
 {
     const char *reason = NULL;
     const char *originator = crm_element_value(msg, PCMK__XA_SRC);
 
     if (cib_legacy_mode()
         && pcmk__str_eq(originator, OUR_NODENAME,
                         pcmk__str_casei|pcmk__str_null_matches)) {
         /* message is from ourselves */
         int bcast_id = 0;
 
         if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
             check_local_notify(bcast_id);
         }
         return;
 
     } else if (crm_peer_cache == NULL) {
         reason = "membership not established";
         goto bail;
     }
 
     if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
         crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
     }
 
     /* crm_log_xml_trace(msg, "Peer[inbound]"); */
     cib_process_request(msg, TRUE, NULL);
     return;
 
   bail:
     if (reason) {
         const char *op = crm_element_value(msg, F_CIB_OPERATION);
 
         crm_warn("Discarding %s message from %s: %s", op, originator, reason);
     }
 }
 
 static gboolean
 cib_force_exit(gpointer data)
 {
     crm_notice("Forcing exit!");
     terminate_cib(__func__, CRM_EX_ERROR);
     return FALSE;
 }
 
 static void
 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
 {
     pcmk__client_t *a_client = value;
 
     crm_err("Can't disconnect client %s: Not implemented",
             pcmk__client_name(a_client));
 }
 
 static void
 initiate_exit(void)
 {
     int active = 0;
     xmlNode *leaving = NULL;
 
     active = crm_active_peers();
     if (active < 2) { // This is the last active node
         terminate_cib(__func__, 0);
         return;
     }
 
     crm_info("Sending shutdown request to %d peers", active);
 
     leaving = create_xml_node(NULL, "exit-notification");
     crm_xml_add(leaving, PCMK__XA_T, T_CIB);
     crm_xml_add(leaving, F_CIB_OPERATION, PCMK__CIB_REQUEST_SHUTDOWN);
 
     send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
     free_xml(leaving);
 
     g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
 }
 
 void
 cib_shutdown(int nsig)
 {
     struct qb_ipcs_stats srv_stats;
 
     if (cib_shutdown_flag == FALSE) {
         int disconnects = 0;
         qb_ipcs_connection_t *c = NULL;
 
         cib_shutdown_flag = TRUE;
 
         c = qb_ipcs_connection_first_get(ipcs_rw);
         while (c != NULL) {
             qb_ipcs_connection_t *last = c;
 
             c = qb_ipcs_connection_next_get(ipcs_rw, last);
 
             crm_debug("Disconnecting r/w client %p...", last);
             qb_ipcs_disconnect(last);
             qb_ipcs_connection_unref(last);
             disconnects++;
         }
 
         c = qb_ipcs_connection_first_get(ipcs_ro);
         while (c != NULL) {
             qb_ipcs_connection_t *last = c;
 
             c = qb_ipcs_connection_next_get(ipcs_ro, last);
 
             crm_debug("Disconnecting r/o client %p...", last);
             qb_ipcs_disconnect(last);
             qb_ipcs_connection_unref(last);
             disconnects++;
         }
 
         c = qb_ipcs_connection_first_get(ipcs_shm);
         while (c != NULL) {
             qb_ipcs_connection_t *last = c;
 
             c = qb_ipcs_connection_next_get(ipcs_shm, last);
 
             crm_debug("Disconnecting non-blocking r/w client %p...", last);
             qb_ipcs_disconnect(last);
             qb_ipcs_connection_unref(last);
             disconnects++;
         }
 
         disconnects += pcmk__ipc_client_count();
 
         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
         crm_info("Disconnected %d clients", disconnects);
     }
 
     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
 
     if (pcmk__ipc_client_count() == 0) {
         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
         initiate_exit();
 
     } else {
         crm_info("Waiting on %d clients to disconnect (%d)",
                  pcmk__ipc_client_count(), srv_stats.active_connections);
     }
 }
 
 extern int remote_fd;
 extern int remote_tls_fd;
 
 /*!
  * \internal
  * \brief Close remote sockets, free the global CIB and quit
  *
  * \param[in] caller           Name of calling function (for log message)
  * \param[in] fast             If -1, skip disconnect; if positive, exit that
  */
 void
 terminate_cib(const char *caller, int fast)
 {
     crm_info("%s: Exiting%s...", caller,
              (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
 
     if (remote_fd > 0) {
         close(remote_fd);
         remote_fd = 0;
     }
     if (remote_tls_fd > 0) {
         close(remote_tls_fd);
         remote_tls_fd = 0;
     }
 
     uninitializeCib();
 
     if (fast > 0) {
         /* Quit fast on error */
         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
         crm_exit(fast);
 
     } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
         /* Quit via returning from the main loop. If fast == -1, we skip the
          * disconnect here, and it will be done when the main loop returns
          * (this allows the peer status callback to avoid messing with the
          * peer caches).
          */
         if (fast == 0) {
             crm_cluster_disconnect(crm_cluster);
         }
         g_main_loop_quit(mainloop);
 
     } else {
         /* Quit via clean exit. Even the peer status callback can disconnect
          * here, because we're not returning control to the caller. */
         crm_cluster_disconnect(crm_cluster);
         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
         crm_exit(CRM_EX_OK);
     }
 }
diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
index 3b632406c9..0ec6e13166 100644
--- a/daemons/fenced/pacemaker-fenced.c
+++ b/daemons/fenced/pacemaker-fenced.c
@@ -1,999 +1,999 @@
 /*
  * Copyright 2009-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/utsname.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>  // PRIu32, PRIx32
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/cmdline_internal.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/output_internal.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <crm/common/mainloop.h>
 
 #include <crm/cib/internal.h>
 
 #include <pacemaker-fenced.h>
 
 #define SUMMARY "daemon for executing fencing devices in a Pacemaker cluster"
 
 char *stonith_our_uname = NULL;
 long long stonith_watchdog_timeout_ms = 0;
 GList *stonith_watchdog_targets = NULL;
 
 static GMainLoop *mainloop = NULL;
 
 gboolean stand_alone = FALSE;
 gboolean stonith_shutdown_flag = FALSE;
 
 static qb_ipcs_service_t *ipcs = NULL;
 static pcmk__output_t *out = NULL;
 
 pcmk__supported_format_t formats[] = {
     PCMK__SUPPORTED_FORMAT_NONE,
     PCMK__SUPPORTED_FORMAT_TEXT,
     PCMK__SUPPORTED_FORMAT_XML,
     { NULL, NULL, NULL }
 };
 
 static struct {
     bool no_cib_connect;
     gchar **log_files;
 } options;
 
 crm_exit_t exit_code = CRM_EX_OK;
 
 static void stonith_cleanup(void);
 
 static int32_t
 st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
 {
     if (stonith_shutdown_flag) {
         crm_info("Ignoring new client [%d] during shutdown",
                  pcmk__client_pid(c));
-        return -EPERM;
+        return -ECONNREFUSED;
     }
 
     if (pcmk__new_client(c, uid, gid) == NULL) {
         return -EIO;
     }
     return 0;
 }
 
 /* Exit code means? */
 static int32_t
 st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     int call_options = 0;
     xmlNode *request = NULL;
     pcmk__client_t *c = pcmk__find_client(qbc);
     const char *op = NULL;
 
     if (c == NULL) {
         crm_info("Invalid client: %p", qbc);
         return 0;
     }
 
     request = pcmk__client_data2xml(c, data, &id, &flags);
     if (request == NULL) {
         pcmk__ipc_send_ack(c, id, flags, PCMK__XE_NACK, NULL, CRM_EX_PROTOCOL);
         return 0;
     }
 
 
     op = crm_element_value(request, PCMK__XA_CRM_TASK);
     if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
         crm_xml_add(request, PCMK__XA_T, T_STONITH_NG);
         crm_xml_add(request, F_STONITH_OPERATION, op);
         crm_xml_add(request, F_STONITH_CLIENTID, c->id);
         crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
         crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);
 
         send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE);
         free_xml(request);
         return 0;
     }
 
     if (c->name == NULL) {
         const char *value = crm_element_value(request, F_STONITH_CLIENTNAME);
 
         c->name = crm_strdup_printf("%s.%u", pcmk__s(value, "unknown"), c->pid);
     }
 
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
     crm_trace("Flags %#08" PRIx32 "/%#08x for command %" PRIu32
               " from client %s", flags, call_options, id, pcmk__client_name(c));
 
     if (pcmk_is_set(call_options, st_opt_sync_call)) {
         CRM_ASSERT(flags & crm_ipc_client_response);
         CRM_LOG_ASSERT(c->request_id == 0);     /* This means the client has two synchronous events in-flight */
         c->request_id = id;     /* Reply only to the last one */
     }
 
     crm_xml_add(request, F_STONITH_CLIENTID, c->id);
     crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
     crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);
 
     crm_log_xml_trace(request, "ipc-received");
     stonith_command(c, id, flags, request, NULL);
 
     free_xml(request);
     return 0;
 }
 
 /* Error code means? */
 static int32_t
 st_ipc_closed(qb_ipcs_connection_t * c)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     if (client == NULL) {
         return 0;
     }
 
     crm_trace("Connection %p closed", c);
     pcmk__free_client(client);
 
     /* 0 means: yes, go ahead and destroy the connection */
     return 0;
 }
 
 static void
 st_ipc_destroy(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p destroyed", c);
     st_ipc_closed(c);
 }
 
 static void
 stonith_peer_callback(xmlNode * msg, void *private_data)
 {
     const char *remote_peer = crm_element_value(msg, PCMK__XA_SRC);
     const char *op = crm_element_value(msg, F_STONITH_OPERATION);
 
     if (pcmk__str_eq(op, "poke", pcmk__str_none)) {
         return;
     }
 
     crm_log_xml_trace(msg, "Peer[inbound]");
     stonith_command(NULL, 0, 0, msg, remote_peer);
 }
 
 #if SUPPORT_COROSYNC
 static void
 stonith_peer_ais_callback(cpg_handle_t handle,
                           const struct cpg_name *groupName,
                           uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
 {
     uint32_t kind = 0;
     xmlNode *xml = NULL;
     const char *from = NULL;
     char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
 
     if(data == NULL) {
         return;
     }
     if (kind == crm_class_cluster) {
         xml = string2xml(data);
         if (xml == NULL) {
             crm_err("Invalid XML: '%.120s'", data);
             free(data);
             return;
         }
         crm_xml_add(xml, PCMK__XA_SRC, from);
         stonith_peer_callback(xml, NULL);
     }
 
     free_xml(xml);
     free(data);
     return;
 }
 
 static void
 stonith_peer_cs_destroy(gpointer user_data)
 {
     crm_crit("Lost connection to cluster layer, shutting down");
     stonith_shutdown(0);
 }
 #endif
 
 void
 do_local_reply(const xmlNode *notify_src, pcmk__client_t *client,
                int call_options)
 {
     /* send callback to originating child */
     int local_rc = pcmk_rc_ok;
     int rid = 0;
     uint32_t ipc_flags = crm_ipc_server_event;
 
     if (pcmk_is_set(call_options, st_opt_sync_call)) {
         CRM_LOG_ASSERT(client->request_id);
         rid = client->request_id;
         client->request_id = 0;
         ipc_flags = crm_ipc_flags_none;
     }
 
     local_rc = pcmk__ipc_send_xml(client, rid, notify_src, ipc_flags);
     if (local_rc == pcmk_rc_ok) {
         crm_trace("Sent response %d to client %s",
                   rid, pcmk__client_name(client));
     } else {
         crm_warn("%synchronous reply to client %s failed: %s",
                  (pcmk_is_set(call_options, st_opt_sync_call)? "S" : "As"),
                  pcmk__client_name(client), pcmk_rc_str(local_rc));
     }
 }
 
 uint64_t
 get_stonith_flag(const char *name)
 {
     if (pcmk__str_eq(name, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) {
         return st_callback_notify_fence;
 
     } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) {
         return st_callback_device_add;
 
     } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) {
         return st_callback_device_del;
 
     } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY, pcmk__str_casei)) {
         return st_callback_notify_history;
 
     } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED, pcmk__str_casei)) {
         return st_callback_notify_history_synced;
 
     }
     return st_callback_unknown;
 }
 
 static void
 stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
 {
 
     const xmlNode *update_msg = user_data;
     pcmk__client_t *client = value;
     const char *type = NULL;
 
     CRM_CHECK(client != NULL, return);
     CRM_CHECK(update_msg != NULL, return);
 
     type = crm_element_value(update_msg, PCMK__XA_SUBT);
     CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);
 
     if (client->ipcs == NULL) {
         crm_trace("Skipping client with NULL channel");
         return;
     }
 
     if (pcmk_is_set(client->flags, get_stonith_flag(type))) {
         int rc = pcmk__ipc_send_xml(client, 0, update_msg,
                                     crm_ipc_server_event);
 
         if (rc != pcmk_rc_ok) {
             crm_warn("%s notification of client %s failed: %s "
                      CRM_XS " id=%.8s rc=%d", type, pcmk__client_name(client),
                      pcmk_rc_str(rc), client->id, rc);
         } else {
             crm_trace("Sent %s notification to client %s",
                       type, pcmk__client_name(client));
         }
     }
 }
 
 void
 do_stonith_async_timeout_update(const char *client_id, const char *call_id, int timeout)
 {
     pcmk__client_t *client = NULL;
     xmlNode *notify_data = NULL;
 
     if (!timeout || !call_id || !client_id) {
         return;
     }
 
     client = pcmk__find_client_by_id(client_id);
     if (!client) {
         return;
     }
 
     notify_data = create_xml_node(NULL, T_STONITH_TIMEOUT_VALUE);
     crm_xml_add(notify_data, PCMK__XA_T, T_STONITH_TIMEOUT_VALUE);
     crm_xml_add(notify_data, F_STONITH_CALLID, call_id);
     crm_xml_add_int(notify_data, F_STONITH_TIMEOUT, timeout);
 
     crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);
 
     if (client) {
         pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event);
     }
 
     free_xml(notify_data);
 }
 
 /*!
  * \internal
  * \brief Notify relevant IPC clients of a fencing operation result
  *
  * \param[in] type     Notification type
  * \param[in] result   Result of fencing operation (assume success if NULL)
  * \param[in] data     If not NULL, add to notification as call data
  */
 void
 fenced_send_notification(const char *type, const pcmk__action_result_t *result,
                          xmlNode *data)
 {
     /* TODO: Standardize the contents of data */
     xmlNode *update_msg = create_xml_node(NULL, "notify");
 
     CRM_LOG_ASSERT(type != NULL);
 
     crm_xml_add(update_msg, PCMK__XA_T, T_STONITH_NOTIFY);
     crm_xml_add(update_msg, PCMK__XA_SUBT, type);
     crm_xml_add(update_msg, F_STONITH_OPERATION, type);
     stonith__xe_set_result(update_msg, result);
 
     if (data != NULL) {
         add_message_xml(update_msg, F_STONITH_CALLDATA, data);
     }
 
     crm_trace("Notifying clients");
     pcmk__foreach_ipc_client(stonith_notify_client, update_msg);
     free_xml(update_msg);
     crm_trace("Notify complete");
 }
 
 /*!
  * \internal
  * \brief Send notifications for a configuration change to subscribed clients
  *
  * \param[in] op      Notification type (STONITH_OP_DEVICE_ADD,
  *                    STONITH_OP_DEVICE_DEL, STONITH_OP_LEVEL_ADD, or
  *                    STONITH_OP_LEVEL_DEL)
  * \param[in] result  Operation result
  * \param[in] desc    Description of what changed
  * \param[in] active  Current number of devices or topologies in use
  */
 static void
 send_config_notification(const char *op, const pcmk__action_result_t *result,
                          const char *desc, int active)
 {
     xmlNode *notify_data = create_xml_node(NULL, op);
 
     CRM_CHECK(notify_data != NULL, return);
 
     crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
     crm_xml_add_int(notify_data, F_STONITH_ACTIVE, active);
 
     fenced_send_notification(op, result, notify_data);
     free_xml(notify_data);
 }
 
 /*!
  * \internal
  * \brief Send notifications for a device change to subscribed clients
  *
  * \param[in] op      Notification type (STONITH_OP_DEVICE_ADD or
  *                    STONITH_OP_DEVICE_DEL)
  * \param[in] result  Operation result
  * \param[in] desc    ID of device that changed
  */
 void
 fenced_send_device_notification(const char *op,
                                 const pcmk__action_result_t *result,
                                 const char *desc)
 {
     send_config_notification(op, result, desc, g_hash_table_size(device_list));
 }
 
 /*!
  * \internal
  * \brief Send notifications for a topology level change to subscribed clients
  *
  * \param[in] op      Notification type (STONITH_OP_LEVEL_ADD or
  *                    STONITH_OP_LEVEL_DEL)
  * \param[in] result  Operation result
  * \param[in] desc    String representation of level (<target>[<level_index>])
  */
 void
 fenced_send_level_notification(const char *op,
                                const pcmk__action_result_t *result,
                                const char *desc)
 {
     send_config_notification(op, result, desc, g_hash_table_size(topology));
 }
 
 /*!
  * \internal
  * \brief Check whether a node does watchdog-fencing
  *
  * \param[in] node    Name of node to check
  *
  * \return TRUE if node found in stonith_watchdog_targets
  *         or stonith_watchdog_targets is empty indicating
  *         all nodes are doing watchdog-fencing
  */
 gboolean
 node_does_watchdog_fencing(const char *node)
 {
     return ((stonith_watchdog_targets == NULL) ||
             pcmk__str_in_list(node, stonith_watchdog_targets, pcmk__str_casei));
 }
 
 void
 stonith_shutdown(int nsig)
 {
     crm_info("Terminating with %d clients", pcmk__ipc_client_count());
     stonith_shutdown_flag = TRUE;
     if (mainloop != NULL && g_main_loop_is_running(mainloop)) {
         g_main_loop_quit(mainloop);
     }
 }
 
 static void
 stonith_cleanup(void)
 {
     fenced_cib_cleanup();
     if (ipcs) {
         qb_ipcs_destroy(ipcs);
     }
 
     crm_peer_destroy();
     pcmk__client_cleanup();
     free_stonith_remote_op_list();
     free_topology_list();
     free_device_list();
     free_metadata_cache();
     fenced_unregister_handlers();
 
     free(stonith_our_uname);
     stonith_our_uname = NULL;
 }
 
 static gboolean
 stand_alone_cpg_cb(const gchar *option_name, const gchar *optarg, gpointer data,
                    GError **error)
 {
     stand_alone = FALSE;
     options.no_cib_connect = true;
     return TRUE;
 }
 
 struct qb_ipcs_service_handlers ipc_callbacks = {
     .connection_accept = st_ipc_accept,
     .connection_created = NULL,
     .msg_process = st_ipc_dispatch,
     .connection_closed = st_ipc_closed,
     .connection_destroyed = st_ipc_destroy
 };
 
 /*!
  * \internal
  * \brief Callback for peer status changes
  *
  * \param[in] type  What changed
  * \param[in] node  What peer had the change
  * \param[in] data  Previous value of what changed
  */
 static void
 st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
 {
     if ((type != crm_status_processes)
         && !pcmk_is_set(node->flags, crm_remote_node)) {
         /*
          * This is a hack until we can send to a nodeid and/or we fix node name lookups
          * These messages are ignored in stonith_peer_callback()
          */
         xmlNode *query = create_xml_node(NULL, "stonith_command");
 
         crm_xml_add(query, PCMK__XA_T, T_STONITH_NG);
         crm_xml_add(query, F_STONITH_OPERATION, "poke");
 
         crm_debug("Broadcasting our uname because of node %u", node->id);
         send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
 
         free_xml(query);
     }
 }
 
 static pcmk__cluster_option_t fencer_options[] = {
     /* name, old name, type, allowed values,
      * default value, validator,
      * context,
      * short description,
      * long description
      */
     {
         PCMK_STONITH_HOST_ARGUMENT, NULL, "string", NULL,
         "port", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "An alternate parameter to supply instead of 'port'"),
         N_("Some devices do not support the standard 'port' parameter or may "
             "provide additional ones. Use this to specify an alternate, device-"
             "specific, parameter that should indicate the machine to be "
             "fenced. A value of \"none\" can be used to tell the cluster not "
             "to supply any additional parameters."),
     },
     {
         PCMK_STONITH_HOST_MAP, NULL, "string", NULL,
         "", NULL,
         pcmk__opt_context_none,
         N_("A mapping of node names to port numbers for devices that do not "
             "support node names."),
         N_("For example, \"node1:1;node2:2,3\" would tell the cluster to use "
             "port 1 for node1 and ports 2 and 3 for node2."),
     },
     {
         PCMK_STONITH_HOST_LIST, NULL, "string", NULL,
         "", NULL,
         pcmk__opt_context_none,
         N_("A list of nodes that can be targeted by this device (optional "
             "unless pcmk_host_list=\"static-list\")"),
         N_("For example, \"node1,node2,node3\"."),
     },
     {
         PCMK_STONITH_HOST_CHECK, NULL, "select",
             "dynamic-list, static-list, status, none",
         "dynamic-list", NULL,
         pcmk__opt_context_none,
         N_("How to determine which nodes can be targeted by the device"),
         N_("Use \"dynamic-list\" to query the device via the 'list' command; "
             "\"static-list\" to check the pcmk_host_list attribute; "
             "\"status\" to query the device via the 'status' command; or "
             "\"none\" to assume every device can fence every node."),
     },
     {
         PCMK_STONITH_DELAY_MAX, NULL, "time", NULL,
         "0s", NULL,
         pcmk__opt_context_none,
         N_("Enable a base delay for fencing actions and specify base delay "
             "value."),
         N_("Enable a delay of no more than the time specified before executing "
             "fencing actions. Pacemaker derives the overall delay by taking "
             "the value of pcmk_delay_base and adding a random delay value such "
             "that the sum is kept below this maximum."),
     },
     {
         PCMK_STONITH_DELAY_BASE, NULL, "string", NULL,
         "0s", NULL,
         pcmk__opt_context_none,
         N_("Enable a base delay for fencing actions and specify base delay "
             "value."),
         N_("This enables a static delay for fencing actions, which can help "
             "avoid \"death matches\" where two nodes try to fence each other "
             "at the same time. If pcmk_delay_max is also used, a random delay "
             "will be added such that the total delay is kept below that value. "
             "This can be set to a single time value to apply to any node "
             "targeted by this device (useful if a separate device is "
             "configured for each target), or to a node map (for example, "
             "\"node1:1s;node2:5\") to set a different value for each target."),
     },
     {
         PCMK_STONITH_ACTION_LIMIT, NULL, "integer", NULL,
         "1", NULL,
         pcmk__opt_context_none,
         N_("The maximum number of actions can be performed in parallel on this "
             "device"),
         N_("Cluster property concurrent-fencing=\"true\" needs to be "
             "configured first. Then use this to specify the maximum number of "
             "actions can be performed in parallel on this device. A value of "
             "-1 means an unlimited number of actions can be performed in "
             "parallel."),
     },
     {
         "pcmk_reboot_action", NULL, "string", NULL,
         PCMK_ACTION_REBOOT, NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "An alternate command to run instead of 'reboot'"),
         N_("Some devices do not support the standard commands or may provide "
             "additional ones. Use this to specify an alternate, device-"
             "specific, command that implements the 'reboot' action."),
     },
     {
         "pcmk_reboot_timeout", NULL, "time", NULL,
         "60s", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "Specify an alternate timeout to use for 'reboot' actions instead "
             "of stonith-timeout"),
         N_("Some devices need much more/less time to complete than normal. "
             "Use this to specify an alternate, device-specific, timeout for "
             "'reboot' actions."),
     },
     {
         "pcmk_reboot_retries", NULL, "integer", NULL,
         "2", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "The maximum number of times to try the 'reboot' command within "
             "the timeout period"),
         N_("Some devices do not support multiple connections. Operations may "
             "\"fail\" if the device is busy with another task. In that case, "
             "Pacemaker will automatically retry the operation if there is time "
             "remaining. Use this option to alter the number of times Pacemaker "
             "tries a 'reboot' action before giving up."),
     },
     {
         "pcmk_off_action", NULL, "string", NULL,
         PCMK_ACTION_OFF, NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "An alternate command to run instead of 'off'"),
         N_("Some devices do not support the standard commands or may provide "
             "additional ones. Use this to specify an alternate, device-"
             "specific, command that implements the 'off' action."),
     },
     {
         "pcmk_off_timeout", NULL, "time", NULL,
         "60s", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "Specify an alternate timeout to use for 'off' actions instead of "
             "stonith-timeout"),
         N_("Some devices need much more/less time to complete than normal. "
             "Use this to specify an alternate, device-specific, timeout for "
             "'off' actions."),
     },
     {
         "pcmk_off_retries", NULL, "integer", NULL,
         "2", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "The maximum number of times to try the 'off' command within the "
             "timeout period"),
         N_("Some devices do not support multiple connections. Operations may "
             "\"fail\" if the device is busy with another task. In that case, "
             "Pacemaker will automatically retry the operation if there is time "
             "remaining. Use this option to alter the number of times Pacemaker "
             "tries a 'off' action before giving up."),
     },
     {
         "pcmk_on_action", NULL, "string", NULL,
         PCMK_ACTION_ON, NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "An alternate command to run instead of 'on'"),
         N_("Some devices do not support the standard commands or may provide "
             "additional ones. Use this to specify an alternate, device-"
             "specific, command that implements the 'on' action."),
     },
     {
         "pcmk_on_timeout", NULL, "time", NULL,
         "60s", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "Specify an alternate timeout to use for 'on' actions instead of "
             "stonith-timeout"),
         N_("Some devices need much more/less time to complete than normal. "
             "Use this to specify an alternate, device-specific, timeout for "
             "'on' actions."),
     },
     {
         "pcmk_on_retries", NULL, "integer", NULL,
         "2", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "The maximum number of times to try the 'on' command within the "
             "timeout period"),
         N_("Some devices do not support multiple connections. Operations may "
             "\"fail\" if the device is busy with another task. In that case, "
             "Pacemaker will automatically retry the operation if there is time "
             "remaining. Use this option to alter the number of times Pacemaker "
             "tries a 'on' action before giving up."),
     },
     {
         "pcmk_list_action", NULL, "string", NULL,
         PCMK_ACTION_LIST, NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "An alternate command to run instead of 'list'"),
         N_("Some devices do not support the standard commands or may provide "
             "additional ones. Use this to specify an alternate, device-"
             "specific, command that implements the 'list' action."),
     },
     {
         "pcmk_list_timeout", NULL, "time", NULL,
         "60s", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "Specify an alternate timeout to use for 'list' actions instead of "
             "stonith-timeout"),
         N_("Some devices need much more/less time to complete than normal. "
             "Use this to specify an alternate, device-specific, timeout for "
             "'list' actions."),
     },
     {
         "pcmk_list_retries", NULL, "integer", NULL,
         "2", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "The maximum number of times to try the 'list' command within the "
             "timeout period"),
         N_("Some devices do not support multiple connections. Operations may "
             "\"fail\" if the device is busy with another task. In that case, "
             "Pacemaker will automatically retry the operation if there is time "
             "remaining. Use this option to alter the number of times Pacemaker "
             "tries a 'list' action before giving up."),
     },
     {
         "pcmk_monitor_action", NULL, "string", NULL,
         PCMK_ACTION_MONITOR, NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "An alternate command to run instead of 'monitor'"),
         N_("Some devices do not support the standard commands or may provide "
             "additional ones. Use this to specify an alternate, device-"
             "specific, command that implements the 'monitor' action."),
     },
     {
         "pcmk_monitor_timeout", NULL, "time", NULL,
         "60s", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "Specify an alternate timeout to use for 'monitor' actions instead "
             "of stonith-timeout"),
         N_("Some devices need much more/less time to complete than normal. "
             "Use this to specify an alternate, device-specific, timeout for "
             "'monitor' actions."),
     },
     {
         "pcmk_monitor_retries", NULL, "integer", NULL,
         "2", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "The maximum number of times to try the 'monitor' command within "
             "the timeout period"),
         N_("Some devices do not support multiple connections. Operations may "
             "\"fail\" if the device is busy with another task. In that case, "
             "Pacemaker will automatically retry the operation if there is time "
             "remaining. Use this option to alter the number of times Pacemaker "
             "tries a 'monitor' action before giving up."),
     },
     {
         "pcmk_status_action", NULL, "string", NULL,
         PCMK_ACTION_STATUS, NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "An alternate command to run instead of 'status'"),
         N_("Some devices do not support the standard commands or may provide "
             "additional ones. Use this to specify an alternate, device-"
             "specific, command that implements the 'status' action."),
     },
     {
         "pcmk_status_timeout", NULL, "time", NULL,
         "60s", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "Specify an alternate timeout to use for 'status' actions instead "
             "of stonith-timeout"),
         N_("Some devices need much more/less time to complete than normal. "
             "Use this to specify an alternate, device-specific, timeout for "
             "'status' actions."),
     },
     {
         "pcmk_status_retries", NULL, "integer", NULL,
         "2", NULL,
         pcmk__opt_context_none,
         N_("*** Advanced Use Only *** "
             "The maximum number of times to try the 'status' command within "
             "the timeout period"),
         N_("Some devices do not support multiple connections. Operations may "
             "\"fail\" if the device is busy with another task. In that case, "
             "Pacemaker will automatically retry the operation if there is time "
             "remaining. Use this option to alter the number of times Pacemaker "
             "tries a 'status' action before giving up."),
     },
 };
 
 void
 fencer_metadata(void)
 {
     const char *name = "pacemaker-fenced";
     const char *desc_short = N_("Instance attributes available for all "
                                 "\"stonith\"-class resources");
     const char *desc_long = N_("Instance attributes available for all "
                                "\"stonith\"-class resources and used by "
                                "Pacemaker's fence daemon, formerly known as "
                                "stonithd");
 
     gchar *s = pcmk__format_option_metadata(name, desc_short, desc_long,
                                             pcmk__opt_context_none,
                                             fencer_options,
                                             PCMK__NELEM(fencer_options));
     printf("%s", s);
     g_free(s);
 }
 
 static GOptionEntry entries[] = {
     { "stand-alone", 's', G_OPTION_FLAG_NONE, G_OPTION_ARG_NONE, &stand_alone,
       N_("Deprecated (will be removed in a future release)"), NULL },
 
     { "stand-alone-w-cpg", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
       stand_alone_cpg_cb, N_("Intended for use in regression testing only"), NULL },
 
     { "logfile", 'l', G_OPTION_FLAG_NONE, G_OPTION_ARG_FILENAME_ARRAY,
       &options.log_files, N_("Send logs to the additional named logfile"), NULL },
 
     { NULL }
 };
 
 static GOptionContext *
 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group)
 {
     GOptionContext *context = NULL;
 
     context = pcmk__build_arg_context(args, "text (default), xml", group,
                                       "[metadata]");
     pcmk__add_main_args(context, entries);
     return context;
 }
 
 int
 main(int argc, char **argv)
 {
     int rc = pcmk_rc_ok;
     crm_cluster_t *cluster = NULL;
     crm_ipc_t *old_instance = NULL;
 
     GError *error = NULL;
 
     GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
     gchar **processed_args = pcmk__cmdline_preproc(argv, "l");
     GOptionContext *context = build_arg_context(args, &output_group);
 
     crm_log_preinit(NULL, argc, argv);
 
     pcmk__register_formats(output_group, formats);
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
         exit_code = CRM_EX_USAGE;
         goto done;
     }
 
     rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
     if (rc != pcmk_rc_ok) {
         exit_code = CRM_EX_ERROR;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                     "Error creating output format %s: %s",
                     args->output_ty, pcmk_rc_str(rc));
         goto done;
     }
 
     if (args->version) {
         out->version(out, false);
         goto done;
     }
 
     if ((g_strv_length(processed_args) >= 2)
         && pcmk__str_eq(processed_args[1], "metadata", pcmk__str_none)) {
         fencer_metadata();
         goto done;
     }
 
     // Open additional log files
     pcmk__add_logfiles(options.log_files, out);
 
     crm_log_init(NULL, LOG_INFO + args->verbosity, TRUE,
                  (args->verbosity > 0), argc, argv, FALSE);
 
     crm_notice("Starting Pacemaker fencer");
 
     old_instance = crm_ipc_new("stonith-ng", 0);
     if (old_instance == NULL) {
         /* crm_ipc_new() will have already logged an error message with
          * crm_err()
          */
         exit_code = CRM_EX_FATAL;
         goto done;
     }
 
     if (pcmk__connect_generic_ipc(old_instance) == pcmk_rc_ok) {
         // IPC endpoint already up
         crm_ipc_close(old_instance);
         crm_ipc_destroy(old_instance);
         crm_err("pacemaker-fenced is already active, aborting startup");
         goto done;
     } else {
         // Not up or not authentic, we'll proceed either way
         crm_ipc_destroy(old_instance);
         old_instance = NULL;
     }
 
     mainloop_add_signal(SIGTERM, stonith_shutdown);
 
     crm_peer_init();
 
     rc = fenced_scheduler_init();
     if (rc != pcmk_rc_ok) {
         exit_code = CRM_EX_FATAL;
         g_set_error(&error, PCMK__EXITC_ERROR, exit_code,
                     "Error initializing scheduler data: %s", pcmk_rc_str(rc));
         goto done;
     }
 
     cluster = pcmk_cluster_new();
 
     if (!stand_alone) {
 #if SUPPORT_COROSYNC
         if (is_corosync_cluster()) {
             cluster->destroy = stonith_peer_cs_destroy;
             cluster->cpg.cpg_deliver_fn = stonith_peer_ais_callback;
             cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
         }
 #endif // SUPPORT_COROSYNC
 
         crm_set_status_callback(&st_peer_update_callback);
 
         if (crm_cluster_connect(cluster) == FALSE) {
             exit_code = CRM_EX_FATAL;
             crm_crit("Cannot sign in to the cluster... terminating");
             goto done;
         }
         pcmk__str_update(&stonith_our_uname, cluster->uname);
 
         if (!options.no_cib_connect) {
             setup_cib();
         }
 
     } else {
         pcmk__str_update(&stonith_our_uname, "localhost");
         crm_warn("Stand-alone mode is deprecated and will be removed "
                  "in a future release");
     }
 
     init_device_list();
     init_topology_list();
 
     pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks);
 
     // Create the mainloop and run it...
     mainloop = g_main_loop_new(NULL, FALSE);
     crm_notice("Pacemaker fencer successfully started and accepting connections");
     g_main_loop_run(mainloop);
 
 done:
     g_strfreev(processed_args);
     pcmk__free_arg_context(context);
 
     g_strfreev(options.log_files);
 
     stonith_cleanup();
     pcmk_cluster_free(cluster);
     fenced_scheduler_cleanup();
 
     pcmk__output_and_clear_error(&error, out);
 
     if (out != NULL) {
         out->finish(out, exit_code, true, NULL);
         pcmk__output_free(out);
     }
 
     pcmk__unregister_formats();
     crm_exit(exit_code);
 }