diff --git a/daemons/attrd/attrd_utils.c b/daemons/attrd/attrd_utils.c
index c311ddc92e..3b8446a9f7 100644
--- a/daemons/attrd/attrd_utils.c
+++ b/daemons/attrd/attrd_utils.c
@@ -1,273 +1,291 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <stdbool.h>
 #include <errno.h>
 #include <glib.h>
 #include <regex.h>
 #include <sys/types.h>
 
 #include <crm/crm.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/mainloop.h>
 
 #include "pacemaker-attrd.h"
 
 cib_t *the_cib = NULL;
 
 static bool shutting_down = FALSE;
 static GMainLoop *mloop = NULL;
 
 /*!
  * \internal
  * \brief Check whether we're currently shutting down
  *
  * \return TRUE if shutting down, FALSE otherwise
  */
 gboolean
 attrd_shutting_down()
 {
     return shutting_down;
 }
 
 /*!
  * \internal
  * \brief  Exit (using mainloop or not, as appropriate)
  *
  * \param[in] nsig  Ignored
  */
 void
 attrd_shutdown(int nsig)
 {
     // Tell various functions not to do anthing
     shutting_down = TRUE;
 
     // Don't respond to signals while shutting down
     mainloop_destroy_signal(SIGTERM);
     mainloop_destroy_signal(SIGCHLD);
     mainloop_destroy_signal(SIGPIPE);
     mainloop_destroy_signal(SIGUSR1);
     mainloop_destroy_signal(SIGUSR2);
     mainloop_destroy_signal(SIGTRAP);
 
     if ((mloop == NULL) || !g_main_loop_is_running(mloop)) {
         /* If there's no main loop active, just exit. This should be possible
          * only if we get SIGTERM in brief windows at start-up and shutdown.
          */
         crm_exit(CRM_EX_OK);
     } else {
         g_main_loop_quit(mloop);
         g_main_loop_unref(mloop);
     }
 }
 
 /*!
  * \internal
  * \brief Create a main loop for attrd
  */
 void
 attrd_init_mainloop()
 {
     mloop = g_main_loop_new(NULL, FALSE);
 }
 
 /*!
  * \internal
  * \brief Run attrd main loop
  */
 void
 attrd_run_mainloop()
 {
     g_main_loop_run(mloop);
 }
 
 /*!
  * \internal
  * \brief Accept a new client IPC connection
  *
  * \param[in] c    New connection
  * \param[in] uid  Client user id
  * \param[in] gid  Client group id
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 static int32_t
 attrd_ipc_accept(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
 {
     crm_trace("New client connection %p", c);
     if (shutting_down) {
         crm_info("Ignoring new connection from pid %d during shutdown",
                  pcmk__client_pid(c));
         return -EPERM;
     }
 
     if (pcmk__new_client(c, uid, gid) == NULL) {
         return -EIO;
     }
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Destroy a client IPC connection
  *
  * \param[in] c  Connection to destroy
  *
  * \return FALSE (i.e. do not re-run this callback)
  */
 static int32_t
 attrd_ipc_closed(qb_ipcs_connection_t *c)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     if (client == NULL) {
         crm_trace("Ignoring request to clean up unknown connection %p", c);
     } else {
         crm_trace("Cleaning up closed client connection %p", c);
         pcmk__free_client(client);
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Destroy a client IPC connection
  *
  * \param[in] c  Connection to destroy
  *
  * \note We handle a destroyed connection the same as a closed one,
  *       but we need a separate handler because the return type is different.
  */
 static void
 attrd_ipc_destroy(qb_ipcs_connection_t *c)
 {
     crm_trace("Destroying client connection %p", c);
     attrd_ipc_closed(c);
 }
 
 /*!
  * \internal
  * \brief Set up attrd IPC communication
  *
  * \param[out] ipcs         Will be set to newly allocated server connection
  * \param[in]  dispatch_fn  Handler for new messages on connection
  */
 void
 attrd_init_ipc(qb_ipcs_service_t **ipcs, qb_ipcs_msg_process_fn dispatch_fn)
 {
 
     static struct qb_ipcs_service_handlers ipc_callbacks = {
         .connection_accept = attrd_ipc_accept,
         .connection_created = NULL,
         .msg_process = NULL,
         .connection_closed = attrd_ipc_closed,
         .connection_destroyed = attrd_ipc_destroy
     };
 
     ipc_callbacks.msg_process = dispatch_fn;
     pcmk__serve_attrd_ipc(ipcs, &ipc_callbacks);
 }
 
 void
 attrd_cib_disconnect()
 {
     CRM_CHECK(the_cib != NULL, return);
+    the_cib->cmds->del_notify_callback(the_cib, T_CIB_REPLACE_NOTIFY, attrd_cib_replaced_cb);
+    the_cib->cmds->del_notify_callback(the_cib, T_CIB_DIFF_NOTIFY, attrd_cib_updated_cb); 
     the_cib->cmds->signoff(the_cib);
     cib_delete(the_cib);
     the_cib = NULL;
 }
 
+void
+attrd_cib_replaced_cb(const char *event, xmlNode * msg)
+{
+    if (attrd_shutting_down()) {
+        return;
+    }
+
+    if (attrd_election_won()) {
+        crm_notice("Updating all attributes after %s event", event);
+        write_attributes(TRUE, FALSE);
+    }
+
+    // Check for changes in alerts
+    mainloop_set_trigger(attrd_config_read);
+}
+
 /* strlen("value") */
 #define plus_plus_len (5)
 
 /*!
  * \internal
  * \brief  Check whether an attribute value should be expanded
  *
  * \param[in] value  Attribute value to check
  *
  * \return TRUE if value needs expansion, FALSE otherwise
  */
 gboolean
 attrd_value_needs_expansion(const char *value)
 {
     return ((strlen(value) >= (plus_plus_len + 2))
            && (value[plus_plus_len] == '+')
            && ((value[plus_plus_len + 1] == '+')
                || (value[plus_plus_len + 1] == '=')));
 }
 
 /*!
  * \internal
  * \brief Expand an increment expression into an integer
  *
  * \param[in] value      Attribute increment expression to expand
  * \param[in] old_value  Previous value of attribute
  *
  * \return Expanded value
  */
 int
 attrd_expand_value(const char *value, const char *old_value)
 {
     int offset = 1;
     int int_value = char2score(old_value);
 
     if (value[plus_plus_len + 1] != '+') {
         const char *offset_s = value + (plus_plus_len + 2);
 
         offset = char2score(offset_s);
     }
     int_value += offset;
 
     if (int_value > INFINITY) {
         int_value = INFINITY;
     }
     return int_value;
 }
 
 /*!
  * \internal
  * \brief Create regular expression matching failure-related attributes
  *
  * \param[out] regex  Where to store created regular expression
  * \param[in]  rsc    Name of resource to clear (or NULL for all)
  * \param[in]  op     Operation to clear if rsc is specified (or NULL for all)
  * \param[in]  interval_ms  Interval of operation to clear if op is specified
  *
  * \return pcmk_ok on success, -EINVAL if arguments are invalid
  *
  * \note The caller is responsible for freeing the result with regfree().
  */
 int
 attrd_failure_regex(regex_t *regex, const char *rsc, const char *op,
                     guint interval_ms)
 {
     char *pattern = NULL;
     int rc;
 
     /* Create a pattern that matches desired attributes */
 
     if (rsc == NULL) {
         pattern = strdup(ATTRD_RE_CLEAR_ALL);
     } else if (op == NULL) {
         pattern = crm_strdup_printf(ATTRD_RE_CLEAR_ONE, rsc);
     } else {
         pattern = crm_strdup_printf(ATTRD_RE_CLEAR_OP, rsc, op, interval_ms);
     }
 
     /* Compile pattern into regular expression */
     crm_trace("Clearing attributes matching %s", pattern);
     rc = regcomp(regex, pattern, REG_EXTENDED|REG_NOSUB);
     free(pattern);
 
     return (rc == 0)? pcmk_ok : -EINVAL;
 }
diff --git a/daemons/attrd/pacemaker-attrd.c b/daemons/attrd/pacemaker-attrd.c
index 5b89d712a7..cefc6eec1e 100644
--- a/daemons/attrd/pacemaker-attrd.c
+++ b/daemons/attrd/pacemaker-attrd.c
@@ -1,449 +1,433 @@
 /*
  * Copyright 2013-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <crm/crm.h>
 #include <crm/cib/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/pengine/rules.h>
 #include <crm/common/iso8601.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/xml.h>
 #include <crm/cluster/internal.h>
 
 #include <crm/common/attrd_internal.h>
 #include "pacemaker-attrd.h"
 
 lrmd_t *the_lrmd = NULL;
 crm_cluster_t *attrd_cluster = NULL;
 crm_trigger_t *attrd_config_read = NULL;
 static crm_exit_t attrd_exit_status = CRM_EX_OK;
 
 static void
 attrd_cpg_dispatch(cpg_handle_t handle,
                  const struct cpg_name *groupName,
                  uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
 {
     uint32_t kind = 0;
     xmlNode *xml = NULL;
     const char *from = NULL;
     char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
 
     if(data == NULL) {
         return;
     }
 
     if (kind == crm_class_cluster) {
         xml = string2xml(data);
     }
 
     if (xml == NULL) {
         crm_err("Bad message of class %d received from %s[%u]: '%.120s'", kind, from, nodeid, data);
     } else {
         crm_node_t *peer = crm_get_peer(nodeid, from);
 
         attrd_peer_message(peer, xml);
     }
 
     free_xml(xml);
     free(data);
 }
 
 static void
 attrd_cpg_destroy(gpointer unused)
 {
     if (attrd_shutting_down()) {
         crm_info("Corosync disconnection complete");
 
     } else {
         crm_crit("Lost connection to cluster layer, shutting down");
         attrd_exit_status = CRM_EX_DISCONNECT;
         attrd_shutdown(0);
     }
 }
 
-static void
-attrd_cib_replaced_cb(const char *event, xmlNode * msg)
-{
-    if (attrd_shutting_down()) {
-        return;
-    }
-
-    if (attrd_election_won()) {
-        crm_notice("Updating all attributes after %s event", event);
-        write_attributes(TRUE, FALSE);
-    }
-
-    // Check for changes in alerts
-    mainloop_set_trigger(attrd_config_read);
-}
-
 static void
 attrd_cib_destroy_cb(gpointer user_data)
 {
     cib_t *conn = user_data;
 
     conn->cmds->signoff(conn);  /* Ensure IPC is cleaned up */
 
     if (attrd_shutting_down()) {
         crm_info("Connection disconnection complete");
 
     } else {
         /* eventually this should trigger a reconnect, not a shutdown */
         crm_crit("Lost connection to the CIB manager, shutting down");
         attrd_exit_status = CRM_EX_DISCONNECT;
         attrd_shutdown(0);
     }
 
     return;
 }
 
 static void
 attrd_erase_cb(xmlNode *msg, int call_id, int rc, xmlNode *output,
                void *user_data)
 {
     do_crm_log_unlikely((rc? LOG_NOTICE : LOG_DEBUG),
                         "Cleared transient attributes: %s "
                         CRM_XS " xpath=%s rc=%d",
                         pcmk_strerror(rc), (char *) user_data, rc);
 }
 
 #define XPATH_TRANSIENT "//node_state[@uname='%s']/" XML_TAG_TRANSIENT_NODEATTRS
 
 /*!
  * \internal
  * \brief Wipe all transient attributes for this node from the CIB
  *
  * Clear any previous transient node attributes from the CIB. This is
  * normally done by the DC's controller when this node leaves the cluster, but
  * this handles the case where the node restarted so quickly that the
  * cluster layer didn't notice.
  *
  * \todo If pacemaker-attrd respawns after crashing (see PCMK_respawned),
  *       ideally we'd skip this and sync our attributes from the writer.
  *       However, currently we reject any values for us that the writer has, in
  *       attrd_peer_update().
  */
 static void
 attrd_erase_attrs(void)
 {
     int call_id;
     char *xpath = crm_strdup_printf(XPATH_TRANSIENT, attrd_cluster->uname);
 
     crm_info("Clearing transient attributes from CIB " CRM_XS " xpath=%s",
              xpath);
 
     call_id = the_cib->cmds->remove(the_cib, xpath, NULL,
                                     cib_quorum_override | cib_xpath);
     the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, xpath,
                                           "attrd_erase_cb", attrd_erase_cb,
                                           free);
 }
 
 static int
 attrd_cib_connect(int max_retry)
 {
     static int attempts = 0;
 
     int rc = -ENOTCONN;
 
     the_cib = cib_new();
     if (the_cib == NULL) {
         return -ENOTCONN;
     }
 
     do {
         if(attempts > 0) {
             sleep(attempts);
         }
 
         attempts++;
         crm_debug("Connection attempt %d to the CIB manager", attempts);
         rc = the_cib->cmds->signon(the_cib, T_ATTRD, cib_command);
 
     } while(rc != pcmk_ok && attempts < max_retry);
 
     if (rc != pcmk_ok) {
         crm_err("Connection to the CIB manager failed: %s " CRM_XS " rc=%d",
                 pcmk_strerror(rc), rc);
         goto cleanup;
     }
 
     crm_debug("Connected to the CIB manager after %d attempts", attempts);
 
     rc = the_cib->cmds->set_connection_dnotify(the_cib, attrd_cib_destroy_cb);
     if (rc != pcmk_ok) {
         crm_err("Could not set disconnection callback");
         goto cleanup;
     }
 
     rc = the_cib->cmds->add_notify_callback(the_cib, T_CIB_REPLACE_NOTIFY, attrd_cib_replaced_cb);
     if(rc != pcmk_ok) {
         crm_err("Could not set CIB notification callback");
         goto cleanup;
     }
 
     rc = the_cib->cmds->add_notify_callback(the_cib, T_CIB_DIFF_NOTIFY, attrd_cib_updated_cb);
     if (rc != pcmk_ok) {
         crm_err("Could not set CIB notification callback (update)");
         goto cleanup;
     }
 
     return pcmk_ok;
 
   cleanup:
     the_cib->cmds->signoff(the_cib);
     cib_delete(the_cib);
     the_cib = NULL;
     return -ENOTCONN;
 }
 
 /*!
  * \internal
  * \brief Prepare the CIB after cluster is connected
  */
 static void
 attrd_cib_init(void)
 {
     // We have no attribute values in memory, wipe the CIB to match
     attrd_erase_attrs();
 
     // Set a trigger for reading the CIB (for the alerts section)
     attrd_config_read = mainloop_add_trigger(G_PRIORITY_HIGH, attrd_read_options, NULL);
 
     // Always read the CIB at start-up
     mainloop_set_trigger(attrd_config_read);
 }
 
 static qb_ipcs_service_t *ipcs = NULL;
 
 static int32_t
 attrd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     pcmk__client_t *client = pcmk__find_client(c);
     xmlNode *xml = NULL;
     const char *op;
 
     // Sanity-check, and parse XML from IPC data
     CRM_CHECK((c != NULL) && (client != NULL), return 0);
     if (data == NULL) {
         crm_debug("No IPC data from PID %d", pcmk__client_pid(c));
         return 0;
     }
     xml = pcmk__client_data2xml(client, data, &id, &flags);
     if (xml == NULL) {
         crm_debug("Unrecognizable IPC data from PID %d", pcmk__client_pid(c));
         return 0;
     }
 
 #if ENABLE_ACL
     CRM_ASSERT(client->user != NULL);
     pcmk__update_acl_user(xml, PCMK__XA_ATTR_USER, client->user);
 #endif
 
     op = crm_element_value(xml, PCMK__XA_TASK);
 
     if (client->name == NULL) {
         const char *value = crm_element_value(xml, F_ORIG);
         client->name = crm_strdup_printf("%s.%d", value?value:"unknown", client->pid);
     }
 
     if (pcmk__str_eq(op, PCMK__ATTRD_CMD_PEER_REMOVE, pcmk__str_casei)) {
         attrd_send_ack(client, id, flags);
         attrd_client_peer_remove(client->name, xml);
 
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_CLEAR_FAILURE, pcmk__str_casei)) {
         attrd_send_ack(client, id, flags);
         attrd_client_clear_failure(xml);
 
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE, pcmk__str_casei)) {
         attrd_send_ack(client, id, flags);
         attrd_client_update(xml);
 
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_BOTH, pcmk__str_casei)) {
         attrd_send_ack(client, id, flags);
         attrd_client_update(xml);
 
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_UPDATE_DELAY, pcmk__str_casei)) {
         attrd_send_ack(client, id, flags);
         attrd_client_update(xml);
 
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_REFRESH, pcmk__str_casei)) {
         attrd_send_ack(client, id, flags);
         attrd_client_refresh();
 
     } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_QUERY, pcmk__str_casei)) {
         /* queries will get reply, so no ack is necessary */
         attrd_client_query(client, id, flags, xml);
 
     } else {
         crm_info("Ignoring request from client %s with unknown operation %s",
                  client->name, op);
     }
 
     free_xml(xml);
     return 0;
 }
 
 void
 attrd_ipc_fini(void)
 {
     if (ipcs != NULL) {
         pcmk__drop_all_clients(ipcs);
         qb_ipcs_destroy(ipcs);
         ipcs = NULL;
     }
 }
 
 static int
 attrd_cluster_connect(void)
 {
     attrd_cluster = calloc(1, sizeof(crm_cluster_t));
 
     attrd_cluster->destroy = attrd_cpg_destroy;
     attrd_cluster->cpg.cpg_deliver_fn = attrd_cpg_dispatch;
     attrd_cluster->cpg.cpg_confchg_fn = pcmk_cpg_membership;
 
     crm_set_status_callback(&attrd_peer_change_cb);
 
     if (crm_cluster_connect(attrd_cluster) == FALSE) {
         crm_err("Cluster connection failed");
         return -ENOTCONN;
     }
     return pcmk_ok;
 }
 
 static pcmk__cli_option_t long_options[] = {
     // long option, argument type, storage, short option, description, flags
     {
         "help",     no_argument, NULL, '?',
         "\tThis text", pcmk__option_default
     },
     {
         "verbose",  no_argument, NULL, 'V',
         "\tIncrease debug output", pcmk__option_default
     },
     { 0, 0, 0, 0 }
 };
 
 int
 main(int argc, char **argv)
 {
     int flag = 0;
     int index = 0;
     int argerr = 0;
     crm_ipc_t *old_instance = NULL;
 
     attrd_init_mainloop();
     crm_log_preinit(NULL, argc, argv);
     pcmk__set_cli_options(NULL, "[options]", long_options,
                           "daemon for managing Pacemaker node attributes");
 
     mainloop_add_signal(SIGTERM, attrd_shutdown);
 
      while (1) {
         flag = pcmk__next_cli_option(argc, argv, &index, NULL);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case 'h':          /* Help message */
                 pcmk__cli_help(flag, CRM_EX_OK);
                 break;
             default:
                 ++argerr;
                 break;
         }
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         pcmk__cli_help('?', CRM_EX_USAGE);
     }
 
     crm_log_init(T_ATTRD, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
     crm_notice("Starting Pacemaker node attribute manager");
 
     old_instance = crm_ipc_new(T_ATTRD, 0);
     if (crm_ipc_connect(old_instance)) {
         /* IPC end-point already up */
         crm_ipc_close(old_instance);
         crm_ipc_destroy(old_instance);
         crm_err("pacemaker-attrd is already active, aborting startup");
         crm_exit(CRM_EX_OK);
     } else {
         /* not up or not authentic, we'll proceed either way */
         crm_ipc_destroy(old_instance);
         old_instance = NULL;
     }
 
     attributes = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_attribute);
 
     /* Connect to the CIB before connecting to the cluster or listening for IPC.
      * This allows us to assume the CIB is connected whenever we process a
      * cluster or IPC message (which also avoids start-up race conditions).
      */
     if (attrd_cib_connect(10) != pcmk_ok) {
         attrd_exit_status = CRM_EX_FATAL;
         goto done;
     }
     crm_info("CIB connection active");
 
     if (attrd_cluster_connect() != pcmk_ok) {
         attrd_exit_status = CRM_EX_FATAL;
         goto done;
     }
     crm_info("Cluster connection active");
 
     // Initialization that requires the cluster to be connected
     attrd_election_init();
     attrd_cib_init();
 
     /* Set a private attribute for ourselves with the protocol version we
      * support. This lets all nodes determine the minimum supported version
      * across all nodes. It also ensures that the writer learns our node name,
      * so it can send our attributes to the CIB.
      */
     attrd_broadcast_protocol();
 
     attrd_init_ipc(&ipcs, attrd_ipc_dispatch);
     crm_notice("Pacemaker node attribute manager successfully started and accepting connections");
     attrd_run_mainloop();
 
   done:
     crm_info("Shutting down attribute manager");
 
     attrd_election_fini();
     attrd_ipc_fini();
     attrd_lrmd_disconnect();
     attrd_cib_disconnect();
     g_hash_table_destroy(attributes);
 
     crm_exit(attrd_exit_status);
 }
diff --git a/daemons/attrd/pacemaker-attrd.h b/daemons/attrd/pacemaker-attrd.h
index bac032cf66..b824555648 100644
--- a/daemons/attrd/pacemaker-attrd.h
+++ b/daemons/attrd/pacemaker-attrd.h
@@ -1,134 +1,135 @@
 /*
  * Copyright 2013-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PACEMAKER_ATTRD__H
 #  define PACEMAKER_ATTRD__H
 
 #include <regex.h>
 #include <glib.h>
 #include <crm/crm.h>
 #include <crm/cluster.h>
 #include <crm/cluster/election.h>
 #include <crm/cib/internal.h>
 
 void attrd_init_mainloop(void);
 void attrd_run_mainloop(void);
 
 gboolean attrd_shutting_down(void);
 void attrd_shutdown(int nsig);
 void attrd_init_ipc(qb_ipcs_service_t **ipcs,
                     qb_ipcs_msg_process_fn dispatch_fn);
 void attrd_ipc_fini(void);
 
 void attrd_cib_disconnect(void);
 
 gboolean attrd_value_needs_expansion(const char *value);
 int attrd_expand_value(const char *value, const char *old_value);
 
 /* regular expression to clear failures of all resources */
 #define ATTRD_RE_CLEAR_ALL \
     "^(" PCMK__FAIL_COUNT_PREFIX "|" PCMK__LAST_FAILURE_PREFIX ")-"
 
 /* regular expression to clear failure of all operations for one resource
  * (format takes resource name)
  *
  * @COMPAT attributes set < 1.1.17:
  * also match older attributes that do not have the operation part
  */
 #define ATTRD_RE_CLEAR_ONE ATTRD_RE_CLEAR_ALL "%s(#.+_[0-9]+)?$"
 
 /* regular expression to clear failure of one operation for one resource
  * (format takes resource name, operation name, and interval)
  *
  * @COMPAT attributes set < 1.1.17:
  * also match older attributes that do not have the operation part
  */
 #define ATTRD_RE_CLEAR_OP ATTRD_RE_CLEAR_ALL "%s(#%s_%u)?$"
 
 int attrd_failure_regex(regex_t *regex, const char *rsc, const char *op,
                         guint interval_ms);
 
 extern cib_t *the_cib;
 
 /* Alerts */
 
 extern lrmd_t *the_lrmd;
 extern crm_trigger_t *attrd_config_read;
 
 void attrd_lrmd_disconnect(void);
 gboolean attrd_read_options(gpointer user_data);
+void attrd_cib_replaced_cb(const char *event, xmlNode * msg);
 void attrd_cib_updated_cb(const char *event, xmlNode *msg);
 int attrd_send_attribute_alert(const char *node, int nodeid,
                                const char *attr, const char *value);
 
 // Elections
 void attrd_election_init(void);
 void attrd_election_fini(void);
 void attrd_start_election_if_needed(void);
 bool attrd_election_won(void);
 void attrd_handle_election_op(const crm_node_t *peer, xmlNode *xml);
 bool attrd_check_for_new_writer(const crm_node_t *peer, const xmlNode *xml);
 void attrd_declare_winner(void);
 void attrd_remove_voter(const crm_node_t *peer);
 void attrd_xml_add_writer(xmlNode *xml);
 
 typedef struct attribute_s {
     char *uuid; /* TODO: Remove if at all possible */
     char *id;
     char *set;
     GHashTable *values;
     int update;
     int timeout_ms;
 
     /* TODO: refactor these three as a bitmask */
     bool changed; /* whether attribute value has changed since last write */
     bool unknown_peer_uuids; /* whether we know we're missing a peer uuid */
     gboolean is_private; /* whether to keep this attribute out of the CIB */
 
     mainloop_timer_t *timer;
 
     char *user;
 
     gboolean force_write; /* Flag for updating attribute by ignoring delay */
 
 } attribute_t;
 
 typedef struct attribute_value_s {
         uint32_t nodeid;
         gboolean is_remote;
         char *nodename;
         char *current;
         char *requested;
         gboolean seen;
 } attribute_value_t;
 
 extern crm_cluster_t *attrd_cluster;
 extern GHashTable *attributes;
 
 #define attrd_send_ack(client, id, flags) \
     pcmk__ipc_send_ack((client), (id), (flags), "ack")
 
 #define CIB_OP_TIMEOUT_S 120
 
 void write_attributes(bool all, bool ignore_delay);
 void attrd_broadcast_protocol(void);
 void attrd_peer_message(crm_node_t *client, xmlNode *msg);
 void attrd_client_peer_remove(const char *client_name, xmlNode *xml);
 void attrd_client_clear_failure(xmlNode *xml);
 void attrd_client_update(xmlNode *xml);
 void attrd_client_refresh(void);
 void attrd_client_query(pcmk__client_t *client, uint32_t id, uint32_t flags,
                         xmlNode *query);
 
 void free_attribute(gpointer data);
 
 gboolean attrd_election_cb(gpointer user_data);
 void attrd_peer_change_cb(enum crm_status_type type, crm_node_t *peer, const void *data);
 
 #endif /* PACEMAKER_ATTRD__H */
diff --git a/daemons/controld/controld_based.c b/daemons/controld/controld_based.c
index dcc7362598..dab7f3711d 100644
--- a/daemons/controld/controld_based.c
+++ b/daemons/controld/controld_based.c
@@ -1,333 +1,334 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <unistd.h>  /* sleep */
 
 #include <crm/common/alerts_internal.h>
 #include <crm/common/xml.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 
 #include <pacemaker-controld.h>
 
 int cib_retries = 0;
 
-static void
+void
 do_cib_updated(const char *event, xmlNode * msg)
 {
     if (pcmk__alert_in_patchset(msg, TRUE)) {
         mainloop_set_trigger(config_read);
     }
 }
 
-static void
+void
 do_cib_replaced(const char *event, xmlNode * msg)
 {
     crm_debug("Updating the CIB after a replace: DC=%s", pcmk__btoa(AM_I_DC));
     if (AM_I_DC == FALSE) {
         return;
 
     } else if ((fsa_state == S_FINALIZE_JOIN)
                && pcmk_is_set(fsa_input_register, R_CIB_ASKED)) {
         /* no need to restart the join - we asked for this replace op */
         return;
     }
 
     /* start the join process again so we get everyone's LRM status */
     populate_cib_nodes(node_update_quick|node_update_all, __func__);
     register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
 }
 
 /* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
 void
 do_cib_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     CRM_ASSERT(fsa_cib_conn != NULL);
 
     if (action & A_CIB_STOP) {
 
         if (fsa_cib_conn->state != cib_disconnected && last_resource_update != 0) {
             crm_info("Waiting for resource update %d to complete", last_resource_update);
             crmd_fsa_stall(FALSE);
             return;
         }
 
         crm_info("Disconnecting from the CIB manager");
         controld_clear_fsa_input_flags(R_CIB_CONNECTED);
 
+        fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced);
         fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY, do_cib_updated);
 
         if (fsa_cib_conn->state != cib_disconnected) {
             fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local);
             fsa_cib_conn->cmds->signoff(fsa_cib_conn);
         }
         crm_notice("Disconnected from the CIB manager");
     }
 
     if (action & A_CIB_START) {
         int rc = pcmk_ok;
 
         if (cur_state == S_STOPPING) {
             crm_err("Ignoring request to connect to the CIB manager after shutdown");
             return;
         }
 
         rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking);
 
         if (rc != pcmk_ok) {
             /* a short wait that usually avoids stalling the FSA */
             sleep(1);
             rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking);
         }
 
         if (rc != pcmk_ok) {
             crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));
 
         } else if (pcmk_ok !=
                    fsa_cib_conn->cmds->set_connection_dnotify(fsa_cib_conn,
                                                               crmd_cib_connection_destroy)) {
             crm_err("Could not set dnotify callback");
 
         } else if (pcmk_ok !=
                    fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY,
                                                            do_cib_replaced)) {
             crm_err("Could not set CIB notification callback (replace)");
 
         } else if (pcmk_ok !=
                    fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY,
                                                            do_cib_updated)) {
             crm_err("Could not set CIB notification callback (update)");
 
         } else {
             controld_set_fsa_input_flags(R_CIB_CONNECTED);
             cib_retries = 0;
         }
 
         if (!pcmk_is_set(fsa_input_register, R_CIB_CONNECTED)) {
 
             cib_retries++;
             crm_warn("Couldn't complete CIB registration %d"
                      " times... pause and retry", cib_retries);
 
             if (cib_retries < 30) {
                 controld_start_timer(wait_timer);
                 crmd_fsa_stall(FALSE);
 
             } else {
                 crm_err("Could not complete CIB"
                         " registration  %d times..." " hard error", cib_retries);
                 register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief Get CIB call options to use local scope if master unavailable
  *
  * \return CIB call options
  */
 int crmd_cib_smart_opt()
 {
     int call_opt = cib_quorum_override;
 
     if (fsa_state == S_ELECTION || fsa_state == S_PENDING) {
         crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state));
         cib__set_call_options(call_opt, "update", cib_scope_local);
     }
     return call_opt;
 }
 
 /*!
  * \internal
  * \brief Check whether an action type should be recorded in the CIB
  *
  * \param[in] action  Action type
  *
  * \return TRUE if action should be recorded, FALSE otherwise
  */
 bool
 controld_action_is_recordable(const char *action)
 {
     return !pcmk__strcase_any_of(action, CRMD_ACTION_CANCEL, CRMD_ACTION_DELETE,
                             CRMD_ACTION_NOTIFY, CRMD_ACTION_METADATA, NULL);
 }
 
 static void
 cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
                     void *user_data)
 {
     char *desc = user_data;
 
     if (rc == 0) {
         crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
     } else {
         crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
                  desc, call_id, pcmk_strerror(rc), rc);
     }
 }
 
 // Searches for various portions of node_state to delete
 
 // Match a particular node's node_state (takes node name 1x)
 #define XPATH_NODE_STATE        "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"
 
 // Node's lrm section (name 1x)
 #define XPATH_NODE_LRM          XPATH_NODE_STATE "/" XML_CIB_TAG_LRM
 
 // Node's lrm_rsc_op entries and lrm_resource entries without lock (name 2x)
 #define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP    \
                                 "|" XPATH_NODE_STATE                        \
                                 "//" XML_LRM_TAG_RESOURCE                   \
                                 "[not(@" XML_CONFIG_ATTR_SHUTDOWN_LOCK ")]"
 
 // Node's transient_attributes section (name 1x)
 #define XPATH_NODE_ATTRS        XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS
 
 // Everything under node_state (name 1x)
 #define XPATH_NODE_ALL          XPATH_NODE_STATE "/*"
 
 // Unlocked history + transient attributes (name 3x)
 #define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "|" XPATH_NODE_ATTRS
 
 /*!
  * \internal
  * \brief Delete subsection of a node's CIB node_state
  *
  * \param[in] uname    Desired node
  * \param[in] section  Subsection of node_state to delete
  * \param[in] options  CIB call options to use
  */
 void
 controld_delete_node_state(const char *uname, enum controld_section_e section,
                            int options)
 {
     char *xpath = NULL;
     char *desc = NULL;
 
     CRM_CHECK(uname != NULL, return);
     switch (section) {
         case controld_section_lrm:
             xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
             desc = crm_strdup_printf("resource history for node %s", uname);
             break;
         case controld_section_lrm_unlocked:
             xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED, uname, uname);
             desc = crm_strdup_printf("resource history (other than shutdown "
                                      "locks) for node %s", uname);
             break;
         case controld_section_attrs:
             xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
             desc = crm_strdup_printf("transient attributes for node %s", uname);
             break;
         case controld_section_all:
             xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
             desc = crm_strdup_printf("all state for node %s", uname);
             break;
         case controld_section_all_unlocked:
             xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
                                       uname, uname, uname);
             desc = crm_strdup_printf("all state (other than shutdown locks) "
                                      "for node %s", uname);
             break;
     }
 
     if (fsa_cib_conn == NULL) {
         crm_warn("Unable to delete %s: no CIB connection", desc);
         free(desc);
     } else {
         int call_id;
 
         cib__set_call_options(options, "node state deletion",
                               cib_quorum_override|cib_xpath|cib_multiple);
         call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, xpath, NULL, options);
         crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
                  desc, call_id, xpath);
         fsa_register_cib_callback(call_id, FALSE, desc, cib_delete_callback);
         // CIB library handles freeing desc
     }
     free(xpath);
 }
 
 // Takes node name and resource ID
 #define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE                       \
                                "[@" XML_ATTR_UNAME "='%s']/"                \
                                XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES    \
                                "/" XML_LRM_TAG_RESOURCE                     \
                                "[@" XML_ATTR_ID "='%s']"
 // @TODO could add "and @XML_CONFIG_ATTR_SHUTDOWN_LOCK" to limit to locks
 
 /*!
  * \internal
  * \brief Clear resource history from CIB for a given resource and node
  *
  * \param[in]  rsc_id        ID of resource to be cleared
  * \param[in]  node          Node whose resource history should be cleared
  * \param[in]  user_name     ACL user name to use
  * \param[in]  call_options  CIB call options
  *
  * \return Standard Pacemaker return code
  */
 int
 controld_delete_resource_history(const char *rsc_id, const char *node,
                                  const char *user_name, int call_options)
 {
     char *desc = NULL;
     char *xpath = NULL;
     int rc = pcmk_rc_ok;
 
     CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);
 
     desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
     if (fsa_cib_conn == NULL) {
         crm_err("Unable to clear %s: no CIB connection", desc);
         free(desc);
         return ENOTCONN;
     }
 
     // Ask CIB to delete the entry
     xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);
     rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, xpath, NULL,
                          NULL, call_options|cib_xpath, user_name);
 
     if (rc < 0) {
         rc = pcmk_legacy2rc(rc);
         crm_err("Could not delete resource status of %s on %s%s%s: %s "
                 CRM_XS " rc=%d", rsc_id, node,
                 (user_name? " for user " : ""), (user_name? user_name : ""),
                 pcmk_rc_str(rc), rc);
         free(desc);
         free(xpath);
         return rc;
     }
 
     if (pcmk_is_set(call_options, cib_sync_call)) {
         if (pcmk_is_set(call_options, cib_dryrun)) {
             crm_debug("Deletion of %s would succeed", desc);
         } else {
             crm_debug("Deletion of %s succeeded", desc);
         }
         free(desc);
 
     } else {
         crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
                  desc, rc, xpath);
         fsa_register_cib_callback(rc, FALSE, desc, cib_delete_callback);
         // CIB library handles freeing desc
     }
 
     free(xpath);
     return pcmk_rc_ok;
 }
diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
index c8c522a90e..5d07c1a6ce 100644
--- a/daemons/controld/controld_control.c
+++ b/daemons/controld/controld_control.c
@@ -1,835 +1,837 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/pengine/rules.h>
 #include <crm/cluster/internal.h>
 #include <crm/cluster/election.h>
 #include <crm/common/ipc_internal.h>
 
 #include <pacemaker-controld.h>
 
 qb_ipcs_service_t *ipcs = NULL;
 
 #if SUPPORT_COROSYNC
 extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
 #endif
 
 void crm_shutdown(int nsig);
 gboolean crm_read_options(gpointer user_data);
 
 gboolean fsa_has_quorum = FALSE;
 crm_trigger_t *fsa_source = NULL;
 crm_trigger_t *config_read = NULL;
 bool no_quorum_suicide_escalation = FALSE;
 bool controld_shutdown_lock_enabled = false;
 
 /*	 A_HA_CONNECT	*/
 void
 do_ha_control(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     gboolean registered = FALSE;
     static crm_cluster_t *cluster = NULL;
 
     if (cluster == NULL) {
         cluster = calloc(1, sizeof(crm_cluster_t));
     }
 
     if (action & A_HA_DISCONNECT) {
         crm_cluster_disconnect(cluster);
         crm_info("Disconnected from the cluster");
 
         controld_set_fsa_input_flags(R_HA_DISCONNECTED);
     }
 
     if (action & A_HA_CONNECT) {
         crm_set_status_callback(&peer_update_callback);
         crm_set_autoreap(FALSE);
 
         if (is_corosync_cluster()) {
 #if SUPPORT_COROSYNC
             registered = crm_connect_corosync(cluster);
 #endif
         }
 
         if (registered == TRUE) {
             controld_election_init(cluster->uname);
             fsa_our_uname = cluster->uname;
             fsa_our_uuid = cluster->uuid;
             if(cluster->uuid == NULL) {
                 crm_err("Could not obtain local uuid");
                 registered = FALSE;
             }
         }
 
         if (registered == FALSE) {
             controld_set_fsa_input_flags(R_HA_DISCONNECTED);
             register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
             return;
         }
 
         populate_cib_nodes(node_update_none, __func__);
         controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
         crm_info("Connected to the cluster");
     }
 
     if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action),
                 __func__);
     }
 }
 
 /*	 A_SHUTDOWN	*/
 void
 do_shutdown(long long action,
             enum crmd_fsa_cause cause,
             enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* just in case */
     controld_set_fsa_input_flags(R_SHUTDOWN);
     controld_disconnect_fencer(FALSE);
 }
 
 /*	 A_SHUTDOWN_REQ	*/
 void
 do_shutdown_req(long long action,
                 enum crmd_fsa_cause cause,
                 enum crmd_fsa_state cur_state,
                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *msg = NULL;
 
     controld_set_fsa_input_flags(R_SHUTDOWN);
     //controld_set_fsa_input_flags(R_STAYDOWN);
     crm_info("Sending shutdown request to all peers (DC is %s)",
              (fsa_our_dc? fsa_our_dc : "not set"));
     msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
 
     if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
     free_xml(msg);
 }
 
 extern char *max_generation_from;
 extern xmlNode *max_generation_xml;
 extern GHashTable *resource_history;
 extern GHashTable *voted;
 
 void
 crmd_fast_exit(crm_exit_t exit_code)
 {
     if (pcmk_is_set(fsa_input_register, R_STAYDOWN)) {
         crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d",
                  exit_code, CRM_EX_FATAL);
         exit_code = CRM_EX_FATAL;
 
     } else if ((exit_code == CRM_EX_OK)
                && pcmk_is_set(fsa_input_register, R_IN_RECOVERY)) {
         crm_err("Could not recover from internal error");
         exit_code = CRM_EX_ERROR;
     }
     crm_exit(exit_code);
 }
 
 crm_exit_t
 crmd_exit(crm_exit_t exit_code)
 {
     GListPtr gIter = NULL;
     GMainLoop *mloop = crmd_mainloop;
 
     static bool in_progress = FALSE;
 
     if (in_progress && (exit_code == CRM_EX_OK)) {
         crm_debug("Exit is already in progress");
         return exit_code;
 
     } else if(in_progress) {
         crm_notice("Error during shutdown process, exiting now with status %d (%s)",
                    exit_code, crm_exit_str(exit_code));
         crm_write_blackbox(SIGTRAP, NULL);
         crmd_fast_exit(exit_code);
     }
 
     in_progress = TRUE;
     crm_trace("Preparing to exit with status %d (%s)",
               exit_code, crm_exit_str(exit_code));
 
     /* Suppress secondary errors resulting from us disconnecting everything */
     controld_set_fsa_input_flags(R_HA_DISCONNECTED);
 
 /* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */
 
     if(ipcs) {
         crm_trace("Closing IPC server");
         mainloop_del_ipc_server(ipcs);
         ipcs = NULL;
     }
 
     controld_close_attrd_ipc();
     pe_subsystem_free();
     controld_disconnect_fencer(TRUE);
 
     if ((exit_code == CRM_EX_OK) && (crmd_mainloop == NULL)) {
         crm_debug("No mainloop detected");
         exit_code = CRM_EX_ERROR;
     }
 
     /* On an error, just get out.
      *
      * Otherwise, make the effort to have mainloop exit gracefully so
      * that it (mostly) cleans up after itself and valgrind has less
      * to report on - allowing real errors stand out
      */
     if (exit_code != CRM_EX_OK) {
         crm_notice("Forcing immediate exit with status %d (%s)",
                    exit_code, crm_exit_str(exit_code));
         crm_write_blackbox(SIGTRAP, NULL);
         crmd_fast_exit(exit_code);
     }
 
 /* Clean up as much memory as possible for valgrind */
 
     for (gIter = fsa_message_queue; gIter != NULL; gIter = gIter->next) {
         fsa_data_t *fsa_data = gIter->data;
 
         crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
                  fsa_input2string(fsa_data->fsa_input),
                  fsa_state2string(fsa_state),
                  fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
         delete_fsa_input(fsa_data);
     }
 
     controld_clear_fsa_input_flags(R_MEMBERSHIP);
     g_list_free(fsa_message_queue); fsa_message_queue = NULL;
 
     metadata_cache_fini();
     controld_election_fini();
 
     /* Tear down the CIB manager connection, but don't free it yet -- it could
      * be used when we drain the mainloop later.
      */
+    fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced);
+    fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY, do_cib_updated);
     cib_free_callbacks(fsa_cib_conn);
     fsa_cib_conn->cmds->signoff(fsa_cib_conn);
 
     verify_stopped(fsa_state, LOG_WARNING);
     controld_clear_fsa_input_flags(R_LRM_CONNECTED);
     lrm_state_destroy_all();
 
     /* This basically will not work, since mainloop has a reference to it */
     mainloop_destroy_trigger(fsa_source); fsa_source = NULL;
 
     mainloop_destroy_trigger(config_read); config_read = NULL;
     mainloop_destroy_trigger(transition_trigger); transition_trigger = NULL;
 
     pcmk__client_cleanup();
     crm_peer_destroy();
 
     controld_free_fsa_timers();
     te_cleanup_stonith_history_sync(NULL, TRUE);
     controld_free_sched_timer();
 
     free(fsa_our_dc_version); fsa_our_dc_version = NULL;
     free(fsa_our_uname); fsa_our_uname = NULL;
     free(fsa_our_uuid); fsa_our_uuid = NULL;
     free(fsa_our_dc); fsa_our_dc = NULL;
 
     free(fsa_cluster_name); fsa_cluster_name = NULL;
 
     free(te_uuid); te_uuid = NULL;
     free(failed_stop_offset); failed_stop_offset = NULL;
     free(failed_start_offset); failed_start_offset = NULL;
 
     free(max_generation_from); max_generation_from = NULL;
     free_xml(max_generation_xml); max_generation_xml = NULL;
 
     mainloop_destroy_signal(SIGPIPE);
     mainloop_destroy_signal(SIGUSR1);
     mainloop_destroy_signal(SIGTERM);
     mainloop_destroy_signal(SIGTRAP);
     /* leave SIGCHLD engaged as we might still want to drain some service-actions */
 
     if (mloop) {
         GMainContext *ctx = g_main_loop_get_context(crmd_mainloop);
 
         /* Don't re-enter this block */
         crmd_mainloop = NULL;
 
         /* no signals on final draining anymore */
         mainloop_destroy_signal(SIGCHLD);
 
         crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
 
         {
             int lpc = 0;
 
             while((g_main_context_pending(ctx) && lpc < 10)) {
                 lpc++;
                 crm_trace("Iteration %d", lpc);
                 g_main_context_dispatch(ctx);
             }
         }
 
         crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
         g_main_loop_quit(mloop);
 
         /* Won't do anything yet, since we're inside it now */
         g_main_loop_unref(mloop);
     } else {
         mainloop_destroy_signal(SIGCHLD);
     }
 
     cib_delete(fsa_cib_conn);
     fsa_cib_conn = NULL;
 
     throttle_fini();
 
     /* Graceful */
     crm_trace("Done preparing for exit with status %d (%s)",
               exit_code, crm_exit_str(exit_code));
     return exit_code;
 }
 
 /*	 A_EXIT_0, A_EXIT_1	*/
 void
 do_exit(long long action,
         enum crmd_fsa_cause cause,
         enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     crm_exit_t exit_code = CRM_EX_OK;
     int log_level = LOG_INFO;
     const char *exit_type = "gracefully";
 
     if (action & A_EXIT_1) {
         log_level = LOG_ERR;
         exit_type = "forcefully";
         exit_code = CRM_EX_ERROR;
     }
 
     verify_stopped(cur_state, LOG_ERR);
     do_crm_log(log_level, "Performing %s - %s exiting the controller",
                fsa_action2string(action), exit_type);
 
     crm_info("[%s] stopped (%d)", crm_system_name, exit_code);
     crmd_exit(exit_code);
 }
 
 static void sigpipe_ignore(int nsig) { return; }
 
 /*	 A_STARTUP	*/
 void
 do_startup(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     crm_debug("Registering Signal Handlers");
     mainloop_add_signal(SIGTERM, crm_shutdown);
     mainloop_add_signal(SIGPIPE, sigpipe_ignore);
 
     fsa_source = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
     config_read = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL);
     transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger, NULL);
 
     crm_debug("Creating CIB manager and executor objects");
     fsa_cib_conn = cib_new();
 
     lrm_state_init_local();
     if (controld_init_fsa_timers() == FALSE) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 // \return libqb error code (0 on success, -errno on error)
 static int32_t
 accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
 {
     crm_trace("Accepting new IPC client connection");
     if (pcmk__new_client(c, uid, gid) == NULL) {
         return -EIO;
     }
     return 0;
 }
 
 // \return libqb error code (0 on success, -errno on error)
 static int32_t
 dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     pcmk__client_t *client = pcmk__find_client(c);
 
     xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);
 
     pcmk__ipc_send_ack(client, id, flags, "ack");
     if (msg == NULL) {
         return 0;
     }
 
 #if ENABLE_ACL
     CRM_ASSERT(client->user != NULL);
     pcmk__update_acl_user(msg, F_CRM_USER, client->user);
 #endif
 
     crm_xml_add(msg, F_CRM_SYS_FROM, client->id);
     if (controld_authorize_ipc_message(msg, client, NULL)) {
         crm_trace("Processing IPC message from %s", pcmk__client_name(client));
         route_message(C_IPC_MESSAGE, msg);
     }
 
     trigger_fsa();
     free_xml(msg);
     return 0;
 }
 
 static int32_t
 crmd_ipc_closed(qb_ipcs_connection_t * c)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     if (client) {
         crm_trace("Disconnecting %sregistered client %s (%p/%p)",
                   (client->userdata? "" : "un"), pcmk__client_name(client),
                   c, client);
         free(client->userdata);
         pcmk__free_client(client);
         trigger_fsa();
     }
     return 0;
 }
 
 static void
 crmd_ipc_destroy(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p", c);
     crmd_ipc_closed(c);
 }
 
 /*	 A_STOP	*/
 void
 do_stop(long long action,
         enum crmd_fsa_cause cause,
         enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     crm_trace("Closing IPC server");
     mainloop_del_ipc_server(ipcs); ipcs = NULL;
     register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
 }
 
 /*	 A_STARTED	*/
 void
 do_started(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     static struct qb_ipcs_service_handlers crmd_callbacks = {
         .connection_accept = accept_controller_client,
         .connection_created = NULL,
         .msg_process = dispatch_controller_ipc,
         .connection_closed = crmd_ipc_closed,
         .connection_destroyed = crmd_ipc_destroy
     };
 
     if (cur_state != S_STARTING) {
         crm_err("Start cancelled... %s", fsa_state2string(cur_state));
         return;
 
     } else if (!pcmk_is_set(fsa_input_register, R_MEMBERSHIP)) {
         crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (!pcmk_is_set(fsa_input_register, R_LRM_CONNECTED)) {
         crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (!pcmk_is_set(fsa_input_register, R_CIB_CONNECTED)) {
         crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (!pcmk_is_set(fsa_input_register, R_READ_CONFIG)) {
         crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (!pcmk_is_set(fsa_input_register, R_PEER_DATA)) {
 
         crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
         crmd_fsa_stall(TRUE);
         return;
     }
 
     crm_debug("Init server comms");
     ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
     if (ipcs == NULL) {
         crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     } else {
         crm_notice("Pacemaker controller successfully started and accepting connections");
     }
     controld_trigger_fencer_connect();
 
     controld_clear_fsa_input_flags(R_STARTING);
     register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
 }
 
 /*	 A_RECOVER	*/
 void
 do_recover(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     controld_set_fsa_input_flags(R_IN_RECOVERY);
     crm_warn("Fast-tracking shutdown in response to errors");
 
     register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
 }
 
 static pcmk__cluster_option_t crmd_opts[] = {
     /* name, old name, type, allowed values,
      * default value, validator,
      * short description,
      * long description
      */
     {
         "dc-version", NULL, "string", NULL, "none", NULL,
         "Pacemaker version on cluster node elected Designated Controller (DC)",
         "Includes a hash which identifies the exact changeset the code was "
             "built from. Used for diagnostic purposes."
     },
     {
         "cluster-infrastructure", NULL, "string", NULL, "corosync", NULL,
         "The messaging stack on which Pacemaker is currently running",
         "Used for informational and diagnostic purposes."
     },
     {
         "cluster-name", NULL, "string", NULL, NULL, NULL,
         "An arbitrary name for the cluster",
         "This optional value is mostly for users' convenience as desired "
             "in administration, but may also be used in Pacemaker "
             "configuration rules via the #cluster-name node attribute, and "
             "by higher-level tools and resource agents."
     },
     {
         XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time",
         NULL, "20s", pcmk__valid_interval_spec,
         "How long to wait for a response from other nodes during start-up",
         "The optimal value will depend on the speed and load of your network "
             "and the type of switches used."
     },
     {
         XML_CONFIG_ATTR_RECHECK, NULL, "time",
         "Zero disables polling, while positive values are an interval in seconds"
             "(unless other units are specified, for example \"5min\")",
         "15min", pcmk__valid_interval_spec,
         "Polling interval to recheck cluster state and evaluate rules "
             "with date specifications",
         "Pacemaker is primarily event-driven, and looks ahead to know when to "
             "recheck cluster state for failure timeouts and most time-based "
             "rules. However, it will also recheck the cluster after this "
             "amount of inactivity, to evaluate rules with date specifications "
             "and serve as a fail-safe for certain types of scheduler bugs."
     },
     {
         "load-threshold", NULL, "percentage", NULL,
         "80%", pcmk__valid_utilization,
         "Maximum amount of system load that should be used by cluster nodes",
         "The cluster will slow down its recovery process when the amount of "
             "system resources used (currently CPU) approaches this limit",
     },
     {
         "node-action-limit", NULL, "integer", NULL,
         "0", pcmk__valid_number,
         "Maximum number of jobs that can be scheduled per node "
             "(defaults to 2x cores)"
     },
     { XML_CONFIG_ATTR_FENCE_REACTION, NULL, "string", NULL, "stop", NULL,
         "How a cluster node should react if notified of its own fencing",
         "A cluster node may receive notification of its own fencing if fencing "
         "is misconfigured, or if fabric fencing is in use that doesn't cut "
         "cluster communication. Allowed values are \"stop\" to attempt to "
         "immediately stop pacemaker and stay stopped, or \"panic\" to attempt "
         "to immediately reboot the local node, falling back to stop on failure."
     },
     {
         XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL,
         "2min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "Declare an election failed if it is not decided within this much "
             "time. If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL,
         "20min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "Exit immediately if shutdown does not complete within this much "
             "time. If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         "join-integration-timeout", "crmd-integration-timeout", "time", NULL,
         "3min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         "join-finalization-timeout", "crmd-finalization-timeout", "time", NULL,
         "30min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         "transition-delay", "crmd-transition-delay", "time", NULL,
         "0s", pcmk__valid_interval_spec,
         "*** Advanced Use Only *** Enabling this option will slow down "
             "cluster recovery under all conditions",
         "Delay cluster recovery for this much time to allow for additional "
             "events to occur. Useful if your configuration is sensitive to "
             "the order in which ping updates arrive."
     },
     {
         "stonith-watchdog-timeout", NULL, "time", NULL,
         "0", pcmk__valid_sbd_timeout,
         "How long to wait before we can assume nodes are safely down "
             "when watchdog-based self-fencing via SBD is in use",
         "If nonzero, along with `have-watchdog=true` automatically set by the "
             "cluster, when fencing is required, watchdog-based self-fencing "
             "will be performed via SBD without requiring a fencing resource "
             "explicitly configured. "
             "If `stonith-watchdog-timeout` is set to a positive value, unseen "
             "nodes are assumed to self-fence within this much time. +WARNING:+ "
             "It must be ensured that this value is larger than the "
             "`SBD_WATCHDOG_TIMEOUT` environment variable on all nodes. "
             "Pacemaker verifies the settings individually on all nodes and "
             "prevents startup or shuts down if configured wrongly on the fly. "
             "It's strongly recommended that `SBD_WATCHDOG_TIMEOUT` is set to "
             "the same value on all nodes. "
             "If `stonith-watchdog-timeout` is set to a negative value, and "
             "`SBD_WATCHDOG_TIMEOUT` is set, twice that value will be used. "
             "+WARNING:+ In this case, it's essential (currently not verified by "
             "pacemaker) that `SBD_WATCHDOG_TIMEOUT` is set to the same value on "
             "all nodes."
     },
     {
         "stonith-max-attempts", NULL, "integer", NULL,
         "10", pcmk__valid_positive_number,
         "How many times fencing can fail before it will no longer be "
             "immediately re-attempted on a target"
     },
 
     // Already documented in libpe_status (other values must be kept identical)
     {
         "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide",
         "stop", pcmk__valid_quorum, NULL, NULL
     },
     {
         XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL,
         "false", pcmk__valid_boolean, NULL, NULL
     },
 };
 
 void
 crmd_metadata(void)
 {
     pcmk__print_option_metadata("pacemaker-controld", "1.0",
                                 "Pacemaker controller options",
                                 "Cluster options used by Pacemaker's "
                                     "controller (formerly called crmd)",
                                 crmd_opts, DIMOF(crmd_opts));
 }
 
 static void
 verify_crmd_options(GHashTable * options)
 {
     pcmk__validate_cluster_options(options, crmd_opts, DIMOF(crmd_opts));
 }
 
 static const char *
 crmd_pref(GHashTable * options, const char *name)
 {
     return pcmk__cluster_option(options, crmd_opts, DIMOF(crmd_opts), name);
 }
 
 static void
 config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     const char *value = NULL;
     GHashTable *config_hash = NULL;
     crm_time_t *now = crm_time_new(NULL);
     xmlNode *crmconfig = NULL;
     xmlNode *alerts = NULL;
 
     if (rc != pcmk_ok) {
         fsa_data_t *msg_data = NULL;
 
         crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 
         if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
             crm_err("The cluster is mis-configured - shutting down and staying down");
             controld_set_fsa_input_flags(R_STAYDOWN);
         }
         goto bail;
     }
 
     crmconfig = output;
     if ((crmconfig) &&
         (crm_element_name(crmconfig)) &&
         (strcmp(crm_element_name(crmconfig), XML_CIB_TAG_CRMCONFIG) != 0)) {
         crmconfig = first_named_child(crmconfig, XML_CIB_TAG_CRMCONFIG);
     }
     if (!crmconfig) {
         fsa_data_t *msg_data = NULL;
 
         crm_err("Local CIB query for " XML_CIB_TAG_CRMCONFIG " section failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         goto bail;
     }
 
     crm_debug("Call %d : Parsing CIB options", call_id);
     config_hash = crm_str_table_new();
     pe_unpack_nvpairs(crmconfig, crmconfig, XML_CIB_TAG_PROPSET, NULL,
                       config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);
 
     verify_crmd_options(config_hash);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
     election_trigger->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, "node-action-limit"); /* Also checks migration-limit */
     throttle_update_job_max(value);
 
     value = crmd_pref(config_hash, "load-threshold");
     if(value) {
         throttle_set_load_target(strtof(value, NULL) / 100.0);
     }
 
     value = crmd_pref(config_hash, "no-quorum-policy");
     if (pcmk__str_eq(value, "suicide", pcmk__str_casei) && pcmk__locate_sbd()) {
         no_quorum_suicide_escalation = TRUE;
     }
 
     set_fence_reaction(crmd_pref(config_hash, XML_CONFIG_ATTR_FENCE_REACTION));
 
     value = crmd_pref(config_hash,"stonith-max-attempts");
     update_stonith_max_attempts(value);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT);
     shutdown_escalation_timer->period_ms = crm_parse_interval_spec(value);
     crm_debug("Shutdown escalation occurs if DC has not responded to request in %ums",
               shutdown_escalation_timer->period_ms);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL);
     controld_set_election_period(value);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK);
     recheck_interval_ms = crm_parse_interval_spec(value);
     crm_debug("Re-run scheduler after %dms of inactivity", recheck_interval_ms);
 
     value = crmd_pref(config_hash, "transition-delay");
     transition_timer->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, "join-integration-timeout");
     integration_timer->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, "join-finalization-timeout");
     finalization_timer->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_SHUTDOWN_LOCK);
     controld_shutdown_lock_enabled = crm_is_true(value);
 
     free(fsa_cluster_name);
     fsa_cluster_name = NULL;
 
     value = g_hash_table_lookup(config_hash, "cluster-name");
     if (value) {
         fsa_cluster_name = strdup(value);
     }
 
     alerts = first_named_child(output, XML_CIB_TAG_ALERTS);
     crmd_unpack_alerts(alerts);
 
     controld_set_fsa_input_flags(R_READ_CONFIG);
     crm_trace("Triggering FSA: %s", __func__);
     mainloop_set_trigger(fsa_source);
 
     g_hash_table_destroy(config_hash);
   bail:
     crm_time_free(now);
 }
 
 gboolean
 crm_read_options(gpointer user_data)
 {
     int call_id =
         fsa_cib_conn->cmds->query(fsa_cib_conn,
             "//" XML_CIB_TAG_CRMCONFIG " | //" XML_CIB_TAG_ALERTS,
             NULL, cib_xpath | cib_scope_local);
 
     fsa_register_cib_callback(call_id, FALSE, NULL, config_query_callback);
     crm_trace("Querying the CIB... call %d", call_id);
     return TRUE;
 }
 
 /*	 A_READCONFIG	*/
 void
 do_read_config(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     throttle_init();
     mainloop_set_trigger(config_read);
 }
 
 void
 crm_shutdown(int nsig)
 {
     if ((crmd_mainloop == NULL) || !g_main_loop_is_running(crmd_mainloop)) {
         crmd_exit(CRM_EX_OK);
         return;
     }
 
     if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
         crm_err("Escalating shutdown");
         register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
         return;
     }
 
     controld_set_fsa_input_flags(R_SHUTDOWN);
     register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
 
     if (shutdown_escalation_timer->period_ms == 0) {
         const char *value = crmd_pref(NULL, XML_CONFIG_ATTR_FORCE_QUIT);
 
         shutdown_escalation_timer->period_ms = crm_parse_interval_spec(value);
     }
 
     crm_notice("Initiating controller shutdown sequence " CRM_XS
                " limit=%ums", shutdown_escalation_timer->period_ms);
     controld_start_timer(shutdown_escalation_timer);
 }
diff --git a/daemons/controld/pacemaker-controld.h b/daemons/controld/pacemaker-controld.h
index 6213c27518..7485d9598f 100644
--- a/daemons/controld/pacemaker-controld.h
+++ b/daemons/controld/pacemaker-controld.h
@@ -1,36 +1,38 @@
 /*
  * Copyright 2004-2019 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef CRMD__H
 #  define CRMD__H
 
 #include <controld_alerts.h>
 #include <controld_callbacks.h>
 #include <controld_fencing.h>
 #include <controld_fsa.h>
 #include <controld_timers.h>
 #include <controld_lrm.h>
 #include <controld_membership.h>
 #include <controld_messages.h>
 #include <controld_metadata.h>
 #include <controld_throttle.h>
 #include <controld_transition.h>
 #include <controld_utils.h>
 
 extern GMainLoop *crmd_mainloop;
 extern bool no_quorum_suicide_escalation;
 
+void do_cib_updated(const char *event, xmlNode * msg);
+void do_cib_replaced(const char *event, xmlNode * msg);
 void crmd_metadata(void);
 void controld_election_init(const char *uname);
 void controld_remove_voter(const char *uname);
 void controld_election_fini(void);
 void controld_set_election_period(const char *value);
 void controld_stop_election_timer(void);
 
 #endif
diff --git a/daemons/fenced/pacemaker-fenced.c b/daemons/fenced/pacemaker-fenced.c
index b8055041a3..66eeb5faa7 100644
--- a/daemons/fenced/pacemaker-fenced.c
+++ b/daemons/fenced/pacemaker-fenced.c
@@ -1,1533 +1,1534 @@
 /*
  * Copyright 2009-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/utsname.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>  /* U32T ~ PRIu32, X32T ~ PRIx32 */
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/cluster/internal.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <crm/common/mainloop.h>
 
 #include <crm/cib/internal.h>
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 
 #include <pacemaker-fenced.h>
 
 char *stonith_our_uname = NULL;
 char *stonith_our_uuid = NULL;
 long stonith_watchdog_timeout_ms = 0;
 
 static GMainLoop *mainloop = NULL;
 
 gboolean stand_alone = FALSE;
 static gboolean no_cib_connect = FALSE;
 static gboolean stonith_shutdown_flag = FALSE;
 
 static qb_ipcs_service_t *ipcs = NULL;
 static xmlNode *local_cib = NULL;
 static pe_working_set_t *fenced_data_set = NULL;
 
 static cib_t *cib_api = NULL;
 
 static void stonith_shutdown(int nsig);
 static void stonith_cleanup(void);
 
 static int32_t
 st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
 {
     if (stonith_shutdown_flag) {
         crm_info("Ignoring new client [%d] during shutdown",
                  pcmk__client_pid(c));
         return -EPERM;
     }
 
     if (pcmk__new_client(c, uid, gid) == NULL) {
         return -EIO;
     }
     return 0;
 }
 
 /* Exit code means? */
 static int32_t
 st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     int call_options = 0;
     xmlNode *request = NULL;
     pcmk__client_t *c = pcmk__find_client(qbc);
     const char *op = NULL;
 
     if (c == NULL) {
         crm_info("Invalid client: %p", qbc);
         return 0;
     }
 
     request = pcmk__client_data2xml(c, data, &id, &flags);
     if (request == NULL) {
         pcmk__ipc_send_ack(c, id, flags, "nack");
         return 0;
     }
 
 
     op = crm_element_value(request, F_CRM_TASK);
     if(pcmk__str_eq(op, CRM_OP_RM_NODE_CACHE, pcmk__str_casei)) {
         crm_xml_add(request, F_TYPE, T_STONITH_NG);
         crm_xml_add(request, F_STONITH_OPERATION, op);
         crm_xml_add(request, F_STONITH_CLIENTID, c->id);
         crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
         crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);
 
         send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE);
         free_xml(request);
         return 0;
     }
 
     if (c->name == NULL) {
         const char *value = crm_element_value(request, F_STONITH_CLIENTNAME);
 
         if (value == NULL) {
             value = "unknown";
         }
         c->name = crm_strdup_printf("%s.%u", value, c->pid);
     }
 
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
     crm_trace("Flags %" X32T "/%u for command %" U32T " from %s",
               flags, call_options, id, pcmk__client_name(c));
 
     if (pcmk_is_set(call_options, st_opt_sync_call)) {
         CRM_ASSERT(flags & crm_ipc_client_response);
         CRM_LOG_ASSERT(c->request_id == 0);     /* This means the client has two synchronous events in-flight */
         c->request_id = id;     /* Reply only to the last one */
     }
 
     crm_xml_add(request, F_STONITH_CLIENTID, c->id);
     crm_xml_add(request, F_STONITH_CLIENTNAME, pcmk__client_name(c));
     crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);
 
     stonith_command(c, id, flags, request, NULL);
 
     free_xml(request);
     return 0;
 }
 
 /* Error code means? */
 static int32_t
 st_ipc_closed(qb_ipcs_connection_t * c)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     if (client == NULL) {
         return 0;
     }
 
     crm_trace("Connection %p closed", c);
     pcmk__free_client(client);
 
     /* 0 means: yes, go ahead and destroy the connection */
     return 0;
 }
 
 static void
 st_ipc_destroy(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p destroyed", c);
     st_ipc_closed(c);
 }
 
 static void
 stonith_peer_callback(xmlNode * msg, void *private_data)
 {
     const char *remote_peer = crm_element_value(msg, F_ORIG);
     const char *op = crm_element_value(msg, F_STONITH_OPERATION);
 
     if (pcmk__str_eq(op, "poke", pcmk__str_none)) {
         return;
     }
 
     crm_log_xml_trace(msg, "Peer[inbound]");
     stonith_command(NULL, 0, 0, msg, remote_peer);
 }
 
 #if SUPPORT_COROSYNC
 static void
 stonith_peer_ais_callback(cpg_handle_t handle,
                           const struct cpg_name *groupName,
                           uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
 {
     uint32_t kind = 0;
     xmlNode *xml = NULL;
     const char *from = NULL;
     char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);
 
     if(data == NULL) {
         return;
     }
     if (kind == crm_class_cluster) {
         xml = string2xml(data);
         if (xml == NULL) {
             crm_err("Invalid XML: '%.120s'", data);
             free(data);
             return;
         }
         crm_xml_add(xml, F_ORIG, from);
         /* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
         stonith_peer_callback(xml, NULL);
     }
 
     free_xml(xml);
     free(data);
     return;
 }
 
 static void
 stonith_peer_cs_destroy(gpointer user_data)
 {
     crm_crit("Lost connection to cluster layer, shutting down");
     stonith_shutdown(0);
 }
 #endif
 
 void
 do_local_reply(xmlNode * notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer)
 {
     /* send callback to originating child */
     pcmk__client_t *client_obj = NULL;
     int local_rc = pcmk_rc_ok;
 
     crm_trace("Sending response");
     client_obj = pcmk__find_client_by_id(client_id);
 
     crm_trace("Sending callback to request originator");
     if (client_obj == NULL) {
         local_rc = EPROTO;
         crm_trace("No client to sent the response to.  F_STONITH_CLIENTID not set.");
 
     } else {
         int rid = 0;
 
         if (sync_reply) {
             CRM_LOG_ASSERT(client_obj->request_id);
 
             rid = client_obj->request_id;
             client_obj->request_id = 0;
 
             crm_trace("Sending response %d to %s %s",
                       rid, client_obj->name, from_peer ? "(originator of delegated request)" : "");
 
         } else {
             crm_trace("Sending an event to %s %s",
                       client_obj->name, from_peer ? "(originator of delegated request)" : "");
         }
 
         local_rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
                                       (sync_reply? crm_ipc_flags_none
                                        : crm_ipc_server_event));
     }
 
     if ((local_rc != pcmk_rc_ok) && (client_obj != NULL)) {
         crm_warn("%s reply to %s failed: %s",
                  (sync_reply? "Synchronous" : "Asynchronous"),
                  (client_obj? client_obj->name : "unknown client"),
                  pcmk_rc_str(local_rc));
     }
 }
 
 uint64_t
 get_stonith_flag(const char *name)
 {
     if (pcmk__str_eq(name, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) {
         return st_callback_notify_fence;
 
     } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_ADD, pcmk__str_casei)) {
         return st_callback_device_add;
 
     } else if (pcmk__str_eq(name, STONITH_OP_DEVICE_DEL, pcmk__str_casei)) {
         return st_callback_device_del;
 
     } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY, pcmk__str_casei)) {
         return st_callback_notify_history;
 
     } else if (pcmk__str_eq(name, T_STONITH_NOTIFY_HISTORY_SYNCED, pcmk__str_casei)) {
         return st_callback_notify_history_synced;
 
     }
     return st_callback_unknown;
 }
 
 static void
 stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
 {
 
     xmlNode *update_msg = user_data;
     pcmk__client_t *client = value;
     const char *type = NULL;
 
     CRM_CHECK(client != NULL, return);
     CRM_CHECK(update_msg != NULL, return);
 
     type = crm_element_value(update_msg, F_SUBTYPE);
     CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);
 
     if (client->ipcs == NULL) {
         crm_trace("Skipping client with NULL channel");
         return;
     }
 
     if (pcmk_is_set(client->flags, get_stonith_flag(type))) {
         int rc = pcmk__ipc_send_xml(client, 0, update_msg,
                                     crm_ipc_server_event|crm_ipc_server_error);
 
         if (rc != pcmk_rc_ok) {
             crm_warn("%s notification of client %s failed: %s "
                      CRM_XS " id=%.8s rc=%d", type, pcmk__client_name(client),
                      pcmk_rc_str(rc), client->id, rc);
         } else {
             crm_trace("Sent %s notification to client %s.%.6s", type,
                       pcmk__client_name(client), client->id);
         }
     }
 }
 
 void
 do_stonith_async_timeout_update(const char *client_id, const char *call_id, int timeout)
 {
     pcmk__client_t *client = NULL;
     xmlNode *notify_data = NULL;
 
     if (!timeout || !call_id || !client_id) {
         return;
     }
 
     client = pcmk__find_client_by_id(client_id);
     if (!client) {
         return;
     }
 
     notify_data = create_xml_node(NULL, T_STONITH_TIMEOUT_VALUE);
     crm_xml_add(notify_data, F_TYPE, T_STONITH_TIMEOUT_VALUE);
     crm_xml_add(notify_data, F_STONITH_CALLID, call_id);
     crm_xml_add_int(notify_data, F_STONITH_TIMEOUT, timeout);
 
     crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);
 
     if (client) {
         pcmk__ipc_send_xml(client, 0, notify_data, crm_ipc_server_event);
     }
 
     free_xml(notify_data);
 }
 
 void
 do_stonith_notify(int options, const char *type, int result, xmlNode * data)
 {
     /* TODO: Standardize the contents of data */
     xmlNode *update_msg = create_xml_node(NULL, "notify");
 
     CRM_CHECK(type != NULL,;);
 
     crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY);
     crm_xml_add(update_msg, F_SUBTYPE, type);
     crm_xml_add(update_msg, F_STONITH_OPERATION, type);
     crm_xml_add_int(update_msg, F_STONITH_RC, result);
 
     if (data != NULL) {
         add_message_xml(update_msg, F_STONITH_CALLDATA, data);
     }
 
     crm_trace("Notifying clients");
     pcmk__foreach_ipc_client(stonith_notify_client, update_msg);
     free_xml(update_msg);
     crm_trace("Notify complete");
 }
 
 static void
 do_stonith_notify_config(int options, const char *op, int rc,
                          const char *desc, int active)
 {
     xmlNode *notify_data = create_xml_node(NULL, op);
 
     CRM_CHECK(notify_data != NULL, return);
 
     crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
     crm_xml_add_int(notify_data, F_STONITH_ACTIVE, active);
 
     do_stonith_notify(options, op, rc, notify_data);
     free_xml(notify_data);
 }
 
 void
 do_stonith_notify_device(int options, const char *op, int rc, const char *desc)
 {
     do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(device_list));
 }
 
 void
 do_stonith_notify_level(int options, const char *op, int rc, const char *desc)
 {
     do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(topology));
 }
 
 static void
 topology_remove_helper(const char *node, int level)
 {
     int rc;
     char *desc = NULL;
     xmlNode *data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);
 
     crm_xml_add(data, F_STONITH_ORIGIN, __func__);
     crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
     crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);
 
     rc = stonith_level_remove(data, &desc);
     do_stonith_notify_level(0, STONITH_OP_LEVEL_DEL, rc, desc);
 
     free_xml(data);
     free(desc);
 }
 
 static void
 remove_cib_device(xmlXPathObjectPtr xpathObj)
 {
     int max = numXpathResults(xpathObj), lpc = 0;
 
     for (lpc = 0; lpc < max; lpc++) {
         const char *rsc_id = NULL;
         const char *standard = NULL;
         xmlNode *match = getXpathResult(xpathObj, lpc);
 
         CRM_LOG_ASSERT(match != NULL);
         if(match != NULL) {
             standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
         }
 
         if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
             continue;
         }
 
         rsc_id = crm_element_value(match, XML_ATTR_ID);
 
         stonith_device_remove(rsc_id, TRUE);
     }
 }
 
 static void
 handle_topology_change(xmlNode *match, bool remove) 
 {
     int rc;
     char *desc = NULL;
 
     CRM_CHECK(match != NULL, return);
     crm_trace("Updating %s", ID(match));
 
     if(remove) {
         int index = 0;
         char *key = stonith_level_key(match, -1);
 
         crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
         topology_remove_helper(key, index);
         free(key);
     }
 
     rc = stonith_level_register(match, &desc);
     do_stonith_notify_level(0, STONITH_OP_LEVEL_ADD, rc, desc);
 
     free(desc);
 }
 
 static void
 remove_fencing_topology(xmlXPathObjectPtr xpathObj)
 {
     int max = numXpathResults(xpathObj), lpc = 0;
 
     for (lpc = 0; lpc < max; lpc++) {
         xmlNode *match = getXpathResult(xpathObj, lpc);
 
         CRM_LOG_ASSERT(match != NULL);
         if (match && crm_element_value(match, XML_DIFF_MARKER)) {
             /* Deletion */
             int index = 0;
             char *target = stonith_level_key(match, -1);
 
             crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
             if (target == NULL) {
                 crm_err("Invalid fencing target in element %s", ID(match));
 
             } else if (index <= 0) {
                 crm_err("Invalid level for %s in element %s", target, ID(match));
 
             } else {
                 topology_remove_helper(target, index);
             }
             /* } else { Deal with modifications during the 'addition' stage */
         }
     }
 }
 
 static void
 register_fencing_topology(xmlXPathObjectPtr xpathObj)
 {
     int max = numXpathResults(xpathObj), lpc = 0;
 
     for (lpc = 0; lpc < max; lpc++) {
         xmlNode *match = getXpathResult(xpathObj, lpc);
 
         handle_topology_change(match, TRUE);
     }
 }
 
 /* Fencing
 <diff crm_feature_set="3.0.6">
   <diff-removed>
     <fencing-topology>
       <fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
       <fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
       <fencing-level devices="disk,network" id="f-p2.1"/>
     </fencing-topology>
   </diff-removed>
   <diff-added>
     <fencing-topology>
       <fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
       <fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
       <fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
     </fencing-topology>
   </diff-added>
 </diff>
 */
 
 static void
 fencing_topology_init(void)
 {
     xmlXPathObjectPtr xpathObj = NULL;
     const char *xpath = "//" XML_TAG_FENCING_LEVEL;
 
     crm_trace("Full topology refresh");
     free_topology_list();
     init_topology_list();
 
     /* Grab everything */
     xpathObj = xpath_search(local_cib, xpath);
     register_fencing_topology(xpathObj);
 
     freeXpathObject(xpathObj);
 }
 
 #define rsc_name(x) x->clone_name?x->clone_name:x->id
 
 /*!
  * \internal
  * \brief Check whether our uname is in a resource's allowed node list
  *
  * \param[in] rsc  Resource to check
  *
  * \return Pointer to node object if found, NULL otherwise
  */
 static pe_node_t *
 our_node_allowed_for(pe_resource_t *rsc)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     if (rsc && stonith_our_uname) {
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
             if (node && strcmp(node->details->uname, stonith_our_uname) == 0) {
                 break;
             }
             node = NULL;
         }
     }
     return node;
 }
 
 /*!
  * \internal
  * \brief If a resource or any of its children are STONITH devices, update their
  *        definitions given a cluster working set.
  *
  * \param[in] rsc       Resource to check
  * \param[in] data_set  Cluster working set with device information
  */
 static void cib_device_update(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     pe_node_t *node = NULL;
     const char *value = NULL;
     const char *rclass = NULL;
     pe_node_t *parent = NULL;
     gboolean remove = TRUE;
 
     /* If this is a complex resource, check children rather than this resource itself.
      * TODO: Mark each installed device and remove if untouched when this process finishes.
      */
     if(rsc->children) {
         GListPtr gIter = NULL;
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             cib_device_update(gIter->data, data_set);
             if(pe_rsc_is_clone(rsc)) {
                 crm_trace("Only processing one copy of the clone %s", rsc->id);
                 break;
             }
         }
         return;
     }
 
     /* We only care about STONITH resources. */
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     if (!pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         return;
     }
 
     /* If this STONITH resource is disabled, just remove it. */
     if (pe__resource_is_disabled(rsc)) {
         crm_info("Device %s has been disabled", rsc->id);
         goto update_done;
     }
 
     /* Check whether our node is allowed for this resource (and its parent if in a group) */
     node = our_node_allowed_for(rsc);
     if (rsc->parent && (rsc->parent->variant == pe_group)) {
         parent = our_node_allowed_for(rsc->parent);
     }
 
     if(node == NULL) {
         /* Our node is disallowed, so remove the device */
         GHashTableIter iter;
 
         crm_info("Device %s has been disabled on %s: unknown", rsc->id, stonith_our_uname);
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
             crm_trace("Available: %s = %d", node->details->uname, node->weight);
         }
 
         goto update_done;
 
     } else if(node->weight < 0 || (parent && parent->weight < 0)) {
         /* Our node (or its group) is disallowed by score, so remove the device */
         char *score = score2char((node->weight < 0) ? node->weight : parent->weight);
 
         crm_info("Device %s has been disabled on %s: score=%s", rsc->id, stonith_our_uname, score);
         free(score);
 
         goto update_done;
 
     } else {
         /* Our node is allowed, so update the device information */
         int rc;
         xmlNode *data;
         GHashTableIter gIter;
         stonith_key_value_t *params = NULL;
 
         const char *name = NULL;
         const char *agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE);
         const char *rsc_provides = NULL;
 
         crm_debug("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight);
         get_rsc_attributes(rsc->parameters, rsc, node, data_set);
         get_meta_attributes(rsc->meta, rsc, node, data_set);
 
         rsc_provides = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROVIDES);
 
         g_hash_table_iter_init(&gIter, rsc->parameters);
         while (g_hash_table_iter_next(&gIter, (gpointer *) & name, (gpointer *) & value)) {
             if (!name || !value) {
                 continue;
             }
             params = stonith_key_value_add(params, name, value);
             crm_trace(" %s=%s", name, value);
         }
 
         remove = FALSE;
         data = create_device_registration_xml(rsc_name(rsc), st_namespace_any,
                                               agent, params, rsc_provides);
         stonith_key_value_freeall(params, 1, 1);
         rc = stonith_device_register(data, NULL, TRUE);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(data);
     }
 
 update_done:
 
     if(remove && g_hash_table_lookup(device_list, rsc_name(rsc))) {
         stonith_device_remove(rsc_name(rsc), TRUE);
     }
 }
 
 /*!
  * \internal
  * \brief Update all STONITH device definitions based on current CIB
  */
 static void
 cib_devices_update(void)
 {
     GListPtr gIter = NULL;
 
     crm_info("Updating devices to version %s.%s.%s",
              crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN),
              crm_element_value(local_cib, XML_ATTR_GENERATION),
              crm_element_value(local_cib, XML_ATTR_NUMUPDATES));
 
     CRM_ASSERT(fenced_data_set != NULL);
     fenced_data_set->input = local_cib;
     fenced_data_set->now = crm_time_new(NULL);
     fenced_data_set->localhost = stonith_our_uname;
     pe__set_working_set_flags(fenced_data_set, pe_flag_quick_location);
 
     cluster_status(fenced_data_set);
     pcmk__schedule_actions(fenced_data_set, NULL, NULL);
 
     for (gIter = fenced_data_set->resources; gIter != NULL; gIter = gIter->next) {
         cib_device_update(gIter->data, fenced_data_set);
     }
     fenced_data_set->input = NULL; // Wasn't a copy, so don't let API free it
     pe_reset_working_set(fenced_data_set);
 }
 
 static void
 update_cib_stonith_devices_v2(const char *event, xmlNode * msg)
 {
     xmlNode *change = NULL;
     char *reason = NULL;
     bool needs_update = FALSE;
     xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     for (change = pcmk__xml_first_child(patchset); change != NULL;
          change = pcmk__xml_next(change)) {
         const char *op = crm_element_value(change, XML_DIFF_OP);
         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
         const char *shortpath = NULL;
 
         if ((op == NULL) ||
             (strcmp(op, "move") == 0) ||
             strstr(xpath, "/"XML_CIB_TAG_STATUS)) {
             continue;
         } else if (pcmk__str_eq(op, "delete", pcmk__str_casei) && strstr(xpath, "/"XML_CIB_TAG_RESOURCE)) {
             const char *rsc_id = NULL;
             char *search = NULL;
             char *mutable = NULL;
 
             if (strstr(xpath, XML_TAG_ATTR_SETS) ||
                 strstr(xpath, XML_TAG_META_SETS)) {
                 needs_update = TRUE;
                 reason = strdup("(meta) attribute deleted from resource");
                 break;
             } 
             mutable = strdup(xpath);
             rsc_id = strstr(mutable, "primitive[@id=\'");
             if (rsc_id != NULL) {
                 rsc_id += strlen("primitive[@id=\'");
                 search = strchr(rsc_id, '\'');
             }
             if (search != NULL) {
                 *search = 0;
                 stonith_device_remove(rsc_id, TRUE);
             } else {
                 crm_warn("Ignoring malformed CIB update (resource deletion)");
             }
             free(mutable);
 
         } else if (strstr(xpath, "/"XML_CIB_TAG_RESOURCES) ||
                    strstr(xpath, "/"XML_CIB_TAG_CONSTRAINTS) ||
                    strstr(xpath, "/"XML_CIB_TAG_RSCCONFIG)) {
             shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
             reason = crm_strdup_printf("%s %s", op, shortpath+1);
             needs_update = TRUE;
             break;
         }
     }
 
     if(needs_update) {
         crm_info("Updating device list from the cib: %s", reason);
         cib_devices_update();
     } else {
         crm_trace("No updates for device list found in cib");
     }
     free(reason);
 }
 
 
 static void
 update_cib_stonith_devices_v1(const char *event, xmlNode * msg)
 {
     const char *reason = "none";
     gboolean needs_update = FALSE;
     xmlXPathObjectPtr xpath_obj = NULL;
 
     /* process new constraints */
     xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_CONS_TAG_RSC_LOCATION);
     if (numXpathResults(xpath_obj) > 0) {
         int max = numXpathResults(xpath_obj), lpc = 0;
 
         /* Safest and simplest to always recompute */
         needs_update = TRUE;
         reason = "new location constraint";
 
         for (lpc = 0; lpc < max; lpc++) {
             xmlNode *match = getXpathResult(xpath_obj, lpc);
 
             crm_log_xml_trace(match, "new constraint");
         }
     }
     freeXpathObject(xpath_obj);
 
     /* process deletions */
     xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_RESOURCE);
     if (numXpathResults(xpath_obj) > 0) {
         remove_cib_device(xpath_obj);
     }
     freeXpathObject(xpath_obj);
 
     /* process additions */
     xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_RESOURCE);
     if (numXpathResults(xpath_obj) > 0) {
         int max = numXpathResults(xpath_obj), lpc = 0;
 
         for (lpc = 0; lpc < max; lpc++) {
             const char *rsc_id = NULL;
             const char *standard = NULL;
             xmlNode *match = getXpathResult(xpath_obj, lpc);
 
             rsc_id = crm_element_value(match, XML_ATTR_ID);
             standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
 
             if (!pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
                 continue;
             }
 
             crm_trace("Fencing resource %s was added or modified", rsc_id);
             reason = "new resource";
             needs_update = TRUE;
         }
     }
     freeXpathObject(xpath_obj);
 
     if(needs_update) {
         crm_info("Updating device list from the cib: %s", reason);
         cib_devices_update();
     }
 }
 
 static void
 update_cib_stonith_devices(const char *event, xmlNode * msg)
 {
     int format = 1;
     xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     CRM_ASSERT(patchset);
     crm_element_value_int(patchset, "format", &format);
     switch(format) {
         case 1:
             update_cib_stonith_devices_v1(event, msg);
             break;
         case 2:
             update_cib_stonith_devices_v2(event, msg);
             break;
         default:
             crm_warn("Unknown patch format: %d", format);
     }
 }
 
 /* Needs to hold node name + attribute name + attribute value + 75 */
 #define XPATH_MAX 512
 
 /*!
  * \internal
  * \brief Check whether a node has a specific attribute name/value
  *
  * \param[in] node    Name of node to check
  * \param[in] name    Name of an attribute to look for
  * \param[in] value   The value the named attribute needs to be set to in order to be considered a match
  *
  * \return TRUE if the locally cached CIB has the specified node attribute
  */
 gboolean
 node_has_attr(const char *node, const char *name, const char *value)
 {
     char xpath[XPATH_MAX];
     xmlNode *match;
     int n;
 
     CRM_CHECK(local_cib != NULL, return FALSE);
 
     /* Search for the node's attributes in the CIB. While the schema allows
      * multiple sets of instance attributes, and allows instance attributes to
      * use id-ref to reference values elsewhere, that is intended for resources,
      * so we ignore that here.
      */
     n = snprintf(xpath, XPATH_MAX, "//" XML_CIB_TAG_NODES
                  "/" XML_CIB_TAG_NODE "[@uname='%s']/" XML_TAG_ATTR_SETS
                  "/" XML_CIB_TAG_NVPAIR "[@name='%s' and @value='%s']",
                  node, name, value);
     match = get_xpath_object(xpath, local_cib, LOG_NEVER);
 
     CRM_CHECK(n < XPATH_MAX, return FALSE);
     return (match != NULL);
 }
 
 static void
 update_fencing_topology(const char *event, xmlNode * msg)
 {
     int format = 1;
     const char *xpath;
     xmlXPathObjectPtr xpathObj = NULL;
     xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     CRM_ASSERT(patchset);
     crm_element_value_int(patchset, "format", &format);
 
     if(format == 1) {
         /* Process deletions (only) */
         xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL;
         xpathObj = xpath_search(msg, xpath);
 
         remove_fencing_topology(xpathObj);
         freeXpathObject(xpathObj);
 
         /* Process additions and changes */
         xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL;
         xpathObj = xpath_search(msg, xpath);
 
         register_fencing_topology(xpathObj);
         freeXpathObject(xpathObj);
 
     } else if(format == 2) {
         xmlNode *change = NULL;
         int add[] = { 0, 0, 0 };
         int del[] = { 0, 0, 0 };
 
         xml_patch_versions(patchset, add, del);
 
         for (change = pcmk__xml_first_child(patchset); change != NULL;
              change = pcmk__xml_next(change)) {
             const char *op = crm_element_value(change, XML_DIFF_OP);
             const char *xpath = crm_element_value(change, XML_DIFF_PATH);
 
             if(op == NULL) {
                 continue;
 
             } else if(strstr(xpath, "/" XML_TAG_FENCING_LEVEL) != NULL) {
                 /* Change to a specific entry */
 
                 crm_trace("Handling %s operation %d.%d.%d for %s", op, add[0], add[1], add[2], xpath);
                 if(strcmp(op, "move") == 0) {
                     continue;
 
                 } else if(strcmp(op, "create") == 0) {
                     handle_topology_change(change->children, FALSE);
 
                 } else if(strcmp(op, "modify") == 0) {
                     xmlNode *match = first_named_child(change, XML_DIFF_RESULT);
 
                     if(match) {
                         handle_topology_change(match->children, TRUE);
                     }
 
                 } else if(strcmp(op, "delete") == 0) {
                     /* Nuclear option, all we have is the path and an id... not enough to remove a specific entry */
                     crm_info("Re-initializing fencing topology after %s operation %d.%d.%d for %s",
                              op, add[0], add[1], add[2], xpath);
                     fencing_topology_init();
                     return;
                 }
 
             } else if (strstr(xpath, "/" XML_TAG_FENCING_TOPOLOGY) != NULL) {
                 /* Change to the topology in general */
                 crm_info("Re-initializing fencing topology after top-level %s operation  %d.%d.%d for %s",
                          op, add[0], add[1], add[2], xpath);
                 fencing_topology_init();
                 return;
 
             } else if (strstr(xpath, "/" XML_CIB_TAG_CONFIGURATION)) {
                 /* Changes to the whole config section, possibly including the topology as a whild */
                 if(first_named_child(change, XML_TAG_FENCING_TOPOLOGY) == NULL) {
                     crm_trace("Nothing for us in %s operation %d.%d.%d for %s.",
                               op, add[0], add[1], add[2], xpath);
 
                 } else if(strcmp(op, "delete") == 0 || strcmp(op, "create") == 0) {
                     crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s.",
                              op, add[0], add[1], add[2], xpath);
                     fencing_topology_init();
                     return;
                 }
 
             } else {
                 crm_trace("Nothing for us in %s operation %d.%d.%d for %s",
                           op, add[0], add[1], add[2], xpath);
             }
         }
 
     } else {
         crm_warn("Unknown patch format: %d", format);
     }
 }
 static bool have_cib_devices = FALSE;
 
 static void
 update_cib_cache_cb(const char *event, xmlNode * msg)
 {
     int rc = pcmk_ok;
     xmlNode *stonith_enabled_xml = NULL;
     xmlNode *stonith_watchdog_xml = NULL;
     const char *stonith_enabled_s = NULL;
     static gboolean stonith_enabled_saved = TRUE;
 
     if(!have_cib_devices) {
         crm_trace("Skipping updates until we get a full dump");
         return;
 
     } else if(msg == NULL) {
         crm_trace("Missing %s update", event);
         return;
     }
 
     /* Maintain a local copy of the CIB so that we have full access
      * to device definitions, location constraints, and node attributes
      */
     if (local_cib != NULL) {
         int rc = pcmk_ok;
         xmlNode *patchset = NULL;
 
         crm_element_value_int(msg, F_CIB_RC, &rc);
         if (rc != pcmk_ok) {
             return;
         }
 
         patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
         xml_log_patchset(LOG_TRACE, "Config update", patchset);
         rc = xml_apply_patchset(local_cib, patchset, TRUE);
         switch (rc) {
             case pcmk_ok:
             case -pcmk_err_old_data:
                 break;
             case -pcmk_err_diff_resync:
             case -pcmk_err_diff_failed:
                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(local_cib);
                 local_cib = NULL;
                 break;
             default:
                 crm_warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(local_cib);
                 local_cib = NULL;
         }
     }
 
     if (local_cib == NULL) {
         crm_trace("Re-requesting the full cib");
         rc = cib_api->cmds->query(cib_api, NULL, &local_cib, cib_scope_local | cib_sync_call);
         if(rc != pcmk_ok) {
             crm_err("Couldn't retrieve the CIB: %s (%d)", pcmk_strerror(rc), rc);
             return;
         }
         CRM_ASSERT(local_cib != NULL);
         stonith_enabled_saved = FALSE; /* Trigger a full refresh below */
     }
 
     crm_peer_caches_refresh(local_cib);
 
     stonith_enabled_xml = get_xpath_object("//nvpair[@name='stonith-enabled']",
                                            local_cib, LOG_NEVER);
     if (stonith_enabled_xml) {
         stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE);
     }
 
     if (stonith_enabled_s == NULL || crm_is_true(stonith_enabled_s)) {
         long timeout_ms = 0;
         const char *value = NULL;
 
         stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']",
                                                 local_cib, LOG_NEVER);
         if (stonith_watchdog_xml) {
             value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE);
         }
 
         if(value) {
             timeout_ms = crm_get_msec(value);
         }
         if (timeout_ms < 0) {
             timeout_ms = pcmk__auto_watchdog_timeout();
         }
 
         if(timeout_ms != stonith_watchdog_timeout_ms) {
             crm_notice("New watchdog timeout %lds (was %lds)", timeout_ms/1000, stonith_watchdog_timeout_ms/1000);
             stonith_watchdog_timeout_ms = timeout_ms;
         }
 
     } else {
         stonith_watchdog_timeout_ms = 0;
     }
 
     if (stonith_enabled_s && crm_is_true(stonith_enabled_s) == FALSE) {
         crm_trace("Ignoring cib updates while stonith is disabled");
         stonith_enabled_saved = FALSE;
         return;
 
     } else if (stonith_enabled_saved == FALSE) {
         crm_info("Updating stonith device and topology lists now that stonith is enabled");
         stonith_enabled_saved = TRUE;
         fencing_topology_init();
         cib_devices_update();
 
     } else {
         update_fencing_topology(event, msg);
         update_cib_stonith_devices(event, msg);
     }
 }
 
 static void
 init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     crm_info("Updating device list from the cib: init");
     have_cib_devices = TRUE;
     local_cib = copy_xml(output);
 
     crm_peer_caches_refresh(local_cib);
 
     fencing_topology_init();
     cib_devices_update();
 }
 
 static void
 stonith_shutdown(int nsig)
 {
     crm_info("Terminating with %d clients", pcmk__ipc_client_count());
     stonith_shutdown_flag = TRUE;
     if (mainloop != NULL && g_main_loop_is_running(mainloop)) {
         g_main_loop_quit(mainloop);
     } else {
         stonith_cleanup();
         crm_exit(CRM_EX_OK);
     }
 }
 
 static void
 cib_connection_destroy(gpointer user_data)
 {
     if (stonith_shutdown_flag) {
         crm_info("Connection to the CIB manager closed");
         return;
     } else {
         crm_crit("Lost connection to the CIB manager, shutting down");
     }
     if (cib_api) {
         cib_api->cmds->signoff(cib_api);
     }
     stonith_shutdown(0);
 }
 
 static void
 stonith_cleanup(void)
 {
     if (cib_api) {
+        cib_api->cmds->del_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb);
         cib_api->cmds->signoff(cib_api);
     }
 
     if (ipcs) {
         qb_ipcs_destroy(ipcs);
     }
 
     crm_peer_destroy();
     pcmk__client_cleanup();
     free_stonith_remote_op_list();
     free_topology_list();
     free_device_list();
     free_metadata_cache();
 
     free(stonith_our_uname);
     stonith_our_uname = NULL;
 
     free_xml(local_cib);
     local_cib = NULL;
 }
 
 static pcmk__cli_option_t long_options[] = {
     // long option, argument type, storage, short option, description, flags
     {
         "stand-alone", no_argument, 0, 's',
         NULL, pcmk__option_default
     },
     {
         "stand-alone-w-cpg", no_argument, 0, 'c',
         NULL, pcmk__option_default
     },
     {
         "logfile", required_argument, 0, 'l',
         NULL, pcmk__option_default
     },
     {
         "verbose", no_argument, 0, 'V',
         NULL, pcmk__option_default
     },
     {
         "version", no_argument, 0, '$',
         NULL, pcmk__option_default
     },
     {
         "help", no_argument, 0, '?',
         NULL, pcmk__option_default
     },
     { 0, 0, 0, 0 }
 };
 
 static void
 setup_cib(void)
 {
     int rc, retries = 0;
 
     cib_api = cib_new();
     if (cib_api == NULL) {
         crm_err("No connection to the CIB manager");
         return;
     }
 
     do {
         sleep(retries);
         rc = cib_api->cmds->signon(cib_api, CRM_SYSTEM_STONITHD, cib_command);
     } while (rc == -ENOTCONN && ++retries < 5);
 
     if (rc != pcmk_ok) {
         crm_err("Could not connect to the CIB manager: %s (%d)", pcmk_strerror(rc), rc);
 
     } else if (pcmk_ok !=
                cib_api->cmds->add_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb)) {
         crm_err("Could not set CIB notification callback");
 
     } else {
         rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_scope_local);
         cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL, "init_cib_cache_cb",
                                          init_cib_cache_cb);
         cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy);
         crm_info("Watching for stonith topology changes");
     }
 }
 
 struct qb_ipcs_service_handlers ipc_callbacks = {
     .connection_accept = st_ipc_accept,
     .connection_created = NULL,
     .msg_process = st_ipc_dispatch,
     .connection_closed = st_ipc_closed,
     .connection_destroyed = st_ipc_destroy
 };
 
 /*!
  * \internal
  * \brief Callback for peer status changes
  *
  * \param[in] type  What changed
  * \param[in] node  What peer had the change
  * \param[in] data  Previous value of what changed
  */
 static void
 st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
 {
     if ((type != crm_status_processes)
         && !pcmk_is_set(node->flags, crm_remote_node)) {
         /*
          * This is a hack until we can send to a nodeid and/or we fix node name lookups
          * These messages are ignored in stonith_peer_callback()
          */
         xmlNode *query = create_xml_node(NULL, "stonith_command");
 
         crm_xml_add(query, F_XML_TAGNAME, "stonith_command");
         crm_xml_add(query, F_TYPE, T_STONITH_NG);
         crm_xml_add(query, F_STONITH_OPERATION, "poke");
 
         crm_debug("Broadcasting our uname because of node %u", node->id);
         send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
 
         free_xml(query);
     }
 }
 
 int
 main(int argc, char **argv)
 {
     int flag;
     int lpc = 0;
     int argerr = 0;
     int option_index = 0;
     crm_cluster_t cluster;
     const char *actions[] = { "reboot", "off", "on", "list", "monitor", "status" };
     crm_ipc_t *old_instance = NULL;
 
     crm_log_preinit(NULL, argc, argv);
     pcmk__set_cli_options(NULL, "[options]", long_options,
                           "daemon for executing fencing devices in a "
                           "Pacemaker cluster");
 
     while (1) {
         flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
         if (flag == -1) {
             break;
         }
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case 'l':
                 crm_add_logfile(optarg);
                 break;
             case 's':
                 stand_alone = TRUE;
                 break;
             case 'c':
                 stand_alone = FALSE;
                 no_cib_connect = TRUE;
                 break;
             case '$':
             case '?':
                 pcmk__cli_help(flag, CRM_EX_OK);
                 break;
             default:
                 ++argerr;
                 break;
         }
     }
 
     if (argc - optind == 1 && pcmk__str_eq("metadata", argv[optind], pcmk__str_casei)) {
         printf("<?xml version=\"1.0\"?><!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n");
         printf("<resource-agent name=\"pacemaker-fenced\">\n");
         printf(" <version>1.0</version>\n");
         printf(" <longdesc lang=\"en\">Instance attributes available for all \"stonith\"-class resources"
                                        " and used by Pacemaker's fence daemon, formerly known as stonithd</longdesc>\n");
         printf(" <shortdesc lang=\"en\">Instance attributes available for all \"stonith\"-class resources</shortdesc>\n");
         printf(" <parameters>\n");
 
 #if 0
         // priority is not implemented yet
         printf("  <parameter name=\"priority\" unique=\"0\">\n");
         printf("    <shortdesc lang=\"en\">Devices that are not in a topology "
                "are tried in order of highest to lowest integer priority</shortdesc>\n");
         printf("    <content type=\"integer\" default=\"0\"/>\n");
         printf("  </parameter>\n");
 #endif
 
         printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTARG);
         printf
             ("    <shortdesc lang=\"en\">Advanced use only: An alternate parameter to supply instead of 'port'</shortdesc>\n");
         printf
             ("    <longdesc lang=\"en\">Some devices do not support the standard 'port' parameter or may provide additional ones.\n"
              "Use this to specify an alternate, device-specific, parameter that should indicate the machine to be fenced.\n"
              "A value of 'none' can be used to tell the cluster not to supply any additional parameters.\n"
              "     </longdesc>\n");
         printf("    <content type=\"string\" default=\"port\"/>\n");
         printf("  </parameter>\n");
 
         printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTMAP);
         printf
             ("    <shortdesc lang=\"en\">A mapping of host names to ports numbers for devices that do not support host names.</shortdesc>\n");
         printf
             ("    <longdesc lang=\"en\">Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2</longdesc>\n");
         printf("    <content type=\"string\" default=\"\"/>\n");
         printf("  </parameter>\n");
 
         printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTLIST);
         printf
             ("    <shortdesc lang=\"en\">A list of machines controlled by this device (Optional unless %s=static-list).</shortdesc>\n",
              STONITH_ATTR_HOSTCHECK);
         printf("    <content type=\"string\" default=\"\"/>\n");
         printf("  </parameter>\n");
 
         printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTCHECK);
         printf
             ("    <shortdesc lang=\"en\">How to determine which machines are controlled by the device.</shortdesc>\n");
         printf("    <longdesc lang=\"en\">Allowed values: dynamic-list "
                "(query the device via the 'list' command), static-list "
                "(check the " STONITH_ATTR_HOSTLIST " attribute), status "
                "(query the device via the 'status' command), none (assume "
                "every device can fence every machine)</longdesc>\n");
         printf("    <content type=\"string\" default=\"dynamic-list\"/>\n");
         printf("  </parameter>\n");
 
         printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_DELAY_MAX);
         printf
             ("    <shortdesc lang=\"en\">Enable a random delay for stonith actions and specify the maximum of random delay.</shortdesc>\n");
         printf
             ("    <longdesc lang=\"en\">This prevents double fencing when using slow devices such as sbd.\n"
              "Use this to enable a random delay for stonith actions.\n"
              "The overall delay is derived from this random delay value adding a static delay so that the sum is kept below the maximum delay.</longdesc>\n");
         printf("    <content type=\"time\" default=\"0s\"/>\n");
         printf("  </parameter>\n");
 
         printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_DELAY_BASE);
         printf
             ("    <shortdesc lang=\"en\">Enable a base delay for stonith actions and specify base delay value.</shortdesc>\n");
         printf
             ("    <longdesc lang=\"en\">This prevents double fencing when different delays are configured on the nodes.\n"
              "Use this to enable a static delay for stonith actions.\n"
              "The overall delay is derived from a random delay value adding this static delay so that the sum is kept below the maximum delay.</longdesc>\n");
         printf("    <content type=\"time\" default=\"0s\"/>\n");
         printf("  </parameter>\n");
 
         printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_ACTION_LIMIT);
         printf
             ("    <shortdesc lang=\"en\">The maximum number of actions can be performed in parallel on this device</shortdesc>\n");
         printf
             ("    <longdesc lang=\"en\">Cluster property concurrent-fencing=true needs to be configured first.\n"
              "Then use this to specify the maximum number of actions can be performed in parallel on this device. -1 is unlimited.</longdesc>\n");
         printf("    <content type=\"integer\" default=\"1\"/>\n");
         printf("  </parameter>\n");
 
 
         for (lpc = 0; lpc < DIMOF(actions); lpc++) {
             printf("  <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
             printf
                 ("    <shortdesc lang=\"en\">Advanced use only: An alternate command to run instead of '%s'</shortdesc>\n",
                  actions[lpc]);
             printf
                 ("    <longdesc lang=\"en\">Some devices do not support the standard commands or may provide additional ones.\n"
                  "Use this to specify an alternate, device-specific, command that implements the '%s' action.</longdesc>\n",
                  actions[lpc]);
             printf("    <content type=\"string\" default=\"%s\"/>\n", actions[lpc]);
             printf("  </parameter>\n");
 
             printf("  <parameter name=\"pcmk_%s_timeout\" unique=\"0\">\n", actions[lpc]);
             printf
                 ("    <shortdesc lang=\"en\">Advanced use only: Specify an alternate timeout to use for %s actions instead of stonith-timeout</shortdesc>\n",
                  actions[lpc]);
             printf
                 ("    <longdesc lang=\"en\">Some devices need much more/less time to complete than normal.\n"
                  "Use this to specify an alternate, device-specific, timeout for '%s' actions.</longdesc>\n",
                  actions[lpc]);
             printf("    <content type=\"time\" default=\"60s\"/>\n");
             printf("  </parameter>\n");
 
             printf("  <parameter name=\"pcmk_%s_retries\" unique=\"0\">\n", actions[lpc]);
             printf
                 ("    <shortdesc lang=\"en\">Advanced use only: The maximum number of times to retry the '%s' command within the timeout period</shortdesc>\n",
                  actions[lpc]);
             printf("    <longdesc lang=\"en\">Some devices do not support multiple connections."
                    " Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining."
                    " Use this option to alter the number of times Pacemaker retries '%s' actions before giving up."
                    "</longdesc>\n", actions[lpc]);
             printf("    <content type=\"integer\" default=\"2\"/>\n");
             printf("  </parameter>\n");
         }
 
         printf(" </parameters>\n");
         printf("</resource-agent>\n");
         return CRM_EX_OK;
     }
 
     if (optind != argc) {
         ++argerr;
     }
 
     if (argerr) {
         pcmk__cli_help('?', CRM_EX_USAGE);
     }
 
     crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
 
     crm_notice("Starting Pacemaker fencer");
 
     old_instance = crm_ipc_new("stonith-ng", 0);
     if (crm_ipc_connect(old_instance)) {
         /* IPC end-point already up */
         crm_ipc_close(old_instance);
         crm_ipc_destroy(old_instance);
         crm_err("pacemaker-fenced is already active, aborting startup");
         crm_exit(CRM_EX_OK);
     } else {
         /* not up or not authentic, we'll proceed either way */
         crm_ipc_destroy(old_instance);
         old_instance = NULL;
     }
 
     mainloop_add_signal(SIGTERM, stonith_shutdown);
 
     crm_peer_init();
 
     fenced_data_set = pe_new_working_set();
     CRM_ASSERT(fenced_data_set != NULL);
     pe__set_working_set_flags(fenced_data_set,
                               pe_flag_no_counts|pe_flag_no_compat);
 
     if (stand_alone == FALSE) {
 
         if (is_corosync_cluster()) {
 #if SUPPORT_COROSYNC
             cluster.destroy = stonith_peer_cs_destroy;
             cluster.cpg.cpg_deliver_fn = stonith_peer_ais_callback;
             cluster.cpg.cpg_confchg_fn = pcmk_cpg_membership;
 #endif
         }
 
         crm_set_status_callback(&st_peer_update_callback);
 
         if (crm_cluster_connect(&cluster) == FALSE) {
             crm_crit("Cannot sign in to the cluster... terminating");
             crm_exit(CRM_EX_FATAL);
         }
         stonith_our_uname = cluster.uname;
         stonith_our_uuid = cluster.uuid;
 
         if (no_cib_connect == FALSE) {
             setup_cib();
         }
 
     } else {
         stonith_our_uname = strdup("localhost");
     }
 
     init_device_list();
     init_topology_list();
 
     if(stonith_watchdog_timeout_ms > 0) {
         int rc;
         xmlNode *xml;
         stonith_key_value_t *params = NULL;
 
         params = stonith_key_value_add(params, STONITH_ATTR_HOSTLIST, stonith_our_uname);
 
         xml = create_device_registration_xml("watchdog", st_namespace_internal,
                                              STONITH_WATCHDOG_AGENT, params,
                                              NULL);
         stonith_key_value_freeall(params, 1, 1);
         rc = stonith_device_register(xml, NULL, FALSE);
         free_xml(xml);
         if (rc != pcmk_ok) {
             crm_crit("Cannot register watchdog pseudo fence agent");
             crm_exit(CRM_EX_FATAL);
         }
     }
 
     pcmk__serve_fenced_ipc(&ipcs, &ipc_callbacks);
 
     /* Create the mainloop and run it... */
     mainloop = g_main_loop_new(NULL, FALSE);
     crm_notice("Pacemaker fencer successfully started and accepting connections");
     g_main_loop_run(mainloop);
 
     stonith_cleanup();
     pe_free_working_set(fenced_data_set);
     crm_exit(CRM_EX_OK);
 }