diff --git a/daemons/controld/controld_attrd.c b/daemons/controld/controld_attrd.c
index cea914af36..059e357fc4 100644
--- a/daemons/controld/controld_attrd.c
+++ b/daemons/controld/controld_attrd.c
@@ -1,184 +1,186 @@
 /*
- * Copyright 2006-2019 the Pacemaker project contributors
+ * Copyright 2006-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/common/attrd_internal.h>
 #include <crm/msg_xml.h>
 
 #include <pacemaker-controld.h>
 
 static crm_ipc_t *attrd_ipc = NULL;
 
 void
 controld_close_attrd_ipc()
 {
     if (attrd_ipc) {
         crm_trace("Closing connection to pacemaker-attrd");
         crm_ipc_close(attrd_ipc);
         crm_ipc_destroy(attrd_ipc);
         attrd_ipc = NULL;
     }
 }
 
 static void
 log_attrd_error(const char *host, const char *name, const char *value,
                 gboolean is_remote, char command, int rc)
 {
     const char *node_type = (is_remote? "Pacemaker Remote" : "cluster");
     gboolean shutting_down = is_set(fsa_input_register, R_SHUTDOWN);
     const char *when = (shutting_down? " at shutdown" : "");
 
     switch (command) {
         case 0:
             crm_err("Could not clear failure attributes for %s on %s node %s%s: %s "
                     CRM_XS " rc=%d", (name? name : "all resources"), node_type,
-                    host, when, pcmk_strerror(rc), rc);
+                    host, when, pcmk_rc_str(rc), rc);
             break;
 
         case 'C':
             crm_err("Could not purge %s node %s in attribute manager%s: %s "
                     CRM_XS " rc=%d",
-                    node_type, host, when, pcmk_strerror(rc), rc);
+                    node_type, host, when, pcmk_rc_str(rc), rc);
             break;
 
         case 'U':
             /* We weren't able to update an attribute after several retries,
              * so something is horribly wrong with the attribute manager or the
              * underlying system.
              */
             do_crm_log(AM_I_DC? LOG_CRIT : LOG_ERR,
                        "Could not update attribute %s=%s for %s node %s%s: %s "
                        CRM_XS " rc=%d", name, value, node_type, host, when,
-                       pcmk_strerror(rc), rc);
+                       pcmk_rc_str(rc), rc);
 
 
             if (AM_I_DC) {
                 /* We are unable to provide accurate information to the
                  * scheduler, so allow another node to take over DC.
                  * @TODO Should we do this unconditionally on any failure?
                  */
                 crmd_exit(CRM_EX_FATAL);
 
             } else if (shutting_down) {
                 // Fast-track shutdown since unable to request via attribute
                 register_fsa_input(C_FSA_INTERNAL, I_FAIL, NULL);
             }
             break;
     }
 }
 
 static void
 update_attrd_helper(const char *host, const char *name, const char *value,
                     const char *interval_spec, const char *user_name,
                     gboolean is_remote_node, char command)
 {
     int rc;
-    int attrd_opts = attrd_opt_none;
+    int attrd_opts = pcmk__node_attr_none;
 
     if (is_remote_node) {
-        attrd_opts |= attrd_opt_remote;
+        attrd_opts |= pcmk__node_attr_remote;
     }
 
     if (attrd_ipc == NULL) {
         attrd_ipc = crm_ipc_new(T_ATTRD, 0);
     }
 
     for (int attempt = 1; attempt <= 4; ++attempt) {
-        rc = pcmk_ok;
+        rc = pcmk_rc_ok;
 
         // If we're not already connected, try to connect
         if (crm_ipc_connected(attrd_ipc) == FALSE) {
             if (attempt == 1) {
                 // Start with a clean slate
                 crm_ipc_close(attrd_ipc);
             }
             if (crm_ipc_connect(attrd_ipc) == FALSE) {
                 rc = errno;
             }
             crm_debug("Attribute manager connection attempt %d of 4: %s (%d)",
-                      attempt, pcmk_strerror(rc), rc);
+                      attempt, pcmk_rc_str(rc), rc);
         }
 
-        if (rc == pcmk_ok) {
-            rc = command?
-                 attrd_update_delegate(attrd_ipc, command, host, name, value,
-                                       XML_CIB_TAG_STATUS, NULL, NULL,
-                                       user_name, attrd_opts)
-
+        if (rc == pcmk_rc_ok) {
+            if (command) {
+                rc = pcmk__node_attr_request(attrd_ipc, command, host, name,
+                                             value, XML_CIB_TAG_STATUS, NULL,
+                                             NULL, user_name, attrd_opts);
+            } else {
                  /* No command means clear fail count (name/value is really
                   * resource/operation)
                   */
-                 : attrd_clear_delegate(attrd_ipc, host, name, value,
-                                        interval_spec, user_name, attrd_opts);
+                 rc = pcmk__node_attr_request_clear(attrd_ipc, host, name,
+                                                    value, interval_spec,
+                                                    user_name, attrd_opts);
+            }
             crm_debug("Attribute manager request attempt %d of 4: %s (%d)",
-                      attempt, pcmk_strerror(rc), rc);
+                      attempt, pcmk_rc_str(rc), rc);
         }
 
-        if (rc == pcmk_ok) {
+        if (rc == pcmk_rc_ok) {
             // Success, we're done
             break;
 
         } else if ((rc != EAGAIN) && (rc != EALREADY)) {
             /* EAGAIN or EALREADY indicates a temporary block, so just try
              * again. Otherwise, close the connection for a clean slate.
              */
             crm_ipc_close(attrd_ipc);
         }
 
         /* @TODO If the attribute manager remains unavailable the entire time,
          * this function takes more than 6 seconds. Maybe set a timer for
          * retries, to let the main loop do other work.
          */
         if (attempt < 4) {
             sleep(attempt);
         }
     }
 
-    if (rc != pcmk_ok) {
+    if (rc != pcmk_rc_ok) {
         log_attrd_error(host, name, value, is_remote_node, command, rc);
     }
 }
 
 void
 update_attrd(const char *host, const char *name, const char *value,
              const char *user_name, gboolean is_remote_node)
 {
     update_attrd_helper(host, name, value, NULL, user_name, is_remote_node,
                         'U');
 }
 
 void
 update_attrd_remote_node_removed(const char *host, const char *user_name)
 {
     crm_trace("Asking attribute manager to purge Pacemaker Remote node %s",
               host);
     update_attrd_helper(host, NULL, NULL, NULL, user_name, TRUE, 'C');
 }
 
 void
 update_attrd_clear_failures(const char *host, const char *rsc, const char *op,
                             const char *interval_spec, gboolean is_remote_node)
 {
     const char *op_desc = NULL;
     const char *interval_desc = NULL;
     const char *node_type = is_remote_node? "Pacemaker Remote" : "cluster";
 
     if (op) {
         interval_desc = interval_spec? interval_spec : "nonrecurring";
         op_desc = op;
     } else {
         interval_desc = "all";
         op_desc = "operations";
     }
     crm_info("Asking pacemaker-attrd to clear failure of %s %s for %s on %s node %s",
              interval_desc, op_desc, rsc, node_type, host);
     update_attrd_helper(host, rsc, op, interval_spec, NULL, is_remote_node, 0);
 }
diff --git a/include/crm/common/attrd_internal.h b/include/crm/common/attrd_internal.h
index 6d9169ac81..0aa5468848 100644
--- a/include/crm/common/attrd_internal.h
+++ b/include/crm/common/attrd_internal.h
@@ -1,37 +1,43 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef CRM_ATTRD__H
 #  define CRM_ATTRD__H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #  include <crm/common/ipc.h>
 
-/* attribute options for clients to use with these functions */
-#define attrd_opt_none    0x000
-#define attrd_opt_remote  0x001
-#define attrd_opt_private 0x002
-
-const char *attrd_get_target(const char *name);
-
-int attrd_update_delegate(crm_ipc_t * ipc, char command, const char *host,
-                          const char *name, const char *value, const char *section,
-                          const char *set, const char *dampen, const char *user_name, int options);
-int attrd_clear_delegate(crm_ipc_t *ipc, const char *host, const char *resource,
-                         const char *operation, const char *interval_spec,
-                         const char *user_name, int options);
+// Options for clients to use with functions below
+enum pcmk__node_attr_opts {
+    pcmk__node_attr_none    = 0,
+    pcmk__node_attr_remote  = (1 << 0),
+    pcmk__node_attr_private = (1 << 1),
+};
+
+int pcmk__node_attr_request(crm_ipc_t * ipc, char command, const char *host,
+                            const char *name, const char *value,
+                            const char *section, const char *set,
+                            const char *dampen, const char *user_name,
+                            int options);
+
+int pcmk__node_attr_request_clear(crm_ipc_t *ipc, const char *host,
+                                  const char *resource, const char *operation,
+                                  const char *interval_spec,
+                                  const char *user_name, int options);
+
+const char *pcmk__node_attr_target(const char *name);
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif
diff --git a/lib/common/attrd_client.c b/lib/common/attrd_client.c
index 750236a00f..56158043c0 100644
--- a/lib/common/attrd_client.c
+++ b/lib/common/attrd_client.c
@@ -1,312 +1,314 @@
 /*
- * Copyright 2011-2019 the Pacemaker project contributors
+ * Copyright 2011-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/attrd_internal.h>
 
 /*!
  * \internal
  * \brief Create a generic pacemaker-attrd operation
  *
  * \param[in] user_name  If not NULL, ACL user to set for operation
  *
  * \return XML of pacemaker-attrd operation
  */
 static xmlNode *
 create_attrd_op(const char *user_name)
 {
     xmlNode *attrd_op = create_xml_node(NULL, __FUNCTION__);
 
     crm_xml_add(attrd_op, F_TYPE, T_ATTRD);
     crm_xml_add(attrd_op, F_ORIG, (crm_system_name? crm_system_name: "unknown"));
 #if ENABLE_ACL
     crm_xml_add(attrd_op, F_ATTRD_USER, user_name);
 #endif
 
     return attrd_op;
 }
 
 /*!
  * \internal
  * \brief Send an operation to pacemaker-attrd via IPC
  *
- * \param[in] ipc       Connection to pacemaker-attrd (or NULL to use a local connection)
+ * \param[in] ipc       Connection to pacemaker-attrd (or create one if NULL)
  * \param[in] attrd_op  XML of pacemaker-attrd operation to send
  *
- * \return pcmk_ok on success, -errno otherwise
+ * \return Standard Pacemaker return code
  */
 static int
 send_attrd_op(crm_ipc_t *ipc, xmlNode *attrd_op)
 {
-    int rc = -ENOTCONN;
+    int rc = -ENOTCONN; // initially handled as legacy return code
     int max = 5;
 
     static gboolean connected = TRUE;
     static crm_ipc_t *local_ipc = NULL;
     static enum crm_ipc_flags flags = crm_ipc_flags_none;
 
     if (ipc == NULL && local_ipc == NULL) {
         local_ipc = crm_ipc_new(T_ATTRD, 0);
         flags |= crm_ipc_client_response;
         connected = FALSE;
     }
 
     if (ipc == NULL) {
         ipc = local_ipc;
     }
 
     while (max > 0) {
         if (connected == FALSE) {
             crm_info("Connecting to cluster... %d retries remaining", max);
             connected = crm_ipc_connect(ipc);
         }
 
         if (connected) {
             rc = crm_ipc_send(ipc, attrd_op, flags, 0, NULL);
         } else {
             crm_perror(LOG_INFO, "Connection to cluster attribute manager failed");
         }
 
         if (ipc != local_ipc) {
             break;
 
         } else if (rc > 0) {
             break;
 
         } else if (rc == -EAGAIN || rc == -EALREADY) {
             sleep(5 - max);
             max--;
 
         } else {
             crm_ipc_close(ipc);
             connected = FALSE;
             sleep(5 - max);
             max--;
         }
     }
 
     if (rc > 0) {
         rc = pcmk_ok;
     }
-    return rc;
+    return pcmk_legacy2rc(rc);
 }
 
 /*!
  * \internal
  * \brief Send a request to pacemaker-attrd
  *
  * \param[in] ipc      Connection to pacemaker-attrd (or NULL to use a local connection)
  * \param[in] command  A character indicating the type of pacemaker-attrd request:
  *                     U or v: update attribute (or refresh if name is NULL)
  *                     u: update attributes matching regular expression in name
  *                     D: delete attribute (value must be NULL)
  *                     R: refresh
  *                     B: update both attribute and its dampening
  *                     Y: update attribute dampening only
  *                     Q: query attribute
  *                     C: remove peer specified by host
  * \param[in] host     Affect only this host (or NULL for all hosts)
  * \param[in] name     Name of attribute to affect
  * \param[in] value    Attribute value to set
  * \param[in] section  Status or nodes
  * \param[in] set      ID of attribute set to use (or NULL to choose first)
  * \param[in] dampen   Attribute dampening to use with B/Y, and U/v if creating
  * \param[in] user_name ACL user to pass to pacemaker-attrd
- * \param[in] options  Bitmask that may include:
- *                     attrd_opt_remote: host is a Pacemaker Remote node
- *                     attrd_opt_private: attribute is private (not kept in CIB)
+ * \param[in] options  Bitmask of pcmk__node_attr_opts
  *
- * \return pcmk_ok if request was successfully submitted to pacemaker-attrd, else -errno
+ * \return Standard Pacemaker return code
  */
 int
-attrd_update_delegate(crm_ipc_t *ipc, char command, const char *host,
-                      const char *name, const char *value, const char *section,
-                      const char *set, const char *dampen,
-                      const char *user_name, int options)
+pcmk__node_attr_request(crm_ipc_t *ipc, char command, const char *host,
+                        const char *name, const char *value,
+                        const char *section, const char *set,
+                        const char *dampen, const char *user_name, int options)
 {
-    int rc = pcmk_ok;
+    int rc = pcmk_rc_ok;
     const char *task = NULL;
     const char *name_as = NULL;
     const char *display_host = (host ? host : "localhost");
     const char *display_command = NULL; /* for commands without name/value */
     xmlNode *update = create_attrd_op(user_name);
 
     /* remap common aliases */
     if (safe_str_eq(section, "reboot")) {
         section = XML_CIB_TAG_STATUS;
 
     } else if (safe_str_eq(section, "forever")) {
         section = XML_CIB_TAG_NODES;
     }
 
     if (name == NULL && command == 'U') {
         command = 'R';
     }
 
     switch (command) {
         case 'u':
             task = ATTRD_OP_UPDATE;
             name_as = F_ATTRD_REGEX;
             break;
         case 'D':
         case 'U':
         case 'v':
             task = ATTRD_OP_UPDATE;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'R':
             task = ATTRD_OP_REFRESH;
             display_command = "refresh";
             break;
         case 'B':
             task = ATTRD_OP_UPDATE_BOTH;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'Y':
             task = ATTRD_OP_UPDATE_DELAY;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'Q':
             task = ATTRD_OP_QUERY;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'C':
             task = ATTRD_OP_PEER_REMOVE;
             display_command = "purge";
             break;
     }
 
     if (name_as != NULL) {
         if (name == NULL) {
-            rc = -EINVAL;
+            rc = EINVAL;
             goto done;
         }
         crm_xml_add(update, name_as, name);
     }
 
     crm_xml_add(update, F_ATTRD_TASK, task);
     crm_xml_add(update, F_ATTRD_VALUE, value);
     crm_xml_add(update, F_ATTRD_DAMPEN, dampen);
     crm_xml_add(update, F_ATTRD_SECTION, section);
     crm_xml_add(update, F_ATTRD_HOST, host);
     crm_xml_add(update, F_ATTRD_SET, set);
-    crm_xml_add_int(update, F_ATTRD_IS_REMOTE, is_set(options, attrd_opt_remote));
-    crm_xml_add_int(update, F_ATTRD_IS_PRIVATE, is_set(options, attrd_opt_private));
+    crm_xml_add_int(update, F_ATTRD_IS_REMOTE,
+                    is_set(options, pcmk__node_attr_remote));
+    crm_xml_add_int(update, F_ATTRD_IS_PRIVATE,
+                    is_set(options, pcmk__node_attr_private));
 
     rc = send_attrd_op(ipc, update);
 
 done:
     free_xml(update);
 
     if (display_command) {
         crm_debug("Asked pacemaker-attrd to %s %s: %s (%d)",
-                  display_command, display_host, pcmk_strerror(rc), rc);
+                  display_command, display_host, pcmk_rc_str(rc), rc);
     } else {
         crm_debug("Asked pacemaker-attrd to update %s=%s for %s: %s (%d)",
-                  name, value, display_host, pcmk_strerror(rc), rc);
+                  name, value, display_host, pcmk_rc_str(rc), rc);
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Send a request to pacemaker-attrd to clear resource failure
  *
  * \param[in] ipc           Connection to pacemaker-attrd (NULL to use local connection)
  * \param[in] host          Affect only this host (or NULL for all hosts)
  * \param[in] resource      Name of resource to clear (or NULL for all)
  * \param[in] operation     Name of operation to clear (or NULL for all)
  * \param[in] interval_spec If operation is not NULL, its interval
  * \param[in] user_name     ACL user to pass to pacemaker-attrd
- * \param[in] options       attrd_opt_remote if host is a Pacemaker Remote node
+ * \param[in] options       Bitmask of pcmk__node_attr_opts
  *
  * \return pcmk_ok if request was successfully submitted to pacemaker-attrd, else -errno
  */
 int
-attrd_clear_delegate(crm_ipc_t *ipc, const char *host, const char *resource,
-                     const char *operation, const char *interval_spec,
-                     const char *user_name, int options)
+pcmk__node_attr_request_clear(crm_ipc_t *ipc, const char *host,
+                              const char *resource, const char *operation,
+                              const char *interval_spec, const char *user_name,
+                              int options)
 {
-    int rc = pcmk_ok;
+    int rc = pcmk_rc_ok;
     xmlNode *clear_op = create_attrd_op(user_name);
     const char *interval_desc = NULL;
     const char *op_desc = NULL;
 
     crm_xml_add(clear_op, F_ATTRD_TASK, ATTRD_OP_CLEAR_FAILURE);
     crm_xml_add(clear_op, F_ATTRD_HOST, host);
     crm_xml_add(clear_op, F_ATTRD_RESOURCE, resource);
     crm_xml_add(clear_op, F_ATTRD_OPERATION, operation);
     crm_xml_add(clear_op, F_ATTRD_INTERVAL, interval_spec);
-    crm_xml_add_int(clear_op, F_ATTRD_IS_REMOTE, is_set(options, attrd_opt_remote));
+    crm_xml_add_int(clear_op, F_ATTRD_IS_REMOTE,
+                    is_set(options, pcmk__node_attr_remote));
 
     rc = send_attrd_op(ipc, clear_op);
     free_xml(clear_op);
 
     if (operation) {
         interval_desc = interval_spec? interval_spec : "nonrecurring";
         op_desc = operation;
     } else {
         interval_desc = "all";
         op_desc = "operations";
     }
     crm_debug("Asked pacemaker-attrd to clear failure of %s %s for %s on %s: %s (%d)",
               interval_desc, op_desc, (resource? resource : "all resources"),
-              (host? host : "all nodes"), pcmk_strerror(rc), rc);
+              (host? host : "all nodes"), pcmk_rc_str(rc), rc);
     return rc;
 }
 
 #define LRM_TARGET_ENV "OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET
 
 /*!
  * \internal
  */
 const char *
-attrd_get_target(const char *name)
+pcmk__node_attr_target(const char *name)
 {
     if(safe_str_eq(name, "auto") || safe_str_eq(name, "localhost")) {
         name = NULL;
     }
 
     if(name != NULL) {
         return name;
 
     } else {
         char *target_var = crm_meta_name(XML_RSC_ATTR_TARGET);
         char *phys_var = crm_meta_name(PCMK_ENV_PHYSICAL_HOST);
         const char *target = getenv(target_var);
         const char *host_physical = getenv(phys_var);
 
         // It is important to use the name by which the scheduler knows us
         if (host_physical && safe_str_eq(target, "host")) {
             name = host_physical;
 
         } else {
             const char *host_pcmk = getenv(LRM_TARGET_ENV);
 
             if (host_pcmk) {
                 name = host_pcmk;
             }
         }
         free(target_var);
         free(phys_var);
     }
 
     // TODO? Call get_local_node_name() if name == NULL
     // (currently would require linkage against libcrmcluster)
     return name;
 }
diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c
index 9ffcb00126..ebba945404 100644
--- a/tools/attrd_updater.c
+++ b/tools/attrd_updater.c
@@ -1,378 +1,380 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <libgen.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 
 #include <crm/common/attrd_internal.h>
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\tThis text"},
     {"version", 0, 0, '$', "\tVersion information"  },
     {"verbose", 0, 0, 'V', "\tIncrease debug output\n"},
 
     {"name",    1, 0, 'n', "The attribute's name"},
 
     {"-spacer-",1, 0, '-', "\nCommands:"},
     {"update",  1, 0, 'U', "Update the attribute's value in pacemaker-attrd. If this causes the value to change, it will also be updated in the cluster configuration"},
     {"update-both", 1, 0, 'B', "Update the attribute's value and time to wait (dampening) in pacemaker-attrd. If this causes the value or dampening to change, the attribute will also be written to the cluster configuration, so be aware that repeatedly changing the dampening reduces its effectiveness."},
     {"update-delay", 0, 0, 'Y', "Update the attribute's dampening in pacemaker-attrd (requires -d/--delay). If this causes the dampening to change, the attribute will also be written to the cluster configuration, so be aware that repeatedly changing the dampening reduces its effectiveness."},
     {"query",   0, 0, 'Q', "\tQuery the attribute's value from pacemaker-attrd"},
     {"delete",  0, 0, 'D', "\tDelete the attribute in pacemaker-attrd.  If a value was previously set, it will also be removed from the cluster configuration"},
     {"refresh", 0, 0, 'R', "\t(Advanced) Force the pacemaker-attrd daemon to resend all current values to the CIB\n"},
 
     {"-spacer-",1, 0, '-', "\nAdditional options:"},
     {"delay",   1, 0, 'd', "The time to wait (dampening) in seconds for further changes before writing"},
     {"set",     1, 0, 's', "(Advanced) The attribute set in which to place the value"},
     {"node",    1, 0, 'N', "Set the attribute for the named node (instead of the local one)"},
     {"all",     0, 0, 'A', "Show values of the attribute for all nodes (query only)"},
     /* lifetime could be implemented if there is sufficient user demand */
     {"lifetime",1, 0, 'l', "(Deprecated) Lifetime of the node attribute (silently ignored by cluster)"},
     {"private", 0, 0, 'p', "\tIf this creates a new attribute, never write the attribute to the CIB"},
 
     /* Legacy options */
     {"quiet",   0, 0, 'q', NULL, pcmk_option_hidden},
     {"update",  1, 0, 'v', NULL, pcmk_option_hidden},
     {"section", 1, 0, 'S', NULL, pcmk_option_hidden},
     {0, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 static int do_query(const char *attr_name, const char *attr_node, gboolean query_all);
 static int do_update(char command, const char *attr_node, const char *attr_name,
                      const char *attr_value, const char *attr_section,
                      const char *attr_set, const char *attr_dampen, int attr_options);
 
 // Free memory at exit to make analyzers happy
 #define cleanup_memory() \
     free(attr_dampen); \
     free(attr_name); \
     free(attr_node); \
     free(attr_section); \
     free(attr_set);
 
 #define set_option(option_var) \
     if (option_var) { \
         free(option_var); \
     } \
     option_var = strdup(optarg);
 
 int
 main(int argc, char **argv)
 {
     int index = 0;
     int argerr = 0;
-    int attr_options = attrd_opt_none;
+    int attr_options = pcmk__node_attr_none;
     int flag;
     crm_exit_t exit_code = CRM_EX_OK;
     char *attr_node = NULL;
     char *attr_name = NULL;
     char *attr_set = NULL;
     char *attr_section = NULL;
     char *attr_dampen = NULL;
     const char *attr_value = NULL;
     char command = 'Q';
 
     gboolean query_all = FALSE;
 
     crm_log_cli_init("attrd_updater");
     crm_set_options(NULL, "command -n attribute [options]", long_options,
                     "Tool for updating cluster node attributes");
 
     if (argc < 2) {
         crm_help('?', CRM_EX_USAGE);
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case '?':
             case '$':
                 cleanup_memory();
                 crm_help(flag, CRM_EX_OK);
                 break;
             case 'n':
                 set_option(attr_name);
                 break;
             case 's':
                 set_option(attr_set);
                 break;
             case 'd':
                 set_option(attr_dampen);
                 break;
             case 'l':
             case 'S':
                 set_option(attr_section);
                 break;
             case 'N':
                 set_option(attr_node);
                 break;
             case 'A':
                 query_all = TRUE;
                 break;
             case 'p':
-                set_bit(attr_options, attrd_opt_private);
+                set_bit(attr_options, pcmk__node_attr_private);
                 break;
             case 'q':
                 break;
             case 'Y':
                 command = flag;
                 crm_log_args(argc, argv); /* Too much? */
                 break;
             case 'Q':
             case 'B':
             case 'R':
             case 'D':
             case 'U':
             case 'v':
                 command = flag;
                 attr_value = optarg;
                 crm_log_args(argc, argv); /* Too much? */
                 break;
             default:
                 ++argerr;
                 break;
         }
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (command != 'R' && attr_name == NULL) {
         ++argerr;
     }
 
     if (argerr) {
         cleanup_memory();
         crm_help('?', CRM_EX_USAGE);
     }
 
     if (command == 'Q') {
         exit_code = crm_errno2exit(do_query(attr_name, attr_node, query_all));
     } else {
         /* @TODO We don't know whether the specified node is a Pacemaker Remote
-         * node or not, so we can't set attrd_opt_remote when appropriate.
+         * node or not, so we can't set pcmk__node_attr_remote when appropriate.
          * However, it's not a big problem, because pacemaker-attrd will learn
          * and remember a node's "remoteness".
          */
-        exit_code = crm_errno2exit(do_update(command,
-                                   attrd_get_target(attr_node), attr_name,
-                                   attr_value, attr_section, attr_set,
-                                   attr_dampen, attr_options));
+        exit_code = pcmk_rc2exitc(do_update(command,
+                                            pcmk__node_attr_target(attr_node),
+                                            attr_name, attr_value,
+                                            attr_section, attr_set,
+                                            attr_dampen, attr_options));
     }
 
     cleanup_memory();
     crm_exit(exit_code);
 }
 
 /*!
  * \internal
  * \brief Submit a query request to pacemaker-attrd and wait for reply
  *
  * \param[in] name    Name of attribute to query
  * \param[in] host    Query applies to this host only (or all hosts if NULL)
  * \param[out] reply  On success, will be set to new XML tree with reply
  *
  * \return pcmk_ok on success, -errno on error
  * \note On success, caller is responsible for freeing result via free_xml(*reply)
  */
 static int
 send_attrd_query(const char *name, const char *host, xmlNode **reply)
 {
     int rc;
     crm_ipc_t *ipc;
     xmlNode *query;
 
     /* Build the query XML */
     query = create_xml_node(NULL, __FUNCTION__);
     if (query == NULL) {
         return -ENOMEM;
     }
     crm_xml_add(query, F_TYPE, T_ATTRD);
     crm_xml_add(query, F_ORIG, crm_system_name);
     crm_xml_add(query, F_ATTRD_HOST, host);
     crm_xml_add(query, F_ATTRD_TASK, ATTRD_OP_QUERY);
     crm_xml_add(query, F_ATTRD_ATTRIBUTE, name);
 
     /* Connect to pacemaker-attrd, send query XML and get reply */
     crm_debug("Sending query for value of %s on %s", name, (host? host : "all nodes"));
     ipc = crm_ipc_new(T_ATTRD, 0);
     if (crm_ipc_connect(ipc) == FALSE) {
         crm_perror(LOG_ERR, "Connection to cluster attribute manager failed");
         rc = -ENOTCONN;
     } else {
         rc = crm_ipc_send(ipc, query, crm_ipc_flags_none|crm_ipc_client_response, 0, reply);
         if (rc > 0) {
             rc = pcmk_ok;
         }
         crm_ipc_close(ipc);
     }
 
     free_xml(query);
     return(rc);
 }
 
 /*!
  * \brief Validate pacemaker-attrd's XML reply to an query
  *
  * param[in] reply      Root of reply XML tree to validate
  * param[in] attr_name  Name of attribute that was queried
  *
  * \return pcmk_ok on success,
  *         -errno on error (-ENXIO = requested attribute does not exist)
  */
 static int
 validate_attrd_reply(xmlNode *reply, const char *attr_name)
 {
     const char *reply_attr;
 
     if (reply == NULL) {
         fprintf(stderr, "Could not query value of %s: reply did not contain valid XML\n",
                 attr_name);
         return -pcmk_err_schema_validation;
     }
     crm_log_xml_trace(reply, "Reply");
 
     reply_attr = crm_element_value(reply, F_ATTRD_ATTRIBUTE);
     if (reply_attr == NULL) {
         fprintf(stderr, "Could not query value of %s: attribute does not exist\n",
                 attr_name);
         return -ENXIO;
     }
 
     if (safe_str_neq(crm_element_value(reply, F_TYPE), T_ATTRD)
         || (crm_element_value(reply, F_ATTRD_VERSION) == NULL)
         || strcmp(reply_attr, attr_name)) {
             fprintf(stderr,
                     "Could not query value of %s: reply did not contain expected identification\n",
                     attr_name);
             return -pcmk_err_schema_validation;
     }
     return pcmk_ok;
 }
 
 /*!
  * \brief Print the attribute values in a pacemaker-attrd XML query reply
  *
  * \param[in] reply     Root of XML tree with query reply
  * \param[in] attr_name Name of attribute that was queried
  *
  * \return TRUE if any values were printed
  */
 static gboolean
 print_attrd_values(xmlNode *reply, const char *attr_name)
 {
     xmlNode *child;
     const char *reply_host, *reply_value;
     gboolean have_values = FALSE;
 
     /* Iterate through reply's XML tags (a node tag for each host-value pair) */
     for (child = __xml_first_child(reply); child != NULL; child = __xml_next(child)) {
         if (safe_str_neq((const char*)child->name, XML_CIB_TAG_NODE)) {
             crm_warn("Ignoring unexpected %s tag in query reply", child->name);
         } else {
             reply_host = crm_element_value(child, F_ATTRD_HOST);
             reply_value = crm_element_value(child, F_ATTRD_VALUE);
 
             if (reply_host == NULL) {
                 crm_warn("Ignoring %s tag without %s attribute in query reply",
                          XML_CIB_TAG_NODE, F_ATTRD_HOST);
             } else {
                 printf("name=\"%s\" host=\"%s\" value=\"%s\"\n",
                        attr_name, reply_host, (reply_value? reply_value : ""));
                 have_values = TRUE;
             }
         }
     }
     return have_values;
 }
 
 /*!
  * \brief Submit a query to pacemaker-attrd and print reply
  *
  * \param[in] attr_name  Name of attribute to be affected by request
  * \param[in] attr_node  Name of host to query for (or NULL for localhost)
  * \param[in] query_all  If TRUE, ignore attr_node and query all nodes instead
  *
  * \return pcmk_ok on success, -errno on error
  */
 static int
 do_query(const char *attr_name, const char *attr_node, gboolean query_all)
 {
     xmlNode *reply = NULL;
     int rc;
 
     /* Decide which node(s) to query */
     if (query_all == TRUE) {
         attr_node = NULL;
     } else {
-        attr_node = attrd_get_target(attr_node);
+        attr_node = pcmk__node_attr_target(attr_node);
     }
 
     /* Build and send pacemaker-attrd request, and get XML reply */
     rc = send_attrd_query(attr_name, attr_node, &reply);
     if (rc != pcmk_ok) {
         fprintf(stderr, "Could not query value of %s: %s (%d)\n", attr_name, pcmk_strerror(rc), rc);
         return rc;
     }
 
     /* Validate the XML reply */
     rc = validate_attrd_reply(reply, attr_name);
     if (rc != pcmk_ok) {
         if (reply != NULL) {
             free_xml(reply);
         }
         return rc;
     }
 
     /* Print the values from the reply */
     if (print_attrd_values(reply, attr_name) == FALSE) {
         fprintf(stderr,
                 "Could not query value of %s: reply had attribute name but no host values\n",
                 attr_name);
         free_xml(reply);
         return -pcmk_err_schema_validation;
     }
 
     return pcmk_ok;
 }
 
 static int
 do_update(char command, const char *attr_node, const char *attr_name,
           const char *attr_value, const char *attr_section,
           const char *attr_set, const char *attr_dampen, int attr_options)
 {
-    int rc = attrd_update_delegate(NULL, command, attr_node, attr_name,
-                                   attr_value, attr_section, attr_set,
-                                   attr_dampen, NULL, attr_options);
-    if (rc != pcmk_ok) {
-        fprintf(stderr, "Could not update %s=%s: %s (%d)\n", attr_name, attr_value, pcmk_strerror(rc), rc);
+    int rc = pcmk__node_attr_request(NULL, command, attr_node, attr_name,
+                                     attr_value, attr_section, attr_set,
+                                     attr_dampen, NULL, attr_options);
+    if (rc != pcmk_rc_ok) {
+        fprintf(stderr, "Could not update %s=%s: %s (%d)\n",
+                attr_name, attr_value, pcmk_rc_str(rc), rc);
     }
     return rc;
 }
diff --git a/tools/crm_attribute.c b/tools/crm_attribute.c
index 5503786741..37274caebc 100644
--- a/tools/crm_attribute.c
+++ b/tools/crm_attribute.c
@@ -1,350 +1,354 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <time.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/util.h>
 #include <crm/cluster.h>
 
 #include <crm/cib.h>
 #include <crm/common/attrd_internal.h>
 #include <sys/utsname.h>
 
 gboolean BE_QUIET = FALSE;
 char command = 'G';
 
 const char *dest_uname = NULL;
 char *dest_node = NULL;
 char *set_name = NULL;
 char *attr_id = NULL;
 char *attr_name = NULL;
 char *attr_pattern = NULL;
 const char *type = NULL;
 const char *rsc_id = NULL;
 const char *attr_value = NULL;
 const char *attr_default = NULL;
 const char *set_type = NULL;
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\tThis text"},
     {"version", 0, 0, '$', "\tVersion information"  },
     {"verbose", 0, 0, 'V', "\tIncrease debug output"},
     {"quiet",   0, 0, 'q', "\tPrint only the value on stdout\n"},
 
     {"name",    1, 0, 'n', "Name of the attribute/option to operate on"},
     {"pattern", 1, 0, 'P', "Pattern matching names of attributes (only with -v/-D and -l reboot)"},
 
     {"-spacer-",    0, 0, '-', "\nCommands:"},
     {"query",       0, 0, 'G', "\tQuery the current value of the attribute/option"},
     {"update",      1, 0, 'v', "Update the value of the attribute/option"},
     {"delete",      0, 0, 'D', "\tDelete the attribute/option"},
 
     {"-spacer-",    0, 0, '-', "\nAdditional Options:"},
     {"node",        1, 0, 'N', "Set an attribute for the named node (instead of a cluster option).  See also: -l"},
     {"type",        1, 0, 't', "Which part of the configuration to update/delete/query the option in"},
     {"-spacer-",    0, 0, '-', "\t\t\tValid values: crm_config, rsc_defaults, op_defaults, tickets"},
     {"lifetime",    1, 0, 'l', "Lifetime of the node attribute"},
     {"-spacer-",    0, 0, '-', "\t\t\tValid values: reboot, forever"},
     {"utilization", 0, 0, 'z', "Set an utilization attribute for the node."},
     {"set-name",    1, 0, 's', "(Advanced) The attribute set in which to place the value"},
     {"id",	    1, 0, 'i', "\t(Advanced) The ID used to identify the attribute"},
     {"default",     1, 0, 'd', "(Advanced) The default value to display if none is found in the configuration"},
 
     {"inhibit-policy-engine", 0, 0, '!', NULL, 1},
 
     /* legacy */
     {"quiet",       0, 0, 'Q', NULL, 1},
     {"node-uname",  1, 0, 'U', NULL, 1},
     {"get-value",   0, 0, 'G', NULL, 1},
     {"delete-attr", 0, 0, 'D', NULL, 1},
     {"attr-value",  1, 0, 'v', NULL, 1},
     {"attr-name",   1, 0, 'n', NULL, 1},
     {"attr-id",     1, 0, 'i', NULL, 1},
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "Add a new node attribute called 'location' with the value of 'office' for host 'myhost':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_attribute --node myhost --name location --update office", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Query the value of the 'location' node attribute for host 'myhost':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_attribute --node myhost --name location --query", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Change the value of the 'location' node attribute for host 'myhost':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_attribute --node myhost --name location --update backoffice", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Delete the 'location' node attribute for host 'myhost':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_attribute --node myhost --name location --delete", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Query the value of the cluster-delay cluster option:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_attribute --type crm_config --name cluster-delay --query", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Query the value of the cluster-delay cluster option. Only print the value:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_attribute --type crm_config --name cluster-delay --query --quiet", pcmk_option_example},
 
     {0, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 int
 main(int argc, char **argv)
 {
     cib_t *the_cib = NULL;
     int rc = pcmk_ok;
 
     int cib_opts = cib_sync_call;
     int argerr = 0;
     int flag;
 
     int option_index = 0;
     int is_remote_node = 0;
 
     bool try_attrd = true;
+    int attrd_opts = pcmk__node_attr_none;
 
     crm_log_cli_init("crm_attribute");
     crm_set_options(NULL, "<command> -n <attribute> [options]", long_options,
                     "Manage node's attributes and cluster options."
                     "\n\nAllows node attributes and cluster options to be queried, modified and deleted.\n");
 
     if (argc < 2) {
         crm_help('?', CRM_EX_USAGE);
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &option_index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case '$':
             case '?':
                 crm_help(flag, CRM_EX_OK);
                 break;
             case 'G':
                 command = flag;
                 attr_value = optarg;
                 break;
             case 'D':
             case 'v':
                 command = flag;
                 attr_value = optarg;
                 crm_log_args(argc, argv);
                 break;
             case 'q':
             case 'Q':
                 BE_QUIET = TRUE;
                 break;
             case 'U':
             case 'N':
                 dest_uname = strdup(optarg);
                 break;
             case 's':
                 set_name = strdup(optarg);
                 break;
             case 'l':
             case 't':
                 type = optarg;
                 break;
             case 'z':
                 type = XML_CIB_TAG_NODES;
                 set_type = XML_TAG_UTILIZATION;
                 break;
             case 'n':
                 attr_name = strdup(optarg);
                 break;
             case 'P':
                 attr_pattern = strdup(optarg);
                 break;
             case 'i':
                 attr_id = strdup(optarg);
                 break;
             case 'r':
                 rsc_id = optarg;
                 break;
             case 'd':
                 attr_default = optarg;
                 break;
             case '!':
                 crm_warn("Inhibiting notifications for this update");
                 cib_opts |= cib_inhibit_notify;
                 break;
             default:
                 printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     if (optind < argc) {
         printf("non-option ARGV-elements: ");
         while (optind < argc)
             printf("%s ", argv[optind++]);
         printf("\n");
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         crm_help('?', CRM_EX_USAGE);
     }
 
     the_cib = cib_new();
     rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
 
     if (rc != pcmk_ok) {
         fprintf(stderr, "Error connecting to the CIB manager: %s\n",
                 pcmk_strerror(rc));
         crm_exit(crm_errno2exit(rc));
     }
 
     if (type == NULL && dest_uname != NULL) {
 	    type = "forever";
     }
 
     if (safe_str_eq(type, "reboot")) {
         type = XML_CIB_TAG_STATUS;
 
     } else if (safe_str_eq(type, "forever")) {
         type = XML_CIB_TAG_NODES;
     }
 
     if (type == NULL && dest_uname == NULL) {
         /* we're updating cluster options - don't populate dest_node */
         type = XML_CIB_TAG_CRMCONFIG;
 
     } else if (safe_str_eq(type, XML_CIB_TAG_CRMCONFIG)) {
     } else if (safe_str_neq(type, XML_CIB_TAG_TICKETS)) {
         /* If we are being called from a resource agent via the cluster,
          * the correct local node name will be passed as an environment
          * variable. Otherwise, we have to ask the cluster.
          */
-        dest_uname = attrd_get_target(dest_uname);
+        dest_uname = pcmk__node_attr_target(dest_uname);
         if (dest_uname == NULL) {
             dest_uname = get_local_node_name();
         }
 
         rc = query_node_uuid(the_cib, dest_uname, &dest_node, &is_remote_node);
         if (pcmk_ok != rc) {
             fprintf(stderr, "Could not map name=%s to a UUID\n", dest_uname);
             the_cib->cmds->signoff(the_cib);
             cib_delete(the_cib);
             crm_exit(crm_errno2exit(rc));
         }
     }
 
     if ((command == 'D') && (attr_name == NULL) && (attr_pattern == NULL)) {
         fprintf(stderr, "Error: must specify attribute name or pattern to delete\n");
         crm_exit(CRM_EX_USAGE);
     }
 
     if (attr_pattern) {
         if (((command != 'v') && (command != 'D'))
             || safe_str_neq(type, XML_CIB_TAG_STATUS)) {
 
             fprintf(stderr, "Error: pattern can only be used with till-reboot update or delete\n");
             crm_exit(CRM_EX_USAGE);
         }
         command = 'u';
         free(attr_name);
         attr_name = attr_pattern;
     }
 
     // Only go through attribute manager for transient attributes
     try_attrd = safe_str_eq(type, XML_CIB_TAG_STATUS);
 
     // Don't try to contact attribute manager if we're using a file as CIB
     if (getenv("CIB_file") || getenv("CIB_shadow")) {
         try_attrd = FALSE;
     }
 
+    if (is_remote_node) {
+        attrd_opts = pcmk__node_attr_remote;
+    }
     if (((command == 'v') || (command == 'D') || (command == 'u')) && try_attrd
-        && pcmk_ok == attrd_update_delegate(NULL, command, dest_uname, attr_name,
-                                            attr_value, type, set_name, NULL, NULL,
-                                            is_remote_node?attrd_opt_remote:attrd_opt_none)) {
+        && (pcmk__node_attr_request(NULL, command, dest_uname, attr_name,
+                                    attr_value, type, set_name, NULL, NULL,
+                                    attrd_opts) == pcmk_rc_ok)) {
         crm_info("Update %s=%s sent via pacemaker-attrd",
                  attr_name, ((command == 'D')? "<none>" : attr_value));
 
     } else if (command == 'D') {
         rc = delete_attr_delegate(the_cib, cib_opts, type, dest_node, set_type, set_name,
                                   attr_id, attr_name, attr_value, TRUE, NULL);
 
         if (rc == -ENXIO) {
             /* Nothing to delete...
              * which means it's not there...
              * which is what the admin wanted
              */
             rc = pcmk_ok;
         }
 
     } else if (command == 'v') {
         CRM_LOG_ASSERT(type != NULL);
         CRM_LOG_ASSERT(attr_name != NULL);
         CRM_LOG_ASSERT(attr_value != NULL);
 
         rc = update_attr_delegate(the_cib, cib_opts, type, dest_node, set_type, set_name,
                                   attr_id, attr_name, attr_value, TRUE, NULL, is_remote_node ? "remote" : NULL);
 
     } else {                    /* query */
 
         char *read_value = NULL;
 
         rc = read_attr_delegate(the_cib, type, dest_node, set_type, set_name,
                                 attr_id, attr_name, &read_value, TRUE, NULL);
 
         if (rc == -ENXIO && attr_default) {
             read_value = strdup(attr_default);
             rc = pcmk_ok;
         }
 
         crm_info("Read %s=%s %s%s",
                  attr_name, crm_str(read_value), set_name ? "in " : "", set_name ? set_name : "");
 
         if (rc == -ENOTUNIQ) {
             // Multiple matches (already displayed) are not error for queries
             rc = pcmk_ok;
 
         } else if (BE_QUIET == FALSE) {
             fprintf(stdout, "%s%s %s%s %s%s value=%s\n",
                     type ? "scope=" : "", type ? type : "",
                     attr_id ? "id=" : "", attr_id ? attr_id : "",
                     attr_name ? "name=" : "", attr_name ? attr_name : "",
                     read_value ? read_value : "(null)");
 
         } else if (read_value != NULL) {
             fprintf(stdout, "%s\n", read_value);
         }
         free(read_value);
     }
 
     if (rc == -ENOTUNIQ) {
         printf("Please choose from one of the matches above and supply the 'id' with --attr-id\n");
 
     } else if (rc != pcmk_ok) {
         fprintf(stderr, "Error performing operation: %s\n", pcmk_strerror(rc));
     }
 
     the_cib->cmds->signoff(the_cib);
     cib_delete(the_cib);
     crm_exit(crm_errno2exit(rc));
 }
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index e1096cc118..d19b440f51 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1,1418 +1,1419 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_resource.h>
 #include <pacemaker-internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/stonith-ng.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <time.h>
 
 bool BE_QUIET = FALSE;
 bool scope_master = FALSE;
 int cib_options = cib_sync_call;
 
 static GMainLoop *mainloop = NULL;
 
 #define MESSAGE_TIMEOUT_S 60
 
 static gboolean
 resource_ipc_timeout(gpointer data)
 {
     fprintf(stderr, "Aborting because no messages received in %d seconds\n",
             MESSAGE_TIMEOUT_S);
     crm_err("No messages received in %d seconds", MESSAGE_TIMEOUT_S);
     crm_exit(CRM_EX_TIMEOUT);
 }
 
 static void
 resource_ipc_connection_destroy(gpointer user_data)
 {
     crm_info("Connection to controller was terminated");
     crm_exit(CRM_EX_DISCONNECT);
 }
 
 static void
 start_mainloop(void)
 {
     if (crmd_replies_needed == 0) {
         return;
     }
 
     mainloop = g_main_loop_new(NULL, FALSE);
     fprintf(stderr, "Waiting for %d repl%s from the controller",
             crmd_replies_needed, (crmd_replies_needed == 1)? "y" : "ies");
     crm_debug("Waiting for %d repl%s from the controller",
               crmd_replies_needed, (crmd_replies_needed == 1)? "y" : "ies");
 
     g_timeout_add(MESSAGE_TIMEOUT_S * 1000, resource_ipc_timeout, NULL);
     g_main_loop_run(mainloop);
 }
 
 static int
 resource_ipc_callback(const char *buffer, ssize_t length, gpointer userdata)
 {
     xmlNode *msg = string2xml(buffer);
 
     fprintf(stderr, ".");
     crm_log_xml_trace(msg, "[inbound]");
 
     crmd_replies_needed--;
     if ((crmd_replies_needed == 0) && mainloop
         && g_main_loop_is_running(mainloop)) {
 
         fprintf(stderr, " OK\n");
         crm_debug("Got all the replies we expected");
         crm_exit(CRM_EX_OK);
     }
 
     free_xml(msg);
     return 0;
 }
 
 static int
 compare_id(gconstpointer a, gconstpointer b)
 {
     return strcmp((const char *)a, (const char *)b);
 }
 
 static GListPtr
 build_constraint_list(xmlNode *root)
 {
     GListPtr retval = NULL;
     xmlNode *cib_constraints = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
     int ndx = 0;
 
     cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, root);
     xpathObj = xpath_search(cib_constraints, "//" XML_CONS_TAG_RSC_LOCATION);
 
     for (ndx = 0; ndx < numXpathResults(xpathObj); ndx++) {
         xmlNode *match = getXpathResult(xpathObj, ndx);
         retval = g_list_insert_sorted(retval, (gpointer) ID(match), compare_id);
     }
 
     freeXpathObject(xpathObj);
     return retval;
 }
 
 struct ipc_client_callbacks crm_callbacks = {
     .dispatch = resource_ipc_callback,
     .destroy = resource_ipc_connection_destroy,
 };
 
 
 /* short option letters still available: eEJkKXyYZ */
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {
         "help", no_argument, NULL, '?',
         "\t\tDisplay this text and exit"
     },
     {
         "version", no_argument, NULL, '$',
         "\t\tDisplay version information and exit"
     },
     {
         "verbose", no_argument, NULL, 'V',
         "\t\tIncrease debug output (may be specified multiple times)"
     },
     {
         "quiet", no_argument, NULL, 'Q',
         "\t\tBe less descriptive in results"
     },
     {
         "resource", required_argument, NULL, 'r',
         "\tResource ID"
     },
 
     { "-spacer-", no_argument, NULL, '-', "\nQueries:" },
     {
         "list", no_argument, NULL, 'L',
         "\t\tList all cluster resources with status"},
     {
         "list-raw", no_argument, NULL, 'l',
         "\t\tList IDs of all instantiated resources (individual members rather than groups etc.)"
     },
     {
         "list-cts", no_argument, NULL, 'c',
         NULL, pcmk_option_hidden
     },
     {
         "list-operations", no_argument, NULL, 'O',
         "\tList active resource operations, optionally filtered by --resource and/or --node"
     },
     {
         "list-all-operations", no_argument, NULL, 'o',
         "List all resource operations, optionally filtered by --resource and/or --node"
     },
     {
         "list-standards", no_argument, NULL, 0,
         "\tList supported standards"
     },
     {
         "list-ocf-providers", no_argument, NULL, 0,
         "List all available OCF providers"
     },
     {
         "list-agents", required_argument, NULL, 0,
         "List all agents available for the named standard and/or provider."
     },
     {
         "list-ocf-alternatives", required_argument, NULL, 0,
         "List all available providers for the named OCF agent"
     },
     {
         "show-metadata", required_argument, NULL, 0,
         "Show the metadata for the named class:provider:agent"
     },
     {
         "query-xml", no_argument, NULL, 'q',
         "\tShow XML configuration of resource (after any template expansion)"
     },
     {
         "query-xml-raw", no_argument, NULL, 'w',
         "\tShow XML configuration of resource (before any template expansion)"
     },
     {
         "get-parameter", required_argument, NULL, 'g',
         "Display named parameter for resource.\n"
         "\t\t\t\tUse instance attribute unless --meta or --utilization is specified"
     },
     {
         "get-property", required_argument, NULL, 'G',
         "Display named property of resource ('class', 'type', or 'provider') (requires --resource)",
         pcmk_option_hidden
     },
     {
         "locate", no_argument, NULL, 'W',
         "\t\tShow node(s) currently running resource"
     },
     {
         "stack", no_argument, NULL, 'A',
         "\t\tDisplay the prerequisites and dependents of a resource"
     },
     {
         "constraints", no_argument, NULL, 'a',
         "\tDisplay the (co)location constraints that apply to a resource"
     },
     {
         "why", no_argument, NULL, 'Y',
         "\t\tShow why resources are not running, optionally filtered by --resource and/or --node"
     },
 
     { "-spacer-", no_argument, NULL, '-', "\nCommands:" },
     {
         "validate", no_argument, NULL, 0,
         "\t\tValidate resource configuration by calling agent's validate-all action.\n"
         "\t\t\t\tThe configuration may be specified either by giving an existing\n"
         "\t\t\t\tresource name with -r, or by specifying --class, --agent, and\n"
         "\t\t\t\t--provider arguments, along with any number of --option arguments."
     },
     {
         "cleanup", no_argument, NULL, 'C',
         "\t\tIf resource has any past failures, clear its history and fail count.\n"
         "\t\t\t\tOptionally filtered by --resource, --node, --operation, and --interval (otherwise all).\n"
         "\t\t\t\t--operation and --interval apply to fail counts, but entire history is always cleared,\n"
         "\t\t\t\tto allow current state to be rechecked. If the named resource is part of a group, or\n"
         "\t\t\t\tone numbered instance of a clone or bundled resource, the clean-up applies to the\n"
         "\t\t\t\twhole collective resource unless --force is given."
     },
     {
         "refresh", no_argument, NULL, 'R',
         "\t\tDelete resource's history (including failures) so its current state is rechecked.\n"
         "\t\t\t\tOptionally filtered by --resource and --node (otherwise all). If the named resource is\n"
         "\t\t\t\tpart of a group, or one numbered instance of a clone or bundled resource, the clean-up\n"
         "applies to the whole collective resource unless --force is given."
     },
     {
         "set-parameter", required_argument, NULL, 'p',
         "Set named parameter for resource (requires -v).\n"
         "\t\t\t\tUse instance attribute unless --meta or --utilization is specified."
     },
     {
         "delete-parameter", required_argument, NULL, 'd',
         "Delete named parameter for resource.\n"
         "\t\t\t\tUse instance attribute unless --meta or --utilization is specified."
     },
     {
         "set-property", required_argument, NULL, 'S',
         "Set named property of resource ('class', 'type', or 'provider') (requires -r, -t, -v)",
         pcmk_option_hidden
     },
 
     { "-spacer-", no_argument, NULL, '-', "\nResource location:" },
     {
         "move", no_argument, NULL, 'M',
         "\t\tCreate a constraint to move resource. If --node is specified, the constraint\n"
         "\t\t\t\twill be to move to that node, otherwise it will be to ban the current node.\n"
         "\t\t\t\tUnless --force is specified, this will return an error if the resource is\n"
         "\t\t\t\talready running on the specified node. If --force is specified, this will\n"
         "\t\t\t\talways ban the current node. Optional: --lifetime, --master.\n"
         "\t\t\t\tNOTE: This may prevent the resource from running on its previous location\n"
         "\t\t\t\tuntil the implicit constraint expires or is removed with --clear."
     },
     {
         "ban", no_argument, NULL, 'B',
         "\t\tCreate a constraint to keep resource off a node. Optional: --node, --lifetime, --master.\n"
         "\t\t\t\tNOTE: This will prevent the resource from running on the affected node\n"
         "\t\t\t\tuntil the implicit constraint expires or is removed with --clear.\n"
         "\t\t\t\tIf --node is not specified, it defaults to the node currently running the resource\n"
         "\t\t\t\tfor primitives and groups, or the master for promotable clones with promoted-max=1\n"
         "\t\t\t\t(all other situations result in an error as there is no sane default).\n"
     },
     {
         "clear", no_argument, NULL, 'U',
         "\t\tRemove all constraints created by the --ban and/or --move commands.\n"
         "\t\t\t\tRequires: --resource. Optional: --node, --master, --expired.\n"
         "\t\t\t\tIf --node is not specified, all constraints created by --ban and --move\n"
         "\t\t\t\twill be removed for the named resource. If --node and --force are specified,\n"
         "\t\t\t\tany constraint created by --move will be cleared, even if it is not for the specified node.\n"
         "\t\t\t\tIf --expired is specified, only those constraints whose lifetimes have expired will\n"
         "\t\t\t\tbe removed.\n"
     },
     {
         "expired", no_argument, NULL, 'e',
         "\t\tModifies the --clear argument to remove constraints with expired lifetimes.\n"
     },
     {
         "lifetime", required_argument, NULL, 'u',
         "\tLifespan (as ISO 8601 duration) of created constraints (with -B, -M)\n"
         "\t\t\t\t(see https://en.wikipedia.org/wiki/ISO_8601#Durations)"
     },
     {
         "master", no_argument, NULL, 0,
         "\t\tLimit scope of command to the Master role (with -B, -M, -U).\n"
         "\t\t\t\tFor -B and -M, the previous master may remain active in the Slave role."
     },
 
     { "-spacer-", no_argument, NULL, '-', "\nAdvanced Commands:" },
     {
         "delete", no_argument, NULL, 'D',
         "\t\t(Advanced) Delete a resource from the CIB. Required: -t"
     },
     {
         "fail", no_argument, NULL, 'F',
         "\t\t(Advanced) Tell the cluster this resource has failed"
     },
     {
         "restart", no_argument, NULL, 0,
         "\t\t(Advanced) Tell the cluster to restart this resource and anything that depends on it"
     },
     {
         "wait", no_argument, NULL, 0,
         "\t\t(Advanced) Wait until the cluster settles into a stable state"
     },
     {
         "force-demote", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and demote a resource on the local node.\n"
         "\t\t\t\tUnless --force is specified, this will refuse to do so if the cluster\n"
         "\t\t\t\tbelieves the resource is a clone instance already running on the local node."
     },
     {
         "force-stop", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and stop a resource on the local node."
     },
     {
         "force-start", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and start a resource on the local node.\n"
         "\t\t\t\tUnless --force is specified, this will refuse to do so if the cluster\n"
         "\t\t\t\tbelieves the resource is a clone instance already running on the local node."
     },
     {
         "force-promote", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and promote a resource on the local node.\n"
         "\t\t\t\tUnless --force is specified, this will refuse to do so if the cluster\n"
         "\t\t\t\tbelieves the resource is a clone instance already running on the local node."
     },
     {
         "force-check", no_argument, NULL, 0,
         "\t(Advanced) Bypass the cluster and check the state of a resource on the local node."
     },
 
     { "-spacer-", no_argument, NULL, '-', "\nValidate Options:" },
     {
         "class", required_argument, NULL, 0,
         "\tThe standard the resource agent confirms to (for example, ocf).\n"
         "\t\t\t\tUse with --agent, --provider, --option, and --validate."
     },
     {
         "agent", required_argument, NULL, 0,
         "\tThe agent to use (for example, IPaddr).\n"
         "\t\t\t\tUse with --class, --provider, --option, and --validate."
     },
     {
         "provider", required_argument, NULL, 0,
         "\tThe vendor that supplies the resource agent (for example, heartbeat).\n"
         "\t\t\t\tuse with --class, --agent, --option, and --validate."
     },
     {
         "option", required_argument, NULL, 0,
         "\tSpecify a device configuration parameter as NAME=VALUE\n"
         "\t\t\t\t(may be specified multiple times).  Use with --validate\n"
         "\t\t\t\tand without the -r option."
     },
 
     { "-spacer-", no_argument, NULL, '-', "\nAdditional Options:" },
     {
         "node", required_argument, NULL, 'N',
         "\tNode name"
     },
     {
         "recursive", no_argument, NULL, 0,
         "\tFollow colocation chains when using --set-parameter"
     },
     {
         "resource-type", required_argument, NULL, 't',
         "Resource XML element (primitive, group, etc.) (with -D)"
     },
     {
         "parameter-value", required_argument, NULL, 'v',
         "Value to use with -p"
     },
     {
         "meta", no_argument, NULL, 'm',
         "\t\tUse resource meta-attribute instead of instance attribute (with -p, -g, -d)"
     },
     {
         "utilization", no_argument, NULL, 'z',
         "\tUse resource utilization attribute instead of instance attribute (with -p, -g, -d)"
     },
     {
         "operation", required_argument, NULL, 'n',
         "\tOperation to clear instead of all (with -C -r)"
     },
     {
         "interval", required_argument, NULL, 'I',
         "\tInterval of operation to clear (default 0) (with -C -r -n)"
     },
     {
         "set-name", required_argument, NULL, 's',
         "\t(Advanced) XML ID of attributes element to use (with -p, -d)"
     },
     {
         "nvpair", required_argument, NULL, 'i',
         "\t(Advanced) XML ID of nvpair element to use (with -p, -d)"
     },
     {
         "timeout", required_argument, NULL, 'T',
         "\t(Advanced) Abort if command does not finish in this time (with --restart, --wait, --force-*)"
     },
     {
         "force", no_argument, NULL, 'f',
         "\t\tIf making CIB changes, do so regardless of quorum.\n"
         "\t\t\t\tSee help for individual commands for additional behavior.\n"
     },
     {
         "xml-file", required_argument, NULL, 'x',
         NULL, pcmk_option_hidden
     },
 
     /* legacy options */
     {"host-uname", required_argument, NULL, 'H', NULL, pcmk_option_hidden},
 
     {"-spacer-", 1, NULL, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', "List the available OCF agents:", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', " crm_resource --list-agents ocf", pcmk_option_example},
     {"-spacer-", 1, NULL, '-', "List the available OCF agents from the linux-ha project:", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', " crm_resource --list-agents ocf:heartbeat", pcmk_option_example},
     {"-spacer-", 1, NULL, '-', "Move 'myResource' to a specific node:", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --move --node altNode", pcmk_option_example},
     {"-spacer-", 1, NULL, '-', "Allow (but not force) 'myResource' to move back to its original location:", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --clear", pcmk_option_example},
     {"-spacer-", 1, NULL, '-', "Stop 'myResource' (and anything that depends on it):", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --set-parameter target-role --meta --parameter-value Stopped", pcmk_option_example},
     {"-spacer-", 1, NULL, '-', "Tell the cluster not to manage 'myResource':", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', "The cluster will not attempt to start or stop the resource under any circumstances."},
     {"-spacer-", 1, NULL, '-', "Useful when performing maintenance tasks on a resource.", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --set-parameter is-managed --meta --parameter-value false", pcmk_option_example},
     {"-spacer-", 1, NULL, '-', "Erase the operation history of 'myResource' on 'aNode':", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', "The cluster will 'forget' the existing resource state (including any errors) and attempt to recover the resource."},
     {"-spacer-", 1, NULL, '-', "Useful when a resource had failed permanently and has been repaired by an administrator.", pcmk_option_paragraph},
     {"-spacer-", 1, NULL, '-', " crm_resource --resource myResource --cleanup --node aNode", pcmk_option_example},
 
     {0, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 
 int
 main(int argc, char **argv)
 {
     char rsc_cmd = 'L';
 
     const char *v_class = NULL;
     const char *v_agent = NULL;
     const char *v_provider = NULL;
     char *name = NULL;
     char *value = NULL;
     GHashTable *validate_options = NULL;
 
     const char *rsc_id = NULL;
     const char *host_uname = NULL;
     const char *prop_name = NULL;
     const char *prop_value = NULL;
     const char *rsc_type = NULL;
     const char *prop_id = NULL;
     const char *prop_set = NULL;
     const char *rsc_long_cmd = NULL;
     const char *longname = NULL;
     const char *operation = NULL;
     const char *interval_spec = NULL;
     const char *cib_file = getenv("CIB_file");
     GHashTable *override_params = NULL;
 
     char *xml_file = NULL;
     crm_ipc_t *crmd_channel = NULL;
     pe_working_set_t *data_set = NULL;
     xmlNode *cib_xml_copy = NULL;
     cib_t *cib_conn = NULL;
     resource_t *rsc = NULL;
     bool recursive = FALSE;
     char *our_pid = NULL;
 
     bool validate_cmdline = FALSE; /* whether we are just validating based on command line options */
     bool require_resource = TRUE; /* whether command requires that resource be specified */
     bool require_dataset = TRUE;  /* whether command requires populated dataset instance */
     bool require_crmd = FALSE;    // whether command requires controller connection
     bool clear_expired = FALSE;
 
     int rc = pcmk_ok;
     int is_ocf_rc = 0;
     int option_index = 0;
     int timeout_ms = 0;
     int argerr = 0;
     int flag;
     int find_flags = 0;           // Flags to use when searching for resource
     crm_exit_t exit_code = CRM_EX_OK;
 
     crm_log_cli_init("crm_resource");
     crm_set_options(NULL, "(query|command) [options]", long_options,
                     "Perform tasks related to cluster resources.\nAllows resources to be queried (definition and location), modified, and moved around the cluster.\n");
 
     validate_options = crm_str_table_new();
 
     while (1) {
         flag = crm_get_option_long(argc, argv, &option_index, &longname);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 0: /* long options with no short equivalent */
                 if (safe_str_eq("master", longname)) {
                     scope_master = TRUE;
 
                 } else if(safe_str_eq(longname, "recursive")) {
                     recursive = TRUE;
 
                 } else if (safe_str_eq("wait", longname)) {
                     rsc_cmd = flag;
                     rsc_long_cmd = longname;
                     require_resource = FALSE;
                     require_dataset = FALSE;
 
                 } else if (
                     safe_str_eq("validate", longname)
                     || safe_str_eq("restart", longname)
                     || safe_str_eq("force-demote",  longname)
                     || safe_str_eq("force-stop",    longname)
                     || safe_str_eq("force-start",   longname)
                     || safe_str_eq("force-promote", longname)
                     || safe_str_eq("force-check",   longname)) {
                     rsc_cmd = flag;
                     rsc_long_cmd = longname;
                     find_flags = pe_find_renamed|pe_find_anon;
                     crm_log_args(argc, argv);
 
                 } else if (safe_str_eq("list-ocf-providers", longname)
                            || safe_str_eq("list-ocf-alternatives", longname)
                            || safe_str_eq("list-standards", longname)) {
                     const char *text = NULL;
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     if (safe_str_eq("list-ocf-providers", longname)
                         || safe_str_eq("list-ocf-alternatives", longname)) {
                         rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, optarg, &list);
                         text = "OCF providers";
 
                     } else if (safe_str_eq("list-standards", longname)) {
                         rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
                         text = "standards";
                     }
 
                     if (rc > 0) {
                         for (iter = list; iter != NULL; iter = iter->next) {
                             printf("%s\n", iter->val);
                         }
                         lrmd_list_freeall(list);
 
                     } else if (optarg) {
                         fprintf(stderr, "No %s found for %s\n", text, optarg);
                         exit_code = CRM_EX_NOSUCH;
 
                     } else {
                         fprintf(stderr, "No %s found\n", text);
                         exit_code = CRM_EX_NOSUCH;
                     }
 
                     lrmd_api_delete(lrmd_conn);
                     crm_exit(exit_code);
 
                 } else if (safe_str_eq("show-metadata", longname)) {
                     char *standard = NULL;
                     char *provider = NULL;
                     char *type = NULL;
                     char *metadata = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     rc = crm_parse_agent_spec(optarg, &standard, &provider, &type);
                     if (rc == pcmk_ok) {
                         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard,
                                                            provider, type,
                                                            &metadata, 0);
                     } else {
                         fprintf(stderr,
                                 "'%s' is not a valid agent specification\n",
                                 optarg);
                         rc = -ENXIO;
                     }
 
                     if (metadata) {
                         printf("%s\n", metadata);
                     } else {
                         fprintf(stderr, "Metadata query for %s failed: %s\n",
                                 optarg, pcmk_strerror(rc));
                         exit_code = crm_errno2exit(rc);
                     }
                     lrmd_api_delete(lrmd_conn);
                     crm_exit(exit_code);
 
                 } else if (safe_str_eq("list-agents", longname)) {
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     char *provider = strchr (optarg, ':');
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     if (provider) {
                         *provider++ = 0;
                     }
                     rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, optarg, provider);
 
                     if (rc > 0) {
                         for (iter = list; iter != NULL; iter = iter->next) {
                             printf("%s\n", iter->val);
                         }
                         lrmd_list_freeall(list);
                     } else {
                         fprintf(stderr, "No agents found for standard=%s, provider=%s\n",
                                 optarg, (provider? provider : "*"));
                         exit_code = CRM_EX_NOSUCH;
                     }
                     lrmd_api_delete(lrmd_conn);
                     crm_exit(exit_code);
 
                 } else if (safe_str_eq("class", longname)) {
                     if (!(pcmk_get_ra_caps(optarg) & pcmk_ra_cap_params)) {
                         if (BE_QUIET == FALSE) {
                             fprintf(stdout, "Standard %s does not support parameters\n",
                                     optarg);
                         }
 
                         crm_exit(exit_code);
                     } else {
                         v_class = optarg;
                     }
 
                     validate_cmdline = TRUE;
                     require_resource = FALSE;
 
                 } else if (safe_str_eq("agent", longname)) {
                     validate_cmdline = TRUE;
                     require_resource = FALSE;
                     v_agent = optarg;
 
                 } else if (safe_str_eq("provider", longname)) {
                     validate_cmdline = TRUE;
                     require_resource = FALSE;
                     v_provider = optarg;
 
                 } else if (safe_str_eq("option", longname)) {
                     crm_info("Scanning: --option %s", optarg);
                     rc = pcmk_scan_nvpair(optarg, &name, &value);
                     if (rc != 2) {
                         fprintf(stderr, "Invalid option: --option %s: %s", optarg, pcmk_strerror(rc));
                         argerr++;
                     } else {
                         crm_info("Got: '%s'='%s'", name, value);
                     }
 
                     g_hash_table_replace(validate_options, name, value);
 
                 } else {
                     crm_err("Unhandled long option: %s", longname);
                 }
                 break;
             case 'V':
                 resource_verbose++;
                 crm_bump_log_level(argc, argv);
                 break;
             case '$':
             case '?':
                 crm_help(flag, CRM_EX_OK);
                 break;
             case 'x':
                 xml_file = strdup(optarg);
                 break;
             case 'Q':
                 BE_QUIET = TRUE;
                 break;
             case 'm':
                 attr_set_type = XML_TAG_META_SETS;
                 break;
             case 'z':
                 attr_set_type = XML_TAG_UTILIZATION;
                 break;
             case 'u':
                 move_lifetime = strdup(optarg);
                 break;
             case 'f':
                 do_force = TRUE;
                 crm_log_args(argc, argv);
                 break;
             case 'i':
                 prop_id = optarg;
                 break;
             case 's':
                 prop_set = optarg;
                 break;
             case 'r':
                 rsc_id = optarg;
                 break;
             case 'v':
                 prop_value = optarg;
                 break;
             case 't':
                 rsc_type = optarg;
                 break;
             case 'T':
                 timeout_ms = crm_get_msec(optarg);
                 break;
             case 'e':
                 clear_expired = TRUE;
                 require_resource = FALSE;
                 break;
 
             case 'C':
             case 'R':
                 crm_log_args(argc, argv);
                 require_resource = FALSE;
                 if (cib_file == NULL) {
                     require_crmd = TRUE;
                 }
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'n':
                 operation = optarg;
                 break;
 
             case 'I':
                 interval_spec = optarg;
                 break;
 
             case 'D':
                 require_dataset = FALSE;
                 crm_log_args(argc, argv);
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'F':
                 require_crmd = TRUE;
                 crm_log_args(argc, argv);
                 rsc_cmd = flag;
                 break;
 
             case 'U':
             case 'B':
             case 'M':
                 crm_log_args(argc, argv);
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'c':
             case 'L':
             case 'l':
             case 'O':
             case 'o':
                 require_resource = FALSE;
                 rsc_cmd = flag;
                 break;
 
             case 'Y':
                 require_resource = FALSE;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'q':
             case 'w':
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'W':
             case 'A':
             case 'a':
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_anon;
                 break;
 
             case 'S':
                 require_dataset = FALSE;
                 crm_log_args(argc, argv);
                 prop_name = optarg;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'p':
             case 'd':
                 crm_log_args(argc, argv);
                 prop_name = optarg;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'G':
             case 'g':
                 prop_name = optarg;
                 rsc_cmd = flag;
                 find_flags = pe_find_renamed|pe_find_any;
                 break;
 
             case 'H':
             case 'N':
                 crm_trace("Option %c => %s", flag, optarg);
                 host_uname = optarg;
                 break;
 
             default:
                 CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     // Catch the case where the user didn't specify a command
     if (rsc_cmd == 'L') {
         require_resource = FALSE;
     }
 
     // --expired without --clear/-U doesn't make sense
     if (clear_expired == TRUE && rsc_cmd != 'U') {
         CMD_ERR("--expired requires --clear or -U");
         argerr++;
     }
 
     if (optind < argc
         && argv[optind] != NULL
         && rsc_cmd == 0
         && rsc_long_cmd) {
 
         override_params = crm_str_table_new();
         while (optind < argc && argv[optind] != NULL) {
             char *name = calloc(1, strlen(argv[optind]));
             char *value = calloc(1, strlen(argv[optind]));
             int rc = sscanf(argv[optind], "%[^=]=%s", name, value);
 
             if(rc == 2) {
                 g_hash_table_replace(override_params, name, value);
 
             } else {
                 CMD_ERR("Error parsing '%s' as a name=value pair for --%s", argv[optind], rsc_long_cmd);
                 free(value);
                 free(name);
                 argerr++;
             }
             optind++;
         }
 
     } else if (optind < argc && argv[optind] != NULL && rsc_cmd == 0) {
         CMD_ERR("non-option ARGV-elements: ");
         while (optind < argc && argv[optind] != NULL) {
             CMD_ERR("[%d of %d] %s ", optind, argc, argv[optind]);
             optind++;
             argerr++;
         }
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     // Sanity check validating from command line parameters.  If everything checks out,
     // go ahead and run the validation.  This way we don't need a CIB connection.
     if (validate_cmdline == TRUE) {
         // -r cannot be used with any of --class, --agent, or --provider
         if (rsc_id != NULL) {
             CMD_ERR("--resource cannot be used with --class, --agent, and --provider");
             argerr++;
 
         // If --class, --agent, or --provider are given, --validate must also be given.
         } else if (!safe_str_eq(rsc_long_cmd, "validate")) {
             CMD_ERR("--class, --agent, and --provider require --validate");
             argerr++;
 
         // Not all of --class, --agent, and --provider need to be given.  Not all
         // classes support the concept of a provider.  Check that what we were given
         // is valid.
         } else if (crm_str_eq(v_class, "stonith", TRUE)) {
             if (v_provider != NULL) {
                 CMD_ERR("stonith does not support providers");
                 argerr++;
 
             } else if (stonith_agent_exists(v_agent, 0) == FALSE) {
                 CMD_ERR("%s is not a known stonith agent", v_agent ? v_agent : "");
                 argerr++;
             }
 
         } else if (resources_agent_exists(v_class, v_provider, v_agent) == FALSE) {
             CMD_ERR("%s:%s:%s is not a known resource",
                     v_class ? v_class : "",
                     v_provider ? v_provider : "",
                     v_agent ? v_agent : "");
             argerr++;
         }
 
         if (argerr == 0) {
             rc = cli_resource_execute_from_params("test", v_class, v_provider, v_agent,
                                                   "validate-all", validate_options,
                                                   override_params, timeout_ms);
             exit_code = crm_errno2exit(rc);
             crm_exit(exit_code);
         }
     }
 
     if (argerr) {
         CMD_ERR("Invalid option(s) supplied, use --help for valid usage");
         crm_exit(CRM_EX_USAGE);
     }
 
     our_pid = crm_getpid_s();
 
     if (do_force) {
         crm_debug("Forcing...");
         cib_options |= cib_quorum_override;
     }
 
     if (require_resource && !rsc_id) {
         CMD_ERR("Must supply a resource id with -r");
         rc = -ENXIO;
         goto bail;
     }
 
     if (find_flags && rsc_id) {
         require_dataset = TRUE;
     }
 
     /* Establish a connection to the CIB manager */
     cib_conn = cib_new();
     rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
     if (rc != pcmk_ok) {
         CMD_ERR("Error connecting to the CIB manager: %s", pcmk_strerror(rc));
         goto bail;
     }
 
     /* Populate working set from XML file if specified or CIB query otherwise */
     if (require_dataset) {
         if (xml_file != NULL) {
             cib_xml_copy = filename2xml(xml_file);
 
         } else {
             rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
         }
 
         if(rc != pcmk_ok) {
             goto bail;
         }
 
         /* Populate the working set instance */
         data_set = pe_new_working_set();
         if (data_set == NULL) {
             rc = -ENOMEM;
             goto bail;
         }
         set_bit(data_set->flags, pe_flag_no_counts);
         rc = update_working_set_xml(data_set, &cib_xml_copy);
         if (rc != pcmk_ok) {
             goto bail;
         }
         cluster_status(data_set);
     }
 
     // If command requires that resource exist if specified, find it
     if (find_flags && rsc_id) {
         rsc = pe_find_resource_with_flags(data_set->resources, rsc_id,
                                           find_flags);
         if (rsc == NULL) {
             CMD_ERR("Resource '%s' not found", rsc_id);
             rc = -ENXIO;
             goto bail;
         }
     }
 
     // Establish a connection to the controller if needed
     if (require_crmd) {
         xmlNode *xml = NULL;
         mainloop_io_t *source =
             mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
         crmd_channel = mainloop_get_ipc_client(source);
 
         if (crmd_channel == NULL) {
             CMD_ERR("Error connecting to the controller");
             rc = -ENOTCONN;
             goto bail;
         }
 
         xml = create_hello_message(our_pid, crm_system_name, "0", "1");
         crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
         free_xml(xml);
     }
 
     /* Handle rsc_cmd appropriately */
     if (rsc_cmd == 'L') {
         rc = pcmk_ok;
         cli_resource_print_list(data_set, FALSE);
 
     } else if (rsc_cmd == 'l') {
         int found = 0;
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             rsc = (resource_t *) lpc->data;
 
             found++;
             cli_resource_print_raw(rsc);
         }
 
         if (found == 0) {
             printf("NO resources configured\n");
             rc = -ENXIO;
         }
 
     } else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "restart")) {
         /* We don't pass data_set because rsc needs to stay valid for the entire
          * lifetime of cli_resource_restart(), but it will reset and update the
          * working set multiple times, so it needs to use its own copy.
          */
         rc = cli_resource_restart(rsc, host_uname, timeout_ms, cib_conn);
 
     } else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "wait")) {
         rc = wait_till_stable(timeout_ms, cib_conn);
 
     } else if (rsc_cmd == 0 && rsc_long_cmd) {
         // validate, force-(stop|start|demote|promote|check)
         rc = cli_resource_execute(rsc, rsc_id, rsc_long_cmd, override_params,
                                   timeout_ms, cib_conn, data_set);
         if (rc >= 0) {
             is_ocf_rc = 1;
         }
 
     } else if (rsc_cmd == 'A' || rsc_cmd == 'a') {
         GListPtr lpc = NULL;
         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
                                                    data_set->input);
 
         unpack_constraints(cib_constraints, data_set);
 
         // Constraints apply to group/clone, not member/instance
         rsc = uber_parent(rsc);
 
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             resource_t *r = (resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         cli_resource_print_colocation(rsc, TRUE, rsc_cmd == 'A', 1);
 
         fprintf(stdout, "* %s\n", rsc->id);
         cli_resource_print_location(rsc, NULL);
 
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             resource_t *r = (resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         cli_resource_print_colocation(rsc, FALSE, rsc_cmd == 'A', 1);
 
     } else if (rsc_cmd == 'c') {
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
             rsc = (resource_t *) lpc->data;
             cli_resource_print_cts(rsc);
         }
         cli_resource_print_cts_constraints(data_set);
 
     } else if (rsc_cmd == 'F') {
         rc = cli_resource_fail(crmd_channel, host_uname, rsc_id, data_set);
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if (rsc_cmd == 'O') {
         rc = cli_resource_print_operations(rsc_id, host_uname, TRUE, data_set);
 
     } else if (rsc_cmd == 'o') {
         rc = cli_resource_print_operations(rsc_id, host_uname, FALSE, data_set);
 
     } else if (rsc_cmd == 'W') {
         rc = cli_resource_search(rsc, rsc_id, data_set);
         if (rc >= 0) {
             rc = pcmk_ok;
         }
 
     } else if (rsc_cmd == 'q') {
         rc = cli_resource_print(rsc, data_set, TRUE);
 
     } else if (rsc_cmd == 'w') {
         rc = cli_resource_print(rsc, data_set, FALSE);
 
     } else if (rsc_cmd == 'Y') {
         node_t *dest = NULL;
 
         if (host_uname) {
             dest = pe_find_node(data_set->nodes, host_uname);
             if (dest == NULL) {
                 rc = -pcmk_err_node_unknown;
                 goto bail;
             }
         }
         cli_resource_why(cib_conn, data_set->resources, rsc, dest);
         rc = pcmk_ok;
 
     } else if (rsc_cmd == 'U') {
         GListPtr before = NULL;
         GListPtr after = NULL;
         GListPtr remaining = NULL;
         GListPtr ele = NULL;
         node_t *dest = NULL;
 
         if (BE_QUIET == FALSE) {
             before = build_constraint_list(data_set->input);
         }
 
         if (clear_expired == TRUE) {
             rc = cli_resource_clear_all_expired(data_set->input, cib_conn, rsc_id, host_uname, scope_master);
 
         } else if (host_uname) {
             dest = pe_find_node(data_set->nodes, host_uname);
             if (dest == NULL) {
                 rc = -pcmk_err_node_unknown;
                 if (BE_QUIET == FALSE) {
                     g_list_free(before);
                 }
                 goto bail;
             }
             rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn, TRUE);
 
         } else {
             rc = cli_resource_clear(rsc_id, NULL, data_set->nodes, cib_conn, TRUE);
         }
 
         if (BE_QUIET == FALSE) {
             rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
             if (rc != pcmk_ok) {
                 CMD_ERR("Could not get modified CIB: %s\n", pcmk_strerror(rc));
                 g_list_free(before);
                 goto bail;
             }
 
             data_set->input = cib_xml_copy;
             cluster_status(data_set);
 
             after = build_constraint_list(data_set->input);
             remaining = subtract_lists(before, after, (GCompareFunc) strcmp);
 
             for (ele = remaining; ele != NULL; ele = ele->next) {
                 printf("Removing constraint: %s\n", (char *) ele->data);
             }
 
             g_list_free(before);
             g_list_free(after);
             g_list_free(remaining);
         }
 
     } else if (rsc_cmd == 'M' && host_uname) {
         rc = cli_resource_move(rsc, rsc_id, host_uname, cib_conn, data_set);
 
     } else if (rsc_cmd == 'B' && host_uname) {
         node_t *dest = pe_find_node(data_set->nodes, host_uname);
 
         if (dest == NULL) {
             rc = -pcmk_err_node_unknown;
             goto bail;
         }
         rc = cli_resource_ban(rsc_id, dest->details->uname, NULL, cib_conn);
 
     } else if (rsc_cmd == 'B' || rsc_cmd == 'M') {
         pe_node_t *current = NULL;
         unsigned int nactive = 0;
 
         current = pe__find_active_requires(rsc, &nactive);
 
         if (nactive == 1) {
             rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);
 
         } else if (is_set(rsc->flags, pe_rsc_promotable)) {
             int count = 0;
             GListPtr iter = NULL;
 
             current = NULL;
             for(iter = rsc->children; iter; iter = iter->next) {
                 resource_t *child = (resource_t *)iter->data;
                 enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
                 if(child_role == RSC_ROLE_MASTER) {
                     count++;
                     current = pe__current_node(child);
                 }
             }
 
             if(count == 1 && current) {
                 rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);
 
             } else {
                 rc = -EINVAL;
                 exit_code = CRM_EX_USAGE;
                 CMD_ERR("Resource '%s' not moved: active in %d locations (promoted in %d).",
                         rsc_id, nactive, count);
                 CMD_ERR("To prevent '%s' from running on a specific location, "
                         "specify a node.", rsc_id);
                 CMD_ERR("To prevent '%s' from being promoted at a specific "
                         "location, specify a node and the master option.",
                         rsc_id);
             }
 
         } else {
             rc = -EINVAL;
             exit_code = CRM_EX_USAGE;
             CMD_ERR("Resource '%s' not moved: active in %d locations.", rsc_id, nactive);
             CMD_ERR("To prevent '%s' from running on a specific location, "
                     "specify a node.", rsc_id);
         }
 
     } else if (rsc_cmd == 'G') {
         rc = cli_resource_print_property(rsc, prop_name, data_set);
 
     } else if (rsc_cmd == 'S') {
         xmlNode *msg_data = NULL;
 
         if ((rsc_type == NULL) || !strlen(rsc_type)) {
             CMD_ERR("Must specify -t with resource type");
             rc = -ENXIO;
             goto bail;
 
         } else if ((prop_value == NULL) || !strlen(prop_value)) {
             CMD_ERR("Must supply -v with new value");
             rc = -EINVAL;
             goto bail;
         }
 
         CRM_LOG_ASSERT(prop_name != NULL);
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
         crm_xml_add(msg_data, prop_name, prop_value);
 
         rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else if (rsc_cmd == 'g') {
         rc = cli_resource_print_attribute(rsc, prop_name, data_set);
 
     } else if (rsc_cmd == 'p') {
         if (prop_value == NULL || strlen(prop_value) == 0) {
             CMD_ERR("You need to supply a value with the -v option");
             rc = -EINVAL;
             goto bail;
         }
 
         /* coverity[var_deref_model] False positive */
         rc = cli_resource_update_attribute(rsc, rsc_id, prop_set, prop_id,
                                            prop_name, prop_value, recursive,
                                            cib_conn, data_set);
 
     } else if (rsc_cmd == 'd') {
         /* coverity[var_deref_model] False positive */
         rc = cli_resource_delete_attribute(rsc, rsc_id, prop_set, prop_id,
                                            prop_name, cib_conn, data_set);
 
     } else if ((rsc_cmd == 'C') && rsc) {
         if (do_force == FALSE) {
             rsc = uber_parent(rsc);
         }
         crmd_replies_needed = 0;
 
         crm_debug("Erasing failures of %s (%s requested) on %s",
                   rsc->id, rsc_id, (host_uname? host_uname: "all nodes"));
         rc = cli_resource_delete(crmd_channel, host_uname, rsc,
                                  operation, interval_spec, TRUE, data_set);
 
         if ((rc == pcmk_ok) && !BE_QUIET) {
             // Show any reasons why resource might stay stopped
             cli_resource_check(cib_conn, rsc);
         }
 
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if (rsc_cmd == 'C') {
         rc = cli_cleanup_all(crmd_channel, host_uname, operation, interval_spec,
                              data_set);
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if ((rsc_cmd == 'R') && rsc) {
         if (do_force == FALSE) {
             rsc = uber_parent(rsc);
         }
         crmd_replies_needed = 0;
 
         crm_debug("Re-checking the state of %s (%s requested) on %s",
                   rsc->id, rsc_id, (host_uname? host_uname: "all nodes"));
         rc = cli_resource_delete(crmd_channel, host_uname, rsc,
                                  NULL, 0, FALSE, data_set);
 
         if ((rc == pcmk_ok) && !BE_QUIET) {
             // Show any reasons why resource might stay stopped
             cli_resource_check(cib_conn, rsc);
         }
 
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if (rsc_cmd == 'R') {
         const char *router_node = host_uname;
         xmlNode *msg_data = NULL;
         xmlNode *cmd = NULL;
-        int attr_options = attrd_opt_none;
+        int attr_options = pcmk__node_attr_none;
 
         if (host_uname) {
             node_t *node = pe_find_node(data_set->nodes, host_uname);
 
             if (pe__is_guest_or_remote_node(node)) {
                 node = pe__current_node(node->details->remote_rsc);
                 if (node == NULL) {
                     CMD_ERR("No cluster connection to Pacemaker Remote node %s detected",
                             host_uname);
                     rc = -ENXIO;
                     goto bail;
                 }
                 router_node = node->details->uname;
-                attr_options |= attrd_opt_remote;
+                attr_options |= pcmk__node_attr_remote;
             }
         }
 
         if (crmd_channel == NULL) {
             printf("Dry run: skipping clean-up of %s due to CIB_file\n",
                    host_uname? host_uname : "all nodes");
             rc = pcmk_ok;
             goto bail;
         }
 
         msg_data = create_xml_node(NULL, "crm-resource-reprobe-op");
         crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
         if (safe_str_neq(router_node, host_uname)) {
             crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
         }
 
         cmd = create_request(CRM_OP_REPROBE, msg_data, router_node,
                              CRM_SYSTEM_CRMD, crm_system_name, our_pid);
         free_xml(msg_data);
 
         crm_debug("Re-checking the state of all resources on %s", host_uname?host_uname:"all nodes");
 
-        rc = attrd_clear_delegate(NULL, host_uname, NULL, NULL, NULL, NULL,
-                                  attr_options);
+        rc = pcmk_rc2legacy(pcmk__node_attr_request_clear(NULL, host_uname,
+                                                          NULL, NULL, NULL,
+                                                          NULL, attr_options));
 
         if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
             start_mainloop();
         }
 
         free_xml(cmd);
 
     } else if (rsc_cmd == 'D') {
         xmlNode *msg_data = NULL;
 
         if (rsc_type == NULL) {
             CMD_ERR("You need to specify a resource type with -t");
             rc = -ENXIO;
             goto bail;
         }
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
 
         rc = cib_conn->cmds->remove(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else {
         CMD_ERR("Unknown command: %c", rsc_cmd);
     }
 
   bail:
 
     free(our_pid);
     pe_free_working_set(data_set);
     if (cib_conn != NULL) {
         cib_conn->cmds->signoff(cib_conn);
         cib_delete(cib_conn);
     }
 
     if (is_ocf_rc) {
         exit_code = rc;
 
     } else if (rc != pcmk_ok) {
         CMD_ERR("Error performing operation: %s", pcmk_strerror(rc));
         if (rc == -pcmk_err_no_quorum) {
             CMD_ERR("To ignore quorum, use the force option");
         }
         if (exit_code == CRM_EX_OK) {
             exit_code = crm_errno2exit(rc);
         }
     }
 
     crm_exit(exit_code);
 }
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 01919f67e2..61eb82a56e 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,2103 +1,2104 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_resource.h>
 
 int resource_verbose = 0;
 bool do_force = FALSE;
 int crmd_replies_needed = 1; /* The welcome message */
 
 const char *attr_set_type = XML_TAG_ATTR_SETS;
 
 static int
 do_find_resource(const char *rsc, resource_t * the_rsc, pe_working_set_t * data_set)
 {
     int found = 0;
     GListPtr lpc = NULL;
 
     for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
         node_t *node = (node_t *) lpc->data;
 
         if (BE_QUIET) {
             fprintf(stdout, "%s\n", node->details->uname);
         } else {
             const char *state = "";
 
             if (!pe_rsc_is_clone(the_rsc) && the_rsc->fns->state(the_rsc, TRUE) == RSC_ROLE_MASTER) {
                 state = "Master";
             }
             fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
         }
 
         found++;
     }
 
     if (BE_QUIET == FALSE && found == 0) {
         fprintf(stderr, "resource %s is NOT running\n", rsc);
     }
 
     return found;
 }
 
 int
 cli_resource_search(resource_t *rsc, const char *requested_name,
                     pe_working_set_t *data_set)
 {
     int found = 0;
     resource_t *parent = uber_parent(rsc);
 
     if (pe_rsc_is_clone(rsc)) {
         for (GListPtr iter = rsc->children; iter != NULL; iter = iter->next) {
             found += do_find_resource(requested_name, iter->data, data_set);
         }
 
     /* The anonymous clone children's common ID is supplied */
     } else if (pe_rsc_is_clone(parent)
                && is_not_set(rsc->flags, pe_rsc_unique)
                && rsc->clone_name
                && safe_str_eq(requested_name, rsc->clone_name)
                && safe_str_neq(requested_name, rsc->id)) {
 
         for (GListPtr iter = parent->children; iter; iter = iter->next) {
             found += do_find_resource(requested_name, iter->data, data_set);
         }
 
     } else {
         found += do_find_resource(requested_name, rsc, data_set);
     }
 
     return found;
 }
 
 #define XPATH_MAX 1024
 
 static int
 find_resource_attr(cib_t * the_cib, const char *attr, const char *rsc, const char *set_type,
                    const char *set_name, const char *attr_id, const char *attr_name, char **value)
 {
     int offset = 0;
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
     char *xpath_string = NULL;
 
     if(value) {
         *value = NULL;
     }
 
     if(the_cib == NULL) {
         return -ENOTCONN;
     }
 
     xpath_string = calloc(1, XPATH_MAX);
     offset +=
         snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", get_object_path("resources"));
 
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//*[@id=\"%s\"]", rsc);
 
     if (set_type) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s", set_type);
         if (set_name) {
             offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "[@id=\"%s\"]", set_name);
         }
     }
 
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "//nvpair[");
     if (attr_id) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@id=\"%s\"", attr_id);
     }
 
     if (attr_name) {
         if (attr_id) {
             offset += snprintf(xpath_string + offset, XPATH_MAX - offset, " and ");
         }
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "@name=\"%s\"", attr_name);
     }
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "]");
     CRM_LOG_ASSERT(offset > 0);
 
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
 
     if (rc != pcmk_ok) {
         goto bail;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         xmlNode *child = NULL;
 
         rc = -EINVAL;
         printf("Multiple attributes match name=%s\n", attr_name);
 
         for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
             printf("  Value: %s \t(id=%s)\n",
                    crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
         }
 
     } else if(value) {
         const char *tmp = crm_element_value(xml_search, attr);
 
         if (tmp) {
             *value = strdup(tmp);
         }
     }
 
   bail:
     free(xpath_string);
     free_xml(xml_search);
     return rc;
 }
 
 /* PRIVATE. Use the find_matching_attr_resources instead. */
 static void
 find_matching_attr_resources_recursive(GList/* <resource_t*> */ ** result, resource_t * rsc, const char * rsc_id, 
 		                       const char * attr_set, const char * attr_id,
                                        const char * attr_name, cib_t * cib, const char * cmd, int depth)
 {
     int rc = pcmk_ok;
     char *lookup_id = clone_strip(rsc->id);
     char *local_attr_id = NULL;
 
     /* visit the children */
     for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
         find_matching_attr_resources_recursive(result, (resource_t*)gIter->data, rsc_id, attr_set, attr_id, attr_name, cib, cmd, depth+1);
         /* do it only once for clones */
         if(pe_clone == rsc->variant) {
             break;
         }
     }
 
     rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
     /* Post-order traversal. 
      * The root is always on the list and it is the last item. */
     if((0 == depth) || (pcmk_ok == rc)) {
         /* push the head */
         *result = g_list_append(*result, rsc);
     }
 
     free(local_attr_id);
     free(lookup_id);
 }
 
 
 /* The result is a linearized pre-ordered tree of resources. */
 static GList/*<resource_t*>*/ *
 find_matching_attr_resources(resource_t * rsc, const char * rsc_id, const char * attr_set, const char * attr_id,
                             const char * attr_name, cib_t * cib, const char * cmd)
 {
     int rc = pcmk_ok;
     char *lookup_id = NULL;
     char *local_attr_id = NULL;
     GList * result = NULL;
     /* If --force is used, update only the requested resource (clone or primitive).
      * Otherwise, if the primitive has the attribute, use that.
      * Otherwise use the clone. */
     if(do_force == TRUE) {
         return g_list_append(result, rsc);
     }
     if(rsc->parent && pe_clone == rsc->parent->variant) {
         int rc = pcmk_ok;
         char *local_attr_id = NULL;
         rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
         free(local_attr_id);
 
         if(rc != pcmk_ok) {
             rsc = rsc->parent;
             if (BE_QUIET == FALSE) {
                 printf("Performing %s of '%s' on '%s', the parent of '%s'\n", cmd, attr_name, rsc->id, rsc_id);
             }
         }
         return g_list_append(result, rsc);
     } else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
         resource_t *child = rsc->children->data;
 
         if(child->variant == pe_native) {
             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
             rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
 
             if(rc == pcmk_ok) {
                 rsc = child;
                 if (BE_QUIET == FALSE) {
                     printf("A value for '%s' already exists in child '%s', performing %s on that instead of '%s'\n", attr_name, lookup_id, cmd, rsc_id);
                 }
             }
 
             free(local_attr_id);
             free(lookup_id);
         }
         return g_list_append(result, rsc);
     }
     /* If the resource is a group ==> children inherit the attribute if defined. */
     find_matching_attr_resources_recursive(&result, rsc, rsc_id, attr_set, attr_id, attr_name, cib, cmd, 0);
     return result;
 }
 
 int
 cli_resource_update_attribute(resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_id,
                               const char *attr_name, const char *attr_value,
                               bool recursive, cib_t *cib,
                               pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
     static bool need_init = TRUE;
 
     char *local_attr_id = NULL;
     char *local_attr_set = NULL;
 
     GList/*<resource_t*>*/ *resources = NULL;
     const char *common_attr_id = attr_id;
 
     if(attr_id == NULL
        && do_force == FALSE
        && find_resource_attr(
            cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == -EINVAL) {
         printf("\n");
     }
 
     if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
         if (do_force == FALSE) {
             rc = find_resource_attr(cib, XML_ATTR_ID, uber_parent(rsc)->id,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &local_attr_id);
             if (rc == pcmk_ok && BE_QUIET == FALSE) {
                 printf("WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)\n",
                        uber_parent(rsc)->id, attr_name, local_attr_id);
                 printf("         Delete '%s' first or use the force option to override\n",
                        local_attr_id);
             }
             free(local_attr_id);
             if (rc == pcmk_ok) {
                 return -ENOTUNIQ;
             }
         }
         resources = g_list_append(resources, rsc);
 
     } else {
         resources = find_matching_attr_resources(rsc, requested_name, attr_set,
                                                  attr_id, attr_name, cib, "update");
     }
 
     /* If either attr_set or attr_id is specified,
      * one clearly intends to modify a single resource.
      * It is the last item on the resource list.*/
     for(GList *gIter = (attr_set||attr_id) ? g_list_last(resources) : resources
             ; gIter; gIter = gIter->next) {
         char *lookup_id = NULL;
 
         xmlNode *xml_top = NULL;
         xmlNode *xml_obj = NULL;
         local_attr_id = NULL;
         local_attr_set = NULL;
 
         rsc = (resource_t*)gIter->data;
         attr_id = common_attr_id;
 
         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
         rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
                                 &local_attr_id);
 
         if (rc == pcmk_ok) {
             crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
             attr_id = local_attr_id;
 
         } else if (rc != -ENXIO) {
             free(lookup_id);
             free(local_attr_id);
             g_list_free(resources);
             return rc;
 
         } else {
             const char *tag = crm_element_name(rsc->xml);
 
             if (attr_set == NULL) {
                 local_attr_set = crm_concat(lookup_id, attr_set_type, '-');
                 attr_set = local_attr_set;
             }
             if (attr_id == NULL) {
                 local_attr_id = crm_concat(attr_set, attr_name, '-');
                 attr_id = local_attr_id;
             }
 
             xml_top = create_xml_node(NULL, tag);
             crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
 
             xml_obj = create_xml_node(xml_top, attr_set_type);
             crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
         }
 
         xml_obj = crm_create_nvpair_xml(xml_obj, attr_id, attr_name, attr_value);
         if (xml_top == NULL) {
             xml_top = xml_obj;
         }
 
         crm_log_xml_debug(xml_top, "Update");
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
         if (rc == pcmk_ok && BE_QUIET == FALSE) {
             printf("Set '%s' option: id=%s%s%s%s%s value=%s\n", lookup_id, local_attr_id,
                    attr_set ? " set=" : "", attr_set ? attr_set : "",
                    attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
         }
 
         free_xml(xml_top);
 
         free(lookup_id);
         free(local_attr_id);
         free(local_attr_set);
 
         if(recursive && safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
             GListPtr lpc = NULL;
 
             if(need_init) {
                 xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
                 need_init = FALSE;
                 unpack_constraints(cib_constraints, data_set);
 
                 for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
                     resource_t *r = (resource_t *) lpc->data;
 
                     clear_bit(r->flags, pe_rsc_allocating);
                 }
             }
 
             crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
             set_bit(rsc->flags, pe_rsc_allocating);
             for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
                 rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
                 resource_t *peer = cons->rsc_lh;
 
                 crm_debug("Checking %s %d", cons->id, cons->score);
                 if (cons->score > 0 && is_not_set(peer->flags, pe_rsc_allocating)) {
                     /* Don't get into colocation loops */
                     crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
                     cli_resource_update_attribute(peer, peer->id, NULL, NULL,
                                                   attr_name, attr_value, recursive,
                                                   cib, data_set);
                 }
             }
         }
     }
     g_list_free(resources);
     return rc;
 }
 
 int
 cli_resource_delete_attribute(resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_id,
                               const char *attr_name, cib_t *cib,
                               pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
     GList/*<resource_t*>*/ *resources = NULL;
 
     if(attr_id == NULL
        && do_force == FALSE
        && find_resource_attr(
            cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) == -EINVAL) {
         printf("\n");
     }
 
     if(safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
         resources = find_matching_attr_resources(rsc, requested_name, attr_set,
                                                  attr_id, attr_name, cib, "delete");
     } else {
         resources = g_list_append(resources, rsc);
     }
 
     for(GList *gIter = resources; gIter; gIter = gIter->next) {
         char *lookup_id = NULL;
         xmlNode *xml_obj = NULL;
         char *local_attr_id = NULL;
 
         rsc = (resource_t*)gIter->data;
 
         lookup_id = clone_strip(rsc->id);
         rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
                                 &local_attr_id);
 
         if (rc == -ENXIO) {
             free(lookup_id);
             rc = pcmk_ok;
             continue;
 
         } else if (rc != pcmk_ok) {
             free(lookup_id);
             g_list_free(resources);
             return rc;
         }
 
         if (attr_id == NULL) {
             attr_id = local_attr_id;
         }
 
         xml_obj = crm_create_nvpair_xml(NULL, attr_id, attr_name, NULL);
         crm_log_xml_debug(xml_obj, "Delete");
 
         CRM_ASSERT(cib);
         rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
 
         if (rc == pcmk_ok && BE_QUIET == FALSE) {
             printf("Deleted '%s' option: id=%s%s%s%s%s\n", lookup_id, local_attr_id,
                    attr_set ? " set=" : "", attr_set ? attr_set : "",
                    attr_name ? " name=" : "", attr_name ? attr_name : "");
         }
 
         free(lookup_id);
         free_xml(xml_obj);
         free(local_attr_id);
     }
     g_list_free(resources);
     return rc;
 }
 
 static int
 send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op,
                 const char *host_uname, const char *rsc_id,
                 bool only_failed, pe_working_set_t * data_set)
 {
     char *our_pid = NULL;
     char *key = NULL;
     int rc = -ECOMM;
     xmlNode *cmd = NULL;
     xmlNode *xml_rsc = NULL;
     const char *value = NULL;
     const char *router_node = host_uname;
     xmlNode *params = NULL;
     xmlNode *msg_data = NULL;
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         CMD_ERR("Resource %s not found", rsc_id);
         return -ENXIO;
 
     } else if (rsc->variant != pe_native) {
         CMD_ERR("We can only process primitive resources, not %s", rsc_id);
         return -EINVAL;
 
     } else if (host_uname == NULL) {
         CMD_ERR("Please specify a node name");
         return -EINVAL;
     } else {
         node_t *node = pe_find_node(data_set->nodes, host_uname);
 
         if (pe__is_guest_or_remote_node(node)) {
             node = pe__current_node(node->details->remote_rsc);
             if (node == NULL) {
                 CMD_ERR("No cluster connection to Pacemaker Remote node %s detected",
                         host_uname);
                 return -ENXIO;
             }
             router_node = node->details->uname;
         }
     }
 
     key = generate_transition_key(0, getpid(), 0, "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
 
     msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
     crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
     free(key);
 
     crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
     if (safe_str_neq(router_node, host_uname)) {
         crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
     }
 
     xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
     if (rsc->clone_name) {
         crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
         crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);
 
     } else {
         crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
     }
 
     value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_ATTR_TYPE);
     if (value == NULL) {
         CMD_ERR("%s has no type!  Aborting...", rsc_id);
         return -ENXIO;
     }
 
     value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_CLASS);
     if (value == NULL) {
         CMD_ERR("%s has no class!  Aborting...", rsc_id);
         return -ENXIO;
     }
 
     crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_PROVIDER);
 
     params = create_xml_node(msg_data, XML_TAG_ATTRS);
     crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 
     key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
     crm_xml_add(params, key, "60000");  /* 1 minute */
     free(key);
 
     our_pid = crm_getpid_s();
     cmd = create_request(op, msg_data, router_node, CRM_SYSTEM_CRMD, crm_system_name, our_pid);
 
 /* 	crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */
     free_xml(msg_data);
 
     if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
         rc = 0;
 
     } else {
         crm_debug("Could not send %s op to the controller", op);
         rc = -ENOTCONN;
     }
 
     free_xml(cmd);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Get resource name as used in failure-related node attributes
  *
  * \param[in] rsc  Resource to check
  *
  * \return Newly allocated string containing resource's fail name
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 rsc_fail_name(resource_t *rsc)
 {
     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 
     return is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
 }
 
 static int
 clear_rsc_history(crm_ipc_t *crmd_channel, const char *host_uname,
                   const char *rsc_id, pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
 
     /* Erase the resource's entire LRM history in the CIB, even if we're only
      * clearing a single operation's fail count. If we erased only entries for a
      * single operation, we might wind up with a wrong idea of the current
      * resource state, and we might not re-probe the resource.
      */
     rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc_id,
                          TRUE, data_set);
     if (rc != pcmk_ok) {
         return rc;
     }
     crmd_replies_needed++;
 
     crm_trace("Processing %d mainloop inputs", crmd_replies_needed);
     while (g_main_context_iteration(NULL, FALSE)) {
         crm_trace("Processed mainloop input, %d still remaining",
                   crmd_replies_needed);
     }
 
     if (crmd_replies_needed < 0) {
         crmd_replies_needed = 0;
     }
     return rc;
 }
 
 static int
 clear_rsc_failures(crm_ipc_t *crmd_channel, const char *node_name,
                    const char *rsc_id, const char *operation,
                    const char *interval_spec, pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
     const char *failed_value = NULL;
     const char *failed_id = NULL;
     const char *interval_ms_s = NULL;
     GHashTable *rscs = NULL;
     GHashTableIter iter;
 
     /* Create a hash table to use as a set of resources to clean. This lets us
      * clean each resource only once (per node) regardless of how many failed
      * operations it has.
      */
     rscs = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
 
     // Normalize interval to milliseconds for comparison to history entry
     if (operation) {
         interval_ms_s = crm_strdup_printf("%u",
                                           crm_parse_interval_spec(interval_spec));
     }
 
     for (xmlNode *xml_op = __xml_first_child(data_set->failed); xml_op != NULL;
          xml_op = __xml_next(xml_op)) {
 
         failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
         if (failed_id == NULL) {
             // Malformed history entry, should never happen
             continue;
         }
 
         // No resource specified means all resources match
         if (rsc_id) {
             resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
                                                                failed_id,
                                                                pe_find_renamed|pe_find_anon);
 
             if (!fail_rsc || safe_str_neq(rsc_id, fail_rsc->id)) {
                 continue;
             }
         }
 
         // Host name should always have been provided by this point
         failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
         if (safe_str_neq(node_name, failed_value)) {
             continue;
         }
 
         // No operation specified means all operations match
         if (operation) {
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
             if (safe_str_neq(operation, failed_value)) {
                 continue;
             }
 
             // Interval (if operation was specified) defaults to 0 (not all)
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
             if (safe_str_neq(interval_ms_s, failed_value)) {
                 continue;
             }
         }
 
         /* not available until glib 2.32
         g_hash_table_add(rscs, (gpointer) failed_id);
         */
         g_hash_table_insert(rscs, (gpointer) failed_id, (gpointer) failed_id);
     }
 
     g_hash_table_iter_init(&iter, rscs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
         rc = clear_rsc_history(crmd_channel, node_name, failed_id, data_set);
         if (rc != pcmk_ok) {
             return rc;
         }
     }
     g_hash_table_destroy(rscs);
     return rc;
 }
 
 static int
 clear_rsc_fail_attrs(resource_t *rsc, const char *operation,
                      const char *interval_spec, node_t *node)
 {
     int rc = pcmk_ok;
-    int attr_options = attrd_opt_none;
+    int attr_options = pcmk__node_attr_none;
     char *rsc_name = rsc_fail_name(rsc);
 
     if (pe__is_guest_or_remote_node(node)) {
-        attr_options |= attrd_opt_remote;
+        attr_options |= pcmk__node_attr_remote;
     }
-    rc = attrd_clear_delegate(NULL, node->details->uname, rsc_name, operation,
-                              interval_spec, NULL, attr_options);
+    rc = pcmk__node_attr_request_clear(NULL, node->details->uname, rsc_name,
+                                       operation, interval_spec, NULL,
+                                       attr_options);
     free(rsc_name);
     return rc;
 }
 
 int
 cli_resource_delete(crm_ipc_t *crmd_channel, const char *host_uname,
                     resource_t *rsc, const char *operation,
                     const char *interval_spec, bool just_failures,
                     pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
     node_t *node = NULL;
 
     if (rsc == NULL) {
         return -ENXIO;
 
     } else if (rsc->children) {
         GListPtr lpc = NULL;
 
         for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
             resource_t *child = (resource_t *) lpc->data;
 
             rc = cli_resource_delete(crmd_channel, host_uname, child, operation,
                                      interval_spec, just_failures, data_set);
             if (rc != pcmk_ok) {
                 return rc;
             }
         }
         return pcmk_ok;
 
     } else if (host_uname == NULL) {
         GListPtr lpc = NULL;
         GListPtr nodes = g_hash_table_get_values(rsc->known_on);
 
         if(nodes == NULL && do_force) {
             nodes = node_list_dup(data_set->nodes, FALSE, FALSE);
 
         } else if(nodes == NULL && rsc->exclusive_discover) {
             GHashTableIter iter;
             pe_node_t *node = NULL;
 
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
                 if(node->weight >= 0) {
                     nodes = g_list_prepend(nodes, node);
                 }
             }
 
         } else if(nodes == NULL) {
             nodes = g_hash_table_get_values(rsc->allowed_nodes);
         }
 
         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
             node = (node_t *) lpc->data;
 
             if (node->details->online) {
                 rc = cli_resource_delete(crmd_channel, node->details->uname,
                                          rsc, operation, interval_spec,
                                          just_failures, data_set);
             }
             if (rc != pcmk_ok) {
                 g_list_free(nodes);
                 return rc;
             }
         }
 
         g_list_free(nodes);
         return pcmk_ok;
     }
 
     node = pe_find_node(data_set->nodes, host_uname);
 
     if (node == NULL) {
         printf("Unable to clean up %s because node %s not found\n",
                rsc->id, host_uname);
         return -ENODEV;
     }
 
     if (!node->details->rsc_discovery_enabled) {
         printf("Unable to clean up %s because resource discovery disabled on %s\n",
                rsc->id, host_uname);
         return -EOPNOTSUPP;
     }
 
     if (crmd_channel == NULL) {
         printf("Dry run: skipping clean-up of %s on %s due to CIB_file\n",
                rsc->id, host_uname);
         return pcmk_ok;
     }
 
     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
-    if (rc != pcmk_ok) {
+    if (rc != pcmk_rc_ok) {
         printf("Unable to clean up %s failures on %s: %s\n",
-                rsc->id, host_uname, pcmk_strerror(rc));
-        return rc;
+                rsc->id, host_uname, pcmk_rc_str(rc));
+        return pcmk_rc2legacy(rc);
     }
 
     if (just_failures) {
         rc = clear_rsc_failures(crmd_channel, host_uname, rsc->id, operation,
                                 interval_spec, data_set);
     } else {
         rc = clear_rsc_history(crmd_channel, host_uname, rsc->id, data_set);
     }
     if (rc != pcmk_ok) {
         printf("Cleaned %s failures on %s, but unable to clean history: %s\n",
                rsc->id, host_uname, pcmk_strerror(rc));
     } else {
         printf("Cleaned up %s on %s\n", rsc->id, host_uname);
     }
     return rc;
 }
 
 int
 cli_cleanup_all(crm_ipc_t *crmd_channel, const char *node_name,
                 const char *operation, const char *interval_spec,
                 pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
-    int attr_options = attrd_opt_none;
+    int attr_options = pcmk__node_attr_none;
     const char *display_name = node_name? node_name : "all nodes";
 
     if (crmd_channel == NULL) {
         printf("Dry run: skipping clean-up of %s due to CIB_file\n",
                display_name);
         return pcmk_ok;
     }
     crmd_replies_needed = 0;
 
     if (node_name) {
         node_t *node = pe_find_node(data_set->nodes, node_name);
 
         if (node == NULL) {
             CMD_ERR("Unknown node: %s", node_name);
             return -ENXIO;
         }
         if (pe__is_guest_or_remote_node(node)) {
-            attr_options |= attrd_opt_remote;
+            attr_options |= pcmk__node_attr_remote;
         }
     }
 
-    rc = attrd_clear_delegate(NULL, node_name, NULL, operation, interval_spec,
-                              NULL, attr_options);
-    if (rc != pcmk_ok) {
+    rc = pcmk__node_attr_request_clear(NULL, node_name, NULL, operation,
+                                       interval_spec, NULL, attr_options);
+    if (rc != pcmk_rc_ok) {
         printf("Unable to clean up all failures on %s: %s\n",
-                display_name, pcmk_strerror(rc));
-        return rc;
+                display_name, pcmk_rc_str(rc));
+        return pcmk_rc2legacy(rc);
     }
 
     if (node_name) {
         rc = clear_rsc_failures(crmd_channel, node_name, NULL,
                                 operation, interval_spec, data_set);
         if (rc != pcmk_ok) {
             printf("Cleaned all resource failures on %s, but unable to clean history: %s\n",
                    node_name, pcmk_strerror(rc));
             return rc;
         }
     } else {
         for (GList *iter = data_set->nodes; iter; iter = iter->next) {
             pe_node_t *node = (pe_node_t *) iter->data;
 
             rc = clear_rsc_failures(crmd_channel, node->details->uname, NULL,
                                     operation, interval_spec, data_set);
             if (rc != pcmk_ok) {
                 printf("Cleaned all resource failures on all nodes, but unable to clean history: %s\n",
                        pcmk_strerror(rc));
                 return rc;
             }
         }
     }
 
     printf("Cleaned up all resources on %s\n", display_name);
     return pcmk_ok;
 }
 
 void
 cli_resource_check(cib_t * cib_conn, resource_t *rsc)
 {
     int need_nl = 0;
     char *role_s = NULL;
     char *managed = NULL;
     resource_t *parent = uber_parent(rsc);
 
     find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);
 
     find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);
 
     if(role_s) {
         enum rsc_role_e role = text2role(role_s);
 
         free(role_s);
         if(role == RSC_ROLE_UNKNOWN) {
             // Treated as if unset
 
         } else if(role == RSC_ROLE_STOPPED) {
             printf("\n  * The configuration specifies that '%s' should remain stopped\n", parent->id);
             need_nl++;
 
         } else if (is_set(parent->flags, pe_rsc_promotable)
                    && (role == RSC_ROLE_SLAVE)) {
             printf("\n  * The configuration specifies that '%s' should not be promoted\n", parent->id);
             need_nl++;
         }
     }
 
     if(managed && crm_is_true(managed) == FALSE) {
         printf("%s  * The configuration prevents the cluster from stopping or starting '%s' (unmanaged)\n", need_nl == 0?"\n":"", parent->id);
         need_nl++;
     }
     free(managed);
 
     if(need_nl) {
         printf("\n");
     }
 }
 
 int
 cli_resource_fail(crm_ipc_t * crmd_channel, const char *host_uname,
              const char *rsc_id, pe_working_set_t * data_set)
 {
     crm_warn("Failing: %s", rsc_id);
     return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set);
 }
 
 static GHashTable *
 generate_resource_params(resource_t * rsc, pe_working_set_t * data_set)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
 
     if (!rsc) {
         crm_err("Resource does not exist in config");
         return NULL;
     }
 
     params = crm_str_table_new();
     meta = crm_str_table_new();
     combined = crm_str_table_new();
 
     get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
     get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);
 
     if (params) {
         char *key = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             g_hash_table_insert(combined, strdup(key), strdup(value));
         }
         g_hash_table_destroy(params);
     }
 
     if (meta) {
         char *key = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
 
             g_hash_table_insert(combined, crm_name, strdup(value));
         }
         g_hash_table_destroy(meta);
     }
 
     return combined;
 }
 
 static bool resource_is_running_on(resource_t *rsc, const char *host) 
 {
     bool found = TRUE;
     GListPtr hIter = NULL;
     GListPtr hosts = NULL;
 
     if(rsc == NULL) {
         return FALSE;
     }
 
     rsc->fns->location(rsc, &hosts, TRUE);
     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
         pe_node_t *node = (pe_node_t *) hIter->data;
 
         if(strcmp(host, node->details->uname) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         } else if(strcmp(host, node->details->id) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         }
     }
 
     if(host != NULL) {
         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
         found = FALSE;
 
     } else if(host == NULL && hosts == NULL) {
         crm_trace("Resource %s is not running\n", rsc->id);
         found = FALSE;
     }
 
   done:
 
     g_list_free(hosts);
     return found;
 }
 
 /*!
  * \internal
  * \brief Create a list of all resources active on host from a given list
  *
  * \param[in] host      Name of host to check whether resources are active
  * \param[in] rsc_list  List of resources to check
  *
  * \return New list of resources from list that are active on host
  */
 static GList *
 get_active_resources(const char *host, GList *rsc_list)
 {
     GList *rIter = NULL;
     GList *active = NULL;
 
     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
         resource_t *rsc = (resource_t *) rIter->data;
 
         /* Expand groups to their members, because if we're restarting a member
          * other than the first, we can't otherwise tell which resources are
          * stopping and starting.
          */
         if (rsc->variant == pe_group) {
             active = g_list_concat(active,
                                    get_active_resources(host, rsc->children));
         } else if (resource_is_running_on(rsc, host)) {
             active = g_list_append(active, strdup(rsc->id));
         }
     }
     return active;
 }
 
 GList*
 subtract_lists(GList *from, GList *items, GCompareFunc cmp)
 {
     GList *item = NULL;
     GList *result = g_list_copy(from);
 
     for (item = items; item != NULL; item = item->next) {
         GList *candidate = NULL;
         for (candidate = from; candidate != NULL; candidate = candidate->next) {
             crm_info("Comparing %s with %s", (const char *) candidate->data,
                      (const char *) item->data);
             if(cmp(candidate->data, item->data) == 0) {
                 result = g_list_remove(result, candidate->data);
                 break;
             }
         }
     }
 
     return result;
 }
 
 static void dump_list(GList *items, const char *tag) 
 {
     int lpc = 0;
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
         lpc++;
     }
 }
 
 static void display_list(GList *items, const char *tag) 
 {
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         fprintf(stdout, "%s%s\n", tag, (const char *)item->data);
     }
 }
 
 /*!
  * \internal
  * \brief Upgrade XML to latest schema version and use it as working set input
  *
  * This also updates the working set timestamp to the current time.
  *
  * \param[in] data_set   Working set instance to update
  * \param[in] xml        XML to use as input
  *
  * \return pcmk_ok on success, -ENOKEY if unable to upgrade XML
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->now.
  * \todo This follows the example of other callers of cli_config_update()
  *       and returns -ENOKEY ("Required key not available") if that fails,
  *       but perhaps -pcmk_err_schema_validation would be better in that case.
  */
 int
 update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
 {
     if (cli_config_update(xml, NULL, FALSE) == FALSE) {
         return -ENOKEY;
     }
     data_set->input = *xml;
     data_set->now = crm_time_new(NULL);
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Update a working set's XML input based on a CIB query
  *
  * \param[in] data_set   Data set instance to initialize
  * \param[in] cib        Connection to the CIB manager
  *
  * \return pcmk_ok on success, -errno on failure
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->input and data_set->now.
  */
 static int
 update_working_set_from_cib(pe_working_set_t * data_set, cib_t *cib)
 {
     xmlNode *cib_xml_copy = NULL;
     int rc;
 
     rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
     if (rc != pcmk_ok) {
         fprintf(stderr, "Could not obtain the current CIB: %s (%d)\n", pcmk_strerror(rc), rc);
         return rc;
     }
     rc = update_working_set_xml(data_set, &cib_xml_copy);
     if (rc != pcmk_ok) {
         fprintf(stderr, "Could not upgrade the current CIB XML\n");
         free_xml(cib_xml_copy);
         return rc;
     }
     return pcmk_ok;
 }
 
 static int
 update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
 {
     char *pid = NULL;
     char *shadow_file = NULL;
     cib_t *shadow_cib = NULL;
     int rc;
 
     pe_reset_working_set(data_set);
     rc = update_working_set_from_cib(data_set, cib);
     if (rc != pcmk_ok) {
         return rc;
     }
 
     if(simulate) {
         pid = crm_getpid_s();
         shadow_cib = cib_shadow_new(pid);
         shadow_file = get_shadow_file(pid);
 
         if (shadow_cib == NULL) {
             fprintf(stderr, "Could not create shadow cib: '%s'\n", pid);
             rc = -ENXIO;
             goto cleanup;
         }
 
         rc = write_xml_file(data_set->input, shadow_file, FALSE);
 
         if (rc < 0) {
             fprintf(stderr, "Could not populate shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
             goto cleanup;
         }
 
         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
         if(rc != pcmk_ok) {
             fprintf(stderr, "Could not connect to shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
             goto cleanup;
         }
 
         pcmk__schedule_actions(data_set, data_set->input, NULL);
         run_simulation(data_set, shadow_cib, NULL, TRUE);
         rc = update_dataset(shadow_cib, data_set, FALSE);
 
     } else {
         cluster_status(data_set);
     }
 
   cleanup:
     /* Do not free data_set->input here, we need rsc->xml to be valid later on */
     cib_delete(shadow_cib);
     free(pid);
 
     if(shadow_file) {
         unlink(shadow_file);
         free(shadow_file);
     }
 
     return rc;
 }
 
 static int
 max_delay_for_resource(pe_working_set_t * data_set, resource_t *rsc) 
 {
     int delay = 0;
     int max_delay = 0;
 
     if(rsc && rsc->children) {
         GList *iter = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             resource_t *child = (resource_t *)iter->data;
 
             delay = max_delay_for_resource(data_set, child);
             if(delay > max_delay) {
                 double seconds = delay / 1000.0;
                 crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
                 max_delay = delay;
             }
         }
 
     } else if(rsc) {
         char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
         action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
         const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
 
         max_delay = crm_int_helper(value, NULL);
         pe_free_action(stop);
     }
 
 
     return max_delay;
 }
 
 static int
 max_delay_in(pe_working_set_t * data_set, GList *resources) 
 {
     int max_delay = 0;
     GList *item = NULL;
 
     for (item = resources; item != NULL; item = item->next) {
         int delay = 0;
         resource_t *rsc = pe_find_resource(data_set->resources, (const char *)item->data);
 
         delay = max_delay_for_resource(data_set, rsc);
 
         if(delay > max_delay) {
             double seconds = delay / 1000.0;
             crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
             max_delay = delay;
         }
     }
 
     return 5 + (max_delay / 1000);
 }
 
 #define waiting_for_starts(d, r, h) ((g_list_length(d) > 0) || \
                                     (resource_is_running_on((r), (h)) == FALSE))
 
 /*!
  * \internal
  * \brief Restart a resource (on a particular host if requested).
  *
  * \param[in] rsc        The resource to restart
  * \param[in] host       The host to restart the resource on (or NULL for all)
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but a two-second
  *                       granularity is actually used; if 0, a timeout will be
  *                       calculated based on the resource timeout)
  * \param[in] cib        Connection to the CIB manager
  *
  * \return pcmk_ok on success, -errno on failure (exits on certain failures)
  */
 int
 cli_resource_restart(pe_resource_t *rsc, const char *host, int timeout_ms,
                      cib_t *cib)
 {
     int rc = 0;
     int lpc = 0;
     int before = 0;
     int step_timeout_s = 0;
     int sleep_interval = 2;
     int timeout = timeout_ms / 1000;
 
     bool stop_via_ban = FALSE;
     char *rsc_id = NULL;
     char *orig_target_role = NULL;
 
     GList *list_delta = NULL;
     GList *target_active = NULL;
     GList *current_active = NULL;
     GList *restart_target_active = NULL;
 
     pe_working_set_t *data_set = NULL;
 
     if(resource_is_running_on(rsc, host) == FALSE) {
         const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
         if(host) {
             printf("%s is not running on %s and so cannot be restarted\n", id, host);
         } else {
             printf("%s is not running anywhere and so cannot be restarted\n", id);
         }
         return -ENXIO;
     }
 
     /* We might set the target-role meta-attribute */
     attr_set_type = XML_TAG_META_SETS;
 
     rsc_id = strdup(rsc->id);
     if ((pe_rsc_is_clone(rsc) || pe_bundle_replicas(rsc)) && host) {
         stop_via_ban = TRUE;
     }
 
     /*
       grab full cib
       determine originally active resources
       disable or ban
       poll cib and watch for affected resources to get stopped
       without --timeout, calculate the stop timeout for each step and wait for that
       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
       if everything stopped, re-enable or un-ban
       poll cib and watch for affected resources to get started
       without --timeout, calculate the start timeout for each step and wait for that
       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
       report success
 
       Optimizations:
       - use constraints to determine ordered list of affected resources
       - Allow a --no-deps option (aka. --force-restart)
     */
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
         crm_perror(LOG_ERR, "Could not allocate working set");
         rc = -ENOMEM;
         goto done;
     }
     set_bit(data_set->flags, pe_flag_no_counts);
     rc = update_dataset(cib, data_set, FALSE);
     if(rc != pcmk_ok) {
         fprintf(stdout, "Could not get new resource list: %s (%d)\n", pcmk_strerror(rc), rc);
         goto done;
     }
 
     restart_target_active = get_active_resources(host, data_set->resources);
     current_active = get_active_resources(host, data_set->resources);
 
     dump_list(current_active, "Origin");
 
     if (stop_via_ban) {
         /* Stop the clone or bundle instance by banning it from the host */
         BE_QUIET = TRUE;
         rc = cli_resource_ban(rsc_id, host, NULL, cib);
 
     } else {
         /* Stop the resource by setting target-role to Stopped.
          * Remember any existing target-role so we can restore it later
          * (though it only makes any difference if it's Slave).
          */
         char *lookup_id = clone_strip(rsc->id);
 
         find_resource_attr(cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
                            NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
         free(lookup_id);
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, NULL,
                                            XML_RSC_ATTR_TARGET_ROLE,
                                            RSC_STOPPED, FALSE, cib, data_set);
     }
     if(rc != pcmk_ok) {
         fprintf(stderr, "Could not set target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
         if (current_active) {
             g_list_free_full(current_active, free);
         }
         if (restart_target_active) {
             g_list_free_full(restart_target_active, free);
         }
         goto done;
     }
 
     rc = update_dataset(cib, data_set, TRUE);
     if(rc != pcmk_ok) {
         fprintf(stderr, "Could not determine which resources would be stopped\n");
         goto failure;
     }
 
     target_active = get_active_resources(host, data_set->resources);
     dump_list(target_active, "Target");
 
     list_delta = subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
     fprintf(stdout, "Waiting for %d resources to stop:\n", g_list_length(list_delta));
     display_list(list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while(g_list_length(list_delta) > 0) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for(lpc = 0; lpc < step_timeout_s && g_list_length(list_delta) > 0; lpc++) {
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
             rc = update_dataset(cib, data_set, FALSE);
             if(rc != pcmk_ok) {
                 fprintf(stderr, "Could not determine which resources were stopped\n");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(host, data_set->resources);
             g_list_free(list_delta);
             list_delta = subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
             fprintf(stderr, "Could not complete shutdown of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
             display_list(list_delta, " * ");
             rc = -ETIME;
             goto failure;
         }
 
     }
 
     if (stop_via_ban) {
         rc = cli_resource_clear(rsc_id, host, NULL, cib, TRUE);
 
     } else if (orig_target_role) {
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, NULL,
                                            XML_RSC_ATTR_TARGET_ROLE,
                                            orig_target_role, FALSE, cib,
                                            data_set);
         free(orig_target_role);
         orig_target_role = NULL;
     } else {
         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, NULL,
                                            XML_RSC_ATTR_TARGET_ROLE, cib,
                                            data_set);
     }
 
     if(rc != pcmk_ok) {
         fprintf(stderr, "Could not unset target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
         goto done;
     }
 
     if (target_active) {
         g_list_free_full(target_active, free);
     }
     target_active = restart_target_active;
     if (list_delta) {
         g_list_free(list_delta);
     }
     list_delta = subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
     fprintf(stdout, "Waiting for %d resources to start again:\n", g_list_length(list_delta));
     display_list(list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (waiting_for_starts(list_delta, rsc, host)) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
 
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
 
             rc = update_dataset(cib, data_set, FALSE);
             if(rc != pcmk_ok) {
                 fprintf(stderr, "Could not determine which resources were started\n");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
 
             /* It's OK if dependent resources moved to a different node,
              * so we check active resources on all nodes.
              */
             current_active = get_active_resources(NULL, data_set->resources);
             g_list_free(list_delta);
             list_delta = subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
             fprintf(stdout, "Could not complete restart of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
             display_list(list_delta, " * ");
             rc = -ETIME;
             goto failure;
         }
 
     }
 
     rc = pcmk_ok;
     goto done;
 
   failure:
     if (stop_via_ban) {
         cli_resource_clear(rsc_id, host, NULL, cib, TRUE);
     } else if (orig_target_role) {
         cli_resource_update_attribute(rsc, rsc_id, NULL, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE,
                                       orig_target_role, FALSE, cib, data_set);
         free(orig_target_role);
     } else {
         cli_resource_delete_attribute(rsc, rsc_id, NULL, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE, cib, data_set);
     }
 
 done:
     if (list_delta) {
         g_list_free(list_delta);
     }
     if (current_active) {
         g_list_free_full(current_active, free);
     }
     if (target_active && (target_active != restart_target_active)) {
         g_list_free_full(target_active, free);
     }
     if (restart_target_active) {
         g_list_free_full(restart_target_active, free);
     }
     free(rsc_id);
     pe_free_working_set(data_set);
     return rc;
 }
 
 static inline int action_is_pending(action_t *action) 
 {
     if(is_set(action->flags, pe_action_optional)) {
         return FALSE;
     } else if(is_set(action->flags, pe_action_runnable) == FALSE) {
         return FALSE;
     } else if(is_set(action->flags, pe_action_pseudo)) {
         return FALSE;
     } else if(safe_str_eq("notify", action->task)) {
         return FALSE;
     }
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Return TRUE if any actions in a list are pending
  *
  * \param[in] actions   List of actions to check
  *
  * \return TRUE if any actions in the list are pending, FALSE otherwise
  */
 static bool
 actions_are_pending(GListPtr actions)
 {
     GListPtr action;
 
     for (action = actions; action != NULL; action = action->next) {
         action_t *a = (action_t *)action->data;
         if (action_is_pending(a)) {
             crm_notice("Waiting for %s (flags=0x%.8x)", a->uuid, a->flags);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Print pending actions to stderr
  *
  * \param[in] actions   List of actions to check
  *
  * \return void
  */
 static void
 print_pending_actions(GListPtr actions)
 {
     GListPtr action;
 
     fprintf(stderr, "Pending actions:\n");
     for (action = actions; action != NULL; action = action->next) {
         action_t *a = (action_t *) action->data;
 
         if (action_is_pending(a)) {
             fprintf(stderr, "\tAction %d: %s", a->id, a->uuid);
             if (a->node) {
                 fprintf(stderr, "\ton %s", a->node->details->uname);
             }
             fprintf(stderr, "\n");
         }
     }
 }
 
 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
 
 /* For --wait, how long to sleep between cluster state checks */
 #define WAIT_SLEEP_S (2)
 
 /*!
  * \internal
  * \brief Wait until all pending cluster actions are complete
  *
  * This waits until either the CIB's transition graph is idle or a timeout is
  * reached.
  *
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but one-second granularity
  *                       is actually used; if 0, a default will be used)
  * \param[in] cib        Connection to the CIB manager
  *
  * \return pcmk_ok on success, -errno on failure
  */
 int
 wait_till_stable(int timeout_ms, cib_t * cib)
 {
     pe_working_set_t *data_set = NULL;
     int rc = -1;
     int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
     time_t expire_time = time(NULL) + timeout_s;
     time_t time_diff;
     bool printed_version_warning = BE_QUIET; // i.e. don't print if quiet
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
         return -ENOMEM;
     }
     set_bit(data_set->flags, pe_flag_no_counts);
 
     do {
 
         /* Abort if timeout is reached */
         time_diff = expire_time - time(NULL);
         if (time_diff > 0) {
             crm_info("Waiting up to %ld seconds for cluster actions to complete", time_diff);
         } else {
             print_pending_actions(data_set->actions);
             pe_free_working_set(data_set);
             return -ETIME;
         }
         if (rc == pcmk_ok) { /* this avoids sleep on first loop iteration */
             sleep(WAIT_SLEEP_S);
         }
 
         /* Get latest transition graph */
         pe_reset_working_set(data_set);
         rc = update_working_set_from_cib(data_set, cib);
         if (rc != pcmk_ok) {
             pe_free_working_set(data_set);
             return rc;
         }
         pcmk__schedule_actions(data_set, data_set->input, NULL);
 
         if (!printed_version_warning) {
             /* If the DC has a different version than the local node, the two
              * could come to different conclusions about what actions need to be
              * done. Warn the user in this case.
              *
              * @TODO A possible long-term solution would be to reimplement the
              * wait as a new controller operation that would be forwarded to the
              * DC. However, that would have potential problems of its own.
              */
             const char *dc_version = g_hash_table_lookup(data_set->config_hash,
                                                          "dc-version");
 
             if (safe_str_neq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION)) {
                 printf("warning: wait option may not work properly in "
                        "mixed-version cluster\n");
                 printed_version_warning = TRUE;
             }
         }
 
     } while (actions_are_pending(data_set->actions));
 
     pe_free_working_set(data_set);
     return pcmk_ok;
 }
 
 int
 cli_resource_execute_from_params(const char *rsc_name, const char *rsc_class,
                                  const char *rsc_prov, const char *rsc_type,
                                  const char *action, GHashTable *params,
                                  GHashTable *override_hash, int timeout_ms)
 {
     GHashTable *params_copy = NULL;
     int rc = pcmk_ok;
     svc_action_t *op = NULL;
 
     if (safe_str_eq(rsc_class, PCMK_RESOURCE_CLASS_STONITH)) {
         CMD_ERR("Sorry, the %s option doesn't support %s resources yet",
                 action, rsc_class);
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     }
 
     /* If no timeout was provided, grab the default. */
     if (timeout_ms == 0) {
         timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
     }
 
     /* add meta_timeout env needed by some resource agents */
     g_hash_table_insert(params, strdup("CRM_meta_timeout"),
                         crm_strdup_printf("%d", timeout_ms));
 
     /* add crm_feature_set env needed by some resource agents */
     g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
 
     /* resources_action_create frees the params hash table it's passed, but we
      * may need to reuse it in a second call to resources_action_create.  Thus
      * we'll make a copy here so that gets freed and the original remains for
      * reuse.
      */
     params_copy = crm_str_table_dup(params);
 
     op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action, 0,
                                  timeout_ms, params_copy, 0);
     if (op == NULL) {
         /* Re-run with stderr enabled so we can display a sane error message */
         crm_enable_stderr(TRUE);
         params_copy = crm_str_table_dup(params);
         op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action, 0,
                                      timeout_ms, params_copy, 0);
 
         /* Callers of cli_resource_execute expect that the params hash table will
          * be freed.  That function uses this one, so for that reason and for
          * making the two act the same, we should free the hash table here too.
          */
         g_hash_table_destroy(params);
 
         /* We know op will be NULL, but this makes static analysis happy */
         services_action_free(op);
         crm_exit(CRM_EX_DATAERR);
     }
 
     setenv("HA_debug", resource_verbose > 0 ? "1" : "0", 1);
     if(resource_verbose > 1) {
         setenv("OCF_TRACE_RA", "1", 1);
     }
 
     /* A resource agent using the standard ocf-shellfuncs library will not print
      * messages to stderr if it doesn't have a controlling terminal (e.g. if
      * crm_resource is called via script or ssh). This forces it to do so.
      */
     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
 
     if (override_hash) {
         GHashTableIter iter;
         char *name = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, override_hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) & name, (gpointer *) & value)) {
             printf("Overriding the cluster configuration for '%s' with '%s' = '%s'\n",
                    rsc_name, name, value);
             g_hash_table_replace(op->params, strdup(name), strdup(value));
         }
     }
 
     if (services_action_sync(op)) {
         int more, lpc, last;
         char *local_copy = NULL;
 
         rc = op->rc;
 
         if (op->status == PCMK_LRM_OP_DONE) {
             printf("Operation %s for %s (%s:%s:%s) returned: '%s' (%d)\n",
                    action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
                    services_ocf_exitcode_str(op->rc), op->rc);
         } else {
             printf("Operation %s for %s (%s:%s:%s) failed: '%s' (%d)\n",
                    action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "", rsc_type,
                    services_lrm_status_str(op->status), op->status);
         }
 
         /* hide output for validate-all if not in verbose */
         if (resource_verbose == 0 && safe_str_eq(action, "validate-all"))
             goto done;
 
         if (op->stdout_data) {
             local_copy = strdup(op->stdout_data);
             more = strlen(local_copy);
             last = 0;
 
             for (lpc = 0; lpc < more; lpc++) {
                 if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
                     local_copy[lpc] = 0;
                     printf(" >  stdout: %s\n", local_copy + last);
                     last = lpc + 1;
                 }
             }
             free(local_copy);
         }
         if (op->stderr_data) {
             local_copy = strdup(op->stderr_data);
             more = strlen(local_copy);
             last = 0;
 
             for (lpc = 0; lpc < more; lpc++) {
                 if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
                     local_copy[lpc] = 0;
                     printf(" >  stderr: %s\n", local_copy + last);
                     last = lpc + 1;
                 }
             }
             free(local_copy);
         }
     } else {
         rc = op->rc == 0 ? pcmk_err_generic : op->rc;
     }
 
 done:
     services_action_free(op);
     /* See comment above about why we free params here. */
     g_hash_table_destroy(params);
     return rc;
 }
 
 int
 cli_resource_execute(resource_t *rsc, const char *requested_name,
                      const char *rsc_action, GHashTable *override_hash,
                      int timeout_ms, cib_t * cib, pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
     const char *rid = NULL;
     const char *rtype = NULL;
     const char *rprov = NULL;
     const char *rclass = NULL;
     const char *action = NULL;
     GHashTable *params = NULL;
 
     if (safe_str_eq(rsc_action, "validate")) {
         action = "validate-all";
 
     } else if (safe_str_eq(rsc_action, "force-check")) {
         action = "monitor";
 
     } else if (safe_str_eq(rsc_action, "force-stop")) {
         action = rsc_action+6;
 
     } else if (safe_str_eq(rsc_action, "force-start")
                || safe_str_eq(rsc_action, "force-demote")
                || safe_str_eq(rsc_action, "force-promote")) {
         action = rsc_action+6;
 
         if(pe_rsc_is_clone(rsc)) {
             rc = cli_resource_search(rsc, requested_name, data_set);
             if(rc > 0 && do_force == FALSE) {
                 CMD_ERR("It is not safe to %s %s here: the cluster claims it is already active",
                         action, rsc->id);
                 CMD_ERR("Try setting target-role=Stopped first or specifying "
                         "the force option");
                 crm_exit(CRM_EX_UNSAFE);
             }
         }
 
     } else {
         action = rsc_action;
     }
 
     if(pe_rsc_is_clone(rsc)) {
         /* Grab the first child resource in the hope it's not a group */
         rsc = rsc->children->data;
     }
 
     if(rsc->variant == pe_group) {
         CMD_ERR("Sorry, the %s option doesn't support group resources",
                 rsc_action);
         crm_exit(CRM_EX_UNIMPLEMENT_FEATURE);
     }
 
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 
     params = generate_resource_params(rsc, data_set);
 
     if (timeout_ms == 0) {
         timeout_ms = pe_get_configured_timeout(rsc, action, data_set);
     }
 
     rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
 
     rc = cli_resource_execute_from_params(rid, rclass, rprov, rtype, action,
                                           params, override_hash, timeout_ms);
     return rc;
 }
 
 int
 cli_resource_move(resource_t *rsc, const char *rsc_id, const char *host_name,
                   cib_t *cib, pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
     unsigned int count = 0;
     node_t *current = NULL;
     node_t *dest = pe_find_node(data_set->nodes, host_name);
     bool cur_is_dest = FALSE;
 
     if (dest == NULL) {
         return -pcmk_err_node_unknown;
     }
 
     if (scope_master && is_not_set(rsc->flags, pe_rsc_promotable)) {
         resource_t *p = uber_parent(rsc);
 
         if (is_set(p->flags, pe_rsc_promotable)) {
             CMD_ERR("Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
             rsc_id = p->id;
             rsc = p;
 
         } else {
             CMD_ERR("Ignoring master option: %s is not promotable", rsc_id);
             scope_master = FALSE;
         }
     }
 
     current = pe__find_active_requires(rsc, &count);
 
     if (is_set(rsc->flags, pe_rsc_promotable)) {
         GListPtr iter = NULL;
         unsigned int master_count = 0;
         pe_node_t *master_node = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             resource_t *child = (resource_t *)iter->data;
             enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
             if(child_role == RSC_ROLE_MASTER) {
                 rsc = child;
                 master_node = pe__current_node(child);
                 master_count++;
             }
         }
         if (scope_master || master_count) {
             count = master_count;
             current = master_node;
         }
 
     }
 
     if (count > 1) {
         if (pe_rsc_is_clone(rsc)) {
             current = NULL;
         } else {
             return -pcmk_err_multiple;
         }
     }
 
     if (current && (current->details == dest->details)) {
         cur_is_dest = TRUE;
         if (do_force) {
             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
                      rsc_id, scope_master?"promoted":"active", dest->details->uname);
         } else {
             return -pcmk_err_already;
         }
     }
 
     /* Clear any previous prefer constraints across all nodes. */
     cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, FALSE);
 
     /* Clear any previous ban constraints on 'dest'. */
     cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib, TRUE);
 
     /* Record an explicit preference for 'dest' */
     rc = cli_resource_prefer(rsc_id, dest->details->uname, cib);
 
     crm_trace("%s%s now prefers node %s%s",
               rsc->id, scope_master?" (master)":"", dest->details->uname, do_force?"(forced)":"");
 
     /* only ban the previous location if current location != destination location.
      * it is possible to use -M to enforce a location without regard of where the
      * resource is currently located */
     if(do_force && (cur_is_dest == FALSE)) {
         /* Ban the original location if possible */
         if(current) {
             (void)cli_resource_ban(rsc_id, current->details->uname, NULL, cib);
 
         } else if(count > 1) {
             CMD_ERR("Resource '%s' is currently %s in %d locations. "
                     "One may now move to %s",
                     rsc_id, (scope_master? "promoted" : "active"),
                     count, dest->details->uname);
             CMD_ERR("To prevent '%s' from being %s at a specific location, "
                     "specify a node.",
                     rsc_id, (scope_master? "promoted" : "active"));
 
         } else {
             crm_trace("Not banning %s from its current location: not active", rsc_id);
         }
     }
 
     return rc;
 }
 
 static void
 cli_resource_why_without_rsc_and_host(cib_t *cib_conn,GListPtr resources)
 {
     GListPtr lpc = NULL;
     GListPtr hosts = NULL;
 
     for (lpc = resources; lpc != NULL; lpc = lpc->next) {
         resource_t *rsc = (resource_t *) lpc->data;
         rsc->fns->location(rsc, &hosts, TRUE);
 
         if (hosts == NULL) {
             printf("Resource %s is not running\n", rsc->id);
         } else {
             printf("Resource %s is running\n", rsc->id);
         }
 
         cli_resource_check(cib_conn, rsc);
         g_list_free(hosts);
         hosts = NULL;
      }
 
 }
 
 static void
 cli_resource_why_with_rsc_and_host(cib_t *cib_conn, GListPtr resources,
                                    resource_t *rsc, const char *host_uname)
 {
     if (resource_is_running_on(rsc, host_uname)) {
         printf("Resource %s is running on host %s\n",rsc->id,host_uname);
     } else {
         printf("Resource %s is not running on host %s\n", rsc->id, host_uname);
     }
     cli_resource_check(cib_conn, rsc);
 }
 
 static void
 cli_resource_why_without_rsc_with_host(cib_t *cib_conn,GListPtr resources,node_t *node)
 {
     const char* host_uname =  node->details->uname;
     GListPtr allResources = node->details->allocated_rsc;
     GListPtr activeResources = node->details->running_rsc;
     GListPtr unactiveResources = subtract_lists(allResources,activeResources,(GCompareFunc) strcmp);
     GListPtr lpc = NULL;
 
     for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
         resource_t *rsc = (resource_t *) lpc->data;
         printf("Resource %s is running on host %s\n",rsc->id,host_uname);
         cli_resource_check(cib_conn,rsc);
     }
 
     for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
         resource_t *rsc = (resource_t *) lpc->data;
         printf("Resource %s is assigned to host %s but not running\n",
                rsc->id, host_uname);
         cli_resource_check(cib_conn,rsc);
      }
 
      g_list_free(allResources);
      g_list_free(activeResources);
      g_list_free(unactiveResources);
 }
 
 static void
 cli_resource_why_with_rsc_without_host(cib_t *cib_conn, GListPtr resources,
                                        resource_t *rsc)
 {
     GListPtr hosts = NULL;
 
     rsc->fns->location(rsc, &hosts, TRUE);
     printf("Resource %s is %srunning\n", rsc->id, (hosts? "" : "not "));
     cli_resource_check(cib_conn, rsc);
     g_list_free(hosts);
 }
 
 void cli_resource_why(cib_t *cib_conn, GListPtr resources, resource_t *rsc,
                       node_t *node)
 {
     const char *host_uname = (node == NULL)? NULL : node->details->uname;
 
     if ((rsc == NULL) && (host_uname == NULL)) {
         cli_resource_why_without_rsc_and_host(cib_conn, resources);
 
     } else if ((rsc != NULL) && (host_uname != NULL)) {
         cli_resource_why_with_rsc_and_host(cib_conn, resources, rsc,
                                            host_uname);
 
     } else if ((rsc == NULL) && (host_uname != NULL)) {
         cli_resource_why_without_rsc_with_host(cib_conn, resources, node);
 
     } else if ((rsc != NULL) && (host_uname == NULL)) {
         cli_resource_why_with_rsc_without_host(cib_conn, resources, rsc);
     }
 }
diff --git a/tools/notifyServicelogEvent.c b/tools/notifyServicelogEvent.c
index 620a7997c8..44a4c01b4c 100644
--- a/tools/notifyServicelogEvent.c
+++ b/tools/notifyServicelogEvent.c
@@ -1,192 +1,193 @@
 /*
  * Original copyright 2009 International Business Machines, IBM, Mark Hamzy
- * Later changes copyright 2009-2019 the Pacemaker project contributors
+ * Later changes copyright 2009-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 /* gcc -o notifyServicelogEvent `pkg-config --cflags servicelog-1` `pkg-config --libs servicelog-1` notifyServicelogEvent.c
 */
 
 #include <crm_internal.h>
 
 #include <string.h>
 #include <stdio.h>
 #include <time.h>
 #include <errno.h>
 #include <servicelog.h>
 #include <syslog.h>
 #include <unistd.h>
 #include <config.h>
 #include <inttypes.h>  /* U64T ~ PRIu64, U64TS ~ SCNu64 */
 
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/common/attrd_internal.h>
 
 typedef enum { STATUS_GREEN = 1, STATUS_YELLOW, STATUS_RED } STATUS;
 
 const char *status2char(STATUS status);
 STATUS event2status(struct sl_event *event);
 
 const char *
 status2char(STATUS status)
 {
     switch (status) {
         default:
         case STATUS_GREEN:
             return "green";
         case STATUS_YELLOW:
             return "yellow";
         case STATUS_RED:
             return "red";
     }
 }
 
 STATUS
 event2status(struct sl_event * event)
 {
     STATUS status = STATUS_GREEN;
 
     crm_debug("Severity = %d, Disposition = %d", event->severity, event->disposition);
 
     /* @TBD */
     if (event->severity == SL_SEV_WARNING) {
         status = STATUS_YELLOW;
     }
 
     if (event->disposition == SL_DISP_UNRECOVERABLE) {
         status = STATUS_RED;
     }
 
     return status;
 }
 
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help", 0, 0, '?', "\tThis text"},
     {"version", 0, 0, '$', "\tVersion information"},
     {"-spacer-", 0, 0, '-', "\nUsage: notifyServicelogEvent event_id"},
     {"-spacer-", 0, 0, '-',
      "\nWhere event_id is unique unsigned event identifier which is then passed into servicelog"},
 
     {0, 0, 0, 0}
 };
 
 int
 main(int argc, char *argv[])
 {
     int argerr = 0;
     int flag;
     int index = 0;
     int rc = 0;
     servicelog *slog = NULL;
     struct sl_event *event = NULL;
     uint64_t event_id = 0;
 
     crm_log_cli_init("notifyServicelogEvent");
     crm_set_options(NULL, "event_id ", long_options,
                     "Gets called upon events written to servicelog database");
 
     if (argc < 2) {
         argerr++;
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case '?':
             case '$':
                 crm_help(flag, CRM_EX_OK);
                 break;
             default:
                 ++argerr;
                 break;
         }
     }
 
     if (argc - optind != 1) {
         ++argerr;
     }
 
     if (argerr) {
         crm_help('?', CRM_EX_USAGE);
     }
 
     openlog("notifyServicelogEvent", LOG_NDELAY, LOG_USER);
 
     if (sscanf(argv[optind], "%" U64TS, &event_id) != 1) {
         crm_err("Error: could not read event_id from args!");
 
         rc = 1;
         goto cleanup;
     }
 
     if (event_id == 0) {
         crm_err("Error: event_id is 0!");
 
         rc = 1;
         goto cleanup;
     }
 
     rc = servicelog_open(&slog, 0);     /* flags is one of SL_FLAG_xxx */
 
     if (!slog) {
         crm_err("Error: servicelog_open failed, rc = %d", rc);
 
         rc = 1;
         goto cleanup;
     }
 
     if (slog) {
         rc = servicelog_event_get(slog, event_id, &event);
     }
 
     if (rc == 0) {
         STATUS status = STATUS_GREEN;
         const char *health_component = "#health-ipmi";
         const char *health_status = NULL;
 
         crm_debug("Event id = %" U64T ", Log timestamp = %s, Event timestamp = %s",
                   event_id, ctime(&(event->time_logged)), ctime(&(event->time_event)));
 
         status = event2status(event);
 
         health_status = status2char(status);
 
         if (health_status) {
-            gboolean rc;
+            int attrd_rc;
 
-            /* @TODO pass attrd_opt_remote when appropriate */
-            rc = (attrd_update_delegate(NULL, 'v', NULL, health_component,
-                                        health_status, NULL, NULL, NULL, NULL,
-                                        attrd_opt_none) > 0);
+            // @TODO pass pcmk__node_attr_remote when appropriate
+            attrd_rc = pcmk__node_attr_request(NULL, 'v', NULL,
+                                               health_component, health_status,
+                                               NULL, NULL, NULL, NULL,
+                                               pcmk__node_attr_none);
             crm_debug("Updating attribute ('%s', '%s') = %d",
-                      health_component, health_status, rc);
+                      health_component, health_status, attrd_rc);
         } else {
             crm_err("Error: status2char failed, status = %d", status);
             rc = 1;
         }
     } else {
         crm_err("Error: servicelog_event_get failed, rc = %d", rc);
     }
 
   cleanup:
     if (event) {
         servicelog_event_free(event);
     }
 
     if (slog) {
         servicelog_close(slog);
     }
 
     closelog();
 
     return rc;
 }