diff --git a/daemons/controld/controld_cib.c b/daemons/controld/controld_cib.c
index c9c54de3eb..948548f0ab 100644
--- a/daemons/controld/controld_cib.c
+++ b/daemons/controld/controld_cib.c
@@ -1,1067 +1,1067 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <unistd.h>  /* sleep */
 
 #include <crm/common/alerts_internal.h>
 #include <crm/common/xml.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/lrmd_internal.h>
 
 #include <pacemaker-controld.h>
 
 // Call ID of the most recent in-progress CIB resource update (or 0 if none)
 static int pending_rsc_update = 0;
 
 /*!
  * \internal
  * \brief Respond to a dropped CIB connection
  *
  * \param[in] user_data  CIB connection that dropped
  */
 static void
 handle_cib_disconnect(gpointer user_data)
 {
     CRM_LOG_ASSERT(user_data == controld_globals.cib_conn);
 
     controld_trigger_fsa();
     controld_globals.cib_conn->state = cib_disconnected;
 
     if (pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
         // @TODO This should trigger a reconnect, not a shutdown
         crm_crit("Lost connection to the CIB manager, shutting down");
         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
         controld_clear_fsa_input_flags(R_CIB_CONNECTED);
 
     } else { // Expected
         crm_info("Disconnected from the CIB manager");
     }
 }
 
 static void
 do_cib_updated(const char *event, xmlNode * msg)
 {
     const xmlNode *patchset = NULL;
     const char *client_name = NULL;
 
     crm_debug("Received CIB diff notification: DC=%s", pcmk__btoa(AM_I_DC));
 
     if (cib__get_notify_patchset(msg, &patchset) != pcmk_rc_ok) {
         return;
     }
 
     if (cib__element_in_patchset(patchset, XML_CIB_TAG_ALERTS)
         || cib__element_in_patchset(patchset, XML_CIB_TAG_CRMCONFIG)) {
 
         controld_trigger_config();
     }
 
     if (!AM_I_DC) {
         // We're not in control of the join sequence
         return;
     }
 
     client_name = crm_element_value(msg, F_CIB_CLIENTNAME);
     if (!cib__client_triggers_refresh(client_name)) {
         // The CIB is still accurate
         return;
     }
 
     if (cib__element_in_patchset(patchset, XML_CIB_TAG_NODES)
         || cib__element_in_patchset(patchset, XML_CIB_TAG_STATUS)) {
 
         /* An unsafe client modified the nodes or status section. Ensure the
          * node list is up-to-date, and start the join process again so we get
          * everyone's current resource history.
          */
         if (client_name == NULL) {
             client_name = crm_element_value(msg, F_CIB_CLIENTID);
         }
         crm_notice("Populating nodes and starting an election after %s event "
                    "triggered by %s",
                    event, pcmk__s(client_name, "(unidentified client)"));
 
         populate_cib_nodes(node_update_quick|node_update_all, __func__);
         register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
     }
 }
 
 void
 controld_disconnect_cib_manager(void)
 {
     cib_t *cib_conn = controld_globals.cib_conn;
 
     CRM_ASSERT(cib_conn != NULL);
 
     crm_debug("Disconnecting from the CIB manager");
 
     controld_clear_fsa_input_flags(R_CIB_CONNECTED);
 
     cib_conn->cmds->del_notify_callback(cib_conn, T_CIB_DIFF_NOTIFY,
                                         do_cib_updated);
     cib_free_callbacks(cib_conn);
 
     if (cib_conn->state != cib_disconnected) {
         cib_conn->cmds->set_secondary(cib_conn,
                                       cib_scope_local|cib_discard_reply);
         cib_conn->cmds->signoff(cib_conn);
     }
 }
 
 /* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */
 void
 do_cib_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     static int cib_retries = 0;
 
     cib_t *cib_conn = controld_globals.cib_conn;
 
     void (*dnotify_fn) (gpointer user_data) = handle_cib_disconnect;
     void (*update_cb) (const char *event, xmlNodePtr msg) = do_cib_updated;
 
     int rc = pcmk_ok;
 
     CRM_ASSERT(cib_conn != NULL);
 
     if (pcmk_is_set(action, A_CIB_STOP)) {
         if ((cib_conn->state != cib_disconnected)
             && (pending_rsc_update != 0)) {
 
             crm_info("Waiting for resource update %d to complete",
                      pending_rsc_update);
             crmd_fsa_stall(FALSE);
             return;
         }
         controld_disconnect_cib_manager();
     }
 
     if (!pcmk_is_set(action, A_CIB_START)) {
         return;
     }
 
     if (cur_state == S_STOPPING) {
         crm_err("Ignoring request to connect to the CIB manager after "
                 "shutdown");
         return;
     }
 
     rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
                                 cib_command_nonblocking);
 
     if (rc != pcmk_ok) {
         // A short wait that usually avoids stalling the FSA
         sleep(1);
         rc = cib_conn->cmds->signon(cib_conn, CRM_SYSTEM_CRMD,
                                     cib_command_nonblocking);
     }
 
     if (rc != pcmk_ok) {
         crm_info("Could not connect to the CIB manager: %s", pcmk_strerror(rc));
 
     } else if (cib_conn->cmds->set_connection_dnotify(cib_conn,
                                                       dnotify_fn) != pcmk_ok) {
         crm_err("Could not set dnotify callback");
 
     } else if (cib_conn->cmds->add_notify_callback(cib_conn,
                                                    T_CIB_DIFF_NOTIFY,
                                                    update_cb) != pcmk_ok) {
         crm_err("Could not set CIB notification callback (update)");
 
     } else {
         controld_set_fsa_input_flags(R_CIB_CONNECTED);
         cib_retries = 0;
     }
 
     if (!pcmk_is_set(controld_globals.fsa_input_register, R_CIB_CONNECTED)) {
         cib_retries++;
 
         if (cib_retries < 30) {
             crm_warn("Couldn't complete CIB registration %d times... "
                      "pause and retry", cib_retries);
             controld_start_wait_timer();
             crmd_fsa_stall(FALSE);
 
         } else {
             crm_err("Could not complete CIB registration %d times... "
                     "hard error", cib_retries);
             register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         }
     }
 }
 
 #define MIN_CIB_OP_TIMEOUT (30)
 
 /*!
  * \internal
  * \brief Get the timeout (in seconds) that should be used with CIB operations
  *
  * \return The maximum of 30 seconds, the value of the PCMK_cib_timeout
  *         environment variable, or 10 seconds times one more than the number of
  *         nodes in the cluster.
  */
 unsigned int
 cib_op_timeout(void)
 {
     // @COMPAT: Drop env_timeout at 3.0.0
     static int env_timeout = -1;
     unsigned int calculated_timeout = 0;
 
     if (env_timeout == -1) {
         const char *env = pcmk__env_option(PCMK__ENV_CIB_TIMEOUT);
 
         pcmk__scan_min_int(env, &env_timeout, MIN_CIB_OP_TIMEOUT);
         crm_trace("Minimum CIB op timeout: %ds (environment: %s)",
                   env_timeout, (env? env : "none"));
     }
 
     calculated_timeout = 1 + crm_active_peers();
     if (crm_remote_peer_cache) {
         calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
     }
     calculated_timeout *= 10;
 
     calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
     crm_trace("Calculated timeout: %us", calculated_timeout);
 
     if (controld_globals.cib_conn) {
         controld_globals.cib_conn->call_timeout = calculated_timeout;
     }
     return calculated_timeout;
 }
 
 /*!
  * \internal
  * \brief Get CIB call options to use local scope if primary is unavailable
  *
  * \return CIB call options
  */
 int
 crmd_cib_smart_opt(void)
 {
     int call_opt = cib_none;
 
     if ((controld_globals.fsa_state == S_ELECTION)
         || (controld_globals.fsa_state == S_PENDING)) {
         crm_info("Sending update to local CIB in state: %s",
                  fsa_state2string(controld_globals.fsa_state));
         cib__set_call_options(call_opt, "update", cib_scope_local);
     }
     return call_opt;
 }
 
 static void
 cib_delete_callback(xmlNode *msg, int call_id, int rc, xmlNode *output,
                     void *user_data)
 {
     char *desc = user_data;
 
     if (rc == 0) {
         crm_debug("Deletion of %s (via CIB call %d) succeeded", desc, call_id);
     } else {
         crm_warn("Deletion of %s (via CIB call %d) failed: %s " CRM_XS " rc=%d",
                  desc, call_id, pcmk_strerror(rc), rc);
     }
 }
 
 // Searches for various portions of node_state to delete
 
 // Match a particular node's node_state (takes node name 1x)
 #define XPATH_NODE_STATE        "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"
 
 // Node's lrm section (name 1x)
 #define XPATH_NODE_LRM          XPATH_NODE_STATE "/" XML_CIB_TAG_LRM
 
 /* Node's lrm_rsc_op entries and lrm_resource entries without unexpired lock
  * (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x)
  */
 #define XPATH_NODE_LRM_UNLOCKED XPATH_NODE_STATE "//" XML_LRM_TAG_RSC_OP    \
                                 "|" XPATH_NODE_STATE                        \
                                 "//" XML_LRM_TAG_RESOURCE                   \
                                 "[not(@" PCMK_OPT_SHUTDOWN_LOCK ") "       \
                                     "or " PCMK_OPT_SHUTDOWN_LOCK "<%lld]"
 
 // Node's transient_attributes section (name 1x)
 #define XPATH_NODE_ATTRS        XPATH_NODE_STATE "/" XML_TAG_TRANSIENT_NODEATTRS
 
 // Everything under node_state (name 1x)
 #define XPATH_NODE_ALL          XPATH_NODE_STATE "/*"
 
 /* Unlocked history + transient attributes
  * (name 2x, (seconds_since_epoch - PCMK_OPT_SHUTDOWN_LOCK_LIMIT) 1x, name 1x)
  */
 #define XPATH_NODE_ALL_UNLOCKED XPATH_NODE_LRM_UNLOCKED "|" XPATH_NODE_ATTRS
 
 /*!
  * \internal
  * \brief Get the XPath and description of a node state section to be deleted
  *
  * \param[in]  uname    Desired node
  * \param[in]  section  Subsection of node_state to be deleted
  * \param[out] xpath    Where to store XPath of \p section
  * \param[out] desc     If not \c NULL, where to store description of \p section
  */
 void
 controld_node_state_deletion_strings(const char *uname,
                                      enum controld_section_e section,
                                      char **xpath, char **desc)
 {
     const char *desc_pre = NULL;
 
     // Shutdown locks that started before this time are expired
     long long expire = (long long) time(NULL)
                        - controld_globals.shutdown_lock_limit;
 
     switch (section) {
         case controld_section_lrm:
             *xpath = crm_strdup_printf(XPATH_NODE_LRM, uname);
             desc_pre = "resource history";
             break;
         case controld_section_lrm_unlocked:
             *xpath = crm_strdup_printf(XPATH_NODE_LRM_UNLOCKED,
                                        uname, uname, expire);
             desc_pre = "resource history (other than shutdown locks)";
             break;
         case controld_section_attrs:
             *xpath = crm_strdup_printf(XPATH_NODE_ATTRS, uname);
             desc_pre = "transient attributes";
             break;
         case controld_section_all:
             *xpath = crm_strdup_printf(XPATH_NODE_ALL, uname);
             desc_pre = "all state";
             break;
         case controld_section_all_unlocked:
             *xpath = crm_strdup_printf(XPATH_NODE_ALL_UNLOCKED,
                                        uname, uname, expire, uname);
             desc_pre = "all state (other than shutdown locks)";
             break;
         default:
             // We called this function incorrectly
             CRM_ASSERT(false);
             break;
     }
 
     if (desc != NULL) {
         *desc = crm_strdup_printf("%s for node %s", desc_pre, uname);
     }
 }
 
 /*!
  * \internal
  * \brief Delete subsection of a node's CIB node_state
  *
  * \param[in] uname    Desired node
  * \param[in] section  Subsection of node_state to delete
  * \param[in] options  CIB call options to use
  */
 void
 controld_delete_node_state(const char *uname, enum controld_section_e section,
                            int options)
 {
     cib_t *cib = controld_globals.cib_conn;
     char *xpath = NULL;
     char *desc = NULL;
     int cib_rc = pcmk_ok;
 
     CRM_ASSERT((uname != NULL) && (cib != NULL));
 
     controld_node_state_deletion_strings(uname, section, &xpath, &desc);
 
     cib__set_call_options(options, "node state deletion",
                           cib_xpath|cib_multiple);
     cib_rc = cib->cmds->remove(cib, xpath, NULL, options);
     fsa_register_cib_callback(cib_rc, desc, cib_delete_callback);
     crm_info("Deleting %s (via CIB call %d) " CRM_XS " xpath=%s",
              desc, cib_rc, xpath);
 
     // CIB library handles freeing desc
     free(xpath);
 }
 
 // Takes node name and resource ID
 #define XPATH_RESOURCE_HISTORY "//" XML_CIB_TAG_STATE                       \
                                "[@" XML_ATTR_UNAME "='%s']/"                \
                                XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES    \
                                "/" XML_LRM_TAG_RESOURCE                     \
                                "[@" PCMK_XA_ID "='%s']"
 // @TODO could add "and @PCMK_OPT_SHUTDOWN_LOCK" to limit to locks
 
 /*!
  * \internal
  * \brief Clear resource history from CIB for a given resource and node
  *
  * \param[in]  rsc_id        ID of resource to be cleared
  * \param[in]  node          Node whose resource history should be cleared
  * \param[in]  user_name     ACL user name to use
  * \param[in]  call_options  CIB call options
  *
  * \return Standard Pacemaker return code
  */
 int
 controld_delete_resource_history(const char *rsc_id, const char *node,
                                  const char *user_name, int call_options)
 {
     char *desc = NULL;
     char *xpath = NULL;
     int rc = pcmk_rc_ok;
     cib_t *cib = controld_globals.cib_conn;
 
     CRM_CHECK((rsc_id != NULL) && (node != NULL), return EINVAL);
 
     desc = crm_strdup_printf("resource history for %s on %s", rsc_id, node);
     if (cib == NULL) {
         crm_err("Unable to clear %s: no CIB connection", desc);
         free(desc);
         return ENOTCONN;
     }
 
     // Ask CIB to delete the entry
     xpath = crm_strdup_printf(XPATH_RESOURCE_HISTORY, node, rsc_id);
 
     cib->cmds->set_user(cib, user_name);
     rc = cib->cmds->remove(cib, xpath, NULL, call_options|cib_xpath);
     cib->cmds->set_user(cib, NULL);
 
     if (rc < 0) {
         rc = pcmk_legacy2rc(rc);
         crm_err("Could not delete resource status of %s on %s%s%s: %s "
                 CRM_XS " rc=%d", rsc_id, node,
                 (user_name? " for user " : ""), (user_name? user_name : ""),
                 pcmk_rc_str(rc), rc);
         free(desc);
         free(xpath);
         return rc;
     }
 
     if (pcmk_is_set(call_options, cib_sync_call)) {
         if (pcmk_is_set(call_options, cib_dryrun)) {
             crm_debug("Deletion of %s would succeed", desc);
         } else {
             crm_debug("Deletion of %s succeeded", desc);
         }
         free(desc);
 
     } else {
         crm_info("Clearing %s (via CIB call %d) " CRM_XS " xpath=%s",
                  desc, rc, xpath);
         fsa_register_cib_callback(rc, desc, cib_delete_callback);
         // CIB library handles freeing desc
     }
 
     free(xpath);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Build XML and string of parameters meeting some criteria, for digest
  *
  * \param[in]  op          Executor event with parameter table to use
  * \param[in]  metadata    Parsed meta-data for executed resource agent
  * \param[in]  param_type  Flag used for selection criteria
  * \param[out] result      Will be set to newly created XML with selected
  *                         parameters as attributes
  *
  * \return Newly allocated space-separated string of parameter names
  * \note Selection criteria varies by param_type: for the restart digest, we
  *       want parameters that are *not* marked reloadable (OCF 1.1) or that
  *       *are* marked unique (pre-1.1), for both string and XML results; for the
  *       secure digest, we want parameters that *are* marked private for the
  *       string, but parameters that are *not* marked private for the XML.
  * \note It is the caller's responsibility to free the string return value with
  *       \p g_string_free() and the XML result with \p free_xml().
  */
 static GString *
 build_parameter_list(const lrmd_event_data_t *op,
                      const struct ra_metadata_s *metadata,
                      enum ra_param_flags_e param_type, xmlNode **result)
 {
     GString *list = NULL;
 
     *result = create_xml_node(NULL, XML_TAG_PARAMS);
 
     /* Consider all parameters only except private ones to be consistent with
      * what scheduler does with calculate_secure_digest().
      */
     if (param_type == ra_param_private
         && compare_version(controld_globals.dc_version, "3.16.0") >= 0) {
         g_hash_table_foreach(op->params, hash2field, *result);
         pcmk__filter_op_for_digest(*result);
     }
 
     for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
         struct ra_param_s *param = (struct ra_param_s *) iter->data;
 
         bool accept_for_list = false;
         bool accept_for_xml = false;
 
         switch (param_type) {
             case ra_param_reloadable:
                 accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
                 accept_for_xml = accept_for_list;
                 break;
 
             case ra_param_unique:
                 accept_for_list = pcmk_is_set(param->rap_flags, param_type);
                 accept_for_xml = accept_for_list;
                 break;
 
             case ra_param_private:
                 accept_for_list = pcmk_is_set(param->rap_flags, param_type);
                 accept_for_xml = !accept_for_list;
                 break;
         }
 
         if (accept_for_list) {
             crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
 
             if (list == NULL) {
                 // We will later search for " WORD ", so start list with a space
                 pcmk__add_word(&list, 256, " ");
             }
             pcmk__add_word(&list, 0, param->rap_name);
 
         } else {
             crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
         }
 
         if (accept_for_xml) {
             const char *v = g_hash_table_lookup(op->params, param->rap_name);
 
             if (v != NULL) {
                 crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
                 crm_xml_add(*result, param->rap_name, v);
             }
 
         } else {
             crm_trace("Removing attr %s from the xml result", param->rap_name);
             xml_remove_prop(*result, param->rap_name);
         }
     }
 
     if (list != NULL) {
         // We will later search for " WORD ", so end list with a space
         pcmk__add_word(&list, 0, " ");
     }
     return list;
 }
 
 static void
 append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                     xmlNode *update, const char *version)
 {
     GString *list = NULL;
     char *digest = NULL;
     xmlNode *restart = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
     if (op->interval_ms > 0) {
         /* monitors are not reloadable */
         return;
     }
 
     if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
         // Add parameters not marked reloadable to the "op-force-restart" list
         list = build_parameter_list(op, metadata, ra_param_reloadable,
                                     &restart);
 
     } else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
         /* @COMPAT pre-OCF-1.1 resource agents
          *
          * Before OCF 1.1, Pacemaker abused "unique=0" to indicate
          * reloadability. Add any parameters with unique="1" to the
          * "op-force-restart" list.
          */
         list = build_parameter_list(op, metadata, ra_param_unique, &restart);
 
     } else {
         // Resource does not support agent reloads
         return;
     }
 
     digest = calculate_operation_digest(restart, version);
     /* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
      * no matter if it actually supports any parameters with unique="1"). */
     crm_xml_add(update, XML_LRM_ATTR_OP_RESTART,
                 (list == NULL)? "" : (const char *) list->str);
     crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
 
     if ((list != NULL) && (list->len > 0)) {
         crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
     } else {
         crm_trace("%s: %s", op->rsc_id, digest);
     }
 
     if (list != NULL) {
         g_string_free(list, TRUE);
     }
     free_xml(restart);
     free(digest);
 }
 
 static void
 append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                    xmlNode *update, const char *version)
 {
     GString *list = NULL;
     char *digest = NULL;
     xmlNode *secure = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
     /*
      * To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
      * secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
      * the insecure ones
      */
     list = build_parameter_list(op, metadata, ra_param_private, &secure);
 
     if (list != NULL) {
         digest = calculate_operation_digest(secure, version);
         crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, (const char *) list->str);
         crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
 
         crm_trace("%s: %s, %s", op->rsc_id, digest, (const char *) list->str);
         g_string_free(list, TRUE);
     } else {
         crm_trace("%s: no secure parameters", op->rsc_id);
     }
 
     free_xml(secure);
     free(digest);
 }
 
 /*!
  * \internal
  * \brief Create XML for a resource history entry
  *
  * \param[in]     func       Function name of caller
  * \param[in,out] parent     XML to add entry to
  * \param[in]     rsc        Affected resource
  * \param[in,out] op         Action to add an entry for (or NULL to do nothing)
  * \param[in]     node_name  Node where action occurred
  */
 void
 controld_add_resource_history_xml_as(const char *func, xmlNode *parent,
                                      const lrmd_rsc_info_t *rsc,
                                      lrmd_event_data_t *op,
                                      const char *node_name)
 {
     int target_rc = 0;
     xmlNode *xml_op = NULL;
     struct ra_metadata_s *metadata = NULL;
     const char *caller_version = NULL;
     lrm_state_t *lrm_state = NULL;
 
     if (op == NULL) {
         return;
     }
 
     target_rc = rsc_op_expected_rc(op);
 
     caller_version = g_hash_table_lookup(op->params, PCMK_XA_CRM_FEATURE_SET);
     CRM_CHECK(caller_version != NULL, caller_version = CRM_FEATURE_SET);
 
     xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
                                       controld_globals.our_nodename, func);
     if (xml_op == NULL) {
         return;
     }
 
     if ((rsc == NULL) || (op->params == NULL)
         || !crm_op_needs_metadata(rsc->standard, op->op_type)) {
 
         crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
                   op->op_type, op->rsc_id, op->params, rsc);
         return;
     }
 
     lrm_state = lrm_state_find(node_name);
     if (lrm_state == NULL) {
         crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
                  " because we have no connection to executor for %s",
                  op->rsc_id, op->op_type, op->interval_ms, node_name);
         return;
     }
 
     /* Ideally the metadata is cached, and the agent is just a fallback.
      *
      * @TODO Go through all callers and ensure they get metadata asynchronously
      * first.
      */
     metadata = controld_get_rsc_metadata(lrm_state, rsc,
                                          controld_metadata_from_agent
                                          |controld_metadata_from_cache);
     if (metadata == NULL) {
         return;
     }
 
     crm_trace("Including additional digests for %s:%s:%s",
               rsc->standard, rsc->provider, rsc->type);
     append_restart_list(op, metadata, xml_op, caller_version);
     append_secure_list(op, metadata, xml_op, caller_version);
 
     return;
 }
 
 /*!
  * \internal
  * \brief Record an action as pending in the CIB, if appropriate
  *
  * \param[in]     node_name  Node where the action is pending
  * \param[in]     rsc        Resource that action is for
  * \param[in,out] op         Pending action
  *
  * \return true if action was recorded in CIB, otherwise false
  */
 bool
 controld_record_pending_op(const char *node_name, const lrmd_rsc_info_t *rsc,
                            lrmd_event_data_t *op)
 {
     const char *record_pending = NULL;
 
     CRM_CHECK((node_name != NULL) && (rsc != NULL) && (op != NULL),
               return false);
 
     // Never record certain operation types as pending
     if ((op->op_type == NULL) || (op->params == NULL)
         || !controld_action_is_recordable(op->op_type)) {
         return false;
     }
 
     // Check action's record-pending meta-attribute (defaults to true)
     record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
     if ((record_pending != NULL) && !crm_is_true(record_pending)) {
         return false;
     }
 
     op->call_id = -1;
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
 
     crm_debug("Recording pending %s-interval %s for %s on %s in the CIB",
               pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
               node_name);
     controld_update_resource_history(node_name, rsc, op, 0);
     return true;
 }
 
 static void
 cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     switch (rc) {
         case pcmk_ok:
         case -pcmk_err_diff_failed:
         case -pcmk_err_diff_resync:
             crm_trace("Resource history update completed (call=%d rc=%d)",
                       call_id, rc);
             break;
         default:
             if (call_id > 0) {
                 crm_warn("Resource history update %d failed: %s "
                          CRM_XS " rc=%d", call_id, pcmk_strerror(rc), rc);
             } else {
                 crm_warn("Resource history update failed: %s " CRM_XS " rc=%d",
                          pcmk_strerror(rc), rc);
             }
     }
 
     if (call_id == pending_rsc_update) {
         pending_rsc_update = 0;
         controld_trigger_fsa();
     }
 }
 
 /* Only successful stops, and probes that found the resource inactive, get locks
  * recorded in the history. This ensures the resource stays locked to the node
  * until it is active there again after the node comes back up.
  */
 static bool
 should_preserve_lock(lrmd_event_data_t *op)
 {
     if (!pcmk_is_set(controld_globals.flags, controld_shutdown_lock_enabled)) {
         return false;
     }
     if (!strcmp(op->op_type, PCMK_ACTION_STOP) && (op->rc == PCMK_OCF_OK)) {
         return true;
     }
     if (!strcmp(op->op_type, PCMK_ACTION_MONITOR)
         && (op->rc == PCMK_OCF_NOT_RUNNING)) {
         return true;
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Request a CIB update
  *
  * \param[in]     section    Section of CIB to update
  * \param[in]     data       New XML of CIB section to update
  * \param[in]     options    CIB call options
  * \param[in]     callback   If not \c NULL, set this as the operation callback
  *
  * \return Standard Pacemaker return code
  *
  * \note If \p callback is \p cib_rsc_callback(), the CIB update's call ID is
  *       stored in \p pending_rsc_update on success.
  */
 int
 controld_update_cib(const char *section, xmlNode *data, int options,
                     void (*callback)(xmlNode *, int, int, xmlNode *, void *))
 {
     cib_t *cib = controld_globals.cib_conn;
     int cib_rc = -ENOTCONN;
 
     CRM_ASSERT(data != NULL);
 
     if (cib != NULL) {
         cib_rc = cib->cmds->modify(cib, section, data, options);
         if (cib_rc >= 0) {
             crm_debug("Submitted CIB update %d for %s section",
                       cib_rc, section);
         }
     }
 
     if (callback == NULL) {
         if (cib_rc < 0) {
             crm_err("Failed to update CIB %s section: %s",
                     section, pcmk_rc_str(pcmk_legacy2rc(cib_rc)));
         }
 
     } else {
         if ((cib_rc >= 0) && (callback == cib_rsc_callback)) {
             /* Checking for a particular callback is a little hacky, but it
              * didn't seem worth adding an output argument for cib_rc for just
              * one use case.
              */
             pending_rsc_update = cib_rc;
         }
         fsa_register_cib_callback(cib_rc, NULL, callback);
     }
 
     return (cib_rc >= 0)? pcmk_rc_ok : pcmk_legacy2rc(cib_rc);
 }
 
 /*!
  * \internal
  * \brief Update resource history entry in CIB
  *
  * \param[in]     node_name  Node where action occurred
  * \param[in]     rsc        Resource that action is for
  * \param[in,out] op         Action to record
  * \param[in]     lock_time  If nonzero, when resource was locked to node
  *
  * \note On success, the CIB update's call ID will be stored in
  *       pending_rsc_update.
  */
 void
 controld_update_resource_history(const char *node_name,
                                  const lrmd_rsc_info_t *rsc,
                                  lrmd_event_data_t *op, time_t lock_time)
 {
     xmlNode *update = NULL;
     xmlNode *xml = NULL;
     int call_opt = crmd_cib_smart_opt();
     const char *node_id = NULL;
     const char *container = NULL;
 
     CRM_CHECK((node_name != NULL) && (op != NULL), return);
 
     if (rsc == NULL) {
         crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
         controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
         return;
     }
 
     // <status>
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
 
     //   <node_state ...>
     xml = create_xml_node(update, XML_CIB_TAG_STATE);
     if (pcmk__str_eq(node_name, controld_globals.our_nodename,
                      pcmk__str_casei)) {
         node_id = controld_globals.our_uuid;
     } else {
         node_id = node_name;
         pcmk__xe_set_bool_attr(xml, XML_NODE_IS_REMOTE, true);
     }
     crm_xml_add(xml, PCMK_XA_ID, node_id);
     crm_xml_add(xml, XML_ATTR_UNAME, node_name);
     crm_xml_add(xml, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);
 
     //     <lrm ...>
     xml = create_xml_node(xml, XML_CIB_TAG_LRM);
     crm_xml_add(xml, PCMK_XA_ID, node_id);
 
     //       <lrm_resources>
     xml = create_xml_node(xml, XML_LRM_TAG_RESOURCES);
 
     //         <lrm_resource ...>
     xml = create_xml_node(xml, XML_LRM_TAG_RESOURCE);
     crm_xml_add(xml, PCMK_XA_ID, op->rsc_id);
     crm_xml_add(xml, XML_AGENT_ATTR_CLASS, rsc->standard);
-    crm_xml_add(xml, XML_AGENT_ATTR_PROVIDER, rsc->provider);
+    crm_xml_add(xml, PCMK_XA_PROVIDER, rsc->provider);
     crm_xml_add(xml, PCMK_XA_TYPE, rsc->type);
     if (lock_time != 0) {
         /* Actions on a locked resource should either preserve the lock by
          * recording it with the action result, or clear it.
          */
         if (!should_preserve_lock(op)) {
             lock_time = 0;
         }
         crm_xml_add_ll(xml, PCMK_OPT_SHUTDOWN_LOCK, (long long) lock_time);
     }
     if (op->params != NULL) {
         container = g_hash_table_lookup(op->params,
                                         CRM_META "_" PCMK__META_CONTAINER);
         if (container != NULL) {
             crm_trace("Resource %s is a part of container resource %s",
                       op->rsc_id, container);
             crm_xml_add(xml, PCMK__META_CONTAINER, container);
         }
     }
 
     //           <lrm_resource_op ...> (possibly more than one)
     controld_add_resource_history_xml(xml, rsc, op, node_name);
 
     /* Update CIB asynchronously. Even if it fails, the resource state should be
      * discovered during the next election. Worst case, the node is wrongly
      * fenced for running a resource it isn't.
      */
     crm_log_xml_trace(update, __func__);
     controld_update_cib(XML_CIB_TAG_STATUS, update, call_opt, cib_rsc_callback);
     free_xml(update);
 }
 
 /*!
  * \internal
  * \brief Erase an LRM history entry from the CIB, given the operation data
  *
  * \param[in] op         Operation whose history should be deleted
  */
 void
 controld_delete_action_history(const lrmd_event_data_t *op)
 {
     xmlNode *xml_top = NULL;
 
     CRM_CHECK(op != NULL, return);
 
     xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
     crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
     crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
 
     if (op->interval_ms > 0) {
         char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
 
         /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
         crm_xml_add(xml_top, PCMK_XA_ID, op_id);
         free(op_id);
     }
 
     crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
               op->rsc_id, op->op_type, op->interval_ms, op->call_id);
 
     controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn,
                                             XML_CIB_TAG_STATUS, xml_top,
                                             cib_none);
     crm_log_xml_trace(xml_top, "op:cancel");
     free_xml(xml_top);
 }
 
 /* Define xpath to find LRM resource history entry by node and resource */
 #define XPATH_HISTORY                                   \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS              \
     "/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"  \
     "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES       \
     "/" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']"   \
     "/" XML_LRM_TAG_RSC_OP
 
 /* ... and also by operation key */
 #define XPATH_HISTORY_ID XPATH_HISTORY "[@" PCMK_XA_ID "='%s']"
 
 /* ... and also by operation key and operation call ID */
 #define XPATH_HISTORY_CALL XPATH_HISTORY \
     "[@" PCMK_XA_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
 
 /* ... and also by operation key and original operation key */
 #define XPATH_HISTORY_ORIG XPATH_HISTORY \
     "[@" PCMK_XA_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
 
 /*!
  * \internal
  * \brief Delete a last_failure resource history entry from the CIB
  *
  * \param[in] rsc_id       Name of resource to clear history for
  * \param[in] node         Name of node to clear history for
  * \param[in] action       If specified, delete only if this was failed action
  * \param[in] interval_ms  If \p action is specified, it has this interval
  */
 void
 controld_cib_delete_last_failure(const char *rsc_id, const char *node,
                                  const char *action, guint interval_ms)
 {
     char *xpath = NULL;
     char *last_failure_key = NULL;
     CRM_CHECK((rsc_id != NULL) && (node != NULL), return);
 
     // Generate XPath to match desired entry
     last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
     if (action == NULL) {
         xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id,
                                   last_failure_key);
     } else {
         char *action_key = pcmk__op_key(rsc_id, action, interval_ms);
 
         xpath = crm_strdup_printf(XPATH_HISTORY_ORIG, node, rsc_id,
                                   last_failure_key, action_key);
         free(action_key);
     }
     free(last_failure_key);
 
     controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
                                             NULL, cib_xpath);
     free(xpath);
 }
 
 /*!
  * \internal
  * \brief Delete resource history entry from the CIB, given operation key
  *
  * \param[in] rsc_id     Name of resource to clear history for
  * \param[in] node       Name of node to clear history for
  * \param[in] key        Operation key of operation to clear history for
  * \param[in] call_id    If specified, delete entry only if it has this call ID
  */
 void
 controld_delete_action_history_by_key(const char *rsc_id, const char *node,
                                       const char *key, int call_id)
 {
     char *xpath = NULL;
 
     CRM_CHECK((rsc_id != NULL) && (node != NULL) && (key != NULL), return);
 
     if (call_id > 0) {
         xpath = crm_strdup_printf(XPATH_HISTORY_CALL, node, rsc_id, key,
                                   call_id);
     } else {
         xpath = crm_strdup_printf(XPATH_HISTORY_ID, node, rsc_id, key);
     }
     controld_globals.cib_conn->cmds->remove(controld_globals.cib_conn, xpath,
                                             NULL, cib_xpath);
     free(xpath);
 }
diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c
index 9c5756126e..77c3c31808 100644
--- a/daemons/controld/controld_execd.c
+++ b/daemons/controld/controld_execd.c
@@ -1,2446 +1,2446 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <regex.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>           // lrmd_event_data_t, lrmd_rsc_info_t, etc.
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/pengine/rules.h>
 #include <crm/lrmd_internal.h>
 
 #include <pacemaker-internal.h>
 #include <pacemaker-controld.h>
 
 #define START_DELAY_THRESHOLD 5 * 60 * 1000
 #define MAX_LRM_REG_FAILS 30
 
 struct delete_event_s {
     int rc;
     const char *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
 
 static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state,
                                        const xmlNode *rsc_op,
                                        const char *rsc_id,
                                        const char *operation);
 static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
                           xmlNode *msg, struct ra_metadata_s *md);
 
 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
                                          int log_level);
 
 static void
 lrm_connection_destroy(void)
 {
     if (pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) {
         crm_crit("Lost connection to local executor");
         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
     }
 }
 
 static char *
 make_stop_id(const char *rsc, int call_id)
 {
     return crm_strdup_printf("%s:%d", rsc, call_id);
 }
 
 static void
 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") == NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 static void
 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") != NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 /*!
  * \internal
  * \brief Remove a recurring operation from a resource's history
  *
  * \param[in,out] history  Resource history to modify
  * \param[in]     op       Operation to remove
  *
  * \return TRUE if the operation was found and removed, FALSE otherwise
  */
 static gboolean
 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_event_data_t *existing = iter->data;
 
         if ((op->interval_ms == existing->interval_ms)
             && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
             && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
 
             history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
             lrmd_free_event(existing);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Free all recurring operations in resource history
  *
  * \param[in,out] history  Resource history to modify
  */
 static void
 history_free_recurring_ops(rsc_history_t *history)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_free_event(iter->data);
     }
     g_list_free(history->recurring_op_list);
     history->recurring_op_list = NULL;
 }
 
 /*!
  * \internal
  * \brief Free resource history
  *
  * \param[in,out] history  Resource history to free
  */
 void
 history_free(gpointer data)
 {
     rsc_history_t *history = (rsc_history_t*)data;
 
     if (history->stop_params) {
         g_hash_table_destroy(history->stop_params);
     }
 
     /* Don't need to free history->rsc.id because it's set to history->id */
     free(history->rsc.type);
     free(history->rsc.standard);
     free(history->rsc.provider);
 
     lrmd_free_event(history->failed);
     lrmd_free_event(history->last);
     free(history->id);
     history_free_recurring_ops(history);
     free(history);
 }
 
 static void
 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
     int target_rc = 0;
     rsc_history_t *entry = NULL;
 
     if (op->rsc_deleted) {
         crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
         controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
                                          NULL, crmd_cib_smart_opt());
         return;
     }
 
     if (pcmk__str_eq(op->op_type, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
         return;
     }
 
     crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
 
     entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
     if (entry == NULL && rsc) {
         entry = calloc(1, sizeof(rsc_history_t));
         entry->id = strdup(op->rsc_id);
         g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
 
         entry->rsc.id = entry->id;
         entry->rsc.type = strdup(rsc->type);
         entry->rsc.standard = strdup(rsc->standard);
         pcmk__str_update(&entry->rsc.provider, rsc->provider);
 
     } else if (entry == NULL) {
         crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
         return;
     }
 
     entry->last_callid = op->call_id;
     target_rc = rsc_op_expected_rc(op);
     if (op->op_status == PCMK_EXEC_CANCELLED) {
         if (op->interval_ms > 0) {
             crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
                       op->rsc_id, op->op_type, op->interval_ms);
             history_remove_recurring_op(entry, op);
             return;
         } else {
             crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
                       op->rsc_id, op->op_type, op->interval_ms, op->rc,
                       op->op_status);
         }
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         /* Store failed monitors here, otherwise the block below will cause them
          * to be forgotten when a stop happens.
          */
         if (entry->failed) {
             lrmd_free_event(entry->failed);
         }
         entry->failed = lrmd_copy_event(op);
 
     } else if (op->interval_ms == 0) {
         if (entry->last) {
             lrmd_free_event(entry->last);
         }
         entry->last = lrmd_copy_event(op);
 
         if (op->params && pcmk__strcase_any_of(op->op_type, PCMK_ACTION_START,
                                                PCMK_ACTION_RELOAD,
                                                PCMK_ACTION_RELOAD_AGENT,
                                                PCMK_ACTION_MONITOR, NULL)) {
             if (entry->stop_params) {
                 g_hash_table_destroy(entry->stop_params);
             }
             entry->stop_params = pcmk__strkey_table(free, free);
 
             g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
         }
     }
 
     if (op->interval_ms > 0) {
         /* Ensure there are no duplicates */
         history_remove_recurring_op(entry, op);
 
         crm_trace("Adding recurring op: " PCMK__OP_FMT,
                   op->rsc_id, op->op_type, op->interval_ms);
         entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
 
     } else if ((entry->recurring_op_list != NULL)
                 && !pcmk__str_eq(op->op_type, PCMK_ACTION_MONITOR,
                                  pcmk__str_casei)) {
         crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
                   g_list_length(entry->recurring_op_list), op->rsc_id,
                   op->op_type, op->interval_ms);
         history_free_recurring_ops(entry);
     }
 }
 
 /*!
  * \internal
  * \brief Send a direct OK ack for a resource task
  *
  * \param[in] lrm_state  LRM connection
  * \param[in] input      Input message being ack'ed
  * \param[in] rsc_id     ID of affected resource
  * \param[in] rsc        Affected resource (if available)
  * \param[in] task       Operation task being ack'ed
  * \param[in] ack_host   Name of host to send ack to
  * \param[in] ack_sys    IPC system name to ack
  */
 static void
 send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input,
                  const char *rsc_id, const lrmd_rsc_info_t *rsc,
                  const char *task, const char *ack_host, const char *ack_sys)
 {
     lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
 
     lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
     controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
     lrmd_free_event(op);
 }
 
 static inline const char *
 op_node_name(lrmd_event_data_t *op)
 {
     return pcmk__s(op->remote_nodename, controld_globals.our_nodename);
 }
 
 void
 lrm_op_callback(lrmd_event_data_t * op)
 {
     CRM_CHECK(op != NULL, return);
     switch (op->type) {
         case lrmd_event_disconnect:
             if (op->remote_nodename == NULL) {
                 /* If this is the local executor IPC connection, set the right
                  * bits in the controller when the connection goes down.
                  */
                 lrm_connection_destroy();
             }
             break;
 
         case lrmd_event_exec_complete:
             {
                 lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
 
                 CRM_ASSERT(lrm_state != NULL);
                 process_lrm_event(lrm_state, op, NULL, NULL);
             }
             break;
 
         default:
             break;
     }
 }
 
 static void
 try_local_executor_connect(long long action, fsa_data_t *msg_data,
                            lrm_state_t *lrm_state)
 {
     int rc = pcmk_rc_ok;
 
     crm_debug("Connecting to the local executor");
 
     // If we can connect, great
     rc = controld_connect_local_executor(lrm_state);
     if (rc == pcmk_rc_ok) {
         controld_set_fsa_input_flags(R_LRM_CONNECTED);
         crm_info("Connection to the local executor established");
         return;
     }
 
     // Otherwise, if we can try again, set a timer to do so
     if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
         crm_warn("Failed to connect to the local executor %d time%s "
                  "(%d max): %s", lrm_state->num_lrm_register_fails,
                  pcmk__plural_s(lrm_state->num_lrm_register_fails),
                  MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
         controld_start_wait_timer();
         crmd_fsa_stall(FALSE);
         return;
     }
 
     // Otherwise give up
     crm_err("Failed to connect to the executor the max allowed "
             "%d time%s: %s", lrm_state->num_lrm_register_fails,
             pcmk__plural_s(lrm_state->num_lrm_register_fails),
             pcmk_rc_str(rc));
     register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 }
 
 /*	 A_LRM_CONNECT	*/
 void
 do_lrm_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* This only pertains to local executor connections. Remote connections are
      * handled as resources within the scheduler. Connecting and disconnecting
      * from remote executor instances is handled differently.
      */
 
     lrm_state_t *lrm_state = NULL;
 
     if (controld_globals.our_nodename == NULL) {
         return; /* Nothing to do */
     }
     lrm_state = lrm_state_find_or_create(controld_globals.our_nodename);
     if (lrm_state == NULL) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (action & A_LRM_DISCONNECT) {
         if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
             if (action == A_LRM_DISCONNECT) {
                 crmd_fsa_stall(FALSE);
                 return;
             }
         }
 
         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
         lrm_state_disconnect(lrm_state);
         lrm_state_reset_tables(lrm_state, FALSE);
     }
 
     if (action & A_LRM_CONNECT) {
         try_local_executor_connect(action, msg_data, lrm_state);
     }
 
     if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action),
                 __func__);
     }
 }
 
 static gboolean
 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
 {
     int counter = 0;
     gboolean rc = TRUE;
     const char *when = "lrm disconnect";
 
     GHashTableIter gIter;
     const char *key = NULL;
     rsc_history_t *entry = NULL;
     active_op_t *pending = NULL;
 
     crm_debug("Checking for active resources before exit");
 
     if (cur_state == S_TERMINATE) {
         log_level = LOG_ERR;
         when = "shutdown";
 
     } else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
         when = "shutdown... waiting";
     }
 
     if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) {
         guint removed = g_hash_table_foreach_remove(lrm_state->active_ops,
                                                     stop_recurring_actions,
                                                     lrm_state);
         guint nremaining = g_hash_table_size(lrm_state->active_ops);
 
         if (removed || nremaining) {
             crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
                        removed, pcmk__plural_s(removed), when, nremaining);
         }
     }
 
     if (lrm_state->active_ops != NULL) {
         g_hash_table_iter_init(&gIter, lrm_state->active_ops);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
             /* Ignore recurring actions in the shutdown calculations */
             if (pending->interval_ms == 0) {
                 counter++;
             }
         }
     }
 
     if (counter > 0) {
         do_crm_log(log_level, "%d pending executor operation%s at %s",
                    counter, pcmk__plural_s(counter), when);
 
         if ((cur_state == S_TERMINATE)
             || !pcmk_is_set(controld_globals.fsa_input_register,
                             R_SENT_RSC_STOP)) {
             g_hash_table_iter_init(&gIter, lrm_state->active_ops);
             while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
                 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
             }
 
         } else {
             rc = FALSE;
         }
         return rc;
     }
 
     if (lrm_state->resource_history == NULL) {
         return rc;
     }
 
     if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
         /* At this point we're not waiting, we're just shutting down */
         when = "shutdown";
     }
 
     counter = 0;
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
         if (is_rsc_active(lrm_state, entry->id) == FALSE) {
             continue;
         }
 
         counter++;
         if (log_level == LOG_ERR) {
             crm_info("Found %s active at %s", entry->id, when);
         } else {
             crm_trace("Found %s active at %s", entry->id, when);
         }
         if (lrm_state->active_ops != NULL) {
             GHashTableIter hIter;
 
             g_hash_table_iter_init(&hIter, lrm_state->active_ops);
             while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
                 if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
                     crm_notice("%sction %s (%s) incomplete at %s",
                                pending->interval_ms == 0 ? "A" : "Recurring a",
                                key, pending->op_key, when);
                 }
             }
         }
     }
 
     if (counter) {
         crm_err("%d resource%s active at %s",
                 counter, (counter == 1)? " was" : "s were", when);
     }
 
     return rc;
 }
 
 static gboolean
 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
 {
     rsc_history_t *entry = NULL;
 
     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     if (entry == NULL || entry->last == NULL) {
         return FALSE;
     }
 
     crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
               entry->last->interval_ms, entry->last->rc);
     if ((entry->last->rc == PCMK_OCF_OK)
         && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_STOP,
                         pcmk__str_casei)) {
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_OK
                && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_MIGRATE_TO,
                                pcmk__str_casei)) {
         // A stricter check is too complex ... leave that to the scheduler
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
         return FALSE;
 
     } else if ((entry->last->interval_ms == 0)
                && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
         /* Badly configured resources can't be reliably stopped */
         return FALSE;
     }
 
     return TRUE;
 }
 
 static gboolean
 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
 {
     GHashTableIter iter;
     rsc_history_t *entry = NULL;
 
     g_hash_table_iter_init(&iter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
 
         GList *gIter = NULL;
         xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
 
         crm_xml_add(xml_rsc, PCMK_XA_ID, entry->id);
         crm_xml_add(xml_rsc, PCMK_XA_TYPE, entry->rsc.type);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
-        crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
+        crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, entry->rsc.provider);
 
         if (entry->last && entry->last->params) {
             static const char *name = CRM_META "_" PCMK__META_CONTAINER;
             const char *container = g_hash_table_lookup(entry->last->params,
                                                         name);
 
             if (container) {
                 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
                 crm_xml_add(xml_rsc, PCMK__META_CONTAINER, container);
             }
         }
         controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed,
                                           lrm_state->node_name);
         controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last,
                                           lrm_state->node_name);
         for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
             controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data,
                                               lrm_state->node_name);
         }
     }
 
     return FALSE;
 }
 
 xmlNode *
 controld_query_executor_state(void)
 {
     xmlNode *xml_state = NULL;
     xmlNode *xml_data = NULL;
     xmlNode *rsc_list = NULL;
     crm_node_t *peer = NULL;
     lrm_state_t *lrm_state = lrm_state_find(controld_globals.our_nodename);
 
     if (!lrm_state) {
         crm_err("Could not find executor state for node %s",
                 controld_globals.our_nodename);
         return NULL;
     }
 
     peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
     CRM_CHECK(peer != NULL, return NULL);
 
     xml_state = create_node_state_update(peer,
                                          node_update_cluster|node_update_peer,
                                          NULL, __func__);
     if (xml_state == NULL) {
         return NULL;
     }
 
     xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
     crm_xml_add(xml_data, PCMK_XA_ID, peer->uuid);
     rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
 
     /* Build a list of active (not always running) resources */
     build_active_RAs(lrm_state, rsc_list);
 
     crm_log_xml_trace(xml_state, "Current executor state");
 
     return xml_state;
 }
 
 /*!
  * \internal
  * \brief Map standard Pacemaker return code to operation status and OCF code
  *
  * \param[out] event  Executor event whose status and return code should be set
  * \param[in]  rc     Standard Pacemaker return code
  */
 void
 controld_rc2event(lrmd_event_data_t *event, int rc)
 {
     /* This is called for cleanup requests from controller peers/clients, not
      * for resource actions, so no exit reason is needed.
      */
     switch (rc) {
         case pcmk_rc_ok:
             lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             break;
         case EACCES:
             lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
                              PCMK_EXEC_ERROR, NULL);
             break;
         default:
             lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              NULL);
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Trigger a new transition after CIB status was deleted
  *
  * If a CIB status delete was not expected (as part of the transition graph),
  * trigger a new transition by updating the (arbitrary) "last-lrm-refresh"
  * cluster property.
  *
  * \param[in] from_sys  IPC name that requested the delete
  * \param[in] rsc_id    Resource whose status was deleted (for logging only)
  */
 void
 controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
 {
     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
         char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
 
         crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
         cib__update_node_attr(controld_globals.logger_out,
                               controld_globals.cib_conn, cib_none,
                               XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
                               "last-lrm-refresh", now_s, NULL, NULL);
         free(now_s);
     }
 }
 
 static void
 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     crm_info("Notifying %s on %s that %s was%s deleted",
              from_sys, (from_host? from_host : "localhost"), rsc_id,
              ((rc == pcmk_ok)? "" : " not"));
     op = construct_op(lrm_state, input->xml, rsc_id, PCMK_ACTION_DELETE);
     controld_rc2event(op, pcmk_legacy2rc(rc));
     controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
     lrmd_free_event(op);
     controld_trigger_delete_refresh(from_sys, rsc_id);
 }
 
 static gboolean
 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     struct delete_event_s *event = user_data;
     struct pending_deletion_op_s *op = value;
 
     if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
         notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
         return TRUE;
     }
     return FALSE;
 }
 
 static gboolean
 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
 {
     const char *rsc = user_data;
     active_op_t *pending = value;
 
     if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
         crm_info("Removing op %s:%d for deleted resource %s",
                  pending->op_key, pending->call_id, rsc);
         return TRUE;
     }
     return FALSE;
 }
 
 static void
 delete_rsc_entry(lrm_state_t *lrm_state, ha_msg_input_t *input,
                  const char *rsc_id, GHashTableIter *rsc_iter, int rc,
                  const char *user_name, bool from_cib)
 {
     struct delete_event_s event;
 
     CRM_CHECK(rsc_id != NULL, return);
 
     if (rc == pcmk_ok) {
         char *rsc_id_copy = strdup(rsc_id);
 
         if (rsc_iter) {
             g_hash_table_iter_remove(rsc_iter);
         } else {
             g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
         }
 
         if (from_cib) {
             controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
                                              user_name, crmd_cib_smart_opt());
         }
         g_hash_table_foreach_remove(lrm_state->active_ops,
                                     lrm_remove_deleted_op, rsc_id_copy);
         free(rsc_id_copy);
     }
 
     if (input) {
         notify_deleted(lrm_state, input, rsc_id, rc);
     }
 
     event.rc = rc;
     event.rsc = rsc_id;
     event.lrm_state = lrm_state;
     g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
 }
 
 static inline gboolean
 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
 {
     if (entry == NULL) {
         return FALSE;
     }
     if (op == NULL) {
         return TRUE;
     }
     return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
             && (interval_ms == entry->failed->interval_ms));
 }
 
 /*!
  * \internal
  * \brief Clear a resource's last failure
  *
  * Erase a resource's last failure on a particular node from both the
  * LRM resource history in the CIB, and the resource history remembered
  * for the LRM state.
  *
  * \param[in] rsc_id      Resource name
  * \param[in] node_name   Node name
  * \param[in] operation   If specified, only clear if matching this operation
  * \param[in] interval_ms If operation is specified, it has this interval
  */
 void
 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
                        const char *operation, guint interval_ms)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
 
     if (lrm_state == NULL) {
         return;
     }
     if (lrm_state->resource_history != NULL) {
         rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
                                                    rsc_id);
 
         if (last_failed_matches_op(entry, operation, interval_ms)) {
             lrmd_free_event(entry->failed);
             entry->failed = NULL;
         }
     }
 }
 
 /* Returns: gboolean - cancellation is in progress */
 static gboolean
 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
 {
     int rc = pcmk_ok;
     char *local_key = NULL;
     active_op_t *pending = NULL;
 
     CRM_CHECK(op != 0, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     if (key == NULL) {
         local_key = make_stop_id(rsc_id, op);
         key = local_key;
     }
     pending = g_hash_table_lookup(lrm_state->active_ops, key);
 
     if (pending) {
         if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
             controld_set_active_op_flags(pending, active_op_remove);
             crm_debug("Scheduling %s for removal", key);
         }
 
         if (pcmk_is_set(pending->flags, active_op_cancelled)) {
             crm_debug("Operation %s already cancelled", key);
             free(local_key);
             return FALSE;
         }
         controld_set_active_op_flags(pending, active_op_cancelled);
 
     } else {
         crm_info("No pending op found for %s", key);
         free(local_key);
         return FALSE;
     }
 
     crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
     rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
                           pending->interval_ms);
     if (rc == pcmk_ok) {
         crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
         free(local_key);
         return TRUE;
     }
 
     crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
     /* The caller needs to make sure the entry is
      * removed from the active operations list
      *
      * Usually by returning TRUE inside the worker function
      * supplied to g_hash_table_foreach_remove()
      *
      * Not removing the entry from active operations will block
      * the node from shutting down
      */
     free(local_key);
     return FALSE;
 }
 
 struct cancel_data {
     gboolean done;
     gboolean remove;
     const char *key;
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct cancel_data *data = user_data;
     active_op_t *op = value;
 
     if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
         data->done = TRUE;
         remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
     }
     return remove;
 }
 
 static gboolean
 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
 {
     guint removed = 0;
     struct cancel_data data;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(key != NULL, return FALSE);
 
     data.key = key;
     data.rsc = rsc;
     data.done = FALSE;
     data.remove = remove;
     data.lrm_state = lrm_state;
 
     removed = g_hash_table_foreach_remove(lrm_state->active_ops,
                                           cancel_action_by_key, &data);
     crm_trace("Removed %u op cache entries, new size: %u",
               removed, g_hash_table_size(lrm_state->active_ops));
     return data.done;
 }
 
 /*!
  * \internal
  * \brief Retrieve resource information from LRM
  *
  * \param[in,out]  lrm_state  Executor connection state to use
  * \param[in]      rsc_xml    XML containing resource configuration
  * \param[in]      do_create  If true, register resource if not already
  * \param[out]     rsc_info   Where to store information obtained from executor
  *
  * \retval pcmk_ok   Success (and rsc_info holds newly allocated result)
  * \retval -EINVAL   Required information is missing from arguments
  * \retval -ENOTCONN No active connection to LRM
  * \retval -ENODEV   Resource not found
  * \retval -errno    Error communicating with executor when registering resource
  *
  * \note Caller is responsible for freeing result on success.
  */
 static int
 get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml,
                  gboolean do_create, lrmd_rsc_info_t **rsc_info)
 {
     const char *id = ID(rsc_xml);
 
     CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
     CRM_CHECK(id, return -EINVAL);
 
     if (lrm_state_is_connected(lrm_state) == FALSE) {
         return -ENOTCONN;
     }
 
     crm_trace("Retrieving resource information for %s from the executor", id);
     *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
 
     // If resource isn't known by ID, try clone name, if provided
     if (!*rsc_info) {
         const char *long_id = crm_element_value(rsc_xml, PCMK__XA_LONG_ID);
 
         if (long_id) {
             *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
         }
     }
 
     if ((*rsc_info == NULL) && do_create) {
         const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
-        const char *provider = crm_element_value(rsc_xml, XML_AGENT_ATTR_PROVIDER);
+        const char *provider = crm_element_value(rsc_xml, PCMK_XA_PROVIDER);
         const char *type = crm_element_value(rsc_xml, PCMK_XA_TYPE);
         int rc;
 
         crm_trace("Registering resource %s with the executor", id);
         rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
                                     lrmd_opt_drop_recurring);
         if (rc != pcmk_ok) {
             fsa_data_t *msg_data = NULL;
 
             crm_err("Could not register resource %s with the executor on %s: %s "
                     CRM_XS " rc=%d",
                     id, lrm_state->node_name, pcmk_strerror(rc), rc);
 
             /* Register this as an internal error if this involves the local
              * executor. Otherwise, we're likely dealing with an unresponsive
              * remote node, which is not an FSA failure.
              */
             if (lrm_state_is_local(lrm_state) == TRUE) {
                 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
             }
             return rc;
         }
 
         *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
     }
     return *rsc_info? pcmk_ok : -ENODEV;
 }
 
 static void
 delete_resource(lrm_state_t *lrm_state, const char *id, lrmd_rsc_info_t *rsc,
                 GHashTableIter *iter, const char *sys, const char *user,
                 ha_msg_input_t *request, bool unregister, bool from_cib)
 {
     int rc = pcmk_ok;
 
     crm_info("Removing resource %s from executor for %s%s%s",
              id, sys, (user? " as " : ""), (user? user : ""));
 
     if (rsc && unregister) {
         rc = lrm_state_unregister_rsc(lrm_state, id, 0);
     }
 
     if (rc == pcmk_ok) {
         crm_trace("Resource %s deleted from executor", id);
     } else if (rc == -EINPROGRESS) {
         crm_info("Deletion of resource '%s' from executor is pending", id);
         if (request) {
             struct pending_deletion_op_s *op = NULL;
             char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
 
             op = calloc(1, sizeof(struct pending_deletion_op_s));
             op->rsc = strdup(rsc->id);
             op->input = copy_ha_msg_input(request);
             g_hash_table_insert(lrm_state->deletion_ops, ref, op);
         }
         return;
     } else {
         crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
                  CRM_XS " rc=%d", id, sys, (user? " as " : ""),
                  (user? user : ""), pcmk_strerror(rc), rc);
     }
 
     delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib);
 }
 
 static int
 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
 {
     int call_id = 999999999;
     rsc_history_t *entry = NULL;
 
     if(lrm_state) {
         entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     }
 
     /* Make sure the call id is greater than the last successful operation,
      * otherwise the failure will not result in a possible recovery of the resource
      * as it could appear the failure occurred before the successful start */
     if (entry) {
         call_id = entry->last_callid + 1;
     }
 
     if (call_id < 0) {
         call_id = 1;
     }
     return call_id;
 }
 
 static void
 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
                enum ocf_exitcode op_exitcode, const char *exit_reason)
 {
     op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
     lrmd__set_result(op, op_exitcode, op_status, exit_reason);
 }
 
 static void
 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
               const char *from_host, const char *user_name,
               gboolean is_remote_node, bool reprobe_all_nodes)
 {
     GHashTableIter gIter;
     rsc_history_t *entry = NULL;
 
     crm_info("Clearing resource history on node %s", lrm_state->node_name);
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
         /* only unregister the resource during a reprobe if it is not a remote connection
          * resource. otherwise unregistering the connection will terminate remote-node
          * membership */
         bool unregister = true;
 
         if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
             unregister = false;
 
             if (reprobe_all_nodes) {
                 lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
 
                 if (remote_lrm_state != NULL) {
                     /* If reprobing all nodes, be sure to reprobe the remote
                      * node before clearing its connection resource
                      */
                     force_reprobe(remote_lrm_state, from_sys, from_host,
                                   user_name, TRUE, reprobe_all_nodes);
                 }
             }
         }
 
         /* Don't delete from the CIB, since we'll delete the whole node's LRM
          * state from the CIB soon
          */
         delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
                         user_name, NULL, unregister, false);
     }
 
     /* Now delete the copy in the CIB */
     controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
                                cib_scope_local);
 
     // @COMPAT DCs < 1.1.14 need this deleted (in case it was explicitly false)
     update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
 }
 
 /*!
  * \internal
  * \brief Fail a requested action without actually executing it
  *
  * For an action that can't be executed, process it similarly to an actual
  * execution result, with specified error status (except for notify actions,
  * which will always be treated as successful).
  *
  * \param[in,out] lrm_state    Executor connection that action is for
  * \param[in]     action       Action XML from request
  * \param[in]     rc           Desired return code to use
  * \param[in]     op_status    Desired operation status to use
  * \param[in]     exit_reason  Human-friendly detail, if error
  */
 static void
 synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action,
                         int op_status, enum ocf_exitcode rc,
                         const char *exit_reason)
 {
     lrmd_event_data_t *op = NULL;
     const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
     const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
     xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
 
     if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) {
         /* @TODO Should we do something else, like direct ack? */
         crm_info("Can't fake %s failure (%d) on %s without resource configuration",
                  crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
                  target_node);
         return;
 
     } else if(operation == NULL) {
         /* This probably came from crm_resource -C, nothing to do */
         crm_info("Can't fake %s failure (%d) on %s without operation",
                  ID(xml_rsc), rc, target_node);
         return;
     }
 
     op = construct_op(lrm_state, action, ID(xml_rsc), operation);
 
     if (pcmk__str_eq(operation, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
         // Notifications can't fail
         fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL);
     } else {
         fake_op_status(lrm_state, op, op_status, rc, exit_reason);
     }
 
     crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
              op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
 
     // Process the result as if it came from the LRM
     process_lrm_event(lrm_state, op, NULL, action);
     lrmd_free_event(op);
 }
 
 /*!
  * \internal
  * \brief Get target of an LRM operation (replacing \p NULL with local node
  *        name)
  *
  * \param[in] xml  LRM operation data XML
  *
  * \return LRM operation target node name (local node or Pacemaker Remote node)
  */
 static const char *
 lrm_op_target(const xmlNode *xml)
 {
     const char *target = NULL;
 
     if (xml) {
         target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
     }
     if (target == NULL) {
         target = controld_globals.our_nodename;
     }
     return target;
 }
 
 static void
 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
                   const char *from_host, const char *from_sys)
 {
     lrmd_event_data_t *op = NULL;
     lrmd_rsc_info_t *rsc = NULL;
     xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);
 
     CRM_CHECK(xml_rsc != NULL, return);
 
     /* The executor simply executes operations and reports the results, without
      * any concept of success or failure, so to fail a resource, we must fake
      * what a failure looks like.
      *
      * To do this, we create a fake executor operation event for the resource,
      * and pass that event to the executor client callback so it will be
      * processed as if it came from the executor.
      */
     op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");
 
     free((char*) op->user_data);
     op->user_data = NULL;
     op->interval_ms = 0;
 
     if (user_name && !pcmk__is_privileged(user_name)) {
         crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
         fake_op_status(lrm_state, op, PCMK_EXEC_ERROR,
                        PCMK_OCF_INSUFFICIENT_PRIV,
                        "Unprivileged user cannot fail resources");
         controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
         lrmd_free_event(op);
         return;
     }
 
 
     if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
         crm_info("Failing resource %s...", rsc->id);
         fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR,
                        "Simulated failure");
         process_lrm_event(lrm_state, op, NULL, xml);
         op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_info("Cannot find/create resource in order to fail it...");
         crm_log_xml_warn(xml, "bad input");
         fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR,
                        "Cannot fail unknown resource");
     }
 
     controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
     lrmd_free_event(op);
 }
 
 static void
 handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
                   const char *from_host, const char *user_name,
                   gboolean is_remote_node, bool reprobe_all_nodes)
 {
     crm_notice("Forcing the status of all resources to be redetected");
     force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node,
                   reprobe_all_nodes);
 
     if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
 
         xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
                                         from_sys, CRM_SYSTEM_LRMD,
                                         controld_globals.our_uuid);
 
         crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_log_xml_err(reply, "Unable to route reply");
         }
         free_xml(reply);
     }
 }
 
 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
 {
     char *op_key = NULL;
     char *meta_key = NULL;
     int call = 0;
     const char *call_id = NULL;
     const char *op_task = NULL;
     guint interval_ms = 0;
     gboolean in_progress = FALSE;
     xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
 
     CRM_CHECK(params != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
     op_task = crm_element_value(params, meta_key);
     free(meta_key);
     CRM_CHECK(op_task != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
     if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
         free(meta_key);
         return FALSE;
     }
     free(meta_key);
 
     op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
     call_id = crm_element_value(params, meta_key);
     free(meta_key);
 
     crm_debug("Scheduler requested op %s (call=%s) be cancelled",
               op_key, (call_id? call_id : "NA"));
     pcmk__scan_min_int(call_id, &call, 0);
     if (call == 0) {
         // Normal case when the scheduler cancels a recurring op
         in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
 
     } else {
         // Normal case when the scheduler cancels an orphan op
         in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
     }
 
     // Acknowledge cancellation operation if for a remote connection resource
     if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         char *op_id = make_stop_id(rsc->id, call);
 
         if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
             crm_info("Nothing known about operation %d for %s", call, op_key);
         }
         controld_delete_action_history_by_key(rsc->id, lrm_state->node_name,
                                               op_key, call);
         send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                          from_host, from_sys);
 
         /* needed at least for cancellation of a remote operation */
         if (lrm_state->active_ops != NULL) {
             g_hash_table_remove(lrm_state->active_ops, op_id);
         }
         free(op_id);
 
     } else {
         /* No ack is needed since abcdaa8, but peers with older versions
          * in a rolling upgrade need one. We didn't bump the feature set
          * at that commit, so we can only compare against the previous
          * CRM version (3.0.8). If any peers have feature set 3.0.9 but
          * not abcdaa8, they will time out waiting for the ack (no
          * released versions of Pacemaker are affected).
          */
         const char *peer_version = crm_element_value(params,
                                                      PCMK_XA_CRM_FEATURE_SET);
 
         if (compare_version(peer_version, "3.0.8") <= 0) {
             crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
                      op_key, from_host, peer_version);
             send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                              from_host, from_sys);
         }
     }
 
     free(op_key);
     return TRUE;
 }
 
 static void
 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
               bool crm_rsc_delete, const char *user_name)
 {
     bool unregister = true;
     int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
                                                   user_name,
                                                   cib_dryrun|cib_sync_call);
 
     if (cib_rc != pcmk_rc_ok) {
         lrmd_event_data_t *op = NULL;
 
         op = construct_op(lrm_state, input->xml, rsc->id, PCMK_ACTION_DELETE);
 
         /* These are resource clean-ups, not actions, so no exit reason is
          * needed.
          */
         lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL);
         controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
         lrmd_free_event(op);
         return;
     }
 
     if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         unregister = false;
     }
 
     delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
                     user_name, input, unregister, true);
 }
 
 // User data for asynchronous metadata execution
 struct metadata_cb_data {
     lrmd_rsc_info_t *rsc;   // Copy of resource information
     xmlNode *input_xml;     // Copy of FSA input XML
 };
 
 static struct metadata_cb_data *
 new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml)
 {
     struct metadata_cb_data *data = NULL;
 
     data = calloc(1, sizeof(struct metadata_cb_data));
     CRM_ASSERT(data != NULL);
     data->input_xml = copy_xml(input_xml);
     data->rsc = lrmd_copy_rsc_info(rsc);
     return data;
 }
 
 static void
 free_metadata_cb_data(struct metadata_cb_data *data)
 {
     lrmd_free_rsc_info(data->rsc);
     free_xml(data->input_xml);
     free(data);
 }
 
 /*!
  * \internal
  * \brief Execute an action after metadata has been retrieved
  *
  * \param[in] pid        Ignored
  * \param[in] result     Result of metadata action
  * \param[in] user_data  Metadata callback data
  */
 static void
 metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data)
 {
     struct metadata_cb_data *data = (struct metadata_cb_data *) user_data;
 
     struct ra_metadata_s *md = NULL;
     lrm_state_t *lrm_state = lrm_state_find(lrm_op_target(data->input_xml));
 
     if ((lrm_state != NULL) && pcmk__result_ok(result)) {
         md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc,
                                      result->action_stdout);
     }
     if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
         do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md);
     }
     free_metadata_cb_data(data);
 }
 
 /*	 A_LRM_INVOKE	*/
 void
 do_lrm_invoke(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     lrm_state_t *lrm_state = NULL;
     const char *crm_op = NULL;
     const char *from_sys = NULL;
     const char *from_host = NULL;
     const char *operation = NULL;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *user_name = NULL;
     const char *target_node = lrm_op_target(input->xml);
     gboolean is_remote_node = FALSE;
     bool crm_rsc_delete = FALSE;
 
     // Message routed to the local node is targeting a specific, non-local node
     is_remote_node = !pcmk__str_eq(target_node, controld_globals.our_nodename,
                                    pcmk__str_casei);
 
     lrm_state = lrm_state_find(target_node);
     if ((lrm_state == NULL) && is_remote_node) {
         crm_err("Failing action because local node has never had connection to remote node %s",
                 target_node);
         synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
                                 PCMK_OCF_UNKNOWN_ERROR,
                                 "Local node has no connection to remote");
         return;
     }
     CRM_ASSERT(lrm_state != NULL);
 
     user_name = pcmk__update_acl_user(input->msg, F_CRM_USER, NULL);
     crm_op = crm_element_value(input->msg, F_CRM_TASK);
     from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
         from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
     }
 
     if (pcmk__str_eq(crm_op, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
         if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
             crm_rsc_delete = TRUE; // from crm_resource
         }
         operation = PCMK_ACTION_DELETE;
 
     } else if (input->xml != NULL) {
         operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
     }
 
     CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return);
 
     crm_trace("'%s' execution request from %s as %s user",
               pcmk__s(crm_op, operation),
               pcmk__s(from_sys, "unknown subsystem"),
               pcmk__s(user_name, "current"));
 
     if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) {
         fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
                           from_sys);
 
     } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_none)) {
         /* @COMPAT This can only be sent by crm_resource --refresh on a
          * Pacemaker Remote node running Pacemaker 1.1.9, which is extremely
          * unlikely. It previously would cause the controller to re-write its
          * resource history to the CIB. Just ignore it.
          */
         crm_notice("Ignoring refresh request from Pacemaker Remote 1.1.9 node");
 
     // @COMPAT DCs <1.1.14 in a rolling upgrade might schedule this op
     } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_none)) {
         update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
                      user_name, is_remote_node);
 
     } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none)
                || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
         const char *raw_target = NULL;
 
         if (input->xml != NULL) {
             // For CRM_OP_REPROBE, a NULL target means we're targeting all nodes
             raw_target = crm_element_value(input->xml, XML_LRM_ATTR_TARGET);
         }
         handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
                           is_remote_node, (raw_target == NULL));
 
     } else if (operation != NULL) {
         lrmd_rsc_info_t *rsc = NULL;
         xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
         gboolean create_rsc = !pcmk__str_eq(operation, PCMK_ACTION_DELETE,
                                             pcmk__str_none);
         int rc;
 
         // We can't return anything meaningful without a resource ID
         CRM_CHECK(xml_rsc && ID(xml_rsc), return);
 
         rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
         if (rc == -ENOTCONN) {
             synthesize_lrmd_failure(lrm_state, input->xml,
                                     PCMK_EXEC_NOT_CONNECTED,
                                     PCMK_OCF_UNKNOWN_ERROR,
                                     "Not connected to remote executor");
             return;
 
         } else if ((rc < 0) && !create_rsc) {
             /* Delete of malformed or nonexistent resource
              * (deleting something that does not exist is a success)
              */
             crm_notice("Not registering resource '%s' for a %s event "
                        CRM_XS " get-rc=%d (%s) transition-key=%s",
                        ID(xml_rsc), operation,
                        rc, pcmk_strerror(rc), ID(input->xml));
             delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
                              user_name, true);
             return;
 
         } else if (rc == -EINVAL) {
             // Resource operation on malformed resource
             crm_err("Invalid resource definition for %s", ID(xml_rsc));
             crm_log_xml_warn(input->msg, "invalid resource");
             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
                                     PCMK_OCF_NOT_CONFIGURED, // fatal error
                                     "Invalid resource definition");
             return;
 
         } else if (rc < 0) {
             // Error communicating with the executor
             crm_err("Could not register resource '%s' with executor: %s "
                     CRM_XS " rc=%d",
                     ID(xml_rsc), pcmk_strerror(rc), rc);
             crm_log_xml_warn(input->msg, "failed registration");
             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
                                     PCMK_OCF_INVALID_PARAM, // hard error
                                     "Could not register resource with executor");
             return;
         }
 
         if (pcmk__str_eq(operation, PCMK_ACTION_CANCEL, pcmk__str_none)) {
             if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
                 crm_log_xml_warn(input->xml, "Bad command");
             }
 
         } else if (pcmk__str_eq(operation, PCMK_ACTION_DELETE,
                                 pcmk__str_none)) {
             do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
                           crm_rsc_delete, user_name);
 
         } else {
             struct ra_metadata_s *md = NULL;
 
             /* Getting metadata from cache is OK except for start actions --
              * always refresh from the agent for those, in case the resource
              * agent was updated.
              *
              * @TODO Only refresh metadata for starts if the agent actually
              * changed (using something like inotify, or a hash or modification
              * time of the agent executable).
              */
             if (strcmp(operation, PCMK_ACTION_START) != 0) {
                 md = controld_get_rsc_metadata(lrm_state, rsc,
                                                controld_metadata_from_cache);
             }
 
             if ((md == NULL) && crm_op_needs_metadata(rsc->standard,
                                                       operation)) {
                 /* Most likely, we'll need the agent metadata to record the
                  * pending operation and the operation result. Get it now rather
                  * than wait until then, so the metadata action doesn't eat into
                  * the real action's timeout.
                  *
                  * @TODO Metadata is retrieved via direct execution of the
                  * agent, which has a couple of related issues: the executor
                  * should execute agents, not the controller; and metadata for
                  * Pacemaker Remote nodes should be collected on those nodes,
                  * not locally.
                  */
                 struct metadata_cb_data *data = NULL;
 
                 data = new_metadata_cb_data(rsc, input->xml);
                 crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously",
                          rsc->id, rsc->standard,
                          ((rsc->provider == NULL)? "" : ":"),
                          ((rsc->provider == NULL)? "" : rsc->provider),
                          rsc->type);
                 (void) lrmd__metadata_async(rsc, metadata_complete,
                                             (void *) data);
             } else {
                 do_lrm_rsc_op(lrm_state, rsc, input->xml, md);
             }
         }
 
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_err("Invalid execution request: unknown command '%s' (bug?)",
                 crm_op);
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 static lrmd_event_data_t *
 construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op,
              const char *rsc_id, const char *operation)
 {
     lrmd_event_data_t *op = NULL;
     const char *op_delay = NULL;
     const char *op_timeout = NULL;
     GHashTable *params = NULL;
 
     xmlNode *primitive = NULL;
     const char *class = NULL;
 
     const char *transition = NULL;
 
     CRM_ASSERT(rsc_id && operation);
 
     op = lrmd_new_event(rsc_id, operation, 0);
     op->type = lrmd_event_exec_complete;
     op->timeout = 0;
     op->start_delay = 0;
     lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
 
     if (rsc_op == NULL) {
         CRM_LOG_ASSERT(pcmk__str_eq(operation, PCMK_ACTION_STOP,
                                     pcmk__str_casei));
         op->user_data = NULL;
         /* the stop_all_resources() case
          * by definition there is no DC (or they'd be shutting
          *   us down).
          * So we should put our version here.
          */
         op->params = pcmk__strkey_table(free, free);
 
         g_hash_table_insert(op->params, strdup(PCMK_XA_CRM_FEATURE_SET),
                             strdup(CRM_FEATURE_SET));
 
         crm_trace("Constructed %s op for %s", operation, rsc_id);
         return op;
     }
 
     params = xml2list(rsc_op);
     g_hash_table_remove(params, CRM_META "_op_target_rc");
 
     op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
     pcmk__scan_min_int(op_delay, &op->start_delay, 0);
 
     op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
     pcmk__scan_min_int(op_timeout, &op->timeout, 0);
 
     if (pcmk__guint_from_hash(params, CRM_META "_" XML_LRM_ATTR_INTERVAL_MS, 0,
                               &(op->interval_ms)) != pcmk_rc_ok) {
         op->interval_ms = 0;
     }
 
     /* Use pcmk_monitor_timeout instead of meta timeout for stonith
        recurring monitor, if set */
     primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE);
     class = crm_element_value(primitive, XML_AGENT_ATTR_CLASS);
 
     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
             && pcmk__str_eq(operation, PCMK_ACTION_MONITOR, pcmk__str_casei)
             && (op->interval_ms > 0)) {
 
         op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
         if (op_timeout != NULL) {
             op->timeout = crm_get_msec(op_timeout);
         }
     }
 
     if (!pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)) {
         op->params = params;
 
     } else {
         rsc_history_t *entry = NULL;
 
         if (lrm_state) {
             entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
         }
 
         /* If we do not have stop parameters cached, use
          * whatever we are given */
         if (!entry || !entry->stop_params) {
             op->params = params;
         } else {
             /* Copy the cached parameter list so that we stop the resource
              * with the old attributes, not the new ones */
             op->params = pcmk__strkey_table(free, free);
 
             g_hash_table_foreach(params, copy_meta_keys, op->params);
             g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
             g_hash_table_destroy(params);
             params = NULL;
         }
     }
 
     /* sanity */
     if (op->timeout <= 0) {
         op->timeout = op->interval_ms;
     }
     if (op->start_delay < 0) {
         op->start_delay = 0;
     }
 
     transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
     CRM_CHECK(transition != NULL, return op);
 
     op->user_data = strdup(transition);
 
     if (op->interval_ms != 0) {
         if (pcmk__strcase_any_of(operation, PCMK_ACTION_START, PCMK_ACTION_STOP,
                                  NULL)) {
             crm_err("Start and Stop actions cannot have an interval: %u",
                     op->interval_ms);
             op->interval_ms = 0;
         }
     }
 
     crm_trace("Constructed %s op for %s: interval=%u",
               operation, rsc_id, op->interval_ms);
 
     return op;
 }
 
 /*!
  * \internal
  * \brief Send a (synthesized) event result
  *
  * Reply with a synthesized event result directly, as opposed to going through
  * the executor.
  *
  * \param[in]     to_host  Host to send result to
  * \param[in]     to_sys   IPC name to send result (NULL for transition engine)
  * \param[in]     rsc      Type information about resource the result is for
  * \param[in,out] op       Event with result to send
  * \param[in]     rsc_id   ID of resource the result is for
  */
 void
 controld_ack_event_directly(const char *to_host, const char *to_sys,
                             const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
                             const char *rsc_id)
 {
     xmlNode *reply = NULL;
     xmlNode *update, *iter;
     crm_node_t *peer = NULL;
 
     CRM_CHECK(op != NULL, return);
     if (op->rsc_id == NULL) {
         CRM_ASSERT(rsc_id != NULL);
         op->rsc_id = strdup(rsc_id);
     }
     if (to_sys == NULL) {
         to_sys = CRM_SYSTEM_TENGINE;
     }
 
     peer = crm_get_peer(0, controld_globals.our_nodename);
     update = create_node_state_update(peer, node_update_none, NULL,
                                       __func__);
 
     iter = create_xml_node(update, XML_CIB_TAG_LRM);
     crm_xml_add(iter, PCMK_XA_ID, controld_globals.our_uuid);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
 
     crm_xml_add(iter, PCMK_XA_ID, op->rsc_id);
 
     controld_add_resource_history_xml(iter, rsc, op,
                                       controld_globals.our_nodename);
     reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
 
     crm_log_xml_trace(update, "[direct ACK]");
 
     crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
               op->rsc_id, op->op_type, op->interval_ms, op->user_data,
               crm_element_value(reply, XML_ATTR_REFERENCE));
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_log_xml_err(reply, "Unable to route reply");
     }
 
     free_xml(update);
     free_xml(reply);
 }
 
 gboolean
 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
 {
     gboolean res = TRUE;
     GList *lrm_state_list = lrm_state_get_list();
     GList *state_entry;
 
     for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
         lrm_state_t *lrm_state = state_entry->data;
 
         if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
             /* keep iterating through all even when false is returned */
             res = FALSE;
         }
     }
 
     controld_set_fsa_input_flags(R_SENT_RSC_STOP);
     g_list_free(lrm_state_list); lrm_state_list = NULL;
     return res;
 }
 
 struct stop_recurring_action_s {
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct stop_recurring_action_s *event = user_data;
     active_op_t *op = value;
 
     if ((op->interval_ms != 0)
         && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
 
         crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
         remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 static gboolean
 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     lrm_state_t *lrm_state = user_data;
     active_op_t *op = value;
 
     if (op->interval_ms != 0) {
         crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
                  (const char *) key);
         remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 /*!
  * \internal
  * \brief Check whether recurring actions should be cancelled before an action
  *
  * \param[in] rsc_id       Resource that action is for
  * \param[in] action       Action being performed
  * \param[in] interval_ms  Operation interval of \p action (in milliseconds)
  *
  * \return true if recurring actions should be cancelled, otherwise false
  */
 static bool
 should_cancel_recurring(const char *rsc_id, const char *action, guint interval_ms)
 {
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0)
         && (strcmp(action, PCMK_ACTION_MIGRATE_TO) == 0)) {
         /* Don't stop monitoring a migrating Pacemaker Remote connection
          * resource until the entire migration has completed. We must detect if
          * the connection is unexpectedly severed, even during a migration.
          */
         return false;
     }
 
     // Cancel recurring actions before changing resource state
     return (interval_ms == 0)
             && !pcmk__str_any_of(action, PCMK_ACTION_MONITOR,
                                  PCMK_ACTION_NOTIFY, NULL);
 }
 
 /*!
  * \internal
  * \brief Check whether an action should not be performed at this time
  *
  * \param[in] operation  Action to be performed
  *
  * \return Readable description of why action should not be performed,
  *         or NULL if it should be performed
  */
 static const char *
 should_nack_action(const char *action)
 {
     if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)
         && pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
 
         register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
         return "Not attempting start due to shutdown in progress";
     }
 
     switch (controld_globals.fsa_state) {
         case S_NOT_DC:
         case S_POLICY_ENGINE:   // Recalculating
         case S_TRANSITION_ENGINE:
             break;
         default:
             if (!pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
                 return "Controller cannot attempt actions at this time";
             }
             break;
     }
     return NULL;
 }
 
 static void
 do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg,
               struct ra_metadata_s *md)
 {
     int rc;
     int call_id = 0;
     char *op_id = NULL;
     lrmd_event_data_t *op = NULL;
     fsa_data_t *msg_data = NULL;
     const char *transition = NULL;
     const char *operation = NULL;
     const char *nack_reason = NULL;
 
     CRM_CHECK((rsc != NULL) && (msg != NULL), return);
 
     operation = crm_element_value(msg, XML_LRM_ATTR_TASK);
     CRM_CHECK(!pcmk__str_empty(operation), return);
 
     transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
     if (pcmk__str_empty(transition)) {
         crm_log_xml_err(msg, "Missing transition number");
     }
 
     if (lrm_state == NULL) {
         // This shouldn't be possible, but provide a failsafe just in case
         crm_err("Cannot execute %s of %s: No executor connection "
                 CRM_XS " transition_key=%s",
                 operation, rsc->id, pcmk__s(transition, ""));
         synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID,
                                 PCMK_OCF_UNKNOWN_ERROR,
                                 "No executor connection");
         return;
     }
 
     if (pcmk__str_any_of(operation, PCMK_ACTION_RELOAD,
                          PCMK_ACTION_RELOAD_AGENT, NULL)) {
         /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs
          * will schedule reload-agent actions only. In either case, we need
          * to map that to whatever the resource agent actually supports.
          * Default to the OCF 1.1 name.
          */
         if ((md != NULL)
             && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
             operation = PCMK_ACTION_RELOAD;
         } else {
             operation = PCMK_ACTION_RELOAD_AGENT;
         }
     }
 
     op = construct_op(lrm_state, msg, rsc->id, operation);
     CRM_CHECK(op != NULL, return);
 
     if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) {
         guint removed = 0;
         struct stop_recurring_action_s data;
 
         data.rsc = rsc;
         data.lrm_state = lrm_state;
         removed = g_hash_table_foreach_remove(lrm_state->active_ops,
                                               stop_recurring_action_by_rsc,
                                               &data);
 
         if (removed) {
             crm_debug("Stopped %u recurring operation%s in preparation for "
                       PCMK__OP_FMT, removed, pcmk__plural_s(removed),
                       rsc->id, operation, op->interval_ms);
         }
     }
 
     /* now do the op */
     crm_notice("Requesting local execution of %s operation for %s on %s "
                CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
                pcmk__readable_action(op->op_type, op->interval_ms), rsc->id,
                lrm_state->node_name, pcmk__s(transition, ""), rsc->id,
                operation, op->interval_ms);
 
     nack_reason = should_nack_action(operation);
     if (nack_reason != NULL) {
         crm_notice("Discarding attempt to perform action %s on %s in state %s "
                    "(shutdown=%s)", operation, rsc->id,
                    fsa_state2string(controld_globals.fsa_state),
                    pcmk__btoa(pcmk_is_set(controld_globals.fsa_input_register,
                                           R_SHUTDOWN)));
 
         lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
                          nack_reason);
         controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
         lrmd_free_event(op);
         free(op_id);
         return;
     }
 
     controld_record_pending_op(lrm_state->node_name, rsc, op);
 
     op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
 
     if (op->interval_ms > 0) {
         /* cancel it so we can then restart it without conflict */
         cancel_op_key(lrm_state, rsc, op_id, FALSE);
     }
 
     rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
                                          op->user_data, op->interval_ms,
                                          op->timeout, op->start_delay,
                                          op->params, &call_id);
     if (rc == pcmk_rc_ok) {
         /* record all operations so we can wait
          * for them to complete during shutdown
          */
         char *call_id_s = make_stop_id(rsc->id, call_id);
         active_op_t *pending = NULL;
 
         pending = calloc(1, sizeof(active_op_t));
         crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
 
         pending->call_id = call_id;
         pending->interval_ms = op->interval_ms;
         pending->op_type = strdup(operation);
         pending->op_key = strdup(op_id);
         pending->rsc_id = strdup(rsc->id);
         pending->start_time = time(NULL);
         pcmk__str_update(&pending->user_data, op->user_data);
         if (crm_element_value_epoch(msg, PCMK_OPT_SHUTDOWN_LOCK,
                                     &(pending->lock_time)) != pcmk_ok) {
             pending->lock_time = 0;
         }
         g_hash_table_replace(lrm_state->active_ops, call_id_s, pending);
 
         if ((op->interval_ms > 0)
             && (op->start_delay > START_DELAY_THRESHOLD)) {
             int target_rc = PCMK_OCF_OK;
 
             crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
             decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
             lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
             controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
         }
 
         pending->params = op->params;
         op->params = NULL;
 
     } else if (lrm_state_is_local(lrm_state)) {
         crm_err("Could not initiate %s action for resource %s locally: %s "
                 CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
         fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
                        PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
         process_lrm_event(lrm_state, op, NULL, NULL);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 
     } else {
         crm_err("Could not initiate %s action for resource %s remotely on %s: "
                 "%s " CRM_XS " rc=%d",
                 operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
         fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
                        PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
         process_lrm_event(lrm_state, op, NULL, NULL);
     }
 
     free(op_id);
     lrmd_free_event(op);
 }
 
 void
 do_lrm_event(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
 {
     CRM_CHECK(FALSE, return);
 }
 
 static char *
 unescape_newlines(const char *string)
 {
     char *pch = NULL;
     char *ret = NULL;
     static const char *escaped_newline = "\\n";
 
     if (!string) {
         return NULL;
     }
 
     ret = strdup(string);
     pch = strstr(ret, escaped_newline);
     while (pch != NULL) {
         /* Replace newline escape pattern with actual newline (and a space so we
          * don't have to shuffle the rest of the buffer)
          */
         pch[0] = '\n';
         pch[1] = ' ';
         pch = strstr(pch, escaped_newline);
     }
 
     return ret;
 }
 
 static bool
 did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
                     const char * op_type, guint interval_ms)
 {
     rsc_history_t *entry = NULL;
 
     CRM_CHECK(lrm_state != NULL, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     CRM_CHECK(op_type != NULL, return FALSE);
 
     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     if (entry == NULL || entry->failed == NULL) {
         return FALSE;
     }
 
     if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
         && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
         && entry->failed->interval_ms == interval_ms) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Log the result of an executor action (actual or synthesized)
  *
  * \param[in] op         Executor action to log result for
  * \param[in] op_key     Operation key for action
  * \param[in] node_name  Name of node action was performed on, if known
  * \param[in] confirmed  Whether to log that graph action was confirmed
  */
 static void
 log_executor_event(const lrmd_event_data_t *op, const char *op_key,
                    const char *node_name, gboolean confirmed)
 {
     int log_level = LOG_ERR;
     GString *str = g_string_sized_new(100); // reasonable starting size
 
     pcmk__g_strcat(str,
                    "Result of ",
                    pcmk__readable_action(op->op_type, op->interval_ms),
                    " operation for ", op->rsc_id, NULL);
 
     if (node_name != NULL) {
         pcmk__g_strcat(str, " on ", node_name, NULL);
     }
 
     switch (op->op_status) {
         case PCMK_EXEC_DONE:
             log_level = LOG_NOTICE;
             pcmk__g_strcat(str, ": ", services_ocf_exitcode_str(op->rc), NULL);
             break;
 
         case PCMK_EXEC_TIMEOUT:
             pcmk__g_strcat(str,
                            ": ", pcmk_exec_status_str(op->op_status), " after ",
                            pcmk__readable_interval(op->timeout), NULL);
             break;
 
         case PCMK_EXEC_CANCELLED:
             log_level = LOG_INFO;
 	    /* order of __attribute__ and Fall through comment is IMPORTANT!
 	     * do not change it without proper testing with both clang and gcc
 	     * in multiple versions.
 	     * the clang check allows to build with all versions of clang.
 	     * the has_c_attribute check is to workaround a bug in clang version
 	     * in rhel7. has_attribute would happily return "YES SIR WE GOT IT"
 	     * and fail the build the next line.
 	     */
 #ifdef __clang__
 #ifdef __has_c_attribute
 #if __has_attribute(fallthrough)
 	    __attribute__((fallthrough));
 #endif
 #endif
 #endif
             // Fall through
         default:
             pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status),
                            NULL);
     }
 
     if ((op->exit_reason != NULL)
         && ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) {
 
         pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL);
     }
 
     g_string_append(str, " " CRM_XS);
     g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s",
                            (confirmed? "" : "un"), op->call_id, op_key);
     if (op->op_status == PCMK_EXEC_DONE) {
         g_string_append_printf(str, " rc=%d", op->rc);
     }
 
     do_crm_log(log_level, "%s", str->str);
     g_string_free(str, TRUE);
 
     /* The services library has already logged the output at info or debug
      * level, so just raise to notice if it looks like a failure.
      */
     if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) {
         char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output",
                                          op->rsc_id, op->op_type,
                                          op->interval_ms, node_name);
 
         crm_log_output(LOG_NOTICE, prefix, op->output);
         free(prefix);
     }
 }
 
 void
 process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
                   active_op_t *pending, const xmlNode *action_xml)
 {
     char *op_id = NULL;
     char *op_key = NULL;
 
     gboolean remove = FALSE;
     gboolean removed = FALSE;
     bool need_direct_ack = FALSE;
     lrmd_rsc_info_t *rsc = NULL;
     const char *node_name = NULL;
 
     CRM_CHECK(op != NULL, return);
     CRM_CHECK(op->rsc_id != NULL, return);
 
     // Remap new status codes for older DCs
     if (compare_version(controld_globals.dc_version, "3.2.0") < 0) {
         switch (op->op_status) {
             case PCMK_EXEC_NOT_CONNECTED:
                 lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
                                  PCMK_EXEC_ERROR, op->exit_reason);
                 break;
             case PCMK_EXEC_INVALID:
                 lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
                                  op->exit_reason);
                 break;
             default:
                 break;
         }
     }
 
     op_id = make_stop_id(op->rsc_id, op->call_id);
     op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
 
     // Get resource info if available (from executor state or action XML)
     if (lrm_state) {
         rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
     }
     if ((rsc == NULL) && action_xml) {
         xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
 
         const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
-        const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
+        const char *provider = crm_element_value(xml, PCMK_XA_PROVIDER);
         const char *type = crm_element_value(xml, PCMK_XA_TYPE);
 
         if (standard && type) {
             crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
                      op->rsc_id, standard,
                      (provider? ":" : ""), (provider? provider : ""), type);
             rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
         } else {
             crm_err("Can't process %s result because %s agent information not cached or in XML",
                     op_key, op->rsc_id);
         }
     }
 
     // Get node name if available (from executor state or action XML)
     if (lrm_state) {
         node_name = lrm_state->node_name;
     } else if (action_xml) {
         node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
     }
 
     if(pending == NULL) {
         remove = TRUE;
         if (lrm_state) {
             pending = g_hash_table_lookup(lrm_state->active_ops, op_id);
         }
     }
 
     if (op->op_status == PCMK_EXEC_ERROR) {
         switch(op->rc) {
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_PROMOTED:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_PROMOTED:
                 // Leave it to the TE/scheduler to decide if this is an error
                 op->op_status = PCMK_EXEC_DONE;
                 break;
             default:
                 /* Nothing to do */
                 break;
         }
     }
 
     if (op->op_status != PCMK_EXEC_CANCELLED) {
         /* We might not record the result, so directly acknowledge it to the
          * originator instead, so it doesn't time out waiting for the result
          * (especially important if part of a transition).
          */
         need_direct_ack = TRUE;
 
         if (controld_action_is_recordable(op->op_type)) {
             if (node_name && rsc) {
                 // We should record the result, and happily, we can
                 time_t lock_time = (pending == NULL)? 0 : pending->lock_time;
 
                 controld_update_resource_history(node_name, rsc, op, lock_time);
                 need_direct_ack = FALSE;
 
             } else if (op->rsc_deleted) {
                 /* We shouldn't record the result (likely the resource was
                  * refreshed, cleaned, or removed while this operation was
                  * in flight).
                  */
                 crm_notice("Not recording %s result in CIB because "
                            "resource information was removed since it was initiated",
                            op_key);
             } else {
                 /* This shouldn't be possible; the executor didn't consider the
                  * resource deleted, but we couldn't find resource or node
                  * information.
                  */
                 crm_err("Unable to record %s result in CIB: %s", op_key,
                         (node_name? "No resource information" : "No node name"));
             }
         }
 
     } else if (op->interval_ms == 0) {
         /* A non-recurring operation was cancelled. Most likely, the
          * never-initiated action was removed from the executor's pending
          * operations list upon resource removal.
          */
         need_direct_ack = TRUE;
 
     } else if (pending == NULL) {
         /* This recurring operation was cancelled, but was not pending. No
          * transition actions are waiting on it, nothing needs to be done.
          */
 
     } else if (op->user_data == NULL) {
         /* This recurring operation was cancelled and pending, but we don't
          * have a transition key. This should never happen.
          */
         crm_err("Recurring operation %s was cancelled without transition information",
                 op_key);
 
     } else if (pcmk_is_set(pending->flags, active_op_remove)) {
         /* This recurring operation was cancelled (by us) and pending, and we
          * have been waiting for it to finish.
          */
         if (lrm_state) {
             controld_delete_action_history(op);
         }
 
         /* Directly acknowledge failed recurring actions here. The above call to
          * controld_delete_action_history() will not erase any corresponding
          * last_failure entry, which means that the DC won't confirm the
          * cancellation via process_op_deletion(), and the transition would
          * otherwise wait for the action timer to pop.
          */
         if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
                                 pending->op_type, pending->interval_ms)) {
             need_direct_ack = TRUE;
         }
 
     } else if (op->rsc_deleted) {
         /* This recurring operation was cancelled (but not by us, and the
          * executor does not have resource information, likely due to resource
          * cleanup, refresh, or removal) and pending.
          */
         crm_debug("Recurring op %s was cancelled due to resource deletion",
                   op_key);
         need_direct_ack = TRUE;
 
     } else {
         /* This recurring operation was cancelled (but not by us, likely by the
          * executor before stopping the resource) and pending. We don't need to
          * do anything special.
          */
     }
 
     if (need_direct_ack) {
         controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
     }
 
     if(remove == FALSE) {
         /* The caller will do this afterwards, but keep the logging consistent */
         removed = TRUE;
 
     } else if (lrm_state && ((op->interval_ms == 0)
                              || (op->op_status == PCMK_EXEC_CANCELLED))) {
 
         gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id);
 
         if (op->interval_ms != 0) {
             removed = TRUE;
         } else if (found) {
             removed = TRUE;
             crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
                       op_key, op->call_id, op_id,
                       g_hash_table_size(lrm_state->active_ops));
         }
     }
 
     log_executor_event(op, op_key, node_name, removed);
 
     if (lrm_state) {
         if (!pcmk__str_eq(op->op_type, PCMK_ACTION_META_DATA,
                           pcmk__str_casei)) {
             crmd_alert_resource_op(lrm_state->node_name, op);
         } else if (rsc && (op->rc == PCMK_OCF_OK)) {
             char *metadata = unescape_newlines(op->output);
 
             controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata);
             free(metadata);
         }
     }
 
     if (op->rsc_deleted) {
         crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
         if (lrm_state) {
             delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL,
                              true);
         }
     }
 
     /* If a shutdown was escalated while operations were pending,
      * then the FSA will be stalled right now... allow it to continue
      */
     controld_trigger_fsa();
     if (lrm_state && rsc) {
         update_history_cache(lrm_state, rsc, op);
     }
 
     lrmd_free_rsc_info(rsc);
     free(op_key);
     free(op_id);
 }
diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c
index dd4d1213a0..7b9bfe91b8 100644
--- a/daemons/controld/controld_te_actions.c
+++ b/daemons/controld/controld_te_actions.c
@@ -1,749 +1,749 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/lrmd.h>               // lrmd_event_data_t, lrmd_free_event()
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cluster.h>
 
 #include <pacemaker-internal.h>
 #include <pacemaker-controld.h>
 
 static GHashTable *te_targets = NULL;
 void send_rsc_command(pcmk__graph_action_t *action);
 static void te_update_job_count(pcmk__graph_action_t *action, int offset);
 
 static void
 te_start_action_timer(const pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     action->timer = g_timeout_add(action->timeout + graph->network_delay,
                                   action_timer_callback, (void *) action);
     CRM_ASSERT(action->timer != 0);
 }
 
 /*!
  * \internal
  * \brief Execute a graph pseudo-action
  *
  * \param[in,out] graph   Transition graph being executed
  * \param[in,out] pseudo  Pseudo-action to execute
  *
  * \return Standard Pacemaker return code
  */
 static int
 execute_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *pseudo)
 {
     const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);
 
     /* send to peers as well? */
     if (pcmk__str_eq(task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_casei)) {
         GHashTableIter iter;
         crm_node_t *node = NULL;
 
         g_hash_table_iter_init(&iter, crm_peer_cache);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
             xmlNode *cmd = NULL;
 
             if (pcmk__str_eq(controld_globals.our_nodename, node->uname,
                              pcmk__str_casei)) {
                 continue;
             }
 
             cmd = create_request(task, pseudo->xml, node->uname,
                                  CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
             send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
             free_xml(cmd);
         }
 
         remote_ra_process_maintenance_nodes(pseudo->xml);
     } else {
         /* Check action for Pacemaker Remote node side effects */
         remote_ra_process_pseudo(pseudo->xml);
     }
 
     crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
               crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
     te_action_confirmed(pseudo, graph);
     return pcmk_rc_ok;
 }
 
 static int
 get_target_rc(pcmk__graph_action_t *action)
 {
     int exit_status;
 
     pcmk__scan_min_int(crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC),
                        &exit_status, 0);
     return exit_status;
 }
 
 /*!
  * \internal
  * \brief Execute a cluster action from a transition graph
  *
  * \param[in,out] graph   Transition graph being executed
  * \param[in,out] action  Cluster action to execute
  *
  * \return Standard Pacemaker return code
  */
 static int
 execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     char *counter = NULL;
     xmlNode *cmd = NULL;
     gboolean is_local = FALSE;
 
     const char *id = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
     const char *router_node = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
 
     id = ID(action->xml);
     CRM_CHECK(!pcmk__str_empty(id), return EPROTO);
 
     task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     CRM_CHECK(!pcmk__str_empty(task), return EPROTO);
 
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown);
 
     router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
     if (router_node == NULL) {
         router_node = on_node;
         if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
             const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);
 
             if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_none)) {
                 router_node = controld_globals.our_nodename;
             }
         }
     }
 
     if (pcmk__str_eq(router_node, controld_globals.our_nodename,
                      pcmk__str_casei)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     crm_info("Handling controller request '%s' (%s on %s)%s%s",
              id, task, on_node, (is_local? " locally" : ""),
              (no_wait? " without waiting" : ""));
 
     if (is_local
         && pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
         /* defer until everything else completes */
         crm_info("Controller request '%s' is a local shutdown", id);
         graph->completion_action = pcmk__graph_shutdown;
         graph->abort_reason = "local shutdown";
         te_action_confirmed(action, graph);
         return pcmk_rc_ok;
 
     } else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
         crm_node_t *peer = crm_get_peer(0, router_node);
 
         pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
     }
 
     cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
 
     counter = pcmk__transition_key(controld_globals.transition_graph->id,
                                    action->id, get_target_rc(action),
                                    controld_globals.te_uuid);
     crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
 
     rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
     free(counter);
     free_xml(cmd);
 
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return ECOMM;
 
     } else if (no_wait) {
         te_action_confirmed(action, graph);
 
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %ums instead",
                     action->id, task, on_node, action->timeout, graph->network_delay);
             action->timeout = (int) graph->network_delay;
         }
         te_start_action_timer(graph, action);
     }
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Synthesize an executor event for a resource action timeout
  *
  * \param[in] action     Resource action that timed out
  * \param[in] target_rc  Expected result of action that timed out
  *
  * Synthesize an executor event for a resource action timeout. (If the executor
  * gets a timeout while waiting for a resource action to complete, that will be
  * reported via the usual callback. This timeout means we didn't hear from the
  * executor itself or the controller that relayed the action to the executor.)
  *
  * \return Newly created executor event for result of \p action
  * \note The caller is responsible for freeing the return value using
  *       lrmd_free_event().
  */
 static lrmd_event_data_t *
 synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *reason = NULL;
     char *dynamic_reason = NULL;
 
     if (pcmk__str_eq(target, get_local_node_name(), pcmk__str_casei)) {
         reason = "Local executor did not return result in time";
     } else {
         const char *router_node = NULL;
 
         router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
         if (router_node == NULL) {
             router_node = target;
         }
         dynamic_reason = crm_strdup_printf("Controller on %s did not return "
                                            "result in time", router_node);
         reason = dynamic_reason;
     }
 
     op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT,
                                        PCMK_OCF_UNKNOWN_ERROR, reason);
     op->call_id = -1;
     op->user_data = pcmk__transition_key(controld_globals.transition_graph->id,
                                          action->id, target_rc,
                                          controld_globals.te_uuid);
     free(dynamic_reason);
     return op;
 }
 
 static void
 controld_record_action_event(pcmk__graph_action_t *action,
                              lrmd_event_data_t *op)
 {
     cib_t *cib_conn = controld_globals.cib_conn;
 
     xmlNode *state = NULL;
     xmlNode *rsc = NULL;
     xmlNode *action_rsc = NULL;
 
     int rc = pcmk_ok;
 
     const char *rsc_id = NULL;
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 
     int target_rc = get_target_rc(action);
 
     action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
     if (action_rsc == NULL) {
         return;
     }
 
     rsc_id = ID(action_rsc);
     CRM_CHECK(rsc_id != NULL,
               crm_log_xml_err(action->xml, "Bad:action"); return);
 
 /*
   update the CIB
 
 <node_state id="hadev">
       <lrm>
         <lrm_resources>
           <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
 */
 
     state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
     crm_xml_add(state, PCMK_XA_ID, target_uuid);
     crm_xml_add(state, XML_ATTR_UNAME, target);
 
     rsc = create_xml_node(state, XML_CIB_TAG_LRM);
     crm_xml_add(rsc, PCMK_XA_ID, target_uuid);
 
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
     crm_xml_add(rsc, PCMK_XA_ID, rsc_id);
 
 
     crm_copy_xml_element(action_rsc, rsc, PCMK_XA_TYPE);
     crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_CLASS);
-    crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_PROVIDER);
+    crm_copy_xml_element(action_rsc, rsc, PCMK_XA_PROVIDER);
 
     pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target,
                              __func__);
 
     rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS, state,
                                 cib_scope_local);
     fsa_register_cib_callback(rc, NULL, cib_action_updated);
     free_xml(state);
 
     crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)",
               rc, action->id, task_uuid, target);
     pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update);
 }
 
 void
 controld_record_action_timeout(pcmk__graph_action_t *action)
 {
     lrmd_event_data_t *op = NULL;
 
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     int target_rc = get_target_rc(action);
 
     crm_warn("%s %d: %s on %s timed out",
              action->xml->name, action->id, task_uuid, target);
 
     op = synthesize_timeout_event(action, target_rc);
     controld_record_action_event(action, op);
     lrmd_free_event(op);
 }
 
 /*!
  * \internal
  * \brief Execute a resource action from a transition graph
  *
  * \param[in,out] graph   Transition graph being executed
  * \param[in,out] action  Resource action to execute
  *
  * \return Standard Pacemaker return code
  */
 static int
 execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     /* never overwrite stop actions in the CIB with
      *   anything other than completed results
      *
      * Writing pending stops makes it look like the
      *   resource is running again
      */
     xmlNode *cmd = NULL;
     xmlNode *rsc_op = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
     gboolean is_local = FALSE;
 
     char *counter = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
     const char *router_node = NULL;
     const char *task_uuid = NULL;
 
     CRM_ASSERT(action != NULL);
     CRM_ASSERT(action->xml != NULL);
 
     pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed);
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     CRM_CHECK(!pcmk__str_empty(on_node),
               crm_err("Corrupted command(id=%s) %s: no node",
                       ID(action->xml), pcmk__s(task, "without task"));
               return pcmk_rc_node_unknown);
 
     rsc_op = action->xml;
     task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
     task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);
 
     if (!router_node) {
         router_node = on_node;
     }
 
     counter = pcmk__transition_key(controld_globals.transition_graph->id,
                                    action->id, get_target_rc(action),
                                    controld_globals.te_uuid);
     crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
 
     if (pcmk__str_eq(router_node, controld_globals.our_nodename,
                      pcmk__str_casei)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
                task, task_uuid, (is_local? " locally" : ""), on_node,
                (no_wait? " without waiting" : ""), action->id);
 
     cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
                          CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
 
     if (is_local) {
         /* shortcut local resource commands */
         ha_msg_input_t data = {
             .msg = cmd,
             .xml = rsc_op,
         };
 
         fsa_data_t msg = {
             .id = 0,
             .data = &data,
             .data_type = fsa_dt_ha_msg,
             .fsa_input = I_NULL,
             .fsa_cause = C_FSA_INTERNAL,
             .actions = A_LRM_INVOKE,
             .origin = __func__,
         };
 
         do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state,
                       I_NULL, &msg);
 
     } else {
         rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
     }
 
     free(counter);
     free_xml(cmd);
 
     pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);
 
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return ECOMM;
 
     } else if (no_wait) {
         /* Just mark confirmed. Don't bump the job count only to immediately
          * decrement it.
          */
         crm_info("Action %d confirmed - no wait", action->id);
         pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
         pcmk__update_graph(controld_globals.transition_graph, action);
         trigger_graph();
 
     } else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
         crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
                   action->id, task, task_uuid, on_node, action->timeout);
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %ums instead",
                     action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
             action->timeout = (int) graph->network_delay;
         }
         te_update_job_count(action, 1);
         te_start_action_timer(graph, action);
     }
 
     return pcmk_rc_ok;
 }
 
 struct te_peer_s
 {
         char *name;
         int jobs;
         int migrate_jobs;
 };
 
 static void te_peer_free(gpointer p)
 {
     struct te_peer_s *peer = p;
 
     free(peer->name);
     free(peer);
 }
 
 void te_reset_job_counts(void)
 {
     GHashTableIter iter;
     struct te_peer_s *peer = NULL;
 
     if(te_targets == NULL) {
         te_targets = pcmk__strkey_table(NULL, te_peer_free);
     }
 
     g_hash_table_iter_init(&iter, te_targets);
     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
         peer->jobs = 0;
         peer->migrate_jobs = 0;
     }
 }
 
 static void
 te_update_job_count_on(const char *target, int offset, bool migrate)
 {
     struct te_peer_s *r = NULL;
 
     if(target == NULL || te_targets == NULL) {
         return;
     }
 
     r = g_hash_table_lookup(te_targets, target);
     if(r == NULL) {
         r = calloc(1, sizeof(struct te_peer_s));
         r->name = strdup(target);
         g_hash_table_insert(te_targets, r->name, r);
     }
 
     r->jobs += offset;
     if(migrate) {
         r->migrate_jobs += offset;
     }
     crm_trace("jobs[%s] = %d", target, r->jobs);
 }
 
 static void
 te_update_job_count(pcmk__graph_action_t *action, int offset)
 {
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     if ((action->type != pcmk__rsc_graph_action) || (target == NULL)) {
         /* No limit on these */
         return;
     }
 
     /* if we have a router node, this means the action is performing
      * on a remote node. For now, we count all actions occurring on a
      * remote node against the job list on the cluster node hosting
      * the connection resources */
     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if ((target == NULL)
         && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
                                 PCMK_ACTION_MIGRATE_FROM, NULL)) {
         const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 
         te_update_job_count_on(t1, offset, TRUE);
         te_update_job_count_on(t2, offset, TRUE);
         return;
     } else if (target == NULL) {
         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     }
 
     te_update_job_count_on(target, offset, FALSE);
 }
 
 /*!
  * \internal
  * \brief Check whether a graph action is allowed to be executed on a node
  *
  * \param[in] graph   Transition graph being executed
  * \param[in] action  Graph action being executed
  * \param[in] target  Name of node where action should be executed
  *
  * \return true if action is allowed, otherwise false
  */
 static bool
 allowed_on_node(const pcmk__graph_t *graph, const pcmk__graph_action_t *action,
                 const char *target)
 {
     int limit = 0;
     struct te_peer_s *r = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     if(target == NULL) {
         /* No limit on these */
         return true;
 
     } else if(te_targets == NULL) {
         return false;
     }
 
     r = g_hash_table_lookup(te_targets, target);
     limit = throttle_get_job_limit(target);
 
     if(r == NULL) {
         r = calloc(1, sizeof(struct te_peer_s));
         r->name = strdup(target);
         g_hash_table_insert(te_targets, r->name, r);
     }
 
     if(limit <= r->jobs) {
         crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
                   target, limit, r->jobs, id);
         return false;
 
     } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
         if (pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
                                  PCMK_ACTION_MIGRATE_FROM, NULL)) {
             crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
                       target, graph->migration_limit, r->migrate_jobs, id);
             return false;
         }
     }
 
     crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);
 
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether a graph action is allowed to be executed
  *
  * \param[in] graph   Transition graph being executed
  * \param[in] action  Graph action being executed
  *
  * \return true if action is allowed, otherwise false
  */
 static bool
 graph_action_allowed(pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     const char *target = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 
     if (action->type != pcmk__rsc_graph_action) {
         /* No limit on these */
         return true;
     }
 
     /* if we have a router node, this means the action is performing
      * on a remote node. For now, we count all actions occurring on a
      * remote node against the job list on the cluster node hosting
      * the connection resources */
     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if ((target == NULL)
         && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
                                 PCMK_ACTION_MIGRATE_FROM, NULL)) {
         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
         if (!allowed_on_node(graph, action, target)) {
             return false;
         }
 
         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 
     } else if (target == NULL) {
         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     }
 
     return allowed_on_node(graph, action, target);
 }
 
 /*!
  * \brief Confirm a graph action (and optionally update graph)
  *
  * \param[in,out] action  Action to confirm
  * \param[in,out] graph   Update and trigger this graph (if non-NULL)
  */
 void
 te_action_confirmed(pcmk__graph_action_t *action, pcmk__graph_t *graph)
 {
     if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
         if ((action->type == pcmk__rsc_graph_action)
             && (crm_element_value(action->xml, XML_LRM_ATTR_TARGET) != NULL)) {
             te_update_job_count(action, -1);
         }
         pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
     }
     if (graph) {
         pcmk__update_graph(graph, action);
         trigger_graph();
     }
 }
 
 static pcmk__graph_functions_t te_graph_fns = {
     execute_pseudo_action,
     execute_rsc_action,
     execute_cluster_action,
     controld_execute_fence_action,
     graph_action_allowed,
 };
 
 /*
  * \internal
  * \brief Register the transitioner's graph functions with \p libpacemaker
  */
 void
 controld_register_graph_functions(void)
 {
     pcmk__set_graph_functions(&te_graph_fns);
 }
 
 void
 notify_crmd(pcmk__graph_t *graph)
 {
     const char *type = "unknown";
     enum crmd_fsa_input event = I_NULL;
 
     crm_debug("Processing transition completion in state %s",
               fsa_state2string(controld_globals.fsa_state));
 
     CRM_CHECK(graph->complete, graph->complete = true);
 
     switch (graph->completion_action) {
         case pcmk__graph_wait:
             type = "stop";
             if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
                 event = I_TE_SUCCESS;
             }
             break;
         case pcmk__graph_done:
             type = "done";
             if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
                 event = I_TE_SUCCESS;
             }
             break;
 
         case pcmk__graph_restart:
             type = "restart";
             if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
                 if (controld_get_period_transition_timer() > 0) {
                     controld_stop_transition_timer();
                     controld_start_transition_timer();
                 } else {
                     event = I_PE_CALC;
                 }
 
             } else if (controld_globals.fsa_state == S_POLICY_ENGINE) {
                 controld_set_fsa_action_flags(A_PE_INVOKE);
                 controld_trigger_fsa();
             }
             break;
 
         case pcmk__graph_shutdown:
             type = "shutdown";
             if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
                 event = I_STOP;
 
             } else {
                 crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
                 event = I_TERMINATE;
             }
     }
 
     crm_debug("Transition %d status: %s - %s", graph->id, type,
               pcmk__s(graph->abort_reason, "unspecified reason"));
 
     graph->abort_reason = NULL;
     graph->completion_action = pcmk__graph_done;
 
     if (event != I_NULL) {
         register_fsa_input(C_FSA_INTERNAL, event, NULL);
     } else {
         controld_trigger_fsa();
     }
 }
diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
index ec533996af..71a67435a2 100644
--- a/include/crm/msg_xml.h
+++ b/include/crm/msg_xml.h
@@ -1,418 +1,419 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__CRM_MSG_XML__H
 #  define PCMK__CRM_MSG_XML__H
 
 #  include <crm/common/xml.h>
 
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
 #include <crm/msg_xml_compat.h>
 #endif
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /* This file defines constants for various XML syntax (mainly element and
  * attribute names).
  *
  * For consistency, new constants should start with "PCMK_", followed by "XE"
  * for XML element names, "XA" for XML attribute names, and "META" for meta
  * attribute names. Old names that don't follow this policy should eventually be
  * deprecated and replaced with names that do.
  *
  * Symbols should be public if the user may specify them somewhere (especially
  * the CIB). They should be internal if they're used only internally to
  * Pacemaker (such as daemon IPC/CPG message XML).
  *
  * For meta-attributes that can be specified as either XML attributes or nvpair
  * names, use "META" unless using both "XA" and "META" constants adds clarity.
  */
 
 /*
  * XML elements
  */
 
 #define PCMK_XE_DATE_EXPRESSION             "date_expression"
 #define PCMK_XE_OP_EXPRESSION               "op_expression"
 
 /* This has been deprecated as a CIB element (an alias for <clone> with
  * PCMK_META_PROMOTABLE set to "true") since 2.0.0.
  */
 #define PCMK_XE_PROMOTABLE_LEGACY           "master"
 
 #define PCMK_XE_RSC_EXPRESSION              "rsc_expression"
 
 
 /*
  * XML attributes
  */
 
 #define PCMK_XA_ADMIN_EPOCH                 "admin_epoch"
 #define PCMK_XA_CIB_LAST_WRITTEN            "cib-last-written"
 #define PCMK_XA_CRM_DEBUG_ORIGIN            "crm-debug-origin"
 #define PCMK_XA_CRM_FEATURE_SET             "crm_feature_set"
 #define PCMK_XA_CRM_TIMESTAMP               "crm-timestamp"
 #define PCMK_XA_DESCRIPTION                 "description"
 #define PCMK_XA_EPOCH                       "epoch"
 #define PCMK_XA_FORMAT                      "format"
 #define PCMK_XA_HAVE_QUORUM                 "have-quorum"
 #define PCMK_XA_ID                          "id"
 #define PCMK_XA_ID_REF                      "id-ref"
 #define PCMK_XA_NAME                        "name"
 #define PCMK_XA_NO_QUORUM_PANIC             "no-quorum-panic"
 #define PCMK_XA_NUM_UPDATES                 "num_updates"
+#define PCMK_XA_PROVIDER                    "provider"
 #define PCMK_XA_TYPE                        "type"
 #define PCMK_XA_VALIDATE_WITH               "validate-with"
 #define PCMK_XA_VALUE                       "value"
 #define PCMK_XA_VERSION                     "version"
 
 
 /*
  * Older constants that don't follow current naming
  */
 
 #  ifndef F_ORIG
 #    define F_ORIG    "src"
 #  endif
 
 #  ifndef F_SEQ
 #    define F_SEQ		"seq"
 #  endif
 
 #  ifndef F_SUBTYPE
 #    define F_SUBTYPE "subt"
 #  endif
 
 #  ifndef F_TYPE
 #    define F_TYPE    "t"
 #  endif
 
 #  ifndef F_CLIENTNAME
 #    define	F_CLIENTNAME	"cn"
 #  endif
 
 #  ifndef F_XML_TAGNAME
 #    define F_XML_TAGNAME	"__name__"
 #  endif
 
 #  ifndef T_CRM
 #    define T_CRM     "crmd"
 #  endif
 
 #  ifndef T_ATTRD
 #    define T_ATTRD     "attrd"
 #  endif
 
 #  define CIB_OPTIONS_FIRST "cib-bootstrap-options"
 
 #  define F_CRM_DATA			"crm_xml"
 #  define F_CRM_TASK			"crm_task"
 #  define F_CRM_HOST_TO			"crm_host_to"
 #  define F_CRM_MSG_TYPE		F_SUBTYPE
 #  define F_CRM_SYS_TO			"crm_sys_to"
 #  define F_CRM_SYS_FROM		"crm_sys_from"
 #  define F_CRM_HOST_FROM		F_ORIG
 #  define F_CRM_REFERENCE		XML_ATTR_REFERENCE
 #  define F_CRM_VERSION			PCMK_XA_VERSION
 #  define F_CRM_ORIGIN			"origin"
 #  define F_CRM_USER			"crm_user"
 #  define F_CRM_JOIN_ID			"join_id"
 #  define F_CRM_DC_LEAVING      "dc-leaving"
 #  define F_CRM_ELECTION_ID		"election-id"
 #  define F_CRM_ELECTION_AGE_S		"election-age-sec"
 #  define F_CRM_ELECTION_AGE_US		"election-age-nano-sec"
 #  define F_CRM_ELECTION_OWNER		"election-owner"
 #  define F_CRM_TGRAPH			"crm-tgraph-file"
 #  define F_CRM_TGRAPH_INPUT		"crm-tgraph-in"
 
 #  define F_CRM_THROTTLE_MODE		"crm-limit-mode"
 #  define F_CRM_THROTTLE_MAX		"crm-limit-max"
 
 /*---- Common tags/attrs */
 #  define XML_DIFF_MARKER		"__crm_diff_marker__"
 #  define XML_TAG_CIB			"cib"
 #  define XML_TAG_FAILED		"failed"
 
 #  define XML_ATTR_TIMEOUT		"timeout"
 #  define XML_ATTR_OP			"op"
 #  define XML_ATTR_DC_UUID		"dc-uuid"
 #  define XML_ATTR_UPDATE_ORIG		"update-origin"
 #  define XML_ATTR_UPDATE_CLIENT	"update-client"
 #  define XML_ATTR_UPDATE_USER		"update-user"
 
 #  define XML_BOOLEAN_TRUE		"true"
 #  define XML_BOOLEAN_FALSE		"false"
 
 #  define XML_TAG_OPTIONS		"options"
 
 /*---- top level tags/attrs */
 #  define XML_ATTR_REQUEST		"request"
 #  define XML_ATTR_RESPONSE		"response"
 
 #  define XML_ATTR_UNAME		"uname"
 #  define XML_ATTR_REFERENCE		"reference"
 
 #  define XML_CRM_TAG_PING		"ping_response"
 #  define XML_PING_ATTR_STATUS		"result"
 #  define XML_PING_ATTR_SYSFROM		"crm_subsystem"
 #  define XML_PING_ATTR_CRMDSTATE   "crmd_state"
 #  define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state"
 #  define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init"
 #  define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons"
 #  define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping"
 #  define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running"
 #  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down"
 #  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete"
 #  define XML_PING_ATTR_PACEMAKERDSTATE_REMOTE "remote"
 
 #  define XML_FAIL_TAG_CIB		"failed_update"
 
 #  define XML_FAILCIB_ATTR_OBJTYPE	"object_type"
 #  define XML_FAILCIB_ATTR_OP		"operation"
 #  define XML_FAILCIB_ATTR_REASON	"reason"
 
 /*---- CIB specific tags/attrs */
 #  define XML_CIB_TAG_SECTION_ALL	"all"
 #  define XML_CIB_TAG_CONFIGURATION	"configuration"
 #  define XML_CIB_TAG_STATUS       	"status"
 #  define XML_CIB_TAG_RESOURCES		"resources"
 #  define XML_CIB_TAG_NODES         	"nodes"
 #  define XML_CIB_TAG_CONSTRAINTS   	"constraints"
 #  define XML_CIB_TAG_CRMCONFIG   	"crm_config"
 #  define XML_CIB_TAG_OPCONFIG		"op_defaults"
 #  define XML_CIB_TAG_RSCCONFIG   	"rsc_defaults"
 #  define XML_CIB_TAG_ACLS   		"acls"
 #  define XML_CIB_TAG_ALERTS    	"alerts"
 #  define XML_CIB_TAG_ALERT   		"alert"
 #  define XML_CIB_TAG_ALERT_RECIPIENT	"recipient"
 #  define XML_CIB_TAG_ALERT_SELECT      "select"
 #  define XML_CIB_TAG_ALERT_ATTRIBUTES  "select_attributes"
 #  define XML_CIB_TAG_ALERT_FENCING     "select_fencing"
 #  define XML_CIB_TAG_ALERT_NODES       "select_nodes"
 #  define XML_CIB_TAG_ALERT_RESOURCES   "select_resources"
 #  define XML_CIB_TAG_ALERT_ATTR        "attribute"
 
 #  define XML_CIB_TAG_STATE         	"node_state"
 #  define XML_CIB_TAG_NODE          	"node"
 #  define XML_CIB_TAG_NVPAIR        	"nvpair"
 
 #  define XML_CIB_TAG_PROPSET	   	"cluster_property_set"
 #  define XML_TAG_ATTR_SETS	   	"instance_attributes"
 #  define XML_TAG_META_SETS	   	"meta_attributes"
 #  define XML_TAG_ATTRS			"attributes"
 #  define XML_TAG_PARAMS		"parameters"
 #  define XML_TAG_PARAM			"param"
 #  define XML_TAG_UTILIZATION		"utilization"
 
 #  define XML_TAG_RESOURCE_REF		"resource_ref"
 #  define XML_CIB_TAG_RESOURCE	  	"primitive"
 #  define XML_CIB_TAG_GROUP	  	"group"
 #  define XML_CIB_TAG_INCARNATION	"clone"
 #  define XML_CIB_TAG_CONTAINER		"bundle"
 
 #  define XML_CIB_TAG_RSC_TEMPLATE	"template"
 
 #  define XML_OP_ATTR_ON_FAIL		"on-fail"
 #  define XML_OP_ATTR_START_DELAY	"start-delay"
 #  define XML_OP_ATTR_ORIGIN		"interval-origin"
 #  define XML_OP_ATTR_PENDING		"record-pending"
 #  define XML_OP_ATTR_DIGESTS_ALL       "digests-all"
 #  define XML_OP_ATTR_DIGESTS_SECURE    "digests-secure"
 
 #  define XML_CIB_TAG_LRM		"lrm"
 #  define XML_LRM_TAG_RESOURCES     	"lrm_resources"
 #  define XML_LRM_TAG_RESOURCE     	"lrm_resource"
 #  define XML_LRM_TAG_RSC_OP		"lrm_rsc_op"
 #  define XML_AGENT_ATTR_CLASS		"class"
 #  define XML_AGENT_ATTR_PROVIDER	"provider"
 
 //! \deprecated Do not use (will be removed in a future release)
 #  define XML_CIB_ATTR_REPLACE       	"replace"
 
 #  define XML_CIB_ATTR_PRIORITY     	"priority"
 
 #  define XML_NODE_IS_REMOTE    	"remote_node"
 #  define XML_NODE_IS_FENCED		"node_fenced"
 #  define XML_NODE_IS_MAINTENANCE   "node_in_maintenance"
 
 #  define XML_CIB_ATTR_SHUTDOWN       	"shutdown"
 
 /* Aside from being an old name for the executor, LRM is a misnomer here because
  * the controller and scheduler use these to track actions, which are not always
  * executor operations.
  */
 
 // XML attribute that takes interval specification (user-facing configuration)
 #  define XML_LRM_ATTR_INTERVAL		"interval"
 
 // XML attribute that takes interval in milliseconds (daemon APIs)
 // (identical value as above, but different constant allows clearer code intent)
 #  define XML_LRM_ATTR_INTERVAL_MS  XML_LRM_ATTR_INTERVAL
 
 #  define XML_LRM_ATTR_TASK		"operation"
 #  define XML_LRM_ATTR_TASK_KEY		"operation_key"
 #  define XML_LRM_ATTR_TARGET		"on_node"
 #  define XML_LRM_ATTR_TARGET_UUID	"on_node_uuid"
 /*! Actions to be executed on Pacemaker Remote nodes are routed through the
  *  controller on the cluster node hosting the remote connection. That cluster
  *  node is considered the router node for the action.
  */
 #  define XML_LRM_ATTR_ROUTER_NODE  "router_node"
 #  define XML_LRM_ATTR_RSCID		"rsc-id"
 #  define XML_LRM_ATTR_OPSTATUS		"op-status"
 #  define XML_LRM_ATTR_RC		"rc-code"
 #  define XML_LRM_ATTR_CALLID		"call-id"
 #  define XML_LRM_ATTR_OP_DIGEST	"op-digest"
 #  define XML_LRM_ATTR_OP_RESTART	"op-force-restart"
 #  define XML_LRM_ATTR_OP_SECURE	"op-secure-params"
 #  define XML_LRM_ATTR_RESTART_DIGEST	"op-restart-digest"
 #  define XML_LRM_ATTR_SECURE_DIGEST	"op-secure-digest"
 #  define XML_LRM_ATTR_EXIT_REASON	"exit-reason"
 
 #  define XML_RSC_OP_LAST_CHANGE        "last-rc-change"
 #  define XML_RSC_OP_T_EXEC             "exec-time"
 #  define XML_RSC_OP_T_QUEUE            "queue-time"
 
 #  define XML_LRM_ATTR_MIGRATE_SOURCE	"migrate_source"
 #  define XML_LRM_ATTR_MIGRATE_TARGET	"migrate_target"
 
 #  define XML_TAG_GRAPH			"transition_graph"
 #  define XML_GRAPH_TAG_RSC_OP		"rsc_op"
 #  define XML_GRAPH_TAG_PSEUDO_EVENT	"pseudo_event"
 #  define XML_GRAPH_TAG_CRM_EVENT	"crm_event"
 #  define XML_GRAPH_TAG_DOWNED            "downed"
 #  define XML_GRAPH_TAG_MAINTENANCE       "maintenance"
 
 #  define XML_TAG_RULE			"rule"
 #  define XML_RULE_ATTR_SCORE		"score"
 #  define XML_RULE_ATTR_SCORE_ATTRIBUTE	"score-attribute"
 #  define XML_RULE_ATTR_ROLE		"role"
 #  define XML_RULE_ATTR_BOOLEAN_OP	"boolean-op"
 
 #  define XML_TAG_EXPRESSION		"expression"
 #  define XML_EXPR_ATTR_ATTRIBUTE	"attribute"
 #  define XML_EXPR_ATTR_OPERATION	"operation"
 #  define XML_EXPR_ATTR_VALUE_SOURCE	"value-source"
 
 #  define XML_CONS_TAG_RSC_DEPEND	"rsc_colocation"
 #  define XML_CONS_TAG_RSC_ORDER	"rsc_order"
 #  define XML_CONS_TAG_RSC_LOCATION	"rsc_location"
 #  define XML_CONS_TAG_RSC_TICKET	"rsc_ticket"
 #  define XML_CONS_TAG_RSC_SET		"resource_set"
 #  define XML_CONS_ATTR_SYMMETRICAL	"symmetrical"
 
 #  define XML_LOCATION_ATTR_DISCOVERY	"resource-discovery"
 
 #  define XML_COLOC_ATTR_SOURCE		"rsc"
 #  define XML_COLOC_ATTR_SOURCE_ROLE	"rsc-role"
 #  define XML_COLOC_ATTR_TARGET		"with-rsc"
 #  define XML_COLOC_ATTR_TARGET_ROLE	"with-rsc-role"
 #  define XML_COLOC_ATTR_NODE_ATTR	"node-attribute"
 #  define XML_COLOC_ATTR_INFLUENCE          "influence"
 
 //! \deprecated Deprecated since 2.1.5
 #  define XML_COLOC_ATTR_SOURCE_INSTANCE	"rsc-instance"
 
 //! \deprecated Deprecated since 2.1.5
 #  define XML_COLOC_ATTR_TARGET_INSTANCE	"with-rsc-instance"
 
 #  define XML_LOC_ATTR_SOURCE           "rsc"
 #  define XML_LOC_ATTR_SOURCE_PATTERN   "rsc-pattern"
 
 #  define XML_ORDER_ATTR_FIRST		"first"
 #  define XML_ORDER_ATTR_THEN		"then"
 #  define XML_ORDER_ATTR_FIRST_ACTION	"first-action"
 #  define XML_ORDER_ATTR_THEN_ACTION	"then-action"
 #  define XML_ORDER_ATTR_KIND		"kind"
 
 //! \deprecated Deprecated since 2.1.5
 #  define XML_ORDER_ATTR_FIRST_INSTANCE	"first-instance"
 
 //! \deprecated Deprecated since 2.1.5
 #  define XML_ORDER_ATTR_THEN_INSTANCE	"then-instance"
 
 #  define XML_TICKET_ATTR_TICKET	"ticket"
 #  define XML_TICKET_ATTR_LOSS_POLICY	"loss-policy"
 
 #  define XML_NODE_ATTR_RSC_DISCOVERY   "resource-discovery-enabled"
 
 #  define XML_ALERT_ATTR_PATH		"path"
 #  define XML_ALERT_ATTR_TIMEOUT	"timeout"
 #  define XML_ALERT_ATTR_TSTAMP_FORMAT	"timestamp-format"
 
 #  define XML_CIB_TAG_GENERATION_TUPPLE	"generation_tuple"
 
 #  define XML_ATTR_TRANSITION_MAGIC	"transition-magic"
 #  define XML_ATTR_TRANSITION_KEY	"transition-key"
 
 #  define XML_ATTR_TE_NOWAIT		"op_no_wait"
 #  define XML_ATTR_TE_TARGET_RC		"op_target_rc"
 #  define XML_TAG_TRANSIENT_NODEATTRS	"transient_attributes"
 
 //! \deprecated Do not use (will be removed in a future release)
 #  define XML_TAG_DIFF_ADDED		"diff-added"
 
 //! \deprecated Do not use (will be removed in a future release)
 #  define XML_TAG_DIFF_REMOVED		"diff-removed"
 
 #  define XML_ACL_TAG_USER		"acl_target"
 #  define XML_ACL_TAG_USERv1		"acl_user"
 #  define XML_ACL_TAG_GROUP		"acl_group"
 #  define XML_ACL_TAG_ROLE		"acl_role"
 #  define XML_ACL_TAG_PERMISSION	"acl_permission"
 #  define XML_ACL_TAG_ROLE_REF 		"role"
 #  define XML_ACL_TAG_ROLE_REFv1	"role_ref"
 #  define XML_ACL_ATTR_KIND		"kind"
 #  define XML_ACL_TAG_READ		"read"
 #  define XML_ACL_TAG_WRITE		"write"
 #  define XML_ACL_TAG_DENY		"deny"
 #  define XML_ACL_ATTR_REF		"reference"
 #  define XML_ACL_ATTR_REFv1		"ref"
 #  define XML_ACL_ATTR_TAG		"object-type"
 #  define XML_ACL_ATTR_TAGv1		"tag"
 #  define XML_ACL_ATTR_XPATH		"xpath"
 #  define XML_ACL_ATTR_ATTRIBUTE	"attribute"
 
 #  define XML_CIB_TAG_TICKETS		"tickets"
 #  define XML_CIB_TAG_TICKET_STATE	"ticket_state"
 
 #  define XML_CIB_TAG_TAGS   		"tags"
 #  define XML_CIB_TAG_TAG   		"tag"
 #  define XML_CIB_TAG_OBJ_REF 		"obj_ref"
 
 #  define XML_TAG_FENCING_TOPOLOGY      "fencing-topology"
 #  define XML_TAG_FENCING_LEVEL         "fencing-level"
 #  define XML_ATTR_STONITH_INDEX        "index"
 #  define XML_ATTR_STONITH_TARGET       "target"
 #  define XML_ATTR_STONITH_TARGET_VALUE     "target-value"
 #  define XML_ATTR_STONITH_TARGET_PATTERN   "target-pattern"
 #  define XML_ATTR_STONITH_TARGET_ATTRIBUTE "target-attribute"
 #  define XML_ATTR_STONITH_DEVICES      "devices"
 
 #  define XML_TAG_DIFF                  "diff"
 #  define XML_DIFF_VERSION              "version"
 #  define XML_DIFF_VSOURCE              "source"
 #  define XML_DIFF_VTARGET              "target"
 #  define XML_DIFF_CHANGE               "change"
 #  define XML_DIFF_LIST                 "change-list"
 #  define XML_DIFF_ATTR                 "change-attr"
 #  define XML_DIFF_RESULT               "change-result"
 #  define XML_DIFF_OP                   "operation"
 #  define XML_DIFF_PATH                 "path"
 #  define XML_DIFF_POSITION             "position"
 
 #  define ID(x) crm_element_value(x, PCMK_XA_ID)
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif
diff --git a/lib/common/ipc_controld.c b/lib/common/ipc_controld.c
index 37ae814f73..408e10afec 100644
--- a/lib/common/ipc_controld.c
+++ b/lib/common/ipc_controld.c
@@ -1,657 +1,657 @@
 /*
  * Copyright 2020-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <stdbool.h>
 #include <errno.h>
 #include <libxml/tree.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/common/ipc_controld.h>
 #include "crmcommon_private.h"
 
 struct controld_api_private_s {
     char *client_uuid;
     unsigned int replies_expected;
 };
 
 /*!
  * \internal
  * \brief Get a string representation of a controller API reply type
  *
  * \param[in] reply  Controller API reply type
  *
  * \return String representation of a controller API reply type
  */
 const char *
 pcmk__controld_api_reply2str(enum pcmk_controld_api_reply reply)
 {
     switch (reply) {
         case pcmk_controld_reply_reprobe:
             return "reprobe";
         case pcmk_controld_reply_info:
             return "info";
         case pcmk_controld_reply_resource:
             return "resource";
         case pcmk_controld_reply_ping:
             return "ping";
         case pcmk_controld_reply_nodes:
             return "nodes";
         default:
             return "unknown";
     }
 }
 
 // \return Standard Pacemaker return code
 static int
 new_data(pcmk_ipc_api_t *api)
 {
     struct controld_api_private_s *private = NULL;
 
     api->api_data = calloc(1, sizeof(struct controld_api_private_s));
 
     if (api->api_data == NULL) {
         return errno;
     }
 
     private = api->api_data;
 
     /* This is set to the PID because that's how it was always done, but PIDs
      * are not unique because clients can be remote. The value appears to be
      * unused other than as part of F_CRM_SYS_FROM in IPC requests, which is
      * only compared against the internal system names (CRM_SYSTEM_TENGINE,
      * etc.), so it shouldn't be a problem.
      */
     private->client_uuid = pcmk__getpid_s();
 
     /* @TODO Implement a call ID model similar to the CIB, executor, and fencer
      *       IPC APIs, so that requests and replies can be matched, and
      *       duplicate replies can be discarded.
      */
     return pcmk_rc_ok;
 }
 
 static void
 free_data(void *data)
 {
     free(((struct controld_api_private_s *) data)->client_uuid);
     free(data);
 }
 
 // \return Standard Pacemaker return code
 static int
 post_connect(pcmk_ipc_api_t *api)
 {
     /* The controller currently requires clients to register via a hello
      * request, but does not reply back.
      */
     struct controld_api_private_s *private = api->api_data;
     const char *client_name = crm_system_name? crm_system_name : "client";
     xmlNode *hello;
     int rc;
 
     hello = create_hello_message(private->client_uuid, client_name,
                                  PCMK__CONTROLD_API_MAJOR,
                                  PCMK__CONTROLD_API_MINOR);
     rc = pcmk__send_ipc_request(api, hello);
     free_xml(hello);
     if (rc != pcmk_rc_ok) {
         crm_info("Could not send IPC hello to %s: %s " CRM_XS " rc=%s",
                  pcmk_ipc_name(api, true), pcmk_rc_str(rc), rc);
     } else {
         crm_debug("Sent IPC hello to %s", pcmk_ipc_name(api, true));
     }
     return rc;
 }
 
 static void
 set_node_info_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
 {
     data->reply_type = pcmk_controld_reply_info;
     if (msg_data == NULL) {
         return;
     }
     data->data.node_info.have_quorum =
         pcmk__xe_attr_is_true(msg_data, PCMK_XA_HAVE_QUORUM);
     data->data.node_info.is_remote = pcmk__xe_attr_is_true(msg_data, XML_NODE_IS_REMOTE);
 
     /* Integer node_info.id is currently valid only for Corosync nodes.
      *
      * @TODO: Improve handling after crm_node_t is refactored to handle layer-
      * specific data better.
      */
     crm_element_value_int(msg_data, PCMK_XA_ID, &(data->data.node_info.id));
 
     data->data.node_info.uuid = crm_element_value(msg_data, PCMK_XA_ID);
     data->data.node_info.uname = crm_element_value(msg_data, XML_ATTR_UNAME);
     data->data.node_info.state = crm_element_value(msg_data, PCMK__XA_CRMD);
 }
 
 static void
 set_ping_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
 {
     data->reply_type = pcmk_controld_reply_ping;
     if (msg_data == NULL) {
         return;
     }
     data->data.ping.sys_from = crm_element_value(msg_data,
                                                  XML_PING_ATTR_SYSFROM);
     data->data.ping.fsa_state = crm_element_value(msg_data,
                                                   XML_PING_ATTR_CRMDSTATE);
     data->data.ping.result = crm_element_value(msg_data, XML_PING_ATTR_STATUS);
 }
 
 static void
 set_nodes_data(pcmk_controld_api_reply_t *data, xmlNode *msg_data)
 {
     pcmk_controld_api_node_t *node_info;
 
     data->reply_type = pcmk_controld_reply_nodes;
     for (xmlNode *node = first_named_child(msg_data, XML_CIB_TAG_NODE);
          node != NULL; node = crm_next_same_xml(node)) {
 
         long long id_ll = 0;
 
         node_info = calloc(1, sizeof(pcmk_controld_api_node_t));
         crm_element_value_ll(node, PCMK_XA_ID, &id_ll);
         if (id_ll > 0) {
             node_info->id = id_ll;
         }
         node_info->uname = crm_element_value(node, XML_ATTR_UNAME);
         node_info->state = crm_element_value(node, PCMK__XA_IN_CCM);
         data->data.nodes = g_list_prepend(data->data.nodes, node_info);
     }
 }
 
 static bool
 reply_expected(pcmk_ipc_api_t *api, const xmlNode *request)
 {
     // We only need to handle commands that API functions can send
     return pcmk__str_any_of(crm_element_value(request, F_CRM_TASK),
                             PCMK__CONTROLD_CMD_NODES,
                             CRM_OP_LRM_DELETE,
                             CRM_OP_LRM_FAIL,
                             CRM_OP_NODE_INFO,
                             CRM_OP_PING,
                             CRM_OP_REPROBE,
                             CRM_OP_RM_NODE_CACHE,
                             NULL);
 }
 
 static bool
 dispatch(pcmk_ipc_api_t *api, xmlNode *reply)
 {
     struct controld_api_private_s *private = api->api_data;
     crm_exit_t status = CRM_EX_OK;
     xmlNode *msg_data = NULL;
     const char *value = NULL;
     pcmk_controld_api_reply_t reply_data = {
         pcmk_controld_reply_unknown, NULL, NULL,
     };
 
     if (pcmk__xe_is(reply, "ack")) {
         /* ACKs are trivial responses that do not count toward expected replies,
          * and do not have all the fields that validation requires, so skip that
          * processing.
          */
         return private->replies_expected > 0;
     }
 
     if (private->replies_expected > 0) {
         private->replies_expected--;
     }
 
     // Do some basic validation of the reply
 
     /* @TODO We should be able to verify that value is always a response, but
      *       currently the controller doesn't always properly set the type. Even
      *       if we fix the controller, we'll still need to handle replies from
      *       old versions (feature set could be used to differentiate).
      */
     value = crm_element_value(reply, F_CRM_MSG_TYPE);
     if (pcmk__str_empty(value)
         || !pcmk__str_any_of(value, XML_ATTR_REQUEST, XML_ATTR_RESPONSE, NULL)) {
         crm_info("Unrecognizable message from controller: "
                  "invalid message type '%s'", pcmk__s(value, ""));
         status = CRM_EX_PROTOCOL;
         goto done;
     }
 
     if (pcmk__str_empty(crm_element_value(reply, XML_ATTR_REFERENCE))) {
         crm_info("Unrecognizable message from controller: no reference");
         status = CRM_EX_PROTOCOL;
         goto done;
     }
 
     value = crm_element_value(reply, F_CRM_TASK);
     if (pcmk__str_empty(value)) {
         crm_info("Unrecognizable message from controller: no command name");
         status = CRM_EX_PROTOCOL;
         goto done;
     }
 
     // Parse useful info from reply
 
     reply_data.feature_set = crm_element_value(reply, PCMK_XA_VERSION);
     reply_data.host_from = crm_element_value(reply, F_CRM_HOST_FROM);
     msg_data = get_message_xml(reply, F_CRM_DATA);
 
     if (!strcmp(value, CRM_OP_REPROBE)) {
         reply_data.reply_type = pcmk_controld_reply_reprobe;
 
     } else if (!strcmp(value, CRM_OP_NODE_INFO)) {
         set_node_info_data(&reply_data, msg_data);
 
     } else if (!strcmp(value, CRM_OP_INVOKE_LRM)) {
         reply_data.reply_type = pcmk_controld_reply_resource;
         reply_data.data.resource.node_state = msg_data;
 
     } else if (!strcmp(value, CRM_OP_PING)) {
         set_ping_data(&reply_data, msg_data);
 
     } else if (!strcmp(value, PCMK__CONTROLD_CMD_NODES)) {
         set_nodes_data(&reply_data, msg_data);
 
     } else {
         crm_info("Unrecognizable message from controller: unknown command '%s'",
                  value);
         status = CRM_EX_PROTOCOL;
     }
 
 done:
     pcmk__call_ipc_callback(api, pcmk_ipc_event_reply, status, &reply_data);
 
     // Free any reply data that was allocated
     if (pcmk__str_eq(value, PCMK__CONTROLD_CMD_NODES, pcmk__str_casei)) {
         g_list_free_full(reply_data.data.nodes, free);
     }
 
     return false; // No further replies needed
 }
 
 pcmk__ipc_methods_t *
 pcmk__controld_api_methods(void)
 {
     pcmk__ipc_methods_t *cmds = calloc(1, sizeof(pcmk__ipc_methods_t));
 
     if (cmds != NULL) {
         cmds->new_data = new_data;
         cmds->free_data = free_data;
         cmds->post_connect = post_connect;
         cmds->reply_expected = reply_expected;
         cmds->dispatch = dispatch;
     }
     return cmds;
 }
 
 /*!
  * \internal
  * \brief Create XML for a controller IPC request
  *
  * \param[in] api       Controller connection
  * \param[in] op        Controller IPC command name
  * \param[in] node      Node name to set as destination host
  * \param[in] msg_data  XML to attach to request as message data
  *
  * \return Newly allocated XML for request
  */
 static xmlNode *
 create_controller_request(const pcmk_ipc_api_t *api, const char *op,
                           const char *node, xmlNode *msg_data)
 {
     struct controld_api_private_s *private = NULL;
     const char *sys_to = NULL;
 
     if (api == NULL) {
         return NULL;
     }
     private = api->api_data;
     if ((node == NULL) && !strcmp(op, CRM_OP_PING)) {
         sys_to = CRM_SYSTEM_DC;
     } else {
         sys_to = CRM_SYSTEM_CRMD;
     }
     return create_request(op, msg_data, node, sys_to,
                           (crm_system_name? crm_system_name : "client"),
                           private->client_uuid);
 }
 
 // \return Standard Pacemaker return code
 static int
 send_controller_request(pcmk_ipc_api_t *api, const xmlNode *request,
                         bool reply_is_expected)
 {
     if (crm_element_value(request, XML_ATTR_REFERENCE) == NULL) {
         return EINVAL;
     }
     if (reply_is_expected) {
         struct controld_api_private_s *private = api->api_data;
 
         private->replies_expected++;
     }
     return pcmk__send_ipc_request(api, request);
 }
 
 static xmlNode *
 create_reprobe_message_data(const char *target_node, const char *router_node)
 {
     xmlNode *msg_data;
 
     msg_data = create_xml_node(NULL, "data_for_" CRM_OP_REPROBE);
     crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
     if ((router_node != NULL) && !pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
         crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
     }
     return msg_data;
 }
 
 /*!
  * \brief Send a reprobe controller operation
  *
  * \param[in,out] api          Controller connection
  * \param[in]     target_node  Name of node to reprobe
  * \param[in]     router_node  Router node for host
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_reprobe.
  */
 int
 pcmk_controld_api_reprobe(pcmk_ipc_api_t *api, const char *target_node,
                           const char *router_node)
 {
     xmlNode *request;
     xmlNode *msg_data;
     int rc = pcmk_rc_ok;
 
     if (api == NULL) {
         return EINVAL;
     }
     if (router_node == NULL) {
         router_node = target_node;
     }
     crm_debug("Sending %s IPC request to reprobe %s via %s",
               pcmk_ipc_name(api, true), pcmk__s(target_node, "local node"),
               pcmk__s(router_node, "local node"));
     msg_data = create_reprobe_message_data(target_node, router_node);
     request = create_controller_request(api, CRM_OP_REPROBE, router_node,
                                         msg_data);
     rc = send_controller_request(api, request, true);
     free_xml(msg_data);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Send a "node info" controller operation
  *
  * \param[in,out] api     Controller connection
  * \param[in]     nodeid  ID of node to get info for (or 0 for local node)
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_info.
  */
 int
 pcmk_controld_api_node_info(pcmk_ipc_api_t *api, uint32_t nodeid)
 {
     xmlNode *request;
     int rc = pcmk_rc_ok;
 
     request = create_controller_request(api, CRM_OP_NODE_INFO, NULL, NULL);
     if (request == NULL) {
         return EINVAL;
     }
     if (nodeid > 0) {
         crm_xml_set_id(request, "%lu", (unsigned long) nodeid);
     }
 
     rc = send_controller_request(api, request, true);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Ask the controller for status
  *
  * \param[in,out] api        Controller connection
  * \param[in]     node_name  Name of node whose status is desired (NULL for DC)
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_ping.
  */
 int
 pcmk_controld_api_ping(pcmk_ipc_api_t *api, const char *node_name)
 {
     xmlNode *request;
     int rc = pcmk_rc_ok;
 
     request = create_controller_request(api, CRM_OP_PING, node_name, NULL);
     if (request == NULL) {
         return EINVAL;
     }
     rc = send_controller_request(api, request, true);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Ask the controller for cluster information
  *
  * \param[in,out] api  Controller connection
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_nodes.
  */
 int
 pcmk_controld_api_list_nodes(pcmk_ipc_api_t *api)
 {
     xmlNode *request;
     int rc = EINVAL;
 
     request = create_controller_request(api, PCMK__CONTROLD_CMD_NODES, NULL,
                                         NULL);
     if (request != NULL) {
         rc = send_controller_request(api, request, true);
         free_xml(request);
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 controller_resource_op(pcmk_ipc_api_t *api, const char *op,
                        const char *target_node, const char *router_node,
                        bool cib_only, const char *rsc_id,
                        const char *rsc_long_id, const char *standard,
                        const char *provider, const char *type)
 {
     int rc = pcmk_rc_ok;
     char *key;
     xmlNode *request, *msg_data, *xml_rsc, *params;
 
     if (api == NULL) {
         return EINVAL;
     }
     if (router_node == NULL) {
         router_node = target_node;
     }
 
     msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
 
     /* The controller logs the transition key from resource op requests, so we
      * need to have *something* for it.
      * @TODO don't use "crm-resource"
      */
     key = pcmk__transition_key(0, getpid(), 0,
                                "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
     crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
     free(key);
 
     crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, target_node);
     if (!pcmk__str_eq(router_node, target_node, pcmk__str_casei)) {
         crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
     }
 
     if (cib_only) {
         // Indicate that only the CIB needs to be cleaned
         crm_xml_add(msg_data, PCMK__XA_MODE, XML_TAG_CIB);
     }
 
     xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
     crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
     crm_xml_add(xml_rsc, PCMK__XA_LONG_ID, rsc_long_id);
     crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, standard);
-    crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, provider);
+    crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, provider);
     crm_xml_add(xml_rsc, PCMK_XA_TYPE, type);
 
     params = create_xml_node(msg_data, XML_TAG_ATTRS);
     crm_xml_add(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
 
     // The controller parses the timeout from the request
     key = crm_meta_name(XML_ATTR_TIMEOUT);
     crm_xml_add(params, key, "60000");  /* 1 minute */ //@TODO pass as arg
     free(key);
 
     request = create_controller_request(api, op, router_node, msg_data);
     rc = send_controller_request(api, request, true);
     free_xml(msg_data);
     free_xml(request);
     return rc;
 }
 
 /*!
  * \brief Ask the controller to fail a resource
  *
  * \param[in,out] api          Controller connection
  * \param[in]     target_node  Name of node resource is on
  * \param[in]     router_node  Router node for target
  * \param[in]     rsc_id       ID of resource to fail
  * \param[in]     rsc_long_id  Long ID of resource (if any)
  * \param[in]     standard     Standard of resource
  * \param[in]     provider     Provider of resource (if any)
  * \param[in]     type         Type of resource to fail
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_resource.
  */
 int
 pcmk_controld_api_fail(pcmk_ipc_api_t *api,
                        const char *target_node, const char *router_node,
                        const char *rsc_id, const char *rsc_long_id,
                        const char *standard, const char *provider,
                        const char *type)
 {
     crm_debug("Sending %s IPC request to fail %s (a.k.a. %s) on %s via %s",
               pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
               pcmk__s(rsc_long_id, "no other names"),
               pcmk__s(target_node, "unspecified node"),
               pcmk__s(router_node, "unspecified node"));
     return controller_resource_op(api, CRM_OP_LRM_FAIL, target_node,
                                   router_node, false, rsc_id, rsc_long_id,
                                   standard, provider, type);
 }
 
 /*!
  * \brief Ask the controller to refresh a resource
  *
  * \param[in,out] api          Controller connection
  * \param[in]     target_node  Name of node resource is on
  * \param[in]     router_node  Router node for target
  * \param[in]     rsc_id       ID of resource to refresh
  * \param[in]     rsc_long_id  Long ID of resource (if any)
  * \param[in]     standard     Standard of resource
  * \param[in]     provider     Provider of resource (if any)
  * \param[in]     type         Type of resource
  * \param[in]     cib_only     If true, clean resource from CIB only
  *
  * \return Standard Pacemaker return code
  * \note Event callback will get a reply of type pcmk_controld_reply_resource.
  */
 int
 pcmk_controld_api_refresh(pcmk_ipc_api_t *api, const char *target_node,
                           const char *router_node,
                           const char *rsc_id, const char *rsc_long_id,
                           const char *standard, const char *provider,
                           const char *type, bool cib_only)
 {
     crm_debug("Sending %s IPC request to refresh %s (a.k.a. %s) on %s via %s",
               pcmk_ipc_name(api, true), pcmk__s(rsc_id, "unknown resource"),
               pcmk__s(rsc_long_id, "no other names"),
               pcmk__s(target_node, "unspecified node"),
               pcmk__s(router_node, "unspecified node"));
     return controller_resource_op(api, CRM_OP_LRM_DELETE, target_node,
                                   router_node, cib_only, rsc_id, rsc_long_id,
                                   standard, provider, type);
 }
 
 /*!
  * \brief Get the number of IPC replies currently expected from the controller
  *
  * \param[in] api  Controller IPC API connection
  *
  * \return Number of replies expected
  */
 unsigned int
 pcmk_controld_api_replies_expected(const pcmk_ipc_api_t *api)
 {
     struct controld_api_private_s *private = api->api_data;
 
     return private->replies_expected;
 }
 
 /*!
  * \brief Create XML for a controller IPC "hello" message
  *
  * \deprecated This function is deprecated as part of the public C API.
  */
 // \todo make this static to this file when breaking API backward compatibility
 xmlNode *
 create_hello_message(const char *uuid, const char *client_name,
                      const char *major_version, const char *minor_version)
 {
     xmlNode *hello_node = NULL;
     xmlNode *hello = NULL;
 
     if (pcmk__str_empty(uuid) || pcmk__str_empty(client_name)
         || pcmk__str_empty(major_version) || pcmk__str_empty(minor_version)) {
         crm_err("Could not create IPC hello message from %s (UUID %s): "
                 "missing information",
                 client_name? client_name : "unknown client",
                 uuid? uuid : "unknown");
         return NULL;
     }
 
     hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
     if (hello_node == NULL) {
         crm_err("Could not create IPC hello message from %s (UUID %s): "
                 "Message data creation failed", client_name, uuid);
         return NULL;
     }
 
     crm_xml_add(hello_node, "major_version", major_version);
     crm_xml_add(hello_node, "minor_version", minor_version);
     crm_xml_add(hello_node, "client_name", client_name);
     crm_xml_add(hello_node, "client_uuid", uuid);
 
     hello = create_request(CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
     if (hello == NULL) {
         crm_err("Could not create IPC hello message from %s (UUID %s): "
                 "Request creation failed", client_name, uuid);
         return NULL;
     }
     free_xml(hello_node);
 
     crm_trace("Created hello message from %s (UUID %s)", client_name, uuid);
     return hello;
 }
diff --git a/lib/lrmd/lrmd_output.c b/lib/lrmd/lrmd_output.c
index b0524c6346..a1204603cb 100644
--- a/lib/lrmd/lrmd_output.c
+++ b/lib/lrmd/lrmd_output.c
@@ -1,146 +1,147 @@
 /*
- * Copyright 2020 the Pacemaker project contributors
+ * Copyright 2020-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <stdarg.h>
 
 #include <crm/lrmd_internal.h>
+#include <crm/msg_xml.h>
 #include <crm/common/output_internal.h>
 
 static int
 default_list(pcmk__output_t *out, lrmd_list_t *list, const char *title) {
     lrmd_list_t *iter = NULL;
 
     out->begin_list(out, NULL, NULL, "%s", title);
 
     for (iter = list; iter != NULL; iter = iter->next) {
         out->list_item(out, NULL, "%s", iter->val);
     }
 
     out->end_list(out);
     lrmd_list_freeall(list);
     return pcmk_rc_ok;
 }
 
 static int
 xml_list(pcmk__output_t *out, lrmd_list_t *list, const char *ele) {
     lrmd_list_t *iter = NULL;
 
     for (iter = list; iter != NULL; iter = iter->next) {
         pcmk__output_create_xml_text_node(out, ele, iter->val);
     }
 
     lrmd_list_freeall(list);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("alternatives-list", "lrmd_list_t *", "const char *")
 static int
 lrmd__alternatives_list_xml(pcmk__output_t *out, va_list args) {
     lrmd_list_t *list = va_arg(args, lrmd_list_t *);
     const char *agent_spec = va_arg(args, const char *);
 
     pcmk__output_xml_create_parent(out, "providers",
                                    "for", agent_spec,
                                    NULL);
     return xml_list(out, list, "provider");
 }
 
 PCMK__OUTPUT_ARGS("alternatives-list", "lrmd_list_t *", "const char *")
 static int
 lrmd__alternatives_list(pcmk__output_t *out, va_list args) {
     lrmd_list_t *list = va_arg(args, lrmd_list_t *);
     const char *agent_spec G_GNUC_UNUSED = va_arg(args, const char *);
 
     return default_list(out, list, "Providers");
 }
 
 PCMK__OUTPUT_ARGS("agents-list", "lrmd_list_t *", "const char *", "const char *")
 static int
 lrmd__agents_list_xml(pcmk__output_t *out, va_list args) {
     lrmd_list_t *list = va_arg(args, lrmd_list_t *);
     const char *agent_spec = va_arg(args, const char *);
     const char *provider = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_xml_create_parent(out, "agents",
                                                      "standard", agent_spec,
                                                      NULL);
 
     if (!pcmk__str_empty(provider)) {
-        crm_xml_add(node, "provider", provider);
+        crm_xml_add(node, PCMK_XA_PROVIDER, provider);
     }
 
     return xml_list(out, list, "agent");
 }
 
 PCMK__OUTPUT_ARGS("agents-list", "lrmd_list_t *", "const char *", "const char *")
 static int
 lrmd__agents_list(pcmk__output_t *out, va_list args) {
     lrmd_list_t *list = va_arg(args, lrmd_list_t *);
     const char *agent_spec = va_arg(args, const char *);
     const char *provider = va_arg(args, const char *);
 
     int rc;
     char *title = crm_strdup_printf("%s agents", pcmk__str_empty(provider) ? agent_spec : provider);
 
     rc = default_list(out, list, title);
     free(title);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("providers-list", "lrmd_list_t *", "const char *")
 static int
 lrmd__providers_list_xml(pcmk__output_t *out, va_list args) {
     lrmd_list_t *list = va_arg(args, lrmd_list_t *);
     const char *agent_spec = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_xml_create_parent(out, "providers",
                                                      "standard", "ocf",
                                                      NULL);
 
     if (agent_spec != NULL) {
         crm_xml_add(node, "agent", agent_spec);
     }
 
     return xml_list(out, list, "provider");
 }
 
 PCMK__OUTPUT_ARGS("providers-list", "lrmd_list_t *", "const char *")
 static int
 lrmd__providers_list(pcmk__output_t *out, va_list args) {
     lrmd_list_t *list = va_arg(args, lrmd_list_t *);
     const char *agent_spec G_GNUC_UNUSED = va_arg(args, const char *);
 
     return default_list(out, list, "Providers");
 }
 
 PCMK__OUTPUT_ARGS("standards-list", "lrmd_list_t *")
 static int
 lrmd__standards_list(pcmk__output_t *out, va_list args) {
     lrmd_list_t *list = va_arg(args, lrmd_list_t *);
 
     return default_list(out, list, "Standards");
 }
 
 static pcmk__message_entry_t fmt_functions[] = {
     { "alternatives-list", "default", lrmd__alternatives_list },
     { "alternatives-list", "xml", lrmd__alternatives_list_xml },
     { "agents-list", "default", lrmd__agents_list },
     { "agents-list", "xml", lrmd__agents_list_xml },
     { "providers-list", "default", lrmd__providers_list },
     { "providers-list", "xml", lrmd__providers_list_xml },
     { "standards-list", "default", lrmd__standards_list },
 
     { NULL, NULL, NULL }
 };
 
 void
 lrmd__register_messages(pcmk__output_t *out) {
     pcmk__register_messages(out, fmt_functions);
 }
diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
index eaef5fd684..4b8dbacf05 100644
--- a/lib/pacemaker/pcmk_graph_producer.c
+++ b/lib/pacemaker/pcmk_graph_producer.c
@@ -1,1097 +1,1097 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 // Convenience macros for logging action properties
 
 #define action_type_str(flags) \
     (pcmk_is_set((flags), pcmk_action_pseudo)? "pseudo-action" : "action")
 
 #define action_optional_str(flags) \
     (pcmk_is_set((flags), pcmk_action_optional)? "optional" : "required")
 
 #define action_runnable_str(flags) \
     (pcmk_is_set((flags), pcmk_action_runnable)? "runnable" : "unrunnable")
 
 #define action_node_str(a) \
     (((a)->node == NULL)? "no node" : (a)->node->details->uname)
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified ID
  *
  * \param[in]     id      Node UUID to add
  * \param[in,out] xml     Parent XML tag to add to
  */
 static xmlNode*
 add_node_to_xml_by_id(const char *id, xmlNode *xml)
 {
     xmlNode *node_xml;
 
     node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
     crm_xml_add(node_xml, PCMK_XA_ID, id);
 
     return node_xml;
 }
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified node
  *
  * \param[in]     node  Node to add
  * \param[in,out] xml   XML to add node to
  */
 static void
 add_node_to_xml(const pcmk_node_t *node, void *xml)
 {
     add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
 }
 
 /*!
  * \internal
  * \brief Count (optionally add to XML) nodes needing maintenance state update
  *
  * \param[in,out] xml        Parent XML tag to add to, if any
  * \param[in]     scheduler  Scheduler data
  *
  * \return Count of nodes added
  * \note Only Pacemaker Remote nodes are considered currently
  */
 static int
 add_maintenance_nodes(xmlNode *xml, const pcmk_scheduler_t *scheduler)
 {
     xmlNode *maintenance = NULL;
     int count = 0;
 
     if (xml != NULL) {
         maintenance = create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE);
     }
     for (const GList *iter = scheduler->nodes;
          iter != NULL; iter = iter->next) {
         const pcmk_node_t *node = iter->data;
 
         if (pe__is_guest_or_remote_node(node) &&
             (node->details->maintenance != node->details->remote_maintenance)) {
 
             if (maintenance != NULL) {
                 crm_xml_add(add_node_to_xml_by_id(node->details->id,
                                                   maintenance),
                             XML_NODE_IS_MAINTENANCE,
                             (node->details->maintenance? "1" : "0"));
             }
             count++;
         }
     }
     crm_trace("%s %d nodes in need of maintenance mode update in state",
               ((maintenance == NULL)? "Counted" : "Added"), count);
     return count;
 }
 
 /*!
  * \internal
  * \brief Add pseudo action with nodes needing maintenance state update
  *
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 add_maintenance_update(pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *action = NULL;
 
     if (add_maintenance_nodes(NULL, scheduler) != 0) {
         action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, scheduler);
         pcmk__set_action_flags(action, pcmk_action_always_in_graph);
     }
 }
 
 /*!
  * \internal
  * \brief Add XML with nodes that an action is expected to bring down
  *
  * If a specified action is expected to bring any nodes down, add an XML block
  * with their UUIDs. When a node is lost, this allows the controller to
  * determine whether it was expected.
  *
  * \param[in,out] xml       Parent XML tag to add to
  * \param[in]     action    Action to check for downed nodes
  */
 static void
 add_downed_nodes(xmlNode *xml, const pcmk_action_t *action)
 {
     CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL),
               return);
 
     if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
 
         /* Shutdown makes the action's node down */
         xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
         add_node_to_xml_by_id(action->node->details->id, downed);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
                             pcmk__str_none)) {
 
         /* Fencing makes the action's node and any hosted guest nodes down */
         const char *fence = g_hash_table_lookup(action->meta, "stonith_action");
 
         if (pcmk__is_fencing_action(fence)) {
             xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
             add_node_to_xml_by_id(action->node->details->id, downed);
             pe_foreach_guest_node(action->node->details->data_set,
                                   action->node, add_node_to_xml, downed);
         }
 
     } else if (action->rsc && action->rsc->is_remote_node
                && pcmk__str_eq(action->task, PCMK_ACTION_STOP,
                                pcmk__str_none)) {
 
         /* Stopping a remote connection resource makes connected node down,
          * unless it's part of a migration
          */
         GList *iter;
         pcmk_action_t *input;
         bool migrating = false;
 
         for (iter = action->actions_before; iter != NULL; iter = iter->next) {
             input = ((pcmk__related_action_t *) iter->data)->action;
             if ((input->rsc != NULL)
                 && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none)
                 && pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM,
                                 pcmk__str_none)) {
                 migrating = true;
                 break;
             }
         }
         if (!migrating) {
             xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
             add_node_to_xml_by_id(action->rsc->id, downed);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Create a transition graph operation key for a clone action
  *
  * \param[in] action       Clone action
  * \param[in] interval_ms  Action interval in milliseconds
  *
  * \return Newly allocated string with transition graph operation key
  */
 static char *
 clone_op_key(const pcmk_action_t *action, guint interval_ms)
 {
     if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
         const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
         const char *n_task = g_hash_table_lookup(action->meta,
                                                  "notify_operation");
 
         CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL));
         return pcmk__notify_key(action->rsc->clone_name, n_type, n_task);
 
     } else if (action->cancel_task != NULL) {
         return pcmk__op_key(action->rsc->clone_name, action->cancel_task,
                             interval_ms);
     } else {
         return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms);
     }
 }
 
 /*!
  * \internal
  * \brief Add node details to transition graph action XML
  *
  * \param[in]     action  Scheduled action
  * \param[in,out] xml     Transition graph action XML for \p action
  */
 static void
 add_node_details(const pcmk_action_t *action, xmlNode *xml)
 {
     pcmk_node_t *router_node = pcmk__connection_host_for_action(action);
 
     crm_xml_add(xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
     crm_xml_add(xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
     if (router_node != NULL) {
         crm_xml_add(xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
     }
 }
 
 /*!
  * \internal
  * \brief Add resource details to transition graph action XML
  *
  * \param[in]     action      Scheduled action
  * \param[in,out] action_xml  Transition graph action XML for \p action
  */
 static void
 add_resource_details(const pcmk_action_t *action, xmlNode *action_xml)
 {
     xmlNode *rsc_xml = NULL;
     const char *attr_list[] = {
         XML_AGENT_ATTR_CLASS,
-        XML_AGENT_ATTR_PROVIDER,
+        PCMK_XA_PROVIDER,
         PCMK_XA_TYPE,
     };
 
     /* If a resource is locked to a node via PCMK_OPT_SHUTDOWN_LOCK, mark its
      * actions so the controller can preserve the lock when the action
      * completes.
      */
     if (pcmk__action_locks_rsc_to_node(action)) {
         crm_xml_add_ll(action_xml, PCMK_OPT_SHUTDOWN_LOCK,
                        (long long) action->rsc->lock_time);
     }
 
     // List affected resource
 
     rsc_xml = create_xml_node(action_xml,
                               (const char *) action->rsc->xml->name);
     if (pcmk_is_set(action->rsc->flags, pcmk_rsc_removed)
         && (action->rsc->clone_name != NULL)) {
         /* Use the numbered instance name here, because if there is more
          * than one instance on a node, we need to make sure the command
          * goes to the right one.
          *
          * This is important even for anonymous clones, because the clone's
          * unique meta-attribute might have just been toggled from on to
          * off.
          */
         crm_debug("Using orphan clone name %s instead of %s",
                   action->rsc->id, action->rsc->clone_name);
         crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->clone_name);
         crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);
 
     } else if (!pcmk_is_set(action->rsc->flags, pcmk_rsc_unique)) {
         const char *xml_id = ID(action->rsc->xml);
 
         crm_debug("Using anonymous clone name %s for %s (aka %s)",
                   xml_id, action->rsc->id, action->rsc->clone_name);
 
         /* ID is what we'd like client to use
          * LONG_ID is what they might know it as instead
          *
          * LONG_ID is only strictly needed /here/ during the
          * transition period until all nodes in the cluster
          * are running the new software /and/ have rebooted
          * once (meaning that they've only ever spoken to a DC
          * supporting this feature).
          *
          * If anyone toggles the unique flag to 'on', the
          * 'instance free' name will correspond to an orphan
          * and fall into the clause above instead
          */
         crm_xml_add(rsc_xml, PCMK_XA_ID, xml_id);
         if ((action->rsc->clone_name != NULL)
             && !pcmk__str_eq(xml_id, action->rsc->clone_name,
                              pcmk__str_none)) {
             crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->clone_name);
         } else {
             crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);
         }
 
     } else {
         CRM_ASSERT(action->rsc->clone_name == NULL);
         crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->id);
     }
 
     for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
         crm_xml_add(rsc_xml, attr_list[lpc],
                     g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
     }
 }
 
 /*!
  * \internal
  * \brief Add action attributes to transition graph action XML
  *
  * \param[in,out] action      Scheduled action
  * \param[in,out] action_xml  Transition graph action XML for \p action
  */
 static void
 add_action_attributes(pcmk_action_t *action, xmlNode *action_xml)
 {
     xmlNode *args_xml = NULL;
 
     /* We create free-standing XML to start, so we can sort the attributes
      * before adding it to action_xml, which keeps the scheduler regression
      * test graphs comparable.
      */
     args_xml = create_xml_node(NULL, XML_TAG_ATTRS);
 
     crm_xml_add(args_xml, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
     g_hash_table_foreach(action->extra, hash2field, args_xml);
 
     if ((action->rsc != NULL) && (action->node != NULL)) {
         // Get the resource instance attributes, evaluated properly for node
         GHashTable *params = pe_rsc_params(action->rsc, action->node,
                                            action->rsc->cluster);
 
         pcmk__substitute_remote_addr(action->rsc, params);
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
 
     } else if ((action->rsc != NULL)
                && (action->rsc->variant <= pcmk_rsc_variant_primitive)) {
         GHashTable *params = pe_rsc_params(action->rsc, NULL,
                                            action->rsc->cluster);
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
     }
 
     g_hash_table_foreach(action->meta, hash2metafield, args_xml);
     if (action->rsc != NULL) {
         pcmk_resource_t *parent = action->rsc;
 
         while (parent != NULL) {
             parent->cmds->add_graph_meta(parent, args_xml);
             parent = parent->parent;
         }
 
         pcmk__add_guest_meta_to_xml(args_xml, action);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)
                && (action->node != NULL)) {
         /* Pass the node's attributes as meta-attributes.
          *
          * @TODO: Determine whether it is still necessary to do this. It was
          * added in 33d99707, probably for the libfence-based implementation in
          * c9a90bd, which is no longer used.
          */
         g_hash_table_foreach(action->node->details->attrs, hash2metafield,
                              args_xml);
     }
 
     sorted_xml(args_xml, action_xml, FALSE);
     free_xml(args_xml);
 }
 
 /*!
  * \internal
  * \brief Create the transition graph XML for a scheduled action
  *
  * \param[in,out] parent        Parent XML element to add action to
  * \param[in,out] action        Scheduled action
  * \param[in]     skip_details  If false, add action details as sub-elements
  * \param[in]     scheduler     Scheduler data
  */
 static void
 create_graph_action(xmlNode *parent, pcmk_action_t *action, bool skip_details,
                     const pcmk_scheduler_t *scheduler)
 {
     bool needs_node_info = true;
     bool needs_maintenance_info = false;
     xmlNode *action_xml = NULL;
 
     if ((action == NULL) || (scheduler == NULL)) {
         return;
     }
 
     // Create the top-level element based on task
 
     if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) {
         /* All fences need node info; guest node fences are pseudo-events */
         if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
             action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
         } else {
             action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
         }
 
     } else if (pcmk__str_any_of(action->task,
                                 PCMK_ACTION_DO_SHUTDOWN,
                                 PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) {
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE,
                             pcmk__str_none)) {
         // CIB-only clean-up for shutdown locks
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
         crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);
 
     } else if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
         if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES,
                          pcmk__str_none)) {
             needs_maintenance_info = true;
         }
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
         needs_node_info = false;
 
     } else {
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP);
     }
 
     crm_xml_add_int(action_xml, PCMK_XA_ID, action->id);
     crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
 
     if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) {
         char *clone_key = NULL;
         guint interval_ms;
 
         if (pcmk__guint_from_hash(action->meta, XML_LRM_ATTR_INTERVAL_MS, 0,
                                   &interval_ms) != pcmk_rc_ok) {
             interval_ms = 0;
         }
         clone_key = clone_op_key(action, interval_ms);
         crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
         crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY,
                     action->uuid);
         free(clone_key);
     } else {
         crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
     }
 
     if (needs_node_info && (action->node != NULL)) {
         add_node_details(action, action_xml);
         g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET),
                             strdup(action->node->details->uname));
         g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID),
                             strdup(action->node->details->id));
     }
 
     if (skip_details) {
         return;
     }
 
     if ((action->rsc != NULL)
         && !pcmk_is_set(action->flags, pcmk_action_pseudo)) {
 
         // This is a real resource action, so add resource details
         add_resource_details(action, action_xml);
     }
 
     /* List any attributes in effect */
     add_action_attributes(action, action_xml);
 
     /* List any nodes this action is expected to make down */
     if (needs_node_info && (action->node != NULL)) {
         add_downed_nodes(action_xml, action);
     }
 
     if (needs_maintenance_info) {
         add_maintenance_nodes(action_xml, scheduler);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether an action should be added to the transition graph
  *
  * \param[in] action  Action to check
  *
  * \return true if action should be added to graph, otherwise false
  */
 static bool
 should_add_action_to_graph(const pcmk_action_t *action)
 {
     if (!pcmk_is_set(action->flags, pcmk_action_runnable)) {
         crm_trace("Ignoring action %s (%d): unrunnable",
                   action->uuid, action->id);
         return false;
     }
 
     if (pcmk_is_set(action->flags, pcmk_action_optional)
         && !pcmk_is_set(action->flags, pcmk_action_always_in_graph)) {
         crm_trace("Ignoring action %s (%d): optional",
                   action->uuid, action->id);
         return false;
     }
 
     /* Actions for unmanaged resources should be excluded from the graph,
      * with the exception of monitors and cancellation of recurring monitors.
      */
     if ((action->rsc != NULL)
         && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
         && !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
 
         const char *interval_ms_s;
 
         /* A cancellation of a recurring monitor will get here because the task
          * is cancel rather than monitor, but the interval can still be used to
          * recognize it. The interval has been normalized to milliseconds by
          * this point, so a string comparison is sufficient.
          */
         interval_ms_s = g_hash_table_lookup(action->meta,
                                             XML_LRM_ATTR_INTERVAL_MS);
         if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
             crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
                       action->uuid, action->id, action->rsc->id);
             return false;
         }
     }
 
     /* Always add pseudo-actions, fence actions, and shutdown actions (already
      * determined to be required and runnable by this point)
      */
     if (pcmk_is_set(action->flags, pcmk_action_pseudo)
         || pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH,
                                 PCMK_ACTION_DO_SHUTDOWN, NULL)) {
         return true;
     }
 
     if (action->node == NULL) {
         pcmk__sched_err("Skipping action %s (%d) "
                         "because it was not assigned to a node (bug?)",
                         action->uuid, action->id);
         pcmk__log_action("Unassigned", action, false);
         return false;
     }
 
     if (pcmk_is_set(action->flags, pcmk_action_on_dc)) {
         crm_trace("Action %s (%d) should be dumped: "
                   "can run on DC instead of %s",
                   action->uuid, action->id, pe__node_name(action->node));
 
     } else if (pe__is_guest_node(action->node)
                && !action->node->details->remote_requires_reset) {
         crm_trace("Action %s (%d) should be dumped: "
                   "assuming will be runnable on guest %s",
                   action->uuid, action->id, pe__node_name(action->node));
 
     } else if (!action->node->details->online) {
         pcmk__sched_err("Skipping action %s (%d) "
                         "because it was scheduled for offline node (bug?)",
                         action->uuid, action->id);
         pcmk__log_action("Offline node", action, false);
         return false;
 
     } else if (action->node->details->unclean) {
         pcmk__sched_err("Skipping action %s (%d) "
                         "because it was scheduled for unclean node (bug?)",
                         action->uuid, action->id);
         pcmk__log_action("Unclean node", action, false);
         return false;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether an ordering's flags can change an action
  *
  * \param[in] ordering  Ordering to check
  *
  * \return true if ordering has flags that can change an action, false otherwise
  */
 static bool
 ordering_can_change_actions(const pcmk__related_action_t *ordering)
 {
     return pcmk_any_flags_set(ordering->type,
                               ~(pcmk__ar_then_implies_first_graphed
                                 |pcmk__ar_first_implies_then_graphed
                                 |pcmk__ar_ordered));
 }
 
 /*!
  * \internal
  * \brief Check whether an action input should be in the transition graph
  *
  * \param[in]     action  Action to check
  * \param[in,out] input   Action input to check
  *
  * \return true if input should be in graph, false otherwise
  * \note This function may not only check an input, but disable it under certian
  *       circumstances (load or anti-colocation orderings that are not needed).
  */
 static bool
 should_add_input_to_graph(const pcmk_action_t *action,
                           pcmk__related_action_t *input)
 {
     if (input->state == pe_link_dumped) {
         return true;
     }
 
     if ((uint32_t) input->type == pcmk__ar_none) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "ordering disabled",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
                && !ordering_can_change_actions(input)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional and input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pcmk_action_runnable)
                && pcmk_is_set(input->type, pcmk__ar_min_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "minimum number of instances required but input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->type, pcmk__ar_unmigratable_then_blocks)
                && !pcmk_is_set(input->action->flags, pcmk_action_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "input blocked if 'then' unmigratable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->type, pcmk__ar_if_first_unmigratable)
                && pcmk_is_set(input->action->flags, pcmk_action_migratable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): ordering applies "
                   "only if input is unmigratable, but it is migratable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (((uint32_t) input->type == pcmk__ar_ordered)
                && pcmk_is_set(input->action->flags, pcmk_action_migratable)
                && pcmk__ends_with(input->action->uuid, "_stop_0")) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional but stop in migration",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if ((uint32_t) input->type == pcmk__ar_if_on_same_node_or_target) {
         pcmk_node_t *input_node = input->action->node;
 
         if ((action->rsc != NULL)
             && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO,
                             pcmk__str_none)) {
 
             pcmk_node_t *assigned = action->rsc->allocated_to;
 
             /* For load_stopped -> migrate_to orderings, we care about where
              * the resource has been assigned, not where migrate_to will be
              * executed.
              */
             if (!pe__same_node(input_node, assigned)) {
                 crm_trace("Ignoring %s (%d) input %s (%d): "
                           "migration target %s is not same as input node %s",
                           action->uuid, action->id,
                           input->action->uuid, input->action->id,
                           (assigned? assigned->details->uname : "<none>"),
                           (input_node? input_node->details->uname : "<none>"));
                 input->type = (enum pe_ordering) pcmk__ar_none;
                 return false;
             }
 
         } else if (!pe__same_node(input_node, action->node)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "not on same node (%s vs %s)",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       (action->node? action->node->details->uname : "<none>"),
                       (input_node? input_node->details->uname : "<none>"));
             input->type = (enum pe_ordering) pcmk__ar_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "ordering optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->type = (enum pe_ordering) pcmk__ar_none;
             return false;
         }
 
     } else if ((uint32_t) input->type == pcmk__ar_if_required_on_same_node) {
         if (input->action->node && action->node
             && !pe__same_node(input->action->node, action->node)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "not on same node (%s vs %s)",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       pe__node_name(action->node),
                       pe__node_name(input->action->node));
             input->type = (enum pe_ordering) pcmk__ar_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pcmk_action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->type = (enum pe_ordering) pcmk__ar_none;
             return false;
         }
 
     } else if (input->action->rsc
                && input->action->rsc != action->rsc
                && pcmk_is_set(input->action->rsc->flags, pcmk_rsc_failed)
                && !pcmk_is_set(input->action->rsc->flags, pcmk_rsc_managed)
                && pcmk__ends_with(input->action->uuid, "_stop_0")
                && action->rsc && pe_rsc_is_clone(action->rsc)) {
         crm_warn("Ignoring requirement that %s complete before %s:"
                  " unmanaged failed resources cannot prevent clone shutdown",
                  input->action->uuid, action->uuid);
         return false;
 
     } else if (pcmk_is_set(input->action->flags, pcmk_action_optional)
                && !pcmk_any_flags_set(input->action->flags,
                                       pcmk_action_always_in_graph
                                       |pcmk_action_added_to_graph)
                && !should_add_action_to_graph(input->action)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "input optional",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
     }
 
     crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
               action->uuid, action->id, action_type_str(input->action->flags),
               input->action->uuid, input->action->id,
               action_node_str(input->action),
               action_runnable_str(input->action->flags),
               action_optional_str(input->action->flags), input->type);
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether an ordering creates an ordering loop
  *
  * \param[in]     init_action  "First" action in ordering
  * \param[in]     action       Callers should always set this the same as
  *                             \p init_action (this function may use a different
  *                             value for recursive calls)
  * \param[in,out] input        Action wrapper for "then" action in ordering
  *
  * \return true if the ordering creates a loop, otherwise false
  */
 bool
 pcmk__graph_has_loop(const pcmk_action_t *init_action,
                      const pcmk_action_t *action, pcmk__related_action_t *input)
 {
     bool has_loop = false;
 
     if (pcmk_is_set(input->action->flags, pcmk_action_detect_loop)) {
         crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->details->uname : "",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   input->type);
         return false;
     }
 
     // Don't need to check inputs that won't be used
     if (!should_add_input_to_graph(action, input)) {
         return false;
     }
 
     if (input->action == init_action) {
         crm_debug("Input loop found in %s@%s ->...-> %s@%s",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   init_action->uuid,
                   init_action->node? init_action->node->details->uname : "");
         return true;
     }
 
     pcmk__set_action_flags(input->action, pcmk_action_detect_loop);
 
     crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
               "for graph loop with %s@%s ",
               action->uuid,
               action->node? action->node->details->uname : "",
               input->action->uuid,
               input->action->node? input->action->node->details->uname : "",
               input->type,
               init_action->uuid,
               init_action->node? init_action->node->details->uname : "");
 
     // Recursively check input itself for loops
     for (GList *iter = input->action->actions_before;
          iter != NULL; iter = iter->next) {
 
         if (pcmk__graph_has_loop(init_action, input->action,
                                  (pcmk__related_action_t *) iter->data)) {
             // Recursive call already logged a debug message
             has_loop = true;
             break;
         }
     }
 
     pcmk__clear_action_flags(input->action, pcmk_action_detect_loop);
 
     if (!has_loop) {
         crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->details->uname : "",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   input->type);
     }
     return has_loop;
 }
 
 /*!
  * \internal
  * \brief Create a synapse XML element for a transition graph
  *
  * \param[in]     action     Action that synapse is for
  * \param[in,out] scheduler  Scheduler data containing graph
  *
  * \return Newly added XML element for new graph synapse
  */
 static xmlNode *
 create_graph_synapse(const pcmk_action_t *action, pcmk_scheduler_t *scheduler)
 {
     int synapse_priority = 0;
     xmlNode *syn = create_xml_node(scheduler->graph, "synapse");
 
     crm_xml_add_int(syn, PCMK_XA_ID, scheduler->num_synapse);
     scheduler->num_synapse++;
 
     if (action->rsc != NULL) {
         synapse_priority = action->rsc->priority;
     }
     if (action->priority > synapse_priority) {
         synapse_priority = action->priority;
     }
     if (synapse_priority > 0) {
         crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
     }
     return syn;
 }
 
 /*!
  * \internal
  * \brief Add an action to the transition graph XML if appropriate
  *
  * \param[in,out] data       Action to possibly add
  * \param[in,out] user_data  Scheduler data
  *
  * \note This will de-duplicate the action inputs, meaning that the
  *       pcmk__related_action_t:type flags can no longer be relied on to retain
  *       their original settings. That means this MUST be called after
  *       pcmk__apply_orderings() is complete, and nothing after this should rely
  *       on those type flags. (For example, some code looks for type equal to
  *       some flag rather than whether the flag is set, and some code looks for
  *       particular combinations of flags -- such code must be done before
  *       pcmk__create_graph().)
  */
 static void
 add_action_to_graph(gpointer data, gpointer user_data)
 {
     pcmk_action_t *action = (pcmk_action_t *) data;
     pcmk_scheduler_t *scheduler = (pcmk_scheduler_t *) user_data;
 
     xmlNode *syn = NULL;
     xmlNode *set = NULL;
     xmlNode *in = NULL;
 
     /* If we haven't already, de-duplicate inputs (even if we won't be adding
      * the action to the graph, so that crm_simulate's dot graphs don't have
      * duplicates).
      */
     if (!pcmk_is_set(action->flags, pcmk_action_inputs_deduplicated)) {
         pcmk__deduplicate_action_inputs(action);
         pcmk__set_action_flags(action, pcmk_action_inputs_deduplicated);
     }
 
     if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)
         || !should_add_action_to_graph(action)) {
         return; // Already added, or shouldn't be
     }
     pcmk__set_action_flags(action, pcmk_action_added_to_graph);
 
     crm_trace("Adding action %d (%s%s%s) to graph",
               action->id, action->uuid,
               ((action->node == NULL)? "" : " on "),
               ((action->node == NULL)? "" : action->node->details->uname));
 
     syn = create_graph_synapse(action, scheduler);
     set = create_xml_node(syn, "action_set");
     in = create_xml_node(syn, "inputs");
 
     create_graph_action(set, action, false, scheduler);
 
     for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
         pcmk__related_action_t *input = lpc->data;
 
         if (should_add_input_to_graph(action, input)) {
             xmlNode *input_xml = create_xml_node(in, "trigger");
 
             input->state = pe_link_dumped;
             create_graph_action(input_xml, input->action, true, scheduler);
         }
     }
 }
 
 static int transition_id = -1;
 
 /*!
  * \internal
  * \brief Log a message after calculating a transition
  *
  * \param[in] filename  Where transition input is stored
  */
 void
 pcmk__log_transition_summary(const char *filename)
 {
     if (was_processing_error || crm_config_error) {
         crm_err("Calculated transition %d (with errors)%s%s",
                 transition_id,
                 (filename == NULL)? "" : ", saving inputs in ",
                 (filename == NULL)? "" : filename);
 
     } else if (was_processing_warning || crm_config_warning) {
         crm_warn("Calculated transition %d (with warnings)%s%s",
                  transition_id,
                  (filename == NULL)? "" : ", saving inputs in ",
                  (filename == NULL)? "" : filename);
 
     } else {
         crm_notice("Calculated transition %d%s%s",
                    transition_id,
                    (filename == NULL)? "" : ", saving inputs in ",
                    (filename == NULL)? "" : filename);
     }
     if (crm_config_error) {
         crm_notice("Configuration errors found during scheduler processing,"
                    "  please run \"crm_verify -L\" to identify issues");
     }
 }
 
 /*!
  * \internal
  * \brief Add a resource's actions to the transition graph
  *
  * \param[in,out] rsc  Resource whose actions should be added
  */
 void
 pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc)
 {
     GList *iter = NULL;
 
     CRM_ASSERT(rsc != NULL);
     pcmk__rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);
 
     // First add the resource's own actions
     g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster);
 
     // Then recursively add its children's actions (appropriate to variant)
     for (iter = rsc->children; iter != NULL; iter = iter->next) {
         pcmk_resource_t *child_rsc = (pcmk_resource_t *) iter->data;
 
         child_rsc->cmds->add_actions_to_graph(child_rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Create a transition graph with all cluster actions needed
  *
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__create_graph(pcmk_scheduler_t *scheduler)
 {
     GList *iter = NULL;
     const char *value = NULL;
     long long limit = 0LL;
 
     transition_id++;
     crm_trace("Creating transition graph %d", transition_id);
 
     scheduler->graph = create_xml_node(NULL, XML_TAG_GRAPH);
 
     value = pe_pref(scheduler->config_hash, PCMK_OPT_CLUSTER_DELAY);
     crm_xml_add(scheduler->graph, PCMK_OPT_CLUSTER_DELAY, value);
 
     value = pe_pref(scheduler->config_hash, PCMK_OPT_STONITH_TIMEOUT);
     crm_xml_add(scheduler->graph, PCMK_OPT_STONITH_TIMEOUT, value);
 
     crm_xml_add(scheduler->graph, "failed-stop-offset", "INFINITY");
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_start_failure_fatal)) {
         crm_xml_add(scheduler->graph, "failed-start-offset", "INFINITY");
     } else {
         crm_xml_add(scheduler->graph, "failed-start-offset", "1");
     }
 
     value = pe_pref(scheduler->config_hash, PCMK_OPT_BATCH_LIMIT);
     crm_xml_add(scheduler->graph, PCMK_OPT_BATCH_LIMIT, value);
 
     crm_xml_add_int(scheduler->graph, "transition_id", transition_id);
 
     value = pe_pref(scheduler->config_hash, PCMK_OPT_MIGRATION_LIMIT);
     if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
         crm_xml_add(scheduler->graph, PCMK_OPT_MIGRATION_LIMIT, value);
     }
 
     if (scheduler->recheck_by > 0) {
         char *recheck_epoch = NULL;
 
         recheck_epoch = crm_strdup_printf("%llu",
                                           (long long) scheduler->recheck_by);
         crm_xml_add(scheduler->graph, "recheck-by", recheck_epoch);
         free(recheck_epoch);
     }
 
     /* The following code will de-duplicate action inputs, so nothing past this
      * should rely on the action input type flags retaining their original
      * values.
      */
 
     // Add resource actions to graph
     for (iter = scheduler->resources; iter != NULL; iter = iter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
         pcmk__rsc_trace(rsc, "Processing actions for %s", rsc->id);
         rsc->cmds->add_actions_to_graph(rsc);
     }
 
     // Add pseudo-action for list of nodes with maintenance state update
     add_maintenance_update(scheduler);
 
     // Add non-resource (node) actions
     for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
         pcmk_action_t *action = (pcmk_action_t *) iter->data;
 
         if ((action->rsc != NULL)
             && (action->node != NULL)
             && action->node->details->shutdown
             && !pcmk_is_set(action->rsc->flags, pcmk_rsc_maintenance)
             && !pcmk_any_flags_set(action->flags,
                                    pcmk_action_optional|pcmk_action_runnable)
             && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
             /* Eventually we should just ignore the 'fence' case, but for now
              * it's the best way to detect (in CTS) when CIB resource updates
              * are being lost.
              */
             if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)
                 || (scheduler->no_quorum_policy == pcmk_no_quorum_ignore)) {
                 const bool managed = pcmk_is_set(action->rsc->flags,
                                                  pcmk_rsc_managed);
                 const bool failed = pcmk_is_set(action->rsc->flags,
                                                 pcmk_rsc_failed);
 
                 crm_crit("Cannot %s %s because of %s:%s%s (%s)",
                          action->node->details->unclean? "fence" : "shut down",
                          pe__node_name(action->node), action->rsc->id,
                          (managed? " blocked" : " unmanaged"),
                          (failed? " failed" : ""), action->uuid);
             }
         }
 
         add_action_to_graph((gpointer) action, (gpointer) scheduler);
     }
 
     crm_log_xml_trace(scheduler->graph, "graph");
 }
diff --git a/lib/pacemaker/pcmk_injections.c b/lib/pacemaker/pcmk_injections.c
index f014de795b..93b18355a1 100644
--- a/lib/pacemaker/pcmk_injections.c
+++ b/lib/pacemaker/pcmk_injections.c
@@ -1,788 +1,788 @@
 /*
  * Copyright 2009-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/cib/internal.h>
 #include <crm/common/util.h>
 #include <crm/common/iso8601.h>
 #include <crm/common/xml_internal.h>
 #include <crm/lrmd_events.h>            // lrmd_event_data_t, etc.
 #include <crm/lrmd_internal.h>
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 bool pcmk__simulate_node_config = false;
 
 #define XPATH_NODE_CONFIG   "//" XML_CIB_TAG_NODE "[@" XML_ATTR_UNAME "='%s']"
 #define XPATH_NODE_STATE    "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"
 #define XPATH_NODE_STATE_BY_ID "//" XML_CIB_TAG_STATE "[@" PCMK_XA_ID "='%s']"
 #define XPATH_RSC_HISTORY   XPATH_NODE_STATE \
                             "//" XML_LRM_TAG_RESOURCE "[@" PCMK_XA_ID "='%s']"
 
 
 /*!
  * \internal
  * \brief Inject a fictitious transient node attribute into scheduler input
  *
  * \param[in,out] out       Output object for displaying error messages
  * \param[in,out] cib_node  node_state XML to inject attribute into
  * \param[in]     name      Transient node attribute name to inject
  * \param[in]     value     Transient node attribute value to inject
  */
 static void
 inject_transient_attr(pcmk__output_t *out, xmlNode *cib_node,
                       const char *name, const char *value)
 {
     xmlNode *attrs = NULL;
     xmlNode *instance_attrs = NULL;
     const char *node_uuid = ID(cib_node);
 
     out->message(out, "inject-attr", name, value, cib_node);
 
     attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
     if (attrs == NULL) {
         attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
         crm_xml_add(attrs, PCMK_XA_ID, node_uuid);
     }
 
     instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
     if (instance_attrs == NULL) {
         instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
         crm_xml_add(instance_attrs, PCMK_XA_ID, node_uuid);
     }
 
     crm_create_nvpair_xml(instance_attrs, NULL, name, value);
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious fail count into a scheduler input
  *
  * \param[in,out] out          Output object for displaying error messages
  * \param[in,out] cib_node     Node state XML to inject into
  * \param[in]     resource     ID of resource for fail count to inject
  * \param[in]     task         Action name for fail count to inject
  * \param[in]     interval_ms  Action interval (in milliseconds) for fail count
  * \param[in]     rc           Action result for fail count to inject (if 0, or
  *                             7 when interval_ms is 0, inject nothing)
  */
 void
 pcmk__inject_failcount(pcmk__output_t *out, xmlNode *cib_node,
                        const char *resource, const char *task,
                        guint interval_ms, int rc)
 {
     if (rc == 0) {
         return;
 
     } else if ((rc == 7) && (interval_ms == 0)) {
         return;
 
     } else {
         char *name = NULL;
         char *now = pcmk__ttoa(time(NULL));
 
         name = pcmk__failcount_name(resource, task, interval_ms);
         inject_transient_attr(out, cib_node, name, "value++");
         free(name);
 
         name = pcmk__lastfailure_name(resource, task, interval_ms);
         inject_transient_attr(out, cib_node, name, now);
         free(name);
 
         free(now);
     }
 }
 
 /*!
  * \internal
  * \brief Create a CIB configuration entry for a fictitious node
  *
  * \param[in,out] cib_conn  CIB object to use
  * \param[in]     node      Node name to use
  */
 static void
 create_node_entry(cib_t *cib_conn, const char *node)
 {
     int rc = pcmk_ok;
     char *xpath = crm_strdup_printf(XPATH_NODE_CONFIG, node);
 
     rc = cib_conn->cmds->query(cib_conn, xpath, NULL,
                                cib_xpath|cib_sync_call|cib_scope_local);
 
     if (rc == -ENXIO) { // Only add if not already existing
         xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);
 
         crm_xml_add(cib_object, PCMK_XA_ID, node); // Use node name as ID
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
                                cib_sync_call|cib_scope_local);
         /* Not bothering with subsequent query to see if it exists,
            we'll bomb out later in the call to query_node_uuid()... */
 
         free_xml(cib_object);
     }
 
     free(xpath);
 }
 
 /*!
  * \internal
  * \brief Synthesize a fake executor event for an action
  *
  * \param[in] cib_resource  XML for any existing resource action history
  * \param[in] task          Name of action to synthesize
  * \param[in] interval_ms   Interval of action to synthesize
  * \param[in] outcome       Result of action to synthesize
  *
  * \return Newly allocated executor event
  * \note It is the caller's responsibility to free the result with
  *       lrmd_free_event().
  */
 static lrmd_event_data_t *
 create_op(const xmlNode *cib_resource, const char *task, guint interval_ms,
           int outcome)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *xop = NULL;
 
     op = lrmd_new_event(ID(cib_resource), task, interval_ms);
     lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
     op->params = NULL; // Not needed for simulation purposes
     op->t_run = (unsigned int) time(NULL);
     op->t_rcchange = op->t_run;
 
     // Use a call ID higher than any existing history entries
     op->call_id = 0;
     for (xop = pcmk__xe_first_child(cib_resource); xop != NULL;
          xop = pcmk__xe_next(xop)) {
 
         int tmp = 0;
 
         crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
         if (tmp > op->call_id) {
             op->call_id = tmp;
         }
     }
     op->call_id++;
 
     return op;
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious resource history entry into a scheduler input
  *
  * \param[in,out] cib_resource  Resource history XML to inject entry into
  * \param[in,out] op            Action result to inject
  * \param[in]     target_rc     Expected result for action to inject
  *
  * \return XML of injected resource history entry
  */
 xmlNode *
 pcmk__inject_action_result(xmlNode *cib_resource, lrmd_event_data_t *op,
                            int target_rc)
 {
     return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
                                     target_rc, NULL, crm_system_name);
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious node into a scheduler input
  *
  * \param[in,out] cib_conn  Scheduler input CIB to inject node into
  * \param[in]     node      Name of node to inject
  * \param[in]     uuid      UUID of node to inject
  *
  * \return XML of node_state entry for new node
  * \note If the global pcmk__simulate_node_config has been set to true, a
  *       node entry in the configuration section will be added, as well as a
  *       node state entry in the status section.
  */
 xmlNode *
 pcmk__inject_node(cib_t *cib_conn, const char *node, const char *uuid)
 {
     int rc = pcmk_ok;
     xmlNode *cib_object = NULL;
     char *xpath = crm_strdup_printf(XPATH_NODE_STATE, node);
     bool duplicate = false;
     char *found_uuid = NULL;
 
     if (pcmk__simulate_node_config) {
         create_node_entry(cib_conn, node);
     }
 
     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                cib_xpath|cib_sync_call|cib_scope_local);
 
     if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
         crm_err("Detected multiple node_state entries for xpath=%s, bailing",
                 xpath);
         duplicate = true;
         goto done;
     }
 
     if (rc == -ENXIO) {
         if (uuid == NULL) {
             query_node_uuid(cib_conn, node, &found_uuid, NULL);
         } else {
             found_uuid = strdup(uuid);
         }
 
         if (found_uuid) {
             char *xpath_by_uuid = crm_strdup_printf(XPATH_NODE_STATE_BY_ID,
                                                     found_uuid);
 
             // It's possible that a node_state entry doesn't have an uname yet.
             rc = cib_conn->cmds->query(cib_conn, xpath_by_uuid, &cib_object,
                                        cib_xpath|cib_sync_call|cib_scope_local);
 
             if ((cib_object != NULL) && (ID(cib_object) == NULL)) {
                 crm_err("Can't inject node state for %s because multiple "
                         "state entries found for ID %s", node, found_uuid);
                 duplicate = true;
                 free(xpath_by_uuid);
                 goto done;
 
             } else if (cib_object != NULL) {
                 crm_xml_add(cib_object, XML_ATTR_UNAME, node);
 
                 rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_STATUS,
                                             cib_object,
                                             cib_sync_call|cib_scope_local);
             }
 
             free(xpath_by_uuid);
         }
     }
 
     if (rc == -ENXIO) {
         cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
         crm_xml_add(cib_object, PCMK_XA_ID, found_uuid);
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
                                cib_sync_call|cib_scope_local);
         free_xml(cib_object);
 
         rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                    cib_xpath|cib_sync_call|cib_scope_local);
         crm_trace("Injecting node state for %s (rc=%d)", node, rc);
     }
 
 done:
     free(found_uuid);
     free(xpath);
 
     if (duplicate) {
         crm_log_xml_warn(cib_object, "Duplicates");
         crm_exit(CRM_EX_SOFTWARE);
         return NULL; // not reached, but makes static analysis happy
     }
 
     CRM_ASSERT(rc == pcmk_ok);
     return cib_object;
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious node state change into a scheduler input
  *
  * \param[in,out] cib_conn  Scheduler input CIB to inject into
  * \param[in]     node      Name of node to inject change for
  * \param[in]     up        If true, change state to online, otherwise offline
  *
  * \return XML of changed (or added) node state entry
  */
 xmlNode *
 pcmk__inject_node_state_change(cib_t *cib_conn, const char *node, bool up)
 {
     xmlNode *cib_node = pcmk__inject_node(cib_conn, node, NULL);
 
     if (up) {
         pcmk__xe_set_props(cib_node,
                            PCMK__XA_IN_CCM, XML_BOOLEAN_TRUE,
                            PCMK__XA_CRMD, ONLINESTATUS,
                            PCMK__XA_JOIN, CRMD_JOINSTATE_MEMBER,
                            PCMK__XA_EXPECTED, CRMD_JOINSTATE_MEMBER,
                            NULL);
     } else {
         pcmk__xe_set_props(cib_node,
                            PCMK__XA_IN_CCM, XML_BOOLEAN_FALSE,
                            PCMK__XA_CRMD, OFFLINESTATUS,
                            PCMK__XA_JOIN, CRMD_JOINSTATE_DOWN,
                            PCMK__XA_EXPECTED, CRMD_JOINSTATE_DOWN,
                            NULL);
     }
     crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, crm_system_name);
     return cib_node;
 }
 
 /*!
  * \internal
  * \brief Check whether a node has history for a given resource
  *
  * \param[in,out] cib_node  Node state XML to check
  * \param[in]     resource  Resource name to check for
  *
  * \return Resource's lrm_resource XML entry beneath \p cib_node if found,
  *         otherwise NULL
  */
 static xmlNode *
 find_resource_xml(xmlNode *cib_node, const char *resource)
 {
     const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
     char *xpath = crm_strdup_printf(XPATH_RSC_HISTORY, node, resource);
     xmlNode *match = get_xpath_object(xpath, cib_node, LOG_TRACE);
 
     free(xpath);
     return match;
 }
 
 /*!
  * \internal
  * \brief Inject a resource history element into a scheduler input
  *
  * \param[in,out] out       Output object for displaying error messages
  * \param[in,out] cib_node  Node state XML to inject resource history entry into
  * \param[in]     resource  ID (in configuration) of resource to inject
  * \param[in]     lrm_name  ID as used in history (could be clone instance)
  * \param[in]     rclass    Resource agent class of resource to inject
  * \param[in]     rtype     Resource agent type of resource to inject
  * \param[in]     rprovider Resource agent provider of resource to inject
  *
  * \return XML of injected resource history element
  * \note If a history element already exists under either \p resource or
  *       \p lrm_name, this will return it rather than injecting a new one.
  */
 xmlNode *
 pcmk__inject_resource_history(pcmk__output_t *out, xmlNode *cib_node,
                               const char *resource, const char *lrm_name,
                               const char *rclass, const char *rtype,
                               const char *rprovider)
 {
     xmlNode *lrm = NULL;
     xmlNode *container = NULL;
     xmlNode *cib_resource = NULL;
 
     cib_resource = find_resource_xml(cib_node, resource);
     if (cib_resource != NULL) {
         /* If an existing LRM history entry uses the resource name,
          * continue using it, even if lrm_name is different.
          */
         return cib_resource;
     }
 
     // Check for history entry under preferred name
     if (strcmp(resource, lrm_name) != 0) {
         cib_resource = find_resource_xml(cib_node, lrm_name);
         if (cib_resource != NULL) {
             return cib_resource;
         }
     }
 
     if ((rclass == NULL) || (rtype == NULL)) {
         // @TODO query configuration for class, provider, type
         out->err(out,
                  "Resource %s not found in the status section of %s "
                  "(supply class and type to continue)",
                  resource, ID(cib_node));
         return NULL;
 
     } else if (!pcmk__strcase_any_of(rclass,
                                      PCMK_RESOURCE_CLASS_OCF,
                                      PCMK_RESOURCE_CLASS_STONITH,
                                      PCMK_RESOURCE_CLASS_SERVICE,
                                      PCMK_RESOURCE_CLASS_UPSTART,
                                      PCMK_RESOURCE_CLASS_SYSTEMD,
                                      PCMK_RESOURCE_CLASS_LSB, NULL)) {
         out->err(out, "Invalid class for %s: %s", resource, rclass);
         return NULL;
 
     } else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
                && (rprovider == NULL)) {
         // @TODO query configuration for provider
         out->err(out, "Please specify the provider for resource %s", resource);
         return NULL;
     }
 
     crm_info("Injecting new resource %s into node state '%s'",
              lrm_name, ID(cib_node));
 
     lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
     if (lrm == NULL) {
         const char *node_uuid = ID(cib_node);
 
         lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
         crm_xml_add(lrm, PCMK_XA_ID, node_uuid);
     }
 
     container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
     if (container == NULL) {
         container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
     }
 
     cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
 
     // If we're creating a new entry, use the preferred name
     crm_xml_add(cib_resource, PCMK_XA_ID, lrm_name);
 
     crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
-    crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
+    crm_xml_add(cib_resource, PCMK_XA_PROVIDER, rprovider);
     crm_xml_add(cib_resource, PCMK_XA_TYPE, rtype);
 
     return cib_resource;
 }
 
 static int
 find_ticket_state(pcmk__output_t *out, cib_t *the_cib, const char *ticket_id,
                   xmlNode **ticket_state_xml)
 {
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
 
     GString *xpath = g_string_sized_new(256);
 
     CRM_ASSERT(ticket_state_xml != NULL);
     *ticket_state_xml = NULL;
 
     g_string_append(xpath,
                     "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS
                     "/" XML_CIB_TAG_TICKETS);
 
     if (ticket_id) {
         pcmk__g_strcat(xpath,
                        "/" XML_CIB_TAG_TICKET_STATE
                        "[@" PCMK_XA_ID "=\"", ticket_id, "\"]", NULL);
     }
     rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
                               cib_sync_call|cib_scope_local|cib_xpath);
     g_string_free(xpath, TRUE);
 
     if (rc != pcmk_ok) {
         return rc;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if ((xml_search->children != NULL) && (ticket_id != NULL)) {
         out->err(out, "Multiple ticket_states match ticket_id=%s", ticket_id);
     }
     *ticket_state_xml = xml_search;
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Inject a ticket attribute into ticket state
  *
  * \param[in,out] out          Output object for displaying error messages
  * \param[in]     ticket_id    Ticket whose state should be changed
  * \param[in]     attr_name    Ticket attribute name to inject
  * \param[in]     attr_value   Boolean value of ticket attribute to inject
  * \param[in,out] cib          CIB object to use
  *
  * \return Standard Pacemaker return code
  */
 static int
 set_ticket_state_attr(pcmk__output_t *out, const char *ticket_id,
                       const char *attr_name, bool attr_value, cib_t *cib)
 {
     int rc = pcmk_rc_ok;
     xmlNode *xml_top = NULL;
     xmlNode *ticket_state_xml = NULL;
 
     // Check for an existing ticket state entry
     rc = find_ticket_state(out, cib, ticket_id, &ticket_state_xml);
     rc = pcmk_legacy2rc(rc);
 
     if (rc == pcmk_rc_ok) { // Ticket state found, use it
         crm_debug("Injecting attribute into existing ticket state %s",
                   ticket_id);
         xml_top = ticket_state_xml;
 
     } else if (rc == ENXIO) { // No ticket state, create it
         xmlNode *xml_obj = NULL;
 
         xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
         xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
         ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
         crm_xml_add(ticket_state_xml, PCMK_XA_ID, ticket_id);
 
     } else { // Error
         return rc;
     }
 
     // Add the attribute to the ticket state
     pcmk__xe_set_bool_attr(ticket_state_xml, attr_name, attr_value);
     crm_log_xml_debug(xml_top, "Update");
 
     // Commit the change to the CIB
     rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top,
                            cib_sync_call|cib_scope_local);
     rc = pcmk_legacy2rc(rc);
 
     free_xml(xml_top);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious action into the cluster
  *
  * \param[in,out] out       Output object for displaying error messages
  * \param[in]     spec      Action specification to inject
  * \param[in,out] cib       CIB object for scheduler input
  * \param[in]     scheduler  Scheduler data
  */
 static void
 inject_action(pcmk__output_t *out, const char *spec, cib_t *cib,
               const pcmk_scheduler_t *scheduler)
 {
     int rc;
     int outcome = PCMK_OCF_OK;
     guint interval_ms = 0;
 
     char *key = NULL;
     char *node = NULL;
     char *task = NULL;
     char *resource = NULL;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *rprovider = NULL;
 
     xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     const pcmk_resource_t *rsc = NULL;
     lrmd_event_data_t *op = NULL;
 
     out->message(out, "inject-spec", spec);
 
     key = calloc(1, strlen(spec) + 1);
     node = calloc(1, strlen(spec) + 1);
     rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
     if (rc != 3) {
         out->err(out, "Invalid operation spec: %s.  Only found %d fields",
                  spec, rc);
         goto done;
     }
 
     parse_op_key(key, &resource, &task, &interval_ms);
 
     rsc = pe_find_resource(scheduler->resources, resource);
     if (rsc == NULL) {
         out->err(out, "Invalid resource name: %s", resource);
         goto done;
     }
 
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
-    rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+    rprovider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
 
     cib_node = pcmk__inject_node(cib, node, NULL);
     CRM_ASSERT(cib_node != NULL);
 
     pcmk__inject_failcount(out, cib_node, resource, task, interval_ms, outcome);
 
     cib_resource = pcmk__inject_resource_history(out, cib_node,
                                                  resource, resource,
                                                  rclass, rtype, rprovider);
     CRM_ASSERT(cib_resource != NULL);
 
     op = create_op(cib_resource, task, interval_ms, outcome);
     CRM_ASSERT(op != NULL);
 
     cib_op = pcmk__inject_action_result(cib_resource, op, 0);
     CRM_ASSERT(cib_op != NULL);
     lrmd_free_event(op);
 
     rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                            cib_sync_call|cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
 done:
     free(task);
     free(node);
     free(key);
 }
 
 /*!
  * \internal
  * \brief Inject fictitious scheduler inputs
  *
  * \param[in,out] scheduler   Scheduler data
  * \param[in,out] cib         CIB object for scheduler input to modify
  * \param[in]     injections  Injections to apply
  */
 void
 pcmk__inject_scheduler_input(pcmk_scheduler_t *scheduler, cib_t *cib,
                              const pcmk_injections_t *injections)
 {
     int rc = pcmk_ok;
     const GList *iter = NULL;
     xmlNode *cib_node = NULL;
     pcmk__output_t *out = scheduler->priv;
 
     out->message(out, "inject-modify-config", injections->quorum,
                  injections->watchdog);
     if (injections->quorum != NULL) {
         xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);
 
         /* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid);      */
         crm_xml_add(top, PCMK_XA_HAVE_QUORUM, injections->quorum);
 
         rc = cib->cmds->modify(cib, NULL, top, cib_sync_call|cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     if (injections->watchdog != NULL) {
         rc = cib__update_node_attr(out, cib, cib_sync_call|cib_scope_local,
                                    XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL,
                                    NULL, PCMK_OPT_HAVE_WATCHDOG,
                                    injections->watchdog, NULL, NULL);
         CRM_ASSERT(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->node_up; iter != NULL; iter = iter->next) {
         const char *node = (const char *) iter->data;
 
         out->message(out, "inject-modify-node", "Online", node);
 
         cib_node = pcmk__inject_node_state_change(cib, node, true);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                cib_sync_call|cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (iter = injections->node_down; iter != NULL; iter = iter->next) {
         const char *node = (const char *) iter->data;
         char *xpath = NULL;
 
         out->message(out, "inject-modify-node", "Offline", node);
 
         cib_node = pcmk__inject_node_state_change(cib, node, false);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                cib_sync_call|cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
 
         xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
                                   node, XML_CIB_TAG_LRM);
         cib->cmds->remove(cib, xpath, NULL,
                           cib_xpath|cib_sync_call|cib_scope_local);
         free(xpath);
 
         xpath = crm_strdup_printf("//node_state[@uname='%s']/%s",
                                   node, XML_TAG_TRANSIENT_NODEATTRS);
         cib->cmds->remove(cib, xpath, NULL,
                           cib_xpath|cib_sync_call|cib_scope_local);
         free(xpath);
     }
 
     for (iter = injections->node_fail; iter != NULL; iter = iter->next) {
         const char *node = (const char *) iter->data;
 
         out->message(out, "inject-modify-node", "Failing", node);
 
         cib_node = pcmk__inject_node_state_change(cib, node, true);
         crm_xml_add(cib_node, PCMK__XA_IN_CCM, XML_BOOLEAN_FALSE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                cib_sync_call|cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (iter = injections->ticket_grant; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Granting", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, "granted", true, cib);
         CRM_ASSERT(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->ticket_revoke; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Revoking", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, "granted", false, cib);
         CRM_ASSERT(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->ticket_standby; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Standby", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, "standby", true, cib);
         CRM_ASSERT(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->ticket_activate; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Activating", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, "standby", false, cib);
         CRM_ASSERT(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->op_inject; iter != NULL; iter = iter->next) {
         inject_action(out, (const char *) iter->data, cib, scheduler);
     }
 
     if (!out->is_quiet(out)) {
         out->end_list(out);
     }
 }
 
 void
 pcmk_free_injections(pcmk_injections_t *injections)
 {
     if (injections == NULL) {
         return;
     }
 
     g_list_free_full(injections->node_up, g_free);
     g_list_free_full(injections->node_down, g_free);
     g_list_free_full(injections->node_fail, g_free);
     g_list_free_full(injections->op_fail, g_free);
     g_list_free_full(injections->op_inject, g_free);
     g_list_free_full(injections->ticket_grant, g_free);
     g_list_free_full(injections->ticket_revoke, g_free);
     g_list_free_full(injections->ticket_standby, g_free);
     g_list_free_full(injections->ticket_activate, g_free);
     free(injections->quorum);
     free(injections->watchdog);
 
     free(injections);
 }
diff --git a/lib/pacemaker/pcmk_sched_resource.c b/lib/pacemaker/pcmk_sched_resource.c
index 563d3a022c..3c600942aa 100644
--- a/lib/pacemaker/pcmk_sched_resource.c
+++ b/lib/pacemaker/pcmk_sched_resource.c
@@ -1,774 +1,774 @@
 /*
  * Copyright 2014-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdlib.h>
 #include <string.h>
 #include <crm/msg_xml.h>
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 // Resource assignment methods by resource variant
 static pcmk_assignment_methods_t assignment_methods[] = {
     {
         pcmk__primitive_assign,
         pcmk__primitive_create_actions,
         pcmk__probe_rsc_on_node,
         pcmk__primitive_internal_constraints,
         pcmk__primitive_apply_coloc_score,
         pcmk__colocated_resources,
         pcmk__with_primitive_colocations,
         pcmk__primitive_with_colocations,
         pcmk__add_colocated_node_scores,
         pcmk__apply_location,
         pcmk__primitive_action_flags,
         pcmk__update_ordered_actions,
         pcmk__output_resource_actions,
         pcmk__add_rsc_actions_to_graph,
         pcmk__primitive_add_graph_meta,
         pcmk__primitive_add_utilization,
         pcmk__primitive_shutdown_lock,
     },
     {
         pcmk__group_assign,
         pcmk__group_create_actions,
         pcmk__probe_rsc_on_node,
         pcmk__group_internal_constraints,
         pcmk__group_apply_coloc_score,
         pcmk__group_colocated_resources,
         pcmk__with_group_colocations,
         pcmk__group_with_colocations,
         pcmk__group_add_colocated_node_scores,
         pcmk__group_apply_location,
         pcmk__group_action_flags,
         pcmk__group_update_ordered_actions,
         pcmk__output_resource_actions,
         pcmk__add_rsc_actions_to_graph,
         pcmk__noop_add_graph_meta,
         pcmk__group_add_utilization,
         pcmk__group_shutdown_lock,
     },
     {
         pcmk__clone_assign,
         pcmk__clone_create_actions,
         pcmk__clone_create_probe,
         pcmk__clone_internal_constraints,
         pcmk__clone_apply_coloc_score,
         pcmk__colocated_resources,
         pcmk__with_clone_colocations,
         pcmk__clone_with_colocations,
         pcmk__add_colocated_node_scores,
         pcmk__clone_apply_location,
         pcmk__clone_action_flags,
         pcmk__instance_update_ordered_actions,
         pcmk__output_resource_actions,
         pcmk__clone_add_actions_to_graph,
         pcmk__clone_add_graph_meta,
         pcmk__clone_add_utilization,
         pcmk__clone_shutdown_lock,
     },
     {
         pcmk__bundle_assign,
         pcmk__bundle_create_actions,
         pcmk__bundle_create_probe,
         pcmk__bundle_internal_constraints,
         pcmk__bundle_apply_coloc_score,
         pcmk__colocated_resources,
         pcmk__with_bundle_colocations,
         pcmk__bundle_with_colocations,
         pcmk__add_colocated_node_scores,
         pcmk__bundle_apply_location,
         pcmk__bundle_action_flags,
         pcmk__instance_update_ordered_actions,
         pcmk__output_bundle_actions,
         pcmk__bundle_add_actions_to_graph,
         pcmk__noop_add_graph_meta,
         pcmk__bundle_add_utilization,
         pcmk__bundle_shutdown_lock,
     }
 };
 
 /*!
  * \internal
  * \brief Check whether a resource's agent standard, provider, or type changed
  *
  * \param[in,out] rsc             Resource to check
  * \param[in,out] node            Node needing unfencing if agent changed
  * \param[in]     rsc_entry       XML with previously known agent information
  * \param[in]     active_on_node  Whether \p rsc is active on \p node
  *
  * \return true if agent for \p rsc changed, otherwise false
  */
 bool
 pcmk__rsc_agent_changed(pcmk_resource_t *rsc, pcmk_node_t *node,
                         const xmlNode *rsc_entry, bool active_on_node)
 {
     bool changed = false;
     const char *attr_list[] = {
         PCMK_XA_TYPE,
         XML_AGENT_ATTR_CLASS,
-        XML_AGENT_ATTR_PROVIDER
+        PCMK_XA_PROVIDER,
     };
 
     for (int i = 0; i < PCMK__NELEM(attr_list); i++) {
         const char *value = crm_element_value(rsc->xml, attr_list[i]);
         const char *old_value = crm_element_value(rsc_entry, attr_list[i]);
 
         if (!pcmk__str_eq(value, old_value, pcmk__str_none)) {
             changed = true;
             trigger_unfencing(rsc, node, "Device definition changed", NULL,
                               rsc->cluster);
             if (active_on_node) {
                 crm_notice("Forcing restart of %s on %s "
                            "because %s changed from '%s' to '%s'",
                            rsc->id, pe__node_name(node), attr_list[i],
                            pcmk__s(old_value, ""), pcmk__s(value, ""));
             }
         }
     }
     if (changed && active_on_node) {
         // Make sure the resource is restarted
         custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
                       rsc->cluster);
         pcmk__set_rsc_flags(rsc, pcmk_rsc_start_pending);
     }
     return changed;
 }
 
 /*!
  * \internal
  * \brief Add resource (and any matching children) to list if it matches ID
  *
  * \param[in] result  List to add resource to
  * \param[in] rsc     Resource to check
  * \param[in] id      ID to match
  *
  * \return (Possibly new) head of list
  */
 static GList *
 add_rsc_if_matching(GList *result, pcmk_resource_t *rsc, const char *id)
 {
     if ((strcmp(rsc->id, id) == 0)
         || ((rsc->clone_name != NULL) && (strcmp(rsc->clone_name, id) == 0))) {
         result = g_list_prepend(result, rsc);
     }
     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         pcmk_resource_t *child = (pcmk_resource_t *) iter->data;
 
         result = add_rsc_if_matching(result, child, id);
     }
     return result;
 }
 
 /*!
  * \internal
  * \brief Find all resources matching a given ID by either ID or clone name
  *
  * \param[in] id         Resource ID to check
  * \param[in] scheduler  Scheduler data
  *
  * \return List of all resources that match \p id
  * \note The caller is responsible for freeing the return value with
  *       g_list_free().
  */
 GList *
 pcmk__rscs_matching_id(const char *id, const pcmk_scheduler_t *scheduler)
 {
     GList *result = NULL;
 
     CRM_CHECK((id != NULL) && (scheduler != NULL), return NULL);
     for (GList *iter = scheduler->resources; iter != NULL; iter = iter->next) {
         result = add_rsc_if_matching(result, (pcmk_resource_t *) iter->data,
                                      id);
     }
     return result;
 }
 
 /*!
  * \internal
  * \brief Set the variant-appropriate assignment methods for a resource
  *
  * \param[in,out] data       Resource to set assignment methods for
  * \param[in]     user_data  Ignored
  */
 static void
 set_assignment_methods_for_rsc(gpointer data, gpointer user_data)
 {
     pcmk_resource_t *rsc = data;
 
     rsc->cmds = &assignment_methods[rsc->variant];
     g_list_foreach(rsc->children, set_assignment_methods_for_rsc, NULL);
 }
 
 /*!
  * \internal
  * \brief Set the variant-appropriate assignment methods for all resources
  *
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__set_assignment_methods(pcmk_scheduler_t *scheduler)
 {
     g_list_foreach(scheduler->resources, set_assignment_methods_for_rsc, NULL);
 }
 
 /*!
  * \internal
  * \brief Wrapper for colocated_resources() method for readability
  *
  * \param[in]      rsc       Resource to add to colocated list
  * \param[in]      orig_rsc  Resource originally requested
  * \param[in,out]  list      Pointer to list to add to
  *
  * \return (Possibly new) head of list
  */
 static inline void
 add_colocated_resources(const pcmk_resource_t *rsc,
                         const pcmk_resource_t *orig_rsc, GList **list)
 {
     *list = rsc->cmds->colocated_resources(rsc, orig_rsc, *list);
 }
 
 // Shared implementation of pcmk_assignment_methods_t:colocated_resources()
 GList *
 pcmk__colocated_resources(const pcmk_resource_t *rsc,
                           const pcmk_resource_t *orig_rsc,
                           GList *colocated_rscs)
 {
     const GList *iter = NULL;
     GList *colocations = NULL;
 
     if (orig_rsc == NULL) {
         orig_rsc = rsc;
     }
 
     if ((rsc == NULL) || (g_list_find(colocated_rscs, rsc) != NULL)) {
         return colocated_rscs;
     }
 
     pcmk__rsc_trace(orig_rsc, "%s is in colocation chain with %s",
                     rsc->id, orig_rsc->id);
     colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);
 
     // Follow colocations where this resource is the dependent resource
     colocations = pcmk__this_with_colocations(rsc);
     for (iter = colocations; iter != NULL; iter = iter->next) {
         const pcmk__colocation_t *constraint = iter->data;
         const pcmk_resource_t *primary = constraint->primary;
 
         if (primary == orig_rsc) {
             continue; // Break colocation loop
         }
 
         if ((constraint->score == INFINITY) &&
             (pcmk__colocation_affects(rsc, primary, constraint,
                                       true) == pcmk__coloc_affects_location)) {
             add_colocated_resources(primary, orig_rsc, &colocated_rscs);
         }
     }
     g_list_free(colocations);
 
     // Follow colocations where this resource is the primary resource
     colocations = pcmk__with_this_colocations(rsc);
     for (iter = colocations; iter != NULL; iter = iter->next) {
         const pcmk__colocation_t *constraint = iter->data;
         const pcmk_resource_t *dependent = constraint->dependent;
 
         if (dependent == orig_rsc) {
             continue; // Break colocation loop
         }
 
         if (pe_rsc_is_clone(rsc) && !pe_rsc_is_clone(dependent)) {
             continue; // We can't be sure whether dependent will be colocated
         }
 
         if ((constraint->score == INFINITY) &&
             (pcmk__colocation_affects(dependent, rsc, constraint,
                                       true) == pcmk__coloc_affects_location)) {
             add_colocated_resources(dependent, orig_rsc, &colocated_rscs);
         }
     }
     g_list_free(colocations);
 
     return colocated_rscs;
 }
 
 // No-op function for variants that don't need to implement add_graph_meta()
 void
 pcmk__noop_add_graph_meta(const pcmk_resource_t *rsc, xmlNode *xml)
 {
 }
 
 /*!
  * \internal
  * \brief Output a summary of scheduled actions for a resource
  *
  * \param[in,out] rsc  Resource to output actions for
  */
 void
 pcmk__output_resource_actions(pcmk_resource_t *rsc)
 {
     pcmk_node_t *next = NULL;
     pcmk_node_t *current = NULL;
     pcmk__output_t *out = NULL;
 
     CRM_ASSERT(rsc != NULL);
 
     out = rsc->cluster->priv;
     if (rsc->children != NULL) {
         for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
             pcmk_resource_t *child = (pcmk_resource_t *) iter->data;
 
             child->cmds->output_actions(child);
         }
         return;
     }
 
     next = rsc->allocated_to;
     if (rsc->running_on) {
         current = pe__current_node(rsc);
         if (rsc->role == pcmk_role_stopped) {
             /* This can occur when resources are being recovered because
              * the current role can change in pcmk__primitive_create_actions()
              */
             rsc->role = pcmk_role_started;
         }
     }
 
     if ((current == NULL) && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
         /* Don't log stopped orphans */
         return;
     }
 
     out->message(out, "rsc-action", rsc, current, next);
 }
 
 /*!
  * \internal
  * \brief Add a resource to a node's list of assigned resources
  *
  * \param[in,out] node  Node to add resource to
  * \param[in]     rsc   Resource to add
  */
 static inline void
 add_assigned_resource(pcmk_node_t *node, pcmk_resource_t *rsc)
 {
     node->details->allocated_rsc = g_list_prepend(node->details->allocated_rsc,
                                                   rsc);
 }
 
 /*!
  * \internal
  * \brief Assign a specified resource (of any variant) to a node
  *
  * Assign a specified resource and its children (if any) to a specified node, if
  * the node can run the resource (or unconditionally, if \p force is true). Mark
  * the resources as no longer provisional.
  *
  * If a resource can't be assigned (or \p node is \c NULL), unassign any
  * previous assignment. If \p stop_if_fail is \c true, set next role to stopped
  * and update any existing actions scheduled for the resource.
  *
  * \param[in,out] rsc           Resource to assign
  * \param[in,out] node          Node to assign \p rsc to
  * \param[in]     force         If true, assign to \p node even if unavailable
  * \param[in]     stop_if_fail  If \c true and either \p rsc can't be assigned
  *                              or \p chosen is \c NULL, set next role to
  *                              stopped and update existing actions (if \p rsc
  *                              is not a primitive, this applies to its
  *                              primitive descendants instead)
  *
  * \return \c true if the assignment of \p rsc changed, or \c false otherwise
  *
  * \note Assigning a resource to the NULL node using this function is different
  *       from calling pcmk__unassign_resource(), in that it may also update any
  *       actions created for the resource.
  * \note The \c pcmk_assignment_methods_t:assign() method is preferred, unless
  *       a resource should be assigned to the \c NULL node or every resource in
  *       a tree should be assigned to the same node.
  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
  *       completely undo the assignment. A successful assignment can be either
  *       undone or left alone as final. A failed assignment has the same effect
  *       as calling pcmk__unassign_resource(); there are no side effects on
  *       roles or actions.
  */
 bool
 pcmk__assign_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool force,
                       bool stop_if_fail)
 {
     bool changed = false;
 
     CRM_ASSERT(rsc != NULL);
 
     if (rsc->children != NULL) {
         for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
             pcmk_resource_t *child_rsc = iter->data;
 
             changed |= pcmk__assign_resource(child_rsc, node, force,
                                              stop_if_fail);
         }
         return changed;
     }
 
     // Assigning a primitive
 
     if (!force && (node != NULL)
         && ((node->weight < 0)
             // Allow graph to assume that guest node connections will come up
             || (!pcmk__node_available(node, true, false)
                 && !pe__is_guest_node(node)))) {
 
         pcmk__rsc_debug(rsc,
                         "All nodes for resource %s are unavailable, unclean or "
                         "shutting down (%s can%s run resources, with score %s)",
                         rsc->id, pe__node_name(node),
                         (pcmk__node_available(node, true, false)? "" : "not"),
                         pcmk_readable_score(node->weight));
 
         if (stop_if_fail) {
             pe__set_next_role(rsc, pcmk_role_stopped, "node availability");
         }
         node = NULL;
     }
 
     if (rsc->allocated_to != NULL) {
         changed = !pe__same_node(rsc->allocated_to, node);
     } else {
         changed = (node != NULL);
     }
     pcmk__unassign_resource(rsc);
     pcmk__clear_rsc_flags(rsc, pcmk_rsc_unassigned);
 
     if (node == NULL) {
         char *rc_stopped = NULL;
 
         pcmk__rsc_debug(rsc, "Could not assign %s to a node", rsc->id);
 
         if (!stop_if_fail) {
             return changed;
         }
         pe__set_next_role(rsc, pcmk_role_stopped, "unable to assign");
 
         for (GList *iter = rsc->actions; iter != NULL; iter = iter->next) {
             pcmk_action_t *op = (pcmk_action_t *) iter->data;
 
             pcmk__rsc_debug(rsc, "Updating %s for %s assignment failure",
                             op->uuid, rsc->id);
 
             if (pcmk__str_eq(op->task, PCMK_ACTION_STOP, pcmk__str_none)) {
                 pcmk__clear_action_flags(op, pcmk_action_optional);
 
             } else if (pcmk__str_eq(op->task, PCMK_ACTION_START,
                                     pcmk__str_none)) {
                 pcmk__clear_action_flags(op, pcmk_action_runnable);
 
             } else {
                 // Cancel recurring actions, unless for stopped state
                 const char *interval_ms_s = NULL;
                 const char *target_rc_s = NULL;
 
                 interval_ms_s = g_hash_table_lookup(op->meta,
                                                     XML_LRM_ATTR_INTERVAL_MS);
                 target_rc_s = g_hash_table_lookup(op->meta,
                                                   XML_ATTR_TE_TARGET_RC);
                 if (rc_stopped == NULL) {
                     rc_stopped = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
                 }
 
                 if (!pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
                     && !pcmk__str_eq(rc_stopped, target_rc_s, pcmk__str_none)) {
 
                     pcmk__clear_action_flags(op, pcmk_action_runnable);
                 }
             }
         }
         free(rc_stopped);
         return changed;
     }
 
     pcmk__rsc_debug(rsc, "Assigning %s to %s", rsc->id, pe__node_name(node));
     rsc->allocated_to = pe__copy_node(node);
 
     add_assigned_resource(node, rsc);
     node->details->num_resources++;
     node->count++;
     pcmk__consume_node_capacity(node->details->utilization, rsc);
 
     if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_show_utilization)) {
         pcmk__output_t *out = rsc->cluster->priv;
 
         out->message(out, "resource-util", rsc, node, __func__);
     }
     return changed;
 }
 
 /*!
  * \internal
  * \brief Remove any node assignment from a specified resource and its children
  *
  * If a specified resource has been assigned to a node, remove that assignment
  * and mark the resource as provisional again.
  *
  * \param[in,out] rsc  Resource to unassign
  *
  * \note This function is called recursively on \p rsc and its children.
  */
 void
 pcmk__unassign_resource(pcmk_resource_t *rsc)
 {
     pcmk_node_t *old = rsc->allocated_to;
 
     if (old == NULL) {
         crm_info("Unassigning %s", rsc->id);
     } else {
         crm_info("Unassigning %s from %s", rsc->id, pe__node_name(old));
     }
 
     pcmk__set_rsc_flags(rsc, pcmk_rsc_unassigned);
 
     if (rsc->children == NULL) {
         if (old == NULL) {
             return;
         }
         rsc->allocated_to = NULL;
 
         /* We're going to free the pcmk_node_t, but its details member is shared
          * and will remain, so update that appropriately first.
          */
         old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc,
                                                     rsc);
         old->details->num_resources--;
         pcmk__release_node_capacity(old->details->utilization, rsc);
         free(old);
         return;
     }
 
     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         pcmk__unassign_resource((pcmk_resource_t *) iter->data);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a resource has reached its migration threshold on a node
  *
  * \param[in,out] rsc       Resource to check
  * \param[in]     node      Node to check
  * \param[out]    failed    If threshold has been reached, this will be set to
  *                          resource that failed (possibly a parent of \p rsc)
  *
  * \return true if the migration threshold has been reached, false otherwise
  */
 bool
 pcmk__threshold_reached(pcmk_resource_t *rsc, const pcmk_node_t *node,
                         pcmk_resource_t **failed)
 {
     int fail_count, remaining_tries;
     pcmk_resource_t *rsc_to_ban = rsc;
 
     // Migration threshold of 0 means never force away
     if (rsc->migration_threshold == 0) {
         return false;
     }
 
     // If we're ignoring failures, also ignore the migration threshold
     if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
         return false;
     }
 
     // If there are no failures, there's no need to force away
     fail_count = pe_get_failcount(node, rsc, NULL,
                                   pcmk__fc_effective|pcmk__fc_fillers, NULL);
     if (fail_count <= 0) {
         return false;
     }
 
     // If failed resource is anonymous clone instance, we'll force clone away
     if (!pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
         rsc_to_ban = uber_parent(rsc);
     }
 
     // How many more times recovery will be tried on this node
     remaining_tries = rsc->migration_threshold - fail_count;
 
     if (remaining_tries <= 0) {
         pcmk__sched_warn("%s cannot run on %s due to reaching migration "
                          "threshold (clean up resource to allow again)"
                          CRM_XS " failures=%d "
                          PCMK_META_MIGRATION_THRESHOLD "=%d",
                          rsc_to_ban->id, pe__node_name(node), fail_count,
                          rsc->migration_threshold);
         if (failed != NULL) {
             *failed = rsc_to_ban;
         }
         return true;
     }
 
     crm_info("%s can fail %d more time%s on "
              "%s before reaching migration threshold (%d)",
              rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries),
              pe__node_name(node), rsc->migration_threshold);
     return false;
 }
 
 /*!
  * \internal
  * \brief Get a node's score
  *
  * \param[in] node     Node with ID to check
  * \param[in] nodes    List of nodes to look for \p node score in
  *
  * \return Node's score, or -INFINITY if not found
  */
 static int
 get_node_score(const pcmk_node_t *node, GHashTable *nodes)
 {
     pcmk_node_t *found_node = NULL;
 
     if ((node != NULL) && (nodes != NULL)) {
         found_node = g_hash_table_lookup(nodes, node->details->id);
     }
     return (found_node == NULL)? -INFINITY : found_node->weight;
 }
 
 /*!
  * \internal
  * \brief Compare two resources according to which should be assigned first
  *
  * \param[in] a     First resource to compare
  * \param[in] b     Second resource to compare
  * \param[in] data  Sorted list of all nodes in cluster
  *
  * \return -1 if \p a should be assigned before \b, 0 if they are equal,
  *         or +1 if \p a should be assigned after \b
  */
 static gint
 cmp_resources(gconstpointer a, gconstpointer b, gpointer data)
 {
     /* GLib insists that this function require gconstpointer arguments, but we
      * make a small, temporary change to each argument (setting the
      * pe_rsc_merging flag) during comparison
      */
     pcmk_resource_t *resource1 = (pcmk_resource_t *) a;
     pcmk_resource_t *resource2 = (pcmk_resource_t *) b;
     const GList *nodes = data;
 
     int rc = 0;
     int r1_score = -INFINITY;
     int r2_score = -INFINITY;
     pcmk_node_t *r1_node = NULL;
     pcmk_node_t *r2_node = NULL;
     GHashTable *r1_nodes = NULL;
     GHashTable *r2_nodes = NULL;
     const char *reason = NULL;
 
     // Resources with highest priority should be assigned first
     reason = "priority";
     r1_score = resource1->priority;
     r2_score = resource2->priority;
     if (r1_score > r2_score) {
         rc = -1;
         goto done;
     }
     if (r1_score < r2_score) {
         rc = 1;
         goto done;
     }
 
     // We need nodes to make any other useful comparisons
     reason = "no node list";
     if (nodes == NULL) {
         goto done;
     }
 
     // Calculate and log node scores
     resource1->cmds->add_colocated_node_scores(resource1, NULL, resource1->id,
                                                &r1_nodes, NULL, 1,
                                                pcmk__coloc_select_this_with);
     resource2->cmds->add_colocated_node_scores(resource2, NULL, resource2->id,
                                                &r2_nodes, NULL, 1,
                                                pcmk__coloc_select_this_with);
     pe__show_node_scores(true, NULL, resource1->id, r1_nodes,
                          resource1->cluster);
     pe__show_node_scores(true, NULL, resource2->id, r2_nodes,
                          resource2->cluster);
 
     // The resource with highest score on its current node goes first
     reason = "current location";
     if (resource1->running_on != NULL) {
         r1_node = pe__current_node(resource1);
     }
     if (resource2->running_on != NULL) {
         r2_node = pe__current_node(resource2);
     }
     r1_score = get_node_score(r1_node, r1_nodes);
     r2_score = get_node_score(r2_node, r2_nodes);
     if (r1_score > r2_score) {
         rc = -1;
         goto done;
     }
     if (r1_score < r2_score) {
         rc = 1;
         goto done;
     }
 
     // Otherwise a higher score on any node will do
     reason = "score";
     for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
         const pcmk_node_t *node = (const pcmk_node_t *) iter->data;
 
         r1_score = get_node_score(node, r1_nodes);
         r2_score = get_node_score(node, r2_nodes);
         if (r1_score > r2_score) {
             rc = -1;
             goto done;
         }
         if (r1_score < r2_score) {
             rc = 1;
             goto done;
         }
     }
 
 done:
     crm_trace("%s (%d)%s%s %c %s (%d)%s%s: %s",
               resource1->id, r1_score,
               ((r1_node == NULL)? "" : " on "),
               ((r1_node == NULL)? "" : r1_node->details->id),
               ((rc < 0)? '>' : ((rc > 0)? '<' : '=')),
               resource2->id, r2_score,
               ((r2_node == NULL)? "" : " on "),
               ((r2_node == NULL)? "" : r2_node->details->id),
               reason);
     if (r1_nodes != NULL) {
         g_hash_table_destroy(r1_nodes);
     }
     if (r2_nodes != NULL) {
         g_hash_table_destroy(r2_nodes);
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Sort resources in the order they should be assigned to nodes
  *
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__sort_resources(pcmk_scheduler_t *scheduler)
 {
     GList *nodes = g_list_copy(scheduler->nodes);
 
     nodes = pcmk__sort_nodes(nodes, NULL);
     scheduler->resources = g_list_sort_with_data(scheduler->resources,
                                                  cmp_resources, nodes);
     g_list_free(nodes);
 }
diff --git a/lib/pacemaker/pcmk_simulate.c b/lib/pacemaker/pcmk_simulate.c
index 264cc2e4b0..8a5b571fa5 100644
--- a/lib/pacemaker/pcmk_simulate.c
+++ b/lib/pacemaker/pcmk_simulate.c
@@ -1,1006 +1,1006 @@
 /*
  * Copyright 2021-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/cib/internal.h>
 #include <crm/common/output.h>
 #include <crm/common/results.h>
 #include <crm/common/scheduler.h>
 #include <pacemaker-internal.h>
 #include <pacemaker.h>
 
 #include <stdint.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include "libpacemaker_private.h"
 
 static pcmk__output_t *out = NULL;
 static cib_t *fake_cib = NULL;
 static GList *fake_resource_list = NULL;
 static const GList *fake_op_fail_list = NULL;
 
 static void set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
                                const char *use_date);
 
 /*!
  * \internal
  * \brief Create an action name for use in a dot graph
  *
  * \param[in] action   Action to create name for
  * \param[in] verbose  If true, add action ID to name
  *
  * \return Newly allocated string with action name
  * \note It is the caller's responsibility to free the result.
  */
 static char *
 create_action_name(const pcmk_action_t *action, bool verbose)
 {
     char *action_name = NULL;
     const char *prefix = "";
     const char *action_host = NULL;
     const char *clone_name = NULL;
     const char *task = action->task;
 
     if (action->node != NULL) {
         action_host = action->node->details->uname;
     } else if (!pcmk_is_set(action->flags, pcmk_action_pseudo)) {
         action_host = "<none>";
     }
 
     if (pcmk__str_eq(action->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
         prefix = "Cancel ";
         task = action->cancel_task;
     }
 
     if (action->rsc != NULL) {
         clone_name = action->rsc->clone_name;
     }
 
     if (clone_name != NULL) {
         char *key = NULL;
         guint interval_ms = 0;
 
         if (pcmk__guint_from_hash(action->meta,
                                   XML_LRM_ATTR_INTERVAL_MS, 0,
                                   &interval_ms) != pcmk_rc_ok) {
             interval_ms = 0;
         }
 
         if (pcmk__strcase_any_of(action->task, PCMK_ACTION_NOTIFY,
                                  PCMK_ACTION_NOTIFIED, NULL)) {
             const char *n_type = g_hash_table_lookup(action->meta,
                                                      "notify_key_type");
             const char *n_task = g_hash_table_lookup(action->meta,
                                                      "notify_key_operation");
 
             CRM_ASSERT(n_type != NULL);
             CRM_ASSERT(n_task != NULL);
             key = pcmk__notify_key(clone_name, n_type, n_task);
         } else {
             key = pcmk__op_key(clone_name, task, interval_ms);
         }
 
         if (action_host != NULL) {
             action_name = crm_strdup_printf("%s%s %s",
                                             prefix, key, action_host);
         } else {
             action_name = crm_strdup_printf("%s%s", prefix, key);
         }
         free(key);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
                             pcmk__str_none)) {
         const char *op = g_hash_table_lookup(action->meta, "stonith_action");
 
         action_name = crm_strdup_printf("%s%s '%s' %s",
                                         prefix, action->task, op, action_host);
 
     } else if (action->rsc && action_host) {
         action_name = crm_strdup_printf("%s%s %s",
                                         prefix, action->uuid, action_host);
 
     } else if (action_host) {
         action_name = crm_strdup_printf("%s%s %s",
                                         prefix, action->task, action_host);
 
     } else {
         action_name = crm_strdup_printf("%s", action->uuid);
     }
 
     if (verbose) {
         char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id);
 
         free(action_name);
         action_name = with_id;
     }
     return action_name;
 }
 
 /*!
  * \internal
  * \brief Display the status of a cluster
  *
  * \param[in,out] scheduler     Scheduler data
  * \param[in]     show_opts     How to modify display (as pcmk_show_opt_e flags)
  * \param[in]     section_opts  Sections to display (as pcmk_section_e flags)
  * \param[in]     title         What to use as list title
  * \param[in]     print_spacer  Whether to display a spacer first
  */
 static void
 print_cluster_status(pcmk_scheduler_t *scheduler, uint32_t show_opts,
                      uint32_t section_opts, const char *title,
                      bool print_spacer)
 {
     pcmk__output_t *out = scheduler->priv;
     GList *all = NULL;
     crm_exit_t stonith_rc = 0;
     enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;
 
     section_opts |= pcmk_section_nodes | pcmk_section_resources;
     show_opts |= pcmk_show_inactive_rscs | pcmk_show_failed_detail;
 
     all = g_list_prepend(all, (gpointer) "*");
 
     PCMK__OUTPUT_SPACER_IF(out, print_spacer);
     out->begin_list(out, NULL, NULL, "%s", title);
     out->message(out, "cluster-status",
                  scheduler, state, stonith_rc, NULL,
                  false, section_opts, show_opts, NULL, all, all);
     out->end_list(out);
 
     g_list_free(all);
 }
 
 /*!
  * \internal
  * \brief Display a summary of all actions scheduled in a transition
  *
  * \param[in,out] scheduler     Scheduler data (fully scheduled)
  * \param[in]     print_spacer  Whether to display a spacer first
  */
 static void
 print_transition_summary(pcmk_scheduler_t *scheduler, bool print_spacer)
 {
     pcmk__output_t *out = scheduler->priv;
 
     PCMK__OUTPUT_SPACER_IF(out, print_spacer);
     out->begin_list(out, NULL, NULL, "Transition Summary");
     pcmk__output_actions(scheduler);
     out->end_list(out);
 }
 
 /*!
  * \internal
  * \brief Reset scheduler input, output, date, and flags
  *
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     input      What to set as cluster input
  * \param[in]     out        What to set as cluster output object
  * \param[in]     use_date   What to set as cluster's current timestamp
  * \param[in]     flags      Group of enum pcmk_scheduler_flags to set
  */
 static void
 reset(pcmk_scheduler_t *scheduler, xmlNodePtr input, pcmk__output_t *out,
       const char *use_date, unsigned int flags)
 {
     scheduler->input = input;
     scheduler->priv = out;
     set_effective_date(scheduler, true, use_date);
     if (pcmk_is_set(flags, pcmk_sim_sanitized)) {
         pcmk__set_scheduler_flags(scheduler, pcmk_sched_sanitized);
     }
     if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
         pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
     }
     if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
         pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
     }
 }
 
 /*!
  * \brief Write out a file in dot(1) format describing the actions that will
  *        be taken by the scheduler in response to an input CIB file.
  *
  * \param[in,out] scheduler    Scheduler data
  * \param[in]     dot_file     The filename to write
  * \param[in]     all_actions  Write all actions, even those that are optional
  *                             or are on unmanaged resources
  * \param[in]     verbose      Add extra information, such as action IDs, to the
  *                             output
  *
  * \return Standard Pacemaker return code
  */
 static int
 write_sim_dotfile(pcmk_scheduler_t *scheduler, const char *dot_file,
                   bool all_actions, bool verbose)
 {
     GList *iter = NULL;
     FILE *dot_strm = fopen(dot_file, "w");
 
     if (dot_strm == NULL) {
         return errno;
     }
 
     fprintf(dot_strm, " digraph \"g\" {\n");
     for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
         pcmk_action_t *action = (pcmk_action_t *) iter->data;
         const char *style = "dashed";
         const char *font = "black";
         const char *color = "black";
         char *action_name = create_action_name(action, verbose);
 
         if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
             font = "orange";
         }
 
         if (pcmk_is_set(action->flags, pcmk_action_added_to_graph)) {
             style = "bold";
             color = "green";
 
         } else if ((action->rsc != NULL)
                    && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)) {
             color = "red";
             font = "purple";
             if (!all_actions) {
                 goto do_not_write;
             }
 
         } else if (pcmk_is_set(action->flags, pcmk_action_optional)) {
             color = "blue";
             if (!all_actions) {
                 goto do_not_write;
             }
 
         } else {
             color = "red";
             CRM_LOG_ASSERT(!pcmk_is_set(action->flags, pcmk_action_runnable));
         }
 
         pcmk__set_action_flags(action, pcmk_action_added_to_graph);
         fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
                 action_name, style, color, font);
   do_not_write:
         free(action_name);
     }
 
     for (iter = scheduler->actions; iter != NULL; iter = iter->next) {
         pcmk_action_t *action = (pcmk_action_t *) iter->data;
 
         for (GList *before_iter = action->actions_before;
              before_iter != NULL; before_iter = before_iter->next) {
 
             pcmk__related_action_t *before = before_iter->data;
 
             char *before_name = NULL;
             char *after_name = NULL;
             const char *style = "dashed";
             bool optional = true;
 
             if (before->state == pe_link_dumped) {
                 optional = false;
                 style = "bold";
             } else if ((uint32_t) before->type == pcmk__ar_none) {
                 continue;
             } else if (pcmk_is_set(before->action->flags,
                                    pcmk_action_added_to_graph)
                        && pcmk_is_set(action->flags, pcmk_action_added_to_graph)
                        && (uint32_t) before->type != pcmk__ar_if_on_same_node_or_target) {
                 optional = false;
             }
 
             if (all_actions || !optional) {
                 before_name = create_action_name(before->action, verbose);
                 after_name = create_action_name(action, verbose);
                 fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
                         before_name, after_name, style);
                 free(before_name);
                 free(after_name);
             }
         }
     }
 
     fprintf(dot_strm, "}\n");
     fflush(dot_strm);
     fclose(dot_strm);
     return pcmk_rc_ok;
 }
 
 /*!
  * \brief Profile the configuration updates and scheduler actions in a single
  *        CIB file, printing the profiling timings.
  *
  * \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
  *       object before this function is called.
  *
  * \param[in]     xml_file   The CIB file to profile
  * \param[in]     repeat     Number of times to run
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     use_date   The date to set the cluster's time to (may be NULL)
  */
 static void
 profile_file(const char *xml_file, long long repeat,
              pcmk_scheduler_t *scheduler, const char *use_date)
 {
     pcmk__output_t *out = scheduler->priv;
     xmlNode *cib_object = NULL;
     clock_t start = 0;
     clock_t end;
     unsigned long long scheduler_flags = pcmk_sched_no_compat;
 
     CRM_ASSERT(out != NULL);
 
     cib_object = filename2xml(xml_file);
     start = clock();
 
     if (pcmk_find_cib_element(cib_object, XML_CIB_TAG_STATUS) == NULL) {
         create_xml_node(cib_object, XML_CIB_TAG_STATUS);
     }
 
     if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
         free_xml(cib_object);
         return;
     }
 
     if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
         free_xml(cib_object);
         return;
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
         scheduler_flags |= pcmk_sched_output_scores;
     }
     if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
         scheduler_flags |= pcmk_sched_show_utilization;
     }
 
     for (int i = 0; i < repeat; ++i) {
         xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object);
 
         scheduler->input = input;
         set_effective_date(scheduler, false, use_date);
         pcmk__schedule_actions(input, scheduler_flags, scheduler);
         pe_reset_working_set(scheduler);
     }
 
     end = clock();
     out->message(out, "profile", xml_file, start, end);
 }
 
 void
 pcmk__profile_dir(const char *dir, long long repeat,
                   pcmk_scheduler_t *scheduler, const char *use_date)
 {
     pcmk__output_t *out = scheduler->priv;
     struct dirent **namelist;
 
     int file_num = scandir(dir, &namelist, 0, alphasort);
 
     CRM_ASSERT(out != NULL);
 
     if (file_num > 0) {
         struct stat prop;
         char buffer[FILENAME_MAX];
 
         out->begin_list(out, NULL, NULL, "Timings");
 
         while (file_num--) {
             if ('.' == namelist[file_num]->d_name[0]) {
                 free(namelist[file_num]);
                 continue;
 
             } else if (!pcmk__ends_with_ext(namelist[file_num]->d_name,
                                             ".xml")) {
                 free(namelist[file_num]);
                 continue;
             }
             snprintf(buffer, sizeof(buffer), "%s/%s",
                      dir, namelist[file_num]->d_name);
             if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
                 profile_file(buffer, repeat, scheduler, use_date);
             }
             free(namelist[file_num]);
         }
         free(namelist);
 
         out->end_list(out);
     }
 }
 
 /*!
  * \brief Set the date of the cluster, either to the value given by
  *        \p use_date, or to the "execution-date" value in the CIB.
  *
  * \note \p scheduler->priv must have been set to a valid \p pcmk__output_t
  *       object before this function is called.
  *
  * \param[in,out] scheduler       Scheduler data
  * \param[in]     print_original  If \p true, the "execution-date" should
  *                                also be printed
  * \param[in]     use_date        The date to set the cluster's time to
  *                                (may be NULL)
  */
 static void
 set_effective_date(pcmk_scheduler_t *scheduler, bool print_original,
                    const char *use_date)
 {
     pcmk__output_t *out = scheduler->priv;
     time_t original_date = 0;
 
     CRM_ASSERT(out != NULL);
 
     crm_element_value_epoch(scheduler->input, "execution-date", &original_date);
 
     if (use_date) {
         scheduler->now = crm_time_new(use_date);
         out->info(out, "Setting effective cluster time: %s", use_date);
         crm_time_log(LOG_NOTICE, "Pretending 'now' is", scheduler->now,
                      crm_time_log_date | crm_time_log_timeofday);
 
     } else if (original_date != 0) {
         scheduler->now = pcmk__copy_timet(original_date);
 
         if (print_original) {
             char *when = crm_time_as_string(scheduler->now,
                             crm_time_log_date|crm_time_log_timeofday);
 
             out->info(out, "Using the original execution date of: %s", when);
             free(when);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Simulate successfully executing a pseudo-action in a graph
  *
  * \param[in,out] graph   Graph to update with pseudo-action result
  * \param[in,out] action  Pseudo-action to simulate executing
  *
  * \return Standard Pacemaker return code
  */
 static int
 simulate_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
     out->message(out, "inject-pseudo-action", node, task);
 
     pcmk__update_graph(graph, action);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Simulate executing a resource action in a graph
  *
  * \param[in,out] graph   Graph to update with resource action result
  * \param[in,out] action  Resource action to simulate executing
  *
  * \return Standard Pacemaker return code
  */
 static int
 simulate_resource_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     int rc;
     lrmd_event_data_t *op = NULL;
     int target_outcome = PCMK_OCF_OK;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *resource = NULL;
     const char *rprovider = NULL;
     const char *resource_config_name = NULL;
     const char *operation = crm_element_value(action->xml, "operation");
     const char *target_rc_s = crm_meta_value(action->params,
                                              XML_ATTR_TE_TARGET_RC);
 
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
     char *uuid = NULL;
     const char *router_node = crm_element_value(action->xml,
                                                 XML_LRM_ATTR_ROUTER_NODE);
 
     // Certain actions don't need to be displayed or history entries
     if (pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
         crm_debug("No history injection for %s op on %s", operation, node);
         goto done; // Confirm action and update graph
     }
 
     if (action_rsc == NULL) { // Shouldn't be possible
         crm_log_xml_err(action->xml, "Bad");
         free(node);
         return EPROTO;
     }
 
     /* A resource might be known by different names in the configuration and in
      * the action (for example, a clone instance). Grab the configuration name
      * (which is preferred when writing history), and if necessary, the instance
      * name.
      */
     resource_config_name = crm_element_value(action_rsc, PCMK_XA_ID);
     if (resource_config_name == NULL) { // Shouldn't be possible
         crm_log_xml_err(action->xml, "No ID");
         free(node);
         return EPROTO;
     }
     resource = resource_config_name;
     if (pe_find_resource(fake_resource_list, resource) == NULL) {
         const char *longname = crm_element_value(action_rsc, PCMK__XA_LONG_ID);
 
         if ((longname != NULL)
             && (pe_find_resource(fake_resource_list, longname) != NULL)) {
             resource = longname;
         }
     }
 
     // Certain actions need to be displayed but don't need history entries
     if (pcmk__strcase_any_of(operation, PCMK_ACTION_DELETE,
                              PCMK_ACTION_META_DATA, NULL)) {
         out->message(out, "inject-rsc-action", resource, operation, node,
                      (guint) 0);
         goto done; // Confirm action and update graph
     }
 
     rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
     rtype = crm_element_value(action_rsc, PCMK_XA_TYPE);
-    rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
+    rprovider = crm_element_value(action_rsc, PCMK_XA_PROVIDER);
 
     pcmk__scan_min_int(target_rc_s, &target_outcome, 0);
 
     CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL,
                                      cib_sync_call|cib_scope_local) == pcmk_ok);
 
     // Ensure the action node is in the CIB
     uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
     cib_node = pcmk__inject_node(fake_cib, node,
                                  ((router_node == NULL)? uuid: node));
     free(uuid);
     CRM_ASSERT(cib_node != NULL);
 
     // Add a history entry for the action
     cib_resource = pcmk__inject_resource_history(out, cib_node, resource,
                                                  resource_config_name,
                                                  rclass, rtype, rprovider);
     if (cib_resource == NULL) {
         crm_err("Could not simulate action %d history for resource %s",
                 action->id, resource);
         free(node);
         free_xml(cib_node);
         return EINVAL;
     }
 
     // Simulate and display an executor event for the action result
     op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
                                        target_outcome, "User-injected result");
     out->message(out, "inject-rsc-action", resource, op->op_type, node,
                  op->interval_ms);
 
     // Check whether action is in a list of desired simulated failures
     for (const GList *iter = fake_op_fail_list;
          iter != NULL; iter = iter->next) {
         const char *spec = (const char *) iter->data;
         char *key = NULL;
         const char *match_name = NULL;
 
         // Allow user to specify anonymous clone with or without instance number
         key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
                                 op->interval_ms, node);
         if (strncasecmp(key, spec, strlen(key)) == 0) {
             match_name = resource;
         }
         free(key);
 
         // If not found, try the resource's name in the configuration
         if ((match_name == NULL)
             && (strcmp(resource, resource_config_name) != 0)) {
 
             key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource_config_name,
                                     op->op_type, op->interval_ms, node);
             if (strncasecmp(key, spec, strlen(key)) == 0) {
                 match_name = resource_config_name;
             }
             free(key);
         }
 
         if (match_name == NULL) {
             continue; // This failed action entry doesn't match
         }
 
         // ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
         rc = sscanf(spec, "%*[^=]=%d", (int *) &op->rc);
         if (rc != 1) {
             out->err(out, "Invalid failed operation '%s' "
                           "(result code must be integer)", spec);
             continue; // Keep checking other list entries
         }
 
         out->info(out, "Pretending action %d failed with rc=%d",
                   action->id, op->rc);
         pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
         graph->abort_priority = INFINITY;
         pcmk__inject_failcount(out, cib_node, match_name, op->op_type,
                                op->interval_ms, op->rc);
         break;
     }
 
     pcmk__inject_action_result(cib_resource, op, target_outcome);
     lrmd_free_event(op);
     rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                 cib_sync_call|cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
   done:
     free(node);
     free_xml(cib_node);
     pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
     pcmk__update_graph(graph, action);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Simulate successfully executing a cluster action
  *
  * \param[in,out] graph   Graph to update with action result
  * \param[in,out] action  Cluster action to simulate
  *
  * \return Standard Pacemaker return code
  */
 static int
 simulate_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
     out->message(out, "inject-cluster-action", node, task, rsc);
     pcmk__update_graph(graph, action);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Simulate successfully executing a fencing action
  *
  * \param[in,out] graph   Graph to update with action result
  * \param[in,out] action  Fencing action to simulate
  *
  * \return Standard Pacemaker return code
  */
 static int
 simulate_fencing_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
 {
     const char *op = crm_meta_value(action->params, "stonith_action");
     char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 
     out->message(out, "inject-fencing-action", target, op);
 
     if (!pcmk__str_eq(op, PCMK_ACTION_ON, pcmk__str_casei)) {
         int rc = pcmk_ok;
         GString *xpath = g_string_sized_new(512);
 
         // Set node state to offline
         xmlNode *cib_node = pcmk__inject_node_state_change(fake_cib, target,
                                                            false);
 
         CRM_ASSERT(cib_node != NULL);
         crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, __func__);
         rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                      cib_sync_call|cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
 
         // Simulate controller clearing node's resource history and attributes
         pcmk__g_strcat(xpath,
                        "//" XML_CIB_TAG_STATE
                        "[@" XML_ATTR_UNAME "='", target, "']/" XML_CIB_TAG_LRM,
                        NULL);
         fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
                                cib_xpath|cib_sync_call|cib_scope_local);
 
         g_string_truncate(xpath, 0);
         pcmk__g_strcat(xpath,
                        "//" XML_CIB_TAG_STATE
                        "[@" XML_ATTR_UNAME "='", target, "']"
                        "/" XML_TAG_TRANSIENT_NODEATTRS, NULL);
         fake_cib->cmds->remove(fake_cib, (const char *) xpath->str, NULL,
                                cib_xpath|cib_sync_call|cib_scope_local);
 
         free_xml(cib_node);
         g_string_free(xpath, TRUE);
     }
 
     pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
     pcmk__update_graph(graph, action);
     free(target);
     return pcmk_rc_ok;
 }
 
 enum pcmk__graph_status
 pcmk__simulate_transition(pcmk_scheduler_t *scheduler, cib_t *cib,
                           const GList *op_fail_list)
 {
     pcmk__graph_t *transition = NULL;
     enum pcmk__graph_status graph_rc;
 
     pcmk__graph_functions_t simulation_fns = {
         simulate_pseudo_action,
         simulate_resource_action,
         simulate_cluster_action,
         simulate_fencing_action,
     };
 
     out = scheduler->priv;
 
     fake_cib = cib;
     fake_op_fail_list = op_fail_list;
 
     if (!out->is_quiet(out)) {
         out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
     }
 
     pcmk__set_graph_functions(&simulation_fns);
     transition = pcmk__unpack_graph(scheduler->graph, crm_system_name);
     pcmk__log_graph(LOG_DEBUG, transition);
 
     fake_resource_list = scheduler->resources;
     do {
         graph_rc = pcmk__execute_graph(transition);
     } while (graph_rc == pcmk__graph_active);
     fake_resource_list = NULL;
 
     if (graph_rc != pcmk__graph_complete) {
         out->err(out, "Transition failed: %s",
                  pcmk__graph_status2text(graph_rc));
         pcmk__log_graph(LOG_ERR, transition);
         out->err(out, "An invalid transition was produced");
     }
     pcmk__free_graph(transition);
 
     if (!out->is_quiet(out)) {
         // If not quiet, we'll need the resulting CIB for later display
         xmlNode *cib_object = NULL;
         int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object,
                                        cib_sync_call|cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
         pe_reset_working_set(scheduler);
         scheduler->input = cib_object;
         out->end_list(out);
     }
     return graph_rc;
 }
 
 int
 pcmk__simulate(pcmk_scheduler_t *scheduler, pcmk__output_t *out,
                const pcmk_injections_t *injections, unsigned int flags,
                uint32_t section_opts, const char *use_date,
                const char *input_file, const char *graph_file,
                const char *dot_file)
 {
     int printed = pcmk_rc_no_output;
     int rc = pcmk_rc_ok;
     xmlNodePtr input = NULL;
     cib_t *cib = NULL;
 
     rc = cib__signon_query(out, &cib, &input);
     if (rc != pcmk_rc_ok) {
         goto simulate_done;
     }
 
     reset(scheduler, input, out, use_date, flags);
     cluster_status(scheduler);
 
     if ((cib->variant == cib_native)
         && pcmk_is_set(section_opts, pcmk_section_times)) {
         if (pcmk__our_nodename == NULL) {
             // Currently used only in the times section
             pcmk__query_node_name(out, 0, &pcmk__our_nodename, 0);
         }
         scheduler->localhost = pcmk__our_nodename;
     }
 
     if (!out->is_quiet(out)) {
         const bool show_pending = pcmk_is_set(flags, pcmk_sim_show_pending);
 
         if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
             printed = out->message(out, "maint-mode", scheduler->flags);
         }
 
         if (scheduler->disabled_resources || scheduler->blocked_resources) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             printed = out->info(out,
                                 "%d of %d resource instances DISABLED and "
                                 "%d BLOCKED from further action due to failure",
                                 scheduler->disabled_resources,
                                 scheduler->ninstances,
                                 scheduler->blocked_resources);
         }
 
         /* Most formatted output headers use caps for each word, but this one
          * only has the first word capitalized for compatibility with pcs.
          */
         print_cluster_status(scheduler, (show_pending? pcmk_show_pending : 0),
                              section_opts, "Current cluster status",
                              (printed == pcmk_rc_ok));
         printed = pcmk_rc_ok;
     }
 
     // If the user requested any injections, handle them
     if ((injections->node_down != NULL)
         || (injections->node_fail != NULL)
         || (injections->node_up != NULL)
         || (injections->op_inject != NULL)
         || (injections->ticket_activate != NULL)
         || (injections->ticket_grant != NULL)
         || (injections->ticket_revoke != NULL)
         || (injections->ticket_standby != NULL)
         || (injections->watchdog != NULL)) {
 
         PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
         pcmk__inject_scheduler_input(scheduler, cib, injections);
         printed = pcmk_rc_ok;
 
         rc = cib->cmds->query(cib, NULL, &input, cib_sync_call);
         if (rc != pcmk_rc_ok) {
             rc = pcmk_legacy2rc(rc);
             goto simulate_done;
         }
 
         cleanup_calculations(scheduler);
         reset(scheduler, input, out, use_date, flags);
         cluster_status(scheduler);
     }
 
     if (input_file != NULL) {
         rc = write_xml_file(input, input_file, FALSE);
         if (rc < 0) {
             rc = pcmk_legacy2rc(rc);
             goto simulate_done;
         }
     }
 
     if (pcmk_any_flags_set(flags, pcmk_sim_process | pcmk_sim_simulate)) {
         pcmk__output_t *logger_out = NULL;
         unsigned long long scheduler_flags = pcmk_sched_no_compat;
 
         if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
             scheduler_flags |= pcmk_sched_output_scores;
         }
         if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
             scheduler_flags |= pcmk_sched_show_utilization;
         }
 
         if (pcmk_all_flags_set(scheduler->flags,
                                pcmk_sched_output_scores
                                |pcmk_sched_show_utilization)) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL,
                             "Assignment Scores and Utilization Information");
             printed = pcmk_rc_ok;
 
         } else if (pcmk_is_set(scheduler->flags, pcmk_sched_output_scores)) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL, "Assignment Scores");
             printed = pcmk_rc_ok;
 
         } else if (pcmk_is_set(scheduler->flags, pcmk_sched_show_utilization)) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL, "Utilization Information");
             printed = pcmk_rc_ok;
 
         } else {
             rc = pcmk__log_output_new(&logger_out);
             if (rc != pcmk_rc_ok) {
                 goto simulate_done;
             }
             pe__register_messages(logger_out);
             pcmk__register_lib_messages(logger_out);
             scheduler->priv = logger_out;
         }
 
         pcmk__schedule_actions(input, scheduler_flags, scheduler);
 
         if (logger_out == NULL) {
             out->end_list(out);
         } else {
             logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
             pcmk__output_free(logger_out);
             scheduler->priv = out;
         }
 
         input = NULL;           /* Don't try and free it twice */
 
         if (graph_file != NULL) {
             rc = write_xml_file(scheduler->graph, graph_file, FALSE);
             if (rc < 0) {
                 rc = pcmk_rc_graph_error;
                 goto simulate_done;
             }
         }
 
         if (dot_file != NULL) {
             rc = write_sim_dotfile(scheduler, dot_file,
                                    pcmk_is_set(flags, pcmk_sim_all_actions),
                                    pcmk_is_set(flags, pcmk_sim_verbose));
             if (rc != pcmk_rc_ok) {
                 rc = pcmk_rc_dot_error;
                 goto simulate_done;
             }
         }
 
         if (!out->is_quiet(out)) {
             print_transition_summary(scheduler, printed == pcmk_rc_ok);
         }
     }
 
     rc = pcmk_rc_ok;
 
     if (!pcmk_is_set(flags, pcmk_sim_simulate)) {
         goto simulate_done;
     }
 
     PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
     if (pcmk__simulate_transition(scheduler, cib, injections->op_fail)
             != pcmk__graph_complete) {
         rc = pcmk_rc_invalid_transition;
     }
 
     if (out->is_quiet(out)) {
         goto simulate_done;
     }
 
     set_effective_date(scheduler, true, use_date);
 
     if (pcmk_is_set(flags, pcmk_sim_show_scores)) {
         pcmk__set_scheduler_flags(scheduler, pcmk_sched_output_scores);
     }
     if (pcmk_is_set(flags, pcmk_sim_show_utilization)) {
         pcmk__set_scheduler_flags(scheduler, pcmk_sched_show_utilization);
     }
 
     cluster_status(scheduler);
     print_cluster_status(scheduler, 0, section_opts, "Revised Cluster Status",
                          true);
 
 simulate_done:
     cib__clean_up_connection(&cib);
     return rc;
 }
 
 int
 pcmk_simulate(xmlNodePtr *xml, pcmk_scheduler_t *scheduler,
               const pcmk_injections_t *injections, unsigned int flags,
               unsigned int section_opts, const char *use_date,
               const char *input_file, const char *graph_file,
               const char *dot_file)
 {
     pcmk__output_t *out = NULL;
     int rc = pcmk_rc_ok;
 
     rc = pcmk__xml_output_new(&out, xml);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     pe__register_messages(out);
     pcmk__register_lib_messages(out);
 
     rc = pcmk__simulate(scheduler, out, injections, flags, section_opts,
                         use_date, input_file, graph_file, dot_file);
     pcmk__xml_output_finish(out, pcmk_rc2exitc(rc), xml);
     return rc;
 }
diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
index 7694f870eb..68fd8a6240 100644
--- a/lib/pengine/bundle.c
+++ b/lib/pengine/bundle.c
@@ -1,2229 +1,2229 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <ctype.h>
 #include <stdint.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/output.h>
 #include <crm/common/xml_internal.h>
 #include <pe_status_private.h>
 
 enum pe__bundle_mount_flags {
     pe__bundle_mount_none       = 0x00,
 
     // mount instance-specific subdirectory rather than source directly
     pe__bundle_mount_subdir     = 0x01
 };
 
 typedef struct {
     char *source;
     char *target;
     char *options;
     uint32_t flags; // bitmask of pe__bundle_mount_flags
 } pe__bundle_mount_t;
 
 typedef struct {
     char *source;
     char *target;
 } pe__bundle_port_t;
 
 enum pe__container_agent {
     PE__CONTAINER_AGENT_UNKNOWN,
     PE__CONTAINER_AGENT_DOCKER,
     PE__CONTAINER_AGENT_RKT,
     PE__CONTAINER_AGENT_PODMAN,
 };
 
 #define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
 #define PE__CONTAINER_AGENT_DOCKER_S  "docker"
 #define PE__CONTAINER_AGENT_RKT_S     "rkt"
 #define PE__CONTAINER_AGENT_PODMAN_S  "podman"
 
 typedef struct pe__bundle_variant_data_s {
         int promoted_max;
         int nreplicas;
         int nreplicas_per_host;
         char *prefix;
         char *image;
         const char *ip_last;
         char *host_network;
         char *host_netmask;
         char *control_port;
         char *container_network;
         char *ip_range_start;
         gboolean add_host;
         gchar *container_host_options;
         char *container_command;
         char *launcher_options;
         const char *attribute_target;
 
         pcmk_resource_t *child;
 
         GList *replicas;    // pcmk__bundle_replica_t *
         GList *ports;       // pe__bundle_port_t *
         GList *mounts;      // pe__bundle_mount_t *
 
         enum pe__container_agent agent_type;
 } pe__bundle_variant_data_t;
 
 #define get_bundle_variant_data(data, rsc)                      \
     CRM_ASSERT(rsc != NULL);                                    \
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_bundle);        \
     CRM_ASSERT(rsc->variant_opaque != NULL);                    \
     data = (pe__bundle_variant_data_t *) rsc->variant_opaque;
 
 /*!
  * \internal
  * \brief Get maximum number of bundle replicas allowed to run
  *
  * \param[in] rsc  Bundle or bundled resource to check
  *
  * \return Maximum replicas for bundle corresponding to \p rsc
  */
 int
 pe__bundle_max(const pcmk_resource_t *rsc)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
     return bundle_data->nreplicas;
 }
 
 /*!
  * \internal
  * \brief Get the resource inside a bundle
  *
  * \param[in] bundle  Bundle to check
  *
  * \return Resource inside \p bundle if any, otherwise NULL
  */
 pcmk_resource_t *
 pe__bundled_resource(const pcmk_resource_t *rsc)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
     return bundle_data->child;
 }
 
 /*!
  * \internal
  * \brief Get containerized resource corresponding to a given bundle container
  *
  * \param[in] instance  Collective instance that might be a bundle container
  *
  * \return Bundled resource instance inside \p instance if it is a bundle
  *         container instance, otherwise NULL
  */
 const pcmk_resource_t *
 pe__get_rsc_in_container(const pcmk_resource_t *instance)
 {
     const pe__bundle_variant_data_t *data = NULL;
     const pcmk_resource_t *top = pe__const_top_resource(instance, true);
 
     if ((top == NULL) || (top->variant != pcmk_rsc_variant_bundle)) {
         return NULL;
     }
     get_bundle_variant_data(data, top);
 
     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
         const pcmk__bundle_replica_t *replica = iter->data;
 
         if (instance == replica->container) {
             return replica->child;
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Check whether a given node is created by a bundle
  *
  * \param[in] bundle  Bundle resource to check
  * \param[in] node    Node to check
  *
  * \return true if \p node is an instance of \p bundle, otherwise false
  */
 bool
 pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
                             const pcmk_node_t *node)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
 
         if (pe__same_node(node, replica->node)) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Get the container of a bundle's first replica
  *
  * \param[in] bundle  Bundle resource to get container for
  *
  * \return Container resource from first replica of \p bundle if any,
  *         otherwise NULL
  */
 pcmk_resource_t *
 pe__first_container(const pcmk_resource_t *bundle)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
     const pcmk__bundle_replica_t *replica = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     if (bundle_data->replicas == NULL) {
         return NULL;
     }
     replica = bundle_data->replicas->data;
     return replica->container;
 }
 
 /*!
  * \internal
  * \brief Iterate over bundle replicas
  *
  * \param[in,out] bundle     Bundle to iterate over
  * \param[in]     fn         Function to call for each replica (its return value
  *                           indicates whether to continue iterating)
  * \param[in,out] user_data  Pointer to pass to \p fn
  */
 void
 pe__foreach_bundle_replica(pcmk_resource_t *bundle,
                            bool (*fn)(pcmk__bundle_replica_t *, void *),
                            void *user_data)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
         if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
             break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Iterate over const bundle replicas
  *
  * \param[in]     bundle     Bundle to iterate over
  * \param[in]     fn         Function to call for each replica (its return value
  *                           indicates whether to continue iterating)
  * \param[in,out] user_data  Pointer to pass to \p fn
  */
 void
 pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
                                  bool (*fn)(const pcmk__bundle_replica_t *,
                                             void *),
                                  void *user_data)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     for (const GList *iter = bundle_data->replicas; iter != NULL;
          iter = iter->next) {
 
         if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
             break;
         }
     }
 }
 
 static char *
 next_ip(const char *last_ip)
 {
     unsigned int oct1 = 0;
     unsigned int oct2 = 0;
     unsigned int oct3 = 0;
     unsigned int oct4 = 0;
     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 
     if (rc != 4) {
         /*@ TODO check for IPv6 */
         return NULL;
 
     } else if (oct3 > 253) {
         return NULL;
 
     } else if (oct4 > 253) {
         ++oct3;
         oct4 = 1;
 
     } else {
         ++oct4;
     }
 
     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 }
 
 static void
 allocate_ip(pe__bundle_variant_data_t *data, pcmk__bundle_replica_t *replica,
             GString *buffer)
 {
     if(data->ip_range_start == NULL) {
         return;
 
     } else if(data->ip_last) {
         replica->ipaddr = next_ip(data->ip_last);
 
     } else {
         replica->ipaddr = strdup(data->ip_range_start);
     }
 
     data->ip_last = replica->ipaddr;
     switch (data->agent_type) {
         case PE__CONTAINER_AGENT_DOCKER:
         case PE__CONTAINER_AGENT_PODMAN:
             if (data->add_host) {
                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
                                        data->prefix, replica->offset,
                                        replica->ipaddr);
             } else {
                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
                                        replica->ipaddr, data->prefix,
                                        replica->offset);
             }
             break;
 
         case PE__CONTAINER_AGENT_RKT:
             g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
                                    replica->ipaddr, data->prefix,
                                    replica->offset);
             break;
 
         default: // PE__CONTAINER_AGENT_UNKNOWN
             break;
     }
 }
 
 static xmlNode *
 create_resource(const char *name, const char *provider, const char *kind)
 {
     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     crm_xml_add(rsc, PCMK_XA_ID, name);
     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
-    crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
+    crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
     crm_xml_add(rsc, PCMK_XA_TYPE, kind);
 
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Check whether cluster can manage resource inside container
  *
  * \param[in,out] data  Container variant data
  *
  * \return TRUE if networking configuration is acceptable, FALSE otherwise
  *
  * \note The resource is manageable if an IP range or control port has been
  *       specified. If a control port is used without an IP range, replicas per
  *       host must be 1.
  */
 static bool
 valid_network(pe__bundle_variant_data_t *data)
 {
     if(data->ip_range_start) {
         return TRUE;
     }
     if(data->control_port) {
         if(data->nreplicas_per_host > 1) {
             pcmk__config_err("Specifying the 'control-port' for %s requires "
                              "'replicas-per-host=1'", data->prefix);
             data->nreplicas_per_host = 1;
             // @TODO to be sure:
             // pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
         }
         return TRUE;
     }
     return FALSE;
 }
 
 static int
 create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
                    pcmk__bundle_replica_t *replica)
 {
     if(data->ip_range_start) {
         char *id = NULL;
         xmlNode *xml_ip = NULL;
         xmlNode *xml_obj = NULL;
 
         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
         crm_xml_sanitize_id(id);
         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
         free(id);
 
         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
         crm_xml_set_id(xml_obj, "%s-attributes-%d",
                        data->prefix, replica->offset);
 
         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
         if(data->host_network) {
             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
         }
 
         if(data->host_netmask) {
             crm_create_nvpair_xml(xml_obj, NULL,
                                   "cidr_netmask", data->host_netmask);
 
         } else {
             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
         }
 
         xml_obj = create_xml_node(xml_ip, "operations");
         crm_create_op_xml(xml_obj, ID(xml_ip), PCMK_ACTION_MONITOR, "60s",
                           NULL);
 
         // TODO: Other ops? Timeouts and intervals from underlying resource?
 
         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
                                 parent->cluster) != pcmk_rc_ok) {
             return pcmk_rc_unpack_error;
         }
 
         parent->children = g_list_append(parent->children, replica->ip);
     }
     return pcmk_rc_ok;
 }
 
 static const char*
 container_agent_str(enum pe__container_agent t)
 {
     switch (t) {
         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
         default: // PE__CONTAINER_AGENT_UNKNOWN
             break;
     }
     return PE__CONTAINER_AGENT_UNKNOWN_S;
 }
 
 static int
 create_container_resource(pcmk_resource_t *parent,
                           const pe__bundle_variant_data_t *data,
                           pcmk__bundle_replica_t *replica)
 {
     char *id = NULL;
     xmlNode *xml_container = NULL;
     xmlNode *xml_obj = NULL;
 
     // Agent-specific
     const char *hostname_opt = NULL;
     const char *env_opt = NULL;
     const char *agent_str = NULL;
     int volid = 0;  // rkt-only
 
     GString *buffer = NULL;
     GString *dbuffer = NULL;
 
     // Where syntax differences are drop-in replacements, set them now
     switch (data->agent_type) {
         case PE__CONTAINER_AGENT_DOCKER:
         case PE__CONTAINER_AGENT_PODMAN:
             hostname_opt = "-h ";
             env_opt = "-e ";
             break;
         case PE__CONTAINER_AGENT_RKT:
             hostname_opt = "--hostname=";
             env_opt = "--environment=";
             break;
         default:    // PE__CONTAINER_AGENT_UNKNOWN
             return pcmk_rc_unpack_error;
     }
     agent_str = container_agent_str(data->agent_type);
 
     buffer = g_string_sized_new(4096);
 
     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
                            replica->offset);
     crm_xml_sanitize_id(id);
     xml_container = create_resource(id, "heartbeat", agent_str);
     free(id);
 
     xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
     crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 
     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
     crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 
     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
         g_string_append(buffer, " --restart=no");
     }
 
     /* Set a container hostname only if we have an IP to map it to. The user can
      * set -h or --uts=host themselves if they want a nicer name for logs, but
      * this makes applications happy who need their  hostname to match the IP
      * they bind to.
      */
     if (data->ip_range_start != NULL) {
         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
                                replica->offset);
     }
     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 
     if (data->container_network != NULL) {
         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
     }
 
     if (data->control_port != NULL) {
         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
                        data->control_port, NULL);
     } else {
         g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
                                env_opt, DEFAULT_REMOTE_PORT);
     }
 
     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
         char *source = NULL;
 
         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
                                        replica->offset);
             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
         }
 
         switch (data->agent_type) {
             case PE__CONTAINER_AGENT_DOCKER:
             case PE__CONTAINER_AGENT_PODMAN:
                 pcmk__g_strcat(buffer,
                                " -v ", pcmk__s(source, mount->source),
                                ":", mount->target, NULL);
 
                 if (mount->options != NULL) {
                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
                 }
                 break;
             case PE__CONTAINER_AGENT_RKT:
                 g_string_append_printf(buffer,
                                        " --volume vol%d,kind=host,"
                                        "source=%s%s%s "
                                        "--mount volume=vol%d,target=%s",
                                        volid, pcmk__s(source, mount->source),
                                        (mount->options != NULL)? "," : "",
                                        pcmk__s(mount->options, ""),
                                        volid, mount->target);
                 volid++;
                 break;
             default:
                 break;
         }
         free(source);
     }
 
     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 
         switch (data->agent_type) {
             case PE__CONTAINER_AGENT_DOCKER:
             case PE__CONTAINER_AGENT_PODMAN:
                 if (replica->ipaddr != NULL) {
                     pcmk__g_strcat(buffer,
                                    " -p ", replica->ipaddr, ":", port->source,
                                    ":", port->target, NULL);
 
                 } else if (!pcmk__str_eq(data->container_network, "host",
                                          pcmk__str_none)) {
                     // No need to do port mapping if net == host
                     pcmk__g_strcat(buffer,
                                    " -p ", port->source, ":", port->target,
                                    NULL);
                 }
                 break;
             case PE__CONTAINER_AGENT_RKT:
                 if (replica->ipaddr != NULL) {
                     pcmk__g_strcat(buffer,
                                    " --port=", port->target,
                                    ":", replica->ipaddr, ":", port->source,
                                    NULL);
                 } else {
                     pcmk__g_strcat(buffer,
                                    " --port=", port->target, ":", port->source,
                                    NULL);
                 }
                 break;
             default:
                 break;
         }
     }
 
     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
      * it would cause restarts during rolling upgrades.
      *
      * In a previous version of the container resource creation logic, if
      * data->launcher_options is not NULL, we append
      * (" %s", data->launcher_options) even if data->launcher_options is an
      * empty string. Likewise for data->container_host_options. Using
      *
      *     pcmk__add_word(buffer, 0, data->launcher_options)
      *
      * removes that extra trailing space, causing a resource definition change.
      */
     if (data->launcher_options != NULL) {
         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
     }
 
     if (data->container_host_options != NULL) {
         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
     }
 
     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
                           (const char *) buffer->str);
     g_string_free(buffer, TRUE);
 
     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
     if (dbuffer != NULL) {
         g_string_free(dbuffer, TRUE);
     }
 
     if (replica->child != NULL) {
         if (data->container_command != NULL) {
             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                   data->container_command);
         } else {
             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                   SBIN_DIR "/pacemaker-remoted");
         }
 
         /* TODO: Allow users to specify their own?
          *
          * We just want to know if the container is alive; we'll monitor the
          * child independently.
          */
         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 #if 0
         /* @TODO Consider supporting the use case where we can start and stop
          * resources, but not proxy local commands (such as setting node
          * attributes), by running the local executor in stand-alone mode.
          * However, this would probably be better done via ACLs as with other
          * Pacemaker Remote nodes.
          */
     } else if ((child != NULL) && data->untrusted) {
         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                               CRM_DAEMON_DIR "/pacemaker-execd");
         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 #endif
     } else {
         if (data->container_command != NULL) {
             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                   data->container_command);
         }
 
         /* TODO: Allow users to specify their own?
          *
          * We don't know what's in the container, so we just want to know if it
          * is alive.
          */
         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
     }
 
     xml_obj = create_xml_node(xml_container, "operations");
     crm_create_op_xml(xml_obj, ID(xml_container), PCMK_ACTION_MONITOR, "60s",
                       NULL);
 
     // TODO: Other ops? Timeouts and intervals from underlying resource?
     if (pe__unpack_resource(xml_container, &replica->container, parent,
                             parent->cluster) != pcmk_rc_ok) {
         return pcmk_rc_unpack_error;
     }
     pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
     parent->children = g_list_append(parent->children, replica->container);
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \brief Ban a node from a resource's (and its children's) allowed nodes list
  *
  * \param[in,out] rsc    Resource to modify
  * \param[in]     uname  Name of node to ban
  */
 static void
 disallow_node(pcmk_resource_t *rsc, const char *uname)
 {
     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 
     if (match) {
         ((pcmk_node_t *) match)->weight = -INFINITY;
         ((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
     }
     if (rsc->children) {
         g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
     }
 }
 
 static int
 create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
                        pcmk__bundle_replica_t *replica)
 {
     if (replica->child && valid_network(data)) {
         GHashTableIter gIter;
         pcmk_node_t *node = NULL;
         xmlNode *xml_remote = NULL;
         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
         char *port_s = NULL;
         const char *uname = NULL;
         const char *connect_name = NULL;
 
         if (pe_find_resource(parent->cluster->resources, id) != NULL) {
             free(id);
             // The biggest hammer we have
             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
                                    replica->child->id, replica->offset);
             //@TODO return error instead of asserting?
             CRM_ASSERT(pe_find_resource(parent->cluster->resources,
                                         id) == NULL);
         }
 
         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
          * connection does not have its own IP is a magic string that we use to
          * support nested remotes (i.e. a bundle running on a remote node).
          */
         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 
         if (data->control_port == NULL) {
             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
         }
 
         /* This sets replica->container as replica->remote's container, which is
          * similar to what happens with guest nodes. This is how the scheduler
          * knows that the bundle node is fenced by recovering the container, and
          * that remote should be ordered relative to the container.
          */
         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
                                           NULL, NULL, NULL,
                                           connect_name, (data->control_port?
                                           data->control_port : port_s));
         free(port_s);
 
         /* Abandon our created ID, and pull the copy from the XML, because we
          * need something that will get freed during scheduler data cleanup to
          * use as the node ID and uname.
          */
         free(id);
         id = NULL;
         uname = ID(xml_remote);
 
         /* Ensure a node has been created for the guest (it may have already
          * been, if it has a permanent node attribute), and ensure its weight is
          * -INFINITY so no other resources can run on it.
          */
         node = pe_find_node(parent->cluster->nodes, uname);
         if (node == NULL) {
             node = pe_create_node(uname, uname, "remote", "-INFINITY",
                                   parent->cluster);
         } else {
             node->weight = -INFINITY;
         }
         node->rsc_discover_mode = pcmk_probe_never;
 
         /* unpack_remote_nodes() ensures that each remote node and guest node
          * has a pcmk_node_t entry. Ideally, it would do the same for bundle
          * nodes. Unfortunately, a bundle has to be mostly unpacked before it's
          * obvious what nodes will be needed, so we do it just above.
          *
          * Worse, that means that the node may have been utilized while
          * unpacking other resources, without our weight correction. The most
          * likely place for this to happen is when pe__unpack_resource() calls
          * resource_location() to set a default score in symmetric clusters.
          * This adds a node *copy* to each resource's allowed nodes, and these
          * copies will have the wrong weight.
          *
          * As a hacky workaround, fix those copies here.
          *
          * @TODO Possible alternative: ensure bundles are unpacked before other
          * resources, so the weight is correct before any copies are made.
          */
         g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
                        (gpointer) uname);
 
         replica->node = pe__copy_node(node);
         replica->node->weight = 500;
         replica->node->rsc_discover_mode = pcmk_probe_exclusive;
 
         /* Ensure the node shows up as allowed and with the correct discovery set */
         if (replica->child->allowed_nodes != NULL) {
             g_hash_table_destroy(replica->child->allowed_nodes);
         }
         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
         g_hash_table_insert(replica->child->allowed_nodes,
                             (gpointer) replica->node->details->id,
                             pe__copy_node(replica->node));
 
         {
             pcmk_node_t *copy = pe__copy_node(replica->node);
             copy->weight = -INFINITY;
             g_hash_table_insert(replica->child->parent->allowed_nodes,
                                 (gpointer) replica->node->details->id, copy);
         }
         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
                                 parent->cluster) != pcmk_rc_ok) {
             return pcmk_rc_unpack_error;
         }
 
         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
             if (pe__is_guest_or_remote_node(node)) {
                 /* Remote resources can only run on 'normal' cluster node */
                 node->weight = -INFINITY;
             }
         }
 
         replica->node->details->remote_rsc = replica->remote;
 
         // Ensure pe__is_guest_node() functions correctly immediately
         replica->remote->container = replica->container;
 
         /* A bundle's #kind is closer to "container" (guest node) than the
          * "remote" set by pe_create_node().
          */
         g_hash_table_insert(replica->node->details->attrs,
                             strdup(CRM_ATTR_KIND), strdup("container"));
 
         /* One effect of this is that setup_container() will add
          * replica->remote to replica->container's fillers, which will make
          * pe__resource_contains_guest_node() true for replica->container.
          *
          * replica->child does NOT get added to replica->container's fillers.
          * The only noticeable effect if it did would be for its fail count to
          * be taken into account when checking replica->container's migration
          * threshold.
          */
         parent->children = g_list_append(parent->children, replica->remote);
     }
     return pcmk_rc_ok;
 }
 
 static int
 create_replica_resources(pcmk_resource_t *parent,
                          pe__bundle_variant_data_t *data,
                          pcmk__bundle_replica_t *replica)
 {
     int rc = pcmk_rc_ok;
 
     rc = create_container_resource(parent, data, replica);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     rc = create_ip_resource(parent, data, replica);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     rc = create_remote_resource(parent, data, replica);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
     }
 
     if (replica->remote != NULL) {
         /*
          * Allow the remote connection resource to be allocated to a
          * different node than the one on which the container is active.
          *
          * This makes it possible to have Pacemaker Remote nodes running
          * containers with pacemaker-remoted inside in order to start
          * services inside those containers.
          */
         pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
     }
     return rc;
 }
 
 static void
 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
           const char *target, const char *options, uint32_t flags)
 {
     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 
     CRM_ASSERT(mount != NULL);
     mount->source = strdup(source);
     mount->target = strdup(target);
     pcmk__str_update(&mount->options, options);
     mount->flags = flags;
     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 }
 
 static void
 mount_free(pe__bundle_mount_t *mount)
 {
     free(mount->source);
     free(mount->target);
     free(mount->options);
     free(mount);
 }
 
 static void
 port_free(pe__bundle_port_t *port)
 {
     free(port->source);
     free(port->target);
     free(port);
 }
 
 static pcmk__bundle_replica_t *
 replica_for_remote(pcmk_resource_t *remote)
 {
     pcmk_resource_t *top = remote;
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     if (top == NULL) {
         return NULL;
     }
 
     while (top->parent != NULL) {
         top = top->parent;
     }
 
     get_bundle_variant_data(bundle_data, top);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         if (replica->remote == remote) {
             return replica;
         }
     }
     CRM_LOG_ASSERT(FALSE);
     return NULL;
 }
 
 bool
 pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
 {
     const char *value;
     GHashTable *params = NULL;
 
     if (rsc == NULL) {
         return false;
     }
 
     // Use NULL node since pcmk__bundle_expand() uses that to set value
     params = pe_rsc_params(rsc, NULL, rsc->cluster);
     value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);
 
     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
            && xml_contains_remote_node(rsc->xml);
 }
 
 const char *
 pe__add_bundle_remote_name(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler,
                            xmlNode *xml, const char *field)
 {
     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 
     pcmk_node_t *node = NULL;
     pcmk__bundle_replica_t *replica = NULL;
 
     if (!pe__bundle_needs_remote_name(rsc)) {
         return NULL;
     }
 
     replica = replica_for_remote(rsc);
     if (replica == NULL) {
         return NULL;
     }
 
     node = replica->container->allocated_to;
     if (node == NULL) {
         /* If it won't be running anywhere after the
          * transition, go with where it's running now.
          */
         node = pe__current_node(replica->container);
     }
 
     if(node == NULL) {
         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
         return NULL;
     }
 
     crm_trace("Setting address for bundle connection %s to bundle host %s",
               rsc->id, pe__node_name(node));
     if(xml != NULL && field != NULL) {
         crm_xml_add(xml, field, node->details->uname);
     }
 
     return node->details->uname;
 }
 
 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
                                    "Bundle mount", ID(mount_xml), flags,    \
                                    (flags_to_set), #flags_to_set);          \
     } while (0)
 
 gboolean
 pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     const char *value = NULL;
     xmlNode *xml_obj = NULL;
     xmlNode *xml_resource = NULL;
     pe__bundle_variant_data_t *bundle_data = NULL;
     bool need_log_mount = TRUE;
 
     CRM_ASSERT(rsc != NULL);
     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 
     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
     rsc->variant_opaque = bundle_data;
     bundle_data->prefix = strdup(rsc->id);
 
     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
     if (xml_obj != NULL) {
         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
     } else {
         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
         if (xml_obj != NULL) {
             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
         } else {
             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
             if (xml_obj != NULL) {
                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
             } else {
                 return FALSE;
             }
         }
     }
 
     // Use 0 for default, minimum, and invalid PCMK_META_PROMOTED_MAX
     value = crm_element_value(xml_obj, PCMK_META_PROMOTED_MAX);
     if (value == NULL) {
         // @COMPAT deprecated since 2.0.0
         value = crm_element_value(xml_obj, "masters");
     }
     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
 
     /* Default replicas to PCMK_META_PROMOTED_MAX if it was specified and 1
      * otherwise
      */
     value = crm_element_value(xml_obj, "replicas");
     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
         bundle_data->nreplicas = bundle_data->promoted_max;
     } else {
         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
     }
 
     /*
      * Communication between containers on the same host via the
      * floating IPs only works if the container is started with:
      *   --userland-proxy=false --ip-masq=false
      */
     value = crm_element_value(xml_obj, "replicas-per-host");
     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
     if (bundle_data->nreplicas_per_host == 1) {
         pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
     }
 
     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
     bundle_data->image = crm_element_value_copy(xml_obj, "image");
     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
 
     xml_obj = first_named_child(rsc->xml, "network");
     if(xml_obj) {
 
         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
         value = crm_element_value(xml_obj, "add-host");
         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
             bundle_data->add_host = TRUE;
         }
 
         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
              xml_child = pcmk__xe_next(xml_child)) {
 
             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
             port->source = crm_element_value_copy(xml_child, "port");
 
             if(port->source == NULL) {
                 port->source = crm_element_value_copy(xml_child, "range");
             } else {
                 port->target = crm_element_value_copy(xml_child, "internal-port");
             }
 
             if(port->source != NULL && strlen(port->source) > 0) {
                 if(port->target == NULL) {
                     port->target = strdup(port->source);
                 }
                 bundle_data->ports = g_list_append(bundle_data->ports, port);
 
             } else {
                 pcmk__config_err("Invalid port directive %s", ID(xml_child));
                 port_free(port);
             }
         }
     }
 
     xml_obj = first_named_child(rsc->xml, "storage");
     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
          xml_child = pcmk__xe_next(xml_child)) {
 
         const char *source = crm_element_value(xml_child, "source-dir");
         const char *target = crm_element_value(xml_child, "target-dir");
         const char *options = crm_element_value(xml_child, "options");
         int flags = pe__bundle_mount_none;
 
         if (source == NULL) {
             source = crm_element_value(xml_child, "source-dir-root");
             pe__set_bundle_mount_flags(xml_child, flags,
                                        pe__bundle_mount_subdir);
         }
 
         if (source && target) {
             mount_add(bundle_data, source, target, options, flags);
             if (strcmp(target, "/var/log") == 0) {
                 need_log_mount = FALSE;
             }
         } else {
             pcmk__config_err("Invalid mount directive %s", ID(xml_child));
         }
     }
 
     xml_obj = first_named_child(rsc->xml, "primitive");
     if (xml_obj && valid_network(bundle_data)) {
         char *value = NULL;
         xmlNode *xml_set = NULL;
 
         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
 
         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
          * part of the resource name, so that bundles don't restart in a rolling
          * upgrade. (It also avoids needing to change regression tests.)
          */
         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
                       (bundle_data->promoted_max? "master"
                       : (const char *)xml_resource->name));
 
         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
 
         crm_create_nvpair_xml(xml_set, NULL,
                               PCMK_META_ORDERED, XML_BOOLEAN_TRUE);
 
         value = pcmk__itoa(bundle_data->nreplicas);
         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
         free(value);
 
         value = pcmk__itoa(bundle_data->nreplicas_per_host);
         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
         free(value);
 
         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
 
         if (bundle_data->promoted_max) {
             crm_create_nvpair_xml(xml_set, NULL,
                                   PCMK_META_PROMOTABLE, XML_BOOLEAN_TRUE);
 
             value = pcmk__itoa(bundle_data->promoted_max);
             crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
             free(value);
         }
 
         //crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
         add_node_copy(xml_resource, xml_obj);
 
     } else if(xml_obj) {
         pcmk__config_err("Cannot control %s inside %s without either "
                          "ip-range-start or control-port",
                          rsc->id, ID(xml_obj));
         return FALSE;
     }
 
     if(xml_resource) {
         int lpc = 0;
         GList *childIter = NULL;
         pe__bundle_port_t *port = NULL;
         GString *buffer = NULL;
 
         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
                                 scheduler) != pcmk_rc_ok) {
             return FALSE;
         }
 
         /* Currently, we always map the default authentication key location
          * into the same location inside the container.
          *
          * Ideally, we would respect the host's PCMK_authkey_location, but:
          * - it may be different on different nodes;
          * - the actual connection will do extra checking to make sure the key
          *   file exists and is readable, that we can't do here on the DC
          * - tools such as crm_resource and crm_simulate may not have the same
          *   environment variables as the cluster, causing operation digests to
          *   differ
          *
          * Always using the default location inside the container is fine,
          * because we control the pacemaker_remote environment, and it avoids
          * having to pass another environment variable to the container.
          *
          * @TODO A better solution may be to have only pacemaker_remote use the
          * environment variable, and have the cluster nodes use a new
          * cluster option for key location. This would introduce the limitation
          * of the location being the same on all cluster nodes, but that's
          * reasonable.
          */
         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
 
         if (need_log_mount) {
             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
                       pe__bundle_mount_subdir);
         }
 
         port = calloc(1, sizeof(pe__bundle_port_t));
         if(bundle_data->control_port) {
             port->source = strdup(bundle_data->control_port);
         } else {
             /* If we wanted to respect PCMK_remote_port, we could use
              * crm_default_remote_port() here and elsewhere in this file instead
              * of DEFAULT_REMOTE_PORT.
              *
              * However, it gains nothing, since we control both the container
              * environment and the connection resource parameters, and the user
              * can use a different port if desired by setting control-port.
              */
             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
         }
         port->target = strdup(port->source);
         bundle_data->ports = g_list_append(bundle_data->ports, port);
 
         buffer = g_string_sized_new(1024);
         for (childIter = bundle_data->child->children; childIter != NULL;
              childIter = childIter->next) {
 
             pcmk__bundle_replica_t *replica = NULL;
 
             replica = calloc(1, sizeof(pcmk__bundle_replica_t));
             replica->child = childIter->data;
             replica->child->exclusive_discover = TRUE;
             replica->offset = lpc++;
 
             // Ensure the child's notify gets set based on the underlying primitive's value
             if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
                 pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
             }
 
             allocate_ip(bundle_data, replica, buffer);
             bundle_data->replicas = g_list_append(bundle_data->replicas,
                                                   replica);
             bundle_data->attribute_target =
                 g_hash_table_lookup(replica->child->meta,
                                     PCMK_META_CONTAINER_ATTR_TARGET);
         }
         bundle_data->container_host_options = g_string_free(buffer, FALSE);
 
         if (bundle_data->attribute_target) {
             g_hash_table_replace(rsc->meta,
                                  strdup(PCMK_META_CONTAINER_ATTR_TARGET),
                                  strdup(bundle_data->attribute_target));
             g_hash_table_replace(bundle_data->child->meta,
                                  strdup(PCMK_META_CONTAINER_ATTR_TARGET),
                                  strdup(bundle_data->attribute_target));
         }
 
     } else {
         // Just a naked container, no pacemaker-remote
         GString *buffer = g_string_sized_new(1024);
 
         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
             pcmk__bundle_replica_t *replica = NULL;
 
             replica = calloc(1, sizeof(pcmk__bundle_replica_t));
             replica->offset = lpc;
             allocate_ip(bundle_data, replica, buffer);
             bundle_data->replicas = g_list_append(bundle_data->replicas,
                                                   replica);
         }
         bundle_data->container_host_options = g_string_free(buffer, FALSE);
     }
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
             pcmk__config_err("Failed unpacking resource %s", rsc->id);
             rsc->fns->free(rsc);
             return FALSE;
         }
 
         /* Utilization needs special handling for bundles. It makes no sense for
          * the inner primitive to have utilization, because it is tied
          * one-to-one to the guest node created by the container resource -- and
          * there's no way to set capacities for that guest node anyway.
          *
          * What the user really wants is to configure utilization for the
          * container. However, the schema only allows utilization for
          * primitives, and the container resource is implicit anyway, so the
          * user can *only* configure utilization for the inner primitive. If
          * they do, move the primitive's utilization values to the container.
          *
          * @TODO This means that bundles without an inner primitive can't have
          * utilization. An alternative might be to allow utilization values in
          * the top-level bundle XML in the schema, and copy those to each
          * container.
          */
         if (replica->child != NULL) {
             GHashTable *empty = replica->container->utilization;
 
             replica->container->utilization = replica->child->utilization;
             replica->child->utilization = empty;
         }
     }
 
     if (bundle_data->child) {
         rsc->children = g_list_append(rsc->children, bundle_data->child);
     }
     return TRUE;
 }
 
 static int
 replica_resource_active(pcmk_resource_t *rsc, gboolean all)
 {
     if (rsc) {
         gboolean child_active = rsc->fns->active(rsc, all);
 
         if (child_active && !all) {
             return TRUE;
         } else if (!child_active && all) {
             return FALSE;
         }
     }
     return -1;
 }
 
 gboolean
 pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     GList *iter = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
         int rsc_active;
 
         rsc_active = replica_resource_active(replica->ip, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->child, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->container, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->remote, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
     }
 
     /* If "all" is TRUE, we've already checked that no resources were inactive,
      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
      * so return FALSE.
      */
     return all;
 }
 
 /*!
  * \internal
  * \brief Find the bundle replica corresponding to a given node
  *
  * \param[in] bundle  Top-level bundle resource
  * \param[in] node    Node to search for
  *
  * \return Bundle replica if found, NULL otherwise
  */
 pcmk_resource_t *
 pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     CRM_ASSERT(bundle && node);
 
     get_bundle_variant_data(bundle_data, bundle);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica && replica->node);
         if (replica->node->details == node->details) {
             return replica->child;
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 static void
 print_rsc_in_list(pcmk_resource_t *rsc, const char *pre_text, long options,
                   void *print_data)
 {
     if (rsc != NULL) {
         if (options & pe_print_html) {
             status_print("<li>");
         }
         rsc->fns->print(rsc, pre_text, options, print_data);
         if (options & pe_print_html) {
             status_print("</li>\n");
         }
     }
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 static void
 bundle_print_xml(pcmk_resource_t *rsc, const char *pre_text, long options,
                  void *print_data)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     char *child_text = NULL;
     CRM_CHECK(rsc != NULL, return);
 
     if (pre_text == NULL) {
         pre_text = "";
     }
     child_text = crm_strdup_printf("%s        ", pre_text);
 
     get_bundle_variant_data(bundle_data, rsc);
 
     status_print("%s<bundle ", pre_text);
     status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
     status_print("image=\"%s\" ", bundle_data->image);
     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_unique));
     status_print("managed=\"%s\" ",
                  pe__rsc_bool_str(rsc, pcmk_rsc_managed));
     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
     status_print(">\n");
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica);
         status_print("%s    <replica " PCMK_XA_ID "=\"%d\">\n",
                      pre_text, replica->offset);
         print_rsc_in_list(replica->ip, child_text, options, print_data);
         print_rsc_in_list(replica->child, child_text, options, print_data);
         print_rsc_in_list(replica->container, child_text, options, print_data);
         print_rsc_in_list(replica->remote, child_text, options, print_data);
         status_print("%s    </replica>\n", pre_text);
     }
     status_print("%s</bundle>\n", pre_text);
     free(child_text);
 }
 
 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__bundle_xml(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     pe__bundle_variant_data_t *bundle_data = NULL;
     int rc = pcmk_rc_no_output;
     gboolean printed_header = FALSE;
     gboolean print_everything = TRUE;
 
     const char *desc = NULL;
 
     CRM_ASSERT(rsc != NULL);
     
     get_bundle_variant_data(bundle_data, rsc);
 
     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
         char *id = NULL;
         gboolean print_ip, print_child, print_ctnr, print_remote;
 
         CRM_ASSERT(replica);
 
         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
             continue;
         }
 
         print_ip = replica->ip != NULL &&
                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
         print_child = replica->child != NULL &&
                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
         print_remote = replica->remote != NULL &&
                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
 
         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
             continue;
         }
 
         if (!printed_header) {
             printed_header = TRUE;
 
             desc = pe__resource_description(rsc, show_opts);
 
             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 8,
                      PCMK_XA_ID, rsc->id,
                      PCMK_XA_TYPE, container_agent_str(bundle_data->agent_type),
                      "image", bundle_data->image,
                      "unique", pe__rsc_bool_str(rsc, pcmk_rsc_unique),
                      "maintenance",
                      pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
                      "managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
                      "failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
                      PCMK_XA_DESCRIPTION, desc);
             CRM_ASSERT(rc == pcmk_rc_ok);
         }
 
         id = pcmk__itoa(replica->offset);
         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, PCMK_XA_ID, id);
         free(id);
         CRM_ASSERT(rc == pcmk_rc_ok);
 
         if (print_ip) {
             out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
                          replica->ip, only_node, only_rsc);
         }
 
         if (print_child) {
             out->message(out, crm_map_element_name(replica->child->xml), show_opts,
                          replica->child, only_node, only_rsc);
         }
 
         if (print_ctnr) {
             out->message(out, crm_map_element_name(replica->container->xml), show_opts,
                          replica->container, only_node, only_rsc);
         }
 
         if (print_remote) {
             out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
                          replica->remote, only_node, only_rsc);
         }
 
         pcmk__output_xml_pop_parent(out); // replica
     }
 
     if (printed_header) {
         pcmk__output_xml_pop_parent(out); // bundle
     }
 
     return rc;
 }
 
 static void
 pe__bundle_replica_output_html(pcmk__output_t *out,
                                pcmk__bundle_replica_t *replica,
                                pcmk_node_t *node, uint32_t show_opts)
 {
     pcmk_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     pe__common_output_html(out, rsc, buffer, node, show_opts);
 }
 
 /*!
  * \internal
  * \brief Get a string describing a resource's unmanaged state or lack thereof
  *
  * \param[in] rsc  Resource to describe
  *
  * \return A string indicating that a resource is in maintenance mode or
  *         otherwise unmanaged, or an empty string otherwise
  */
 static const char *
 get_unmanaged_str(const pcmk_resource_t *rsc)
 {
     if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
         return " (maintenance)";
     }
     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
         return " (unmanaged)";
     }
     return "";
 }
 
 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__bundle_html(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     const char *desc = NULL;
     pe__bundle_variant_data_t *bundle_data = NULL;
     int rc = pcmk_rc_no_output;
     gboolean print_everything = TRUE;
 
     CRM_ASSERT(rsc != NULL);
 
     get_bundle_variant_data(bundle_data, rsc);
 
     desc = pe__resource_description(rsc, show_opts);
 
     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
         gboolean print_ip, print_child, print_ctnr, print_remote;
 
         CRM_ASSERT(replica);
 
         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
             continue;
         }
 
         print_ip = replica->ip != NULL &&
                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
         print_child = replica->child != NULL &&
                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
         print_remote = replica->remote != NULL &&
                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
 
         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
             /* The text output messages used below require pe_print_implicit to
              * be set to do anything.
              */
             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
 
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
             }
 
             if (print_ip) {
                 out->message(out, crm_map_element_name(replica->ip->xml),
                              new_show_opts, replica->ip, only_node, only_rsc);
             }
 
             if (print_child) {
                 out->message(out, crm_map_element_name(replica->child->xml),
                              new_show_opts, replica->child, only_node, only_rsc);
             }
 
             if (print_ctnr) {
                 out->message(out, crm_map_element_name(replica->container->xml),
                              new_show_opts, replica->container, only_node, only_rsc);
             }
 
             if (print_remote) {
                 out->message(out, crm_map_element_name(replica->remote->xml),
                              new_show_opts, replica->remote, only_node, only_rsc);
             }
 
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 out->end_list(out);
             }
         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
             continue;
         } else {
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
                                            show_opts);
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 static void
 pe__bundle_replica_output_text(pcmk__output_t *out,
                                pcmk__bundle_replica_t *replica,
                                pcmk_node_t *node, uint32_t show_opts)
 {
     const pcmk_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     pe__common_output_text(out, rsc, buffer, node, show_opts);
 }
 
 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__bundle_text(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     const char *desc = NULL;
     pe__bundle_variant_data_t *bundle_data = NULL;
     int rc = pcmk_rc_no_output;
     gboolean print_everything = TRUE;
 
     desc = pe__resource_description(rsc, show_opts);
     
     get_bundle_variant_data(bundle_data, rsc);
 
     CRM_ASSERT(rsc != NULL);
 
     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
         gboolean print_ip, print_child, print_ctnr, print_remote;
 
         CRM_ASSERT(replica);
 
         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
             continue;
         }
 
         print_ip = replica->ip != NULL &&
                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
         print_child = replica->child != NULL &&
                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
         print_remote = replica->remote != NULL &&
                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
 
         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
             /* The text output messages used below require pe_print_implicit to
              * be set to do anything.
              */
             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
 
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
             }
 
             out->begin_list(out, NULL, NULL, NULL);
 
             if (print_ip) {
                 out->message(out, crm_map_element_name(replica->ip->xml),
                              new_show_opts, replica->ip, only_node, only_rsc);
             }
 
             if (print_child) {
                 out->message(out, crm_map_element_name(replica->child->xml),
                              new_show_opts, replica->child, only_node, only_rsc);
             }
 
             if (print_ctnr) {
                 out->message(out, crm_map_element_name(replica->container->xml),
                              new_show_opts, replica->container, only_node, only_rsc);
             }
 
             if (print_remote) {
                 out->message(out, crm_map_element_name(replica->remote->xml),
                              new_show_opts, replica->remote, only_node, only_rsc);
             }
 
             out->end_list(out);
         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
             continue;
         } else {
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
                                            show_opts);
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 static void
 print_bundle_replica(pcmk__bundle_replica_t *replica, const char *pre_text,
                      long options, void *print_data)
 {
     pcmk_node_t *node = NULL;
     pcmk_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     node = pe__current_node(replica->container);
     common_print(rsc, pre_text, buffer, node, options, print_data);
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 void
 pe__print_bundle(pcmk_resource_t *rsc, const char *pre_text, long options,
                  void *print_data)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     char *child_text = NULL;
     CRM_CHECK(rsc != NULL, return);
 
     if (options & pe_print_xml) {
         bundle_print_xml(rsc, pre_text, options, print_data);
         return;
     }
 
     get_bundle_variant_data(bundle_data, rsc);
 
     if (pre_text == NULL) {
         pre_text = " ";
     }
 
     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
                  rsc->id, bundle_data->image,
                  pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
                  pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
     if (options & pe_print_html) {
         status_print("<br />\n<ul>\n");
     }
 
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         CRM_ASSERT(replica);
         if (options & pe_print_html) {
             status_print("<li>");
         }
 
         if (pcmk_is_set(options, pe_print_implicit)) {
             child_text = crm_strdup_printf("     %s", pre_text);
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
             }
             if (options & pe_print_html) {
                 status_print("<br />\n<ul>\n");
             }
             print_rsc_in_list(replica->ip, child_text, options, print_data);
             print_rsc_in_list(replica->container, child_text, options, print_data);
             print_rsc_in_list(replica->remote, child_text, options, print_data);
             print_rsc_in_list(replica->child, child_text, options, print_data);
             if (options & pe_print_html) {
                 status_print("</ul>\n");
             }
         } else {
             child_text = crm_strdup_printf("%s  ", pre_text);
             print_bundle_replica(replica, child_text, options, print_data);
         }
         free(child_text);
 
         if (options & pe_print_html) {
             status_print("</li>\n");
         }
     }
     if (options & pe_print_html) {
         status_print("</ul>\n");
     }
 }
 
 static void
 free_bundle_replica(pcmk__bundle_replica_t *replica)
 {
     if (replica == NULL) {
         return;
     }
 
     if (replica->node) {
         free(replica->node);
         replica->node = NULL;
     }
 
     if (replica->ip) {
         free_xml(replica->ip->xml);
         replica->ip->xml = NULL;
         replica->ip->fns->free(replica->ip);
         replica->ip = NULL;
     }
     if (replica->container) {
         free_xml(replica->container->xml);
         replica->container->xml = NULL;
         replica->container->fns->free(replica->container);
         replica->container = NULL;
     }
     if (replica->remote) {
         free_xml(replica->remote->xml);
         replica->remote->xml = NULL;
         replica->remote->fns->free(replica->remote);
         replica->remote = NULL;
     }
     free(replica->ipaddr);
     free(replica);
 }
 
 void
 pe__free_bundle(pcmk_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     CRM_CHECK(rsc != NULL, return);
 
     get_bundle_variant_data(bundle_data, rsc);
     pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
 
     free(bundle_data->prefix);
     free(bundle_data->image);
     free(bundle_data->control_port);
     free(bundle_data->host_network);
     free(bundle_data->host_netmask);
     free(bundle_data->ip_range_start);
     free(bundle_data->container_network);
     free(bundle_data->launcher_options);
     free(bundle_data->container_command);
     g_free(bundle_data->container_host_options);
 
     g_list_free_full(bundle_data->replicas,
                      (GDestroyNotify) free_bundle_replica);
     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
     g_list_free(rsc->children);
 
     if(bundle_data->child) {
         free_xml(bundle_data->child->xml);
         bundle_data->child->xml = NULL;
         bundle_data->child->fns->free(bundle_data->child);
     }
     common_free(rsc);
 }
 
 enum rsc_role_e
 pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
 {
     enum rsc_role_e container_role = pcmk_role_unknown;
     return container_role;
 }
 
 /*!
  * \brief Get the number of configured replicas in a bundle
  *
  * \param[in] rsc  Bundle resource
  *
  * \return Number of configured replicas, or 0 on error
  */
 int
 pe_bundle_replicas(const pcmk_resource_t *rsc)
 {
     if ((rsc == NULL) || (rsc->variant != pcmk_rsc_variant_bundle)) {
         return 0;
     } else {
         pe__bundle_variant_data_t *bundle_data = NULL;
 
         get_bundle_variant_data(bundle_data, rsc);
         return bundle_data->nreplicas;
     }
 }
 
 void
 pe__count_bundle(pcmk_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
         pcmk__bundle_replica_t *replica = item->data;
 
         if (replica->ip) {
             replica->ip->fns->count(replica->ip);
         }
         if (replica->child) {
             replica->child->fns->count(replica->child);
         }
         if (replica->container) {
             replica->container->fns->count(replica->container);
         }
         if (replica->remote) {
             replica->remote->fns->count(replica->remote);
         }
     }
 }
 
 gboolean
 pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                        gboolean check_parent)
 {
     gboolean passes = FALSE;
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
         passes = TRUE;
     } else {
         get_bundle_variant_data(bundle_data, rsc);
 
         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
             pcmk__bundle_replica_t *replica = gIter->data;
 
             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
                 passes = TRUE;
                 break;
             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
                 passes = TRUE;
                 break;
             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
                 passes = TRUE;
                 break;
             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
                 passes = TRUE;
                 break;
             }
         }
     }
 
     return !passes;
 }
 
 /*!
  * \internal
  * \brief Get a list of a bundle's containers
  *
  * \param[in] bundle  Bundle resource
  *
  * \return Newly created list of \p bundle's containers
  * \note It is the caller's responsibility to free the result with
  *       g_list_free().
  */
 GList *
 pe__bundle_containers(const pcmk_resource_t *bundle)
 {
     GList *containers = NULL;
     const pe__bundle_variant_data_t *data = NULL;
 
     get_bundle_variant_data(data, bundle);
     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
 
         containers = g_list_append(containers, replica->container);
     }
     return containers;
 }
 
 // Bundle implementation of pcmk_rsc_methods_t:active_node()
 pcmk_node_t *
 pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
                        unsigned int *count_clean)
 {
     pcmk_node_t *active = NULL;
     pcmk_node_t *node = NULL;
     pcmk_resource_t *container = NULL;
     GList *containers = NULL;
     GList *iter = NULL;
     GHashTable *nodes = NULL;
     const pe__bundle_variant_data_t *data = NULL;
 
     if (count_all != NULL) {
         *count_all = 0;
     }
     if (count_clean != NULL) {
         *count_clean = 0;
     }
     if (rsc == NULL) {
         return NULL;
     }
 
     /* For the purposes of this method, we only care about where the bundle's
      * containers are active, so build a list of active containers.
      */
     get_bundle_variant_data(data, rsc);
     for (iter = data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
 
         if (replica->container->running_on != NULL) {
             containers = g_list_append(containers, replica->container);
         }
     }
     if (containers == NULL) {
         return NULL;
     }
 
     /* If the bundle has only a single active container, just use that
      * container's method. If live migration is ever supported for bundle
      * containers, this will allow us to prefer the migration source when there
      * is only one container and it is migrating. For now, this just lets us
      * avoid creating the nodes table.
      */
     if (pcmk__list_of_1(containers)) {
         container = containers->data;
         node = container->fns->active_node(container, count_all, count_clean);
         g_list_free(containers);
         return node;
     }
 
     // Add all containers' active nodes to a hash table (for uniqueness)
     nodes = g_hash_table_new(NULL, NULL);
     for (iter = containers; iter != NULL; iter = iter->next) {
         container = iter->data;
 
         for (GList *node_iter = container->running_on; node_iter != NULL;
              node_iter = node_iter->next) {
             node = node_iter->data;
 
             // If insert returns true, we haven't counted this node yet
             if (g_hash_table_insert(nodes, (gpointer) node->details,
                                     (gpointer) node)
                 && !pe__count_active_node(rsc, node, &active, count_all,
                                           count_clean)) {
                 goto done;
             }
         }
     }
 
 done:
     g_list_free(containers);
     g_hash_table_destroy(nodes);
     return active;
 }
 
 /*!
  * \internal
  * \brief Get maximum bundle resource instances per node
  *
  * \param[in] rsc  Bundle resource to check
  *
  * \return Maximum number of \p rsc instances that can be active on one node
  */
 unsigned int
 pe__bundle_max_per_node(const pcmk_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
     return (unsigned int) bundle_data->nreplicas_per_host;
 }
diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
index ec377c6aca..2942bb2035 100644
--- a/lib/pengine/complex.c
+++ b/lib/pengine/complex.c
@@ -1,1202 +1,1202 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml_internal.h>
 #include <crm/common/scheduler_internal.h>
 
 #include "pe_status_private.h"
 
 void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);
 
 static pcmk_node_t *active_node(const pcmk_resource_t *rsc,
                                 unsigned int *count_all,
                                 unsigned int *count_clean);
 
 pcmk_rsc_methods_t resource_class_functions[] = {
     {
          native_unpack,
          native_find_rsc,
          native_parameter,
          native_print,
          native_active,
          native_resource_state,
          native_location,
          native_free,
          pe__count_common,
          pe__native_is_filtered,
          active_node,
          pe__primitive_max_per_node,
     },
     {
          group_unpack,
          native_find_rsc,
          native_parameter,
          group_print,
          group_active,
          group_resource_state,
          native_location,
          group_free,
          pe__count_common,
          pe__group_is_filtered,
          active_node,
          pe__group_max_per_node,
     },
     {
          clone_unpack,
          native_find_rsc,
          native_parameter,
          clone_print,
          clone_active,
          clone_resource_state,
          native_location,
          clone_free,
          pe__count_common,
          pe__clone_is_filtered,
          active_node,
          pe__clone_max_per_node,
     },
     {
          pe__unpack_bundle,
          native_find_rsc,
          native_parameter,
          pe__print_bundle,
          pe__bundle_active,
          pe__bundle_resource_state,
          native_location,
          pe__free_bundle,
          pe__count_bundle,
          pe__bundle_is_filtered,
          pe__bundle_active_node,
          pe__bundle_max_per_node,
     }
 };
 
 static enum pe_obj_types
 get_resource_type(const char *name)
 {
     if (pcmk__str_eq(name, XML_CIB_TAG_RESOURCE, pcmk__str_casei)) {
         return pcmk_rsc_variant_primitive;
 
     } else if (pcmk__str_eq(name, XML_CIB_TAG_GROUP, pcmk__str_casei)) {
         return pcmk_rsc_variant_group;
 
     } else if (pcmk__str_eq(name, XML_CIB_TAG_INCARNATION, pcmk__str_casei)) {
         return pcmk_rsc_variant_clone;
 
     } else if (pcmk__str_eq(name, PCMK_XE_PROMOTABLE_LEGACY, pcmk__str_casei)) {
         // @COMPAT deprecated since 2.0.0
         return pcmk_rsc_variant_clone;
 
     } else if (pcmk__str_eq(name, XML_CIB_TAG_CONTAINER, pcmk__str_casei)) {
         return pcmk_rsc_variant_bundle;
     }
 
     return pcmk_rsc_variant_unknown;
 }
 
 static void
 dup_attr(gpointer key, gpointer value, gpointer user_data)
 {
     add_hash_param(user_data, key, value);
 }
 
 static void
 expand_parents_fixed_nvpairs(pcmk_resource_t *rsc,
                              pe_rule_eval_data_t *rule_data,
                              GHashTable *meta_hash, pcmk_scheduler_t *scheduler)
 {
     GHashTable *parent_orig_meta = pcmk__strkey_table(free, free);
     pcmk_resource_t *p = rsc->parent;
 
     if (p == NULL) {
         return ;
     }
 
     /* Search all parent resources, get the fixed value of "meta_attributes" set only in the original xml, and stack it in the hash table. */
     /* The fixed value of the lower parent resource takes precedence and is not overwritten. */
     while(p != NULL) {
         /* A hash table for comparison is generated, including the id-ref. */
         pe__unpack_dataset_nvpairs(p->xml, XML_TAG_META_SETS, rule_data,
                                    parent_orig_meta, NULL, FALSE, scheduler);
         p = p->parent; 
     }
 
     /* If there is a fixed value of "meta_attributes" of the parent resource, it will be processed. */
     if (parent_orig_meta != NULL) {
         GHashTableIter iter;
         char *key = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, parent_orig_meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
             /* Parameters set in the original xml of the parent resource will also try to overwrite the child resource. */
             /* Attributes that already exist in the child lease are not updated. */
             dup_attr(key, value, meta_hash);
         }
     }
 
     if (parent_orig_meta != NULL) {
         g_hash_table_destroy(parent_orig_meta);
     }
     
     return ;
 
 }
 void
 get_meta_attributes(GHashTable * meta_hash, pcmk_resource_t * rsc,
                     pcmk_node_t *node, pcmk_scheduler_t *scheduler)
 {
     pe_rsc_eval_data_t rsc_rule_data = {
         .standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
-        .provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
+        .provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
         .agent = crm_element_value(rsc->xml, PCMK_XA_TYPE)
     };
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = scheduler->now,
         .match_data = NULL,
         .rsc_data = &rsc_rule_data,
         .op_data = NULL
     };
 
     if (node) {
         rule_data.node_hash = node->details->attrs;
     }
 
     for (xmlAttrPtr a = pcmk__xe_first_attr(rsc->xml); a != NULL; a = a->next) {
         const char *prop_name = (const char *) a->name;
         const char *prop_value = pcmk__xml_attr_value(a);
 
         add_hash_param(meta_hash, prop_name, prop_value);
     }
 
     pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, &rule_data,
                                meta_hash, NULL, FALSE, scheduler);
 
     /* Set the "meta_attributes" explicitly set in the parent resource to the hash table of the child resource. */
     /* If it is already explicitly set as a child, it will not be overwritten. */
     if (rsc->parent != NULL) {
         expand_parents_fixed_nvpairs(rsc, &rule_data, meta_hash, scheduler);
     }
 
     /* check the defaults */
     pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_META_SETS,
                                &rule_data, meta_hash, NULL, FALSE, scheduler);
 
     /* If there is "meta_attributes" that the parent resource has not explicitly set, set a value that is not set from rsc_default either. */
     /* The values already set up to this point will not be overwritten. */
     if (rsc->parent) {
         g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
     }
 }
 
 void
 get_rsc_attributes(GHashTable *meta_hash, const pcmk_resource_t *rsc,
                    const pcmk_node_t *node, pcmk_scheduler_t *scheduler)
 {
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = scheduler->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     if (node) {
         rule_data.node_hash = node->details->attrs;
     }
 
     pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, &rule_data,
                                meta_hash, NULL, FALSE, scheduler);
 
     /* set anything else based on the parent */
     if (rsc->parent != NULL) {
         get_rsc_attributes(meta_hash, rsc->parent, node, scheduler);
 
     } else {
         /* and finally check the defaults */
         pe__unpack_dataset_nvpairs(scheduler->rsc_defaults, XML_TAG_ATTR_SETS,
                                    &rule_data, meta_hash, NULL, FALSE,
                                    scheduler);
     }
 }
 
 static char *
 template_op_key(xmlNode * op)
 {
     const char *name = crm_element_value(op, PCMK_XA_NAME);
     const char *role = crm_element_value(op, "role");
     char *key = NULL;
 
     if ((role == NULL)
         || pcmk__strcase_any_of(role, PCMK__ROLE_STARTED, PCMK__ROLE_UNPROMOTED,
                                 PCMK__ROLE_UNPROMOTED_LEGACY, NULL)) {
         role = PCMK__ROLE_UNKNOWN;
     }
 
     key = crm_strdup_printf("%s-%s", name, role);
     return key;
 }
 
 static gboolean
 unpack_template(xmlNode *xml_obj, xmlNode **expanded_xml,
                 pcmk_scheduler_t *scheduler)
 {
     xmlNode *cib_resources = NULL;
     xmlNode *template = NULL;
     xmlNode *new_xml = NULL;
     xmlNode *child_xml = NULL;
     xmlNode *rsc_ops = NULL;
     xmlNode *template_ops = NULL;
     const char *template_ref = NULL;
     const char *id = NULL;
 
     if (xml_obj == NULL) {
         pcmk__config_err("No resource object for template unpacking");
         return FALSE;
     }
 
     template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
     if (template_ref == NULL) {
         return TRUE;
     }
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("'%s' object must have a id", xml_obj->name);
         return FALSE;
     }
 
     if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
         pcmk__config_err("The resource object '%s' should not reference itself",
                          id);
         return FALSE;
     }
 
     cib_resources = get_xpath_object("//" XML_CIB_TAG_RESOURCES,
                                      scheduler->input, LOG_TRACE);
     if (cib_resources == NULL) {
         pcmk__config_err("No resources configured");
         return FALSE;
     }
 
     template = pcmk__xe_match(cib_resources, XML_CIB_TAG_RSC_TEMPLATE,
                               PCMK_XA_ID, template_ref);
     if (template == NULL) {
         pcmk__config_err("No template named '%s'", template_ref);
         return FALSE;
     }
 
     new_xml = copy_xml(template);
     xmlNodeSetName(new_xml, xml_obj->name);
     crm_xml_add(new_xml, PCMK_XA_ID, id);
     crm_xml_add(new_xml, PCMK__META_CLONE_INSTANCE_NUM,
                 crm_element_value(xml_obj, PCMK__META_CLONE_INSTANCE_NUM));
 
     template_ops = find_xml_node(new_xml, "operations", FALSE);
 
     for (child_xml = pcmk__xe_first_child(xml_obj); child_xml != NULL;
          child_xml = pcmk__xe_next(child_xml)) {
         xmlNode *new_child = NULL;
 
         new_child = add_node_copy(new_xml, child_xml);
 
         if (pcmk__str_eq((const char *)new_child->name, "operations", pcmk__str_none)) {
             rsc_ops = new_child;
         }
     }
 
     if (template_ops && rsc_ops) {
         xmlNode *op = NULL;
         GHashTable *rsc_ops_hash = pcmk__strkey_table(free, NULL);
 
         for (op = pcmk__xe_first_child(rsc_ops); op != NULL;
              op = pcmk__xe_next(op)) {
 
             char *key = template_op_key(op);
 
             g_hash_table_insert(rsc_ops_hash, key, op);
         }
 
         for (op = pcmk__xe_first_child(template_ops); op != NULL;
              op = pcmk__xe_next(op)) {
 
             char *key = template_op_key(op);
 
             if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
                 add_node_copy(rsc_ops, op);
             }
 
             free(key);
         }
 
         if (rsc_ops_hash) {
             g_hash_table_destroy(rsc_ops_hash);
         }
 
         free_xml(template_ops);
     }
 
     /*free_xml(*expanded_xml); */
     *expanded_xml = new_xml;
 
 #if 0 /* Disable multi-level templates for now */
     if (!unpack_template(new_xml, expanded_xml, scheduler)) {
        free_xml(*expanded_xml);
        *expanded_xml = NULL;
        return FALSE;
     }
 #endif
 
     return TRUE;
 }
 
 static gboolean
 add_template_rsc(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
 {
     const char *template_ref = NULL;
     const char *id = NULL;
 
     if (xml_obj == NULL) {
         pcmk__config_err("No resource object for processing resource list "
                          "of template");
         return FALSE;
     }
 
     template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
     if (template_ref == NULL) {
         return TRUE;
     }
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("'%s' object must have a id", xml_obj->name);
         return FALSE;
     }
 
     if (pcmk__str_eq(template_ref, id, pcmk__str_none)) {
         pcmk__config_err("The resource object '%s' should not reference itself",
                          id);
         return FALSE;
     }
 
     if (add_tag_ref(scheduler->template_rsc_sets, template_ref, id) == FALSE) {
         return FALSE;
     }
 
     return TRUE;
 }
 
 static bool
 detect_promotable(pcmk_resource_t *rsc)
 {
     const char *promotable = g_hash_table_lookup(rsc->meta,
                                                  PCMK_META_PROMOTABLE);
 
     if (crm_is_true(promotable)) {
         return TRUE;
     }
 
     // @COMPAT deprecated since 2.0.0
     if (pcmk__xe_is(rsc->xml, PCMK_XE_PROMOTABLE_LEGACY)) {
         /* @TODO in some future version, pcmk__warn_once() here,
          *       then drop support in even later version
          */
         g_hash_table_insert(rsc->meta, strdup(PCMK_META_PROMOTABLE),
                             strdup(XML_BOOLEAN_TRUE));
         return TRUE;
     }
     return FALSE;
 }
 
 static void
 free_params_table(gpointer data)
 {
     g_hash_table_destroy((GHashTable *) data);
 }
 
 /*!
  * \brief Get a table of resource parameters
  *
  * \param[in,out] rsc        Resource to query
  * \param[in]     node       Node for evaluating rules (NULL for defaults)
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Hash table containing resource parameter names and values
  *         (or NULL if \p rsc or \p scheduler is NULL)
  * \note The returned table will be destroyed when the resource is freed, so
  *       callers should not destroy it.
  */
 GHashTable *
 pe_rsc_params(pcmk_resource_t *rsc, const pcmk_node_t *node,
               pcmk_scheduler_t *scheduler)
 {
     GHashTable *params_on_node = NULL;
 
     /* A NULL node is used to request the resource's default parameters
      * (not evaluated for node), but we always want something non-NULL
      * as a hash table key.
      */
     const char *node_name = "";
 
     // Sanity check
     if ((rsc == NULL) || (scheduler == NULL)) {
         return NULL;
     }
     if ((node != NULL) && (node->details->uname != NULL)) {
         node_name = node->details->uname;
     }
 
     // Find the parameter table for given node
     if (rsc->parameter_cache == NULL) {
         rsc->parameter_cache = pcmk__strikey_table(free, free_params_table);
     } else {
         params_on_node = g_hash_table_lookup(rsc->parameter_cache, node_name);
     }
 
     // If none exists yet, create one with parameters evaluated for node
     if (params_on_node == NULL) {
         params_on_node = pcmk__strkey_table(free, free);
         get_rsc_attributes(params_on_node, rsc, node, scheduler);
         g_hash_table_insert(rsc->parameter_cache, strdup(node_name),
                             params_on_node);
     }
     return params_on_node;
 }
 
 /*!
  * \internal
  * \brief Unpack a resource's "requires" meta-attribute
  *
  * \param[in,out] rsc         Resource being unpacked
  * \param[in]     value       Value of "requires" meta-attribute
  * \param[in]     is_default  Whether \p value was selected by default
  */
 static void
 unpack_requires(pcmk_resource_t *rsc, const char *value, bool is_default)
 {
     if (pcmk__str_eq(value, PCMK__VALUE_NOTHING, pcmk__str_casei)) {
 
     } else if (pcmk__str_eq(value, PCMK__VALUE_QUORUM, pcmk__str_casei)) {
         pcmk__set_rsc_flags(rsc, pcmk_rsc_needs_quorum);
 
     } else if (pcmk__str_eq(value, PCMK__VALUE_FENCING, pcmk__str_casei)) {
         pcmk__set_rsc_flags(rsc, pcmk_rsc_needs_fencing);
         if (!pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
             pcmk__config_warn("%s requires fencing but fencing is disabled",
                               rsc->id);
         }
 
     } else if (pcmk__str_eq(value, PCMK__VALUE_UNFENCING, pcmk__str_casei)) {
         if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
             pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
                               "to \"" PCMK__VALUE_QUORUM "\" because fencing "
                               "devices cannot require unfencing", rsc->id);
             unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
             return;
 
         } else if (!pcmk_is_set(rsc->cluster->flags,
                                 pcmk_sched_fencing_enabled)) {
             pcmk__config_warn("Resetting \"" PCMK_META_REQUIRES "\" for %s "
                               "to \"" PCMK__VALUE_QUORUM "\" because fencing "
                               "is disabled", rsc->id);
             unpack_requires(rsc, PCMK__VALUE_QUORUM, true);
             return;
 
         } else {
             pcmk__set_rsc_flags(rsc, pcmk_rsc_needs_fencing
                                      |pcmk_rsc_needs_unfencing);
         }
 
     } else {
         const char *orig_value = value;
 
         if (pcmk_is_set(rsc->flags, pcmk_rsc_fence_device)) {
             value = PCMK__VALUE_QUORUM;
 
         } else if ((rsc->variant == pcmk_rsc_variant_primitive)
                    && xml_contains_remote_node(rsc->xml)) {
             value = PCMK__VALUE_QUORUM;
 
         } else if (pcmk_is_set(rsc->cluster->flags,
                                pcmk_sched_enable_unfencing)) {
             value = PCMK__VALUE_UNFENCING;
 
         } else if (pcmk_is_set(rsc->cluster->flags,
                                pcmk_sched_fencing_enabled)) {
             value = PCMK__VALUE_FENCING;
 
         } else if (rsc->cluster->no_quorum_policy == pcmk_no_quorum_ignore) {
             value = PCMK__VALUE_NOTHING;
 
         } else {
             value = PCMK__VALUE_QUORUM;
         }
 
         if (orig_value != NULL) {
             pcmk__config_err("Resetting '" PCMK_META_REQUIRES "' for %s "
                              "to '%s' because '%s' is not valid",
                               rsc->id, value, orig_value);
         }
         unpack_requires(rsc, value, true);
         return;
     }
 
     pcmk__rsc_trace(rsc, "\tRequired to start: %s%s", value,
                     (is_default? " (default)" : ""));
 }
 
 #ifndef PCMK__COMPAT_2_0
 static void
 warn_about_deprecated_classes(pcmk_resource_t *rsc)
 {
     const char *std = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
     if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_none)) {
         pcmk__warn_once(pcmk__wo_upstart,
                         "Support for Upstart resources (such as %s) is "
                         "deprecated and will be removed in a future release",
                         rsc->id);
 
     } else if (pcmk__str_eq(std, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_none)) {
         pcmk__warn_once(pcmk__wo_nagios,
                         "Support for Nagios resources (such as %s) is "
                         "deprecated and will be removed in a future release",
                         rsc->id);
     }
 }
 #endif
 
 /*!
  * \internal
  * \brief Unpack configuration XML for a given resource
  *
  * Unpack the XML object containing a resource's configuration into a new
  * \c pcmk_resource_t object.
  *
  * \param[in]     xml_obj    XML node containing the resource's configuration
  * \param[out]    rsc        Where to store the unpacked resource information
  * \param[in]     parent     Resource's parent, if any
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Standard Pacemaker return code
  * \note If pcmk_rc_ok is returned, \p *rsc is guaranteed to be non-NULL, and
  *       the caller is responsible for freeing it using its variant-specific
  *       free() method. Otherwise, \p *rsc is guaranteed to be NULL.
  */
 int
 pe__unpack_resource(xmlNode *xml_obj, pcmk_resource_t **rsc,
                     pcmk_resource_t *parent, pcmk_scheduler_t *scheduler)
 {
     xmlNode *expanded_xml = NULL;
     xmlNode *ops = NULL;
     const char *value = NULL;
     const char *id = NULL;
     bool guest_node = false;
     bool remote_node = false;
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = NULL,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     CRM_CHECK(rsc != NULL, return EINVAL);
     CRM_CHECK((xml_obj != NULL) && (scheduler != NULL),
               *rsc = NULL;
               return EINVAL);
 
     rule_data.now = scheduler->now;
 
     crm_log_xml_trace(xml_obj, "[raw XML]");
 
     id = crm_element_value(xml_obj, PCMK_XA_ID);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> configuration without " PCMK_XA_ID,
                          xml_obj->name);
         return pcmk_rc_unpack_error;
     }
 
     if (unpack_template(xml_obj, &expanded_xml, scheduler) == FALSE) {
         return pcmk_rc_unpack_error;
     }
 
     *rsc = calloc(1, sizeof(pcmk_resource_t));
     if (*rsc == NULL) {
         pcmk__sched_err("Unable to allocate memory for resource '%s'", id);
         return ENOMEM;
     }
     (*rsc)->cluster = scheduler;
 
     if (expanded_xml) {
         crm_log_xml_trace(expanded_xml, "[expanded XML]");
         (*rsc)->xml = expanded_xml;
         (*rsc)->orig_xml = xml_obj;
 
     } else {
         (*rsc)->xml = xml_obj;
         (*rsc)->orig_xml = NULL;
     }
 
     /* Do not use xml_obj from here on, use (*rsc)->xml in case templates are involved */
 
     (*rsc)->parent = parent;
 
     ops = find_xml_node((*rsc)->xml, "operations", FALSE);
     (*rsc)->ops_xml = expand_idref(ops, scheduler->input);
 
     (*rsc)->variant = get_resource_type((const char *) (*rsc)->xml->name);
     if ((*rsc)->variant == pcmk_rsc_variant_unknown) {
         pcmk__config_err("Ignoring resource '%s' of unknown type '%s'",
                          id, (*rsc)->xml->name);
         common_free(*rsc);
         *rsc = NULL;
         return pcmk_rc_unpack_error;
     }
 
 #ifndef PCMK__COMPAT_2_0
     warn_about_deprecated_classes(*rsc);
 #endif
 
     (*rsc)->meta = pcmk__strkey_table(free, free);
     (*rsc)->allowed_nodes = pcmk__strkey_table(NULL, free);
     (*rsc)->known_on = pcmk__strkey_table(NULL, free);
 
     value = crm_element_value((*rsc)->xml, PCMK__META_CLONE_INSTANCE_NUM);
     if (value) {
         (*rsc)->id = crm_strdup_printf("%s:%s", id, value);
         add_hash_param((*rsc)->meta, PCMK__META_CLONE_INSTANCE_NUM, value);
 
     } else {
         (*rsc)->id = strdup(id);
     }
 
     (*rsc)->fns = &resource_class_functions[(*rsc)->variant];
 
     get_meta_attributes((*rsc)->meta, *rsc, NULL, scheduler);
     (*rsc)->parameters = pe_rsc_params(*rsc, NULL, scheduler); // \deprecated
 
     (*rsc)->flags = 0;
     pcmk__set_rsc_flags(*rsc, pcmk_rsc_runnable|pcmk_rsc_unassigned);
 
     if (!pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_managed);
     }
 
     (*rsc)->rsc_cons = NULL;
     (*rsc)->rsc_tickets = NULL;
     (*rsc)->actions = NULL;
     (*rsc)->role = pcmk_role_stopped;
     (*rsc)->next_role = pcmk_role_unknown;
 
     (*rsc)->recovery_type = pcmk_multiply_active_restart;
     (*rsc)->stickiness = 0;
     (*rsc)->migration_threshold = INFINITY;
     (*rsc)->failure_timeout = 0;
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_PRIORITY);
     (*rsc)->priority = char2score(value);
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_CRITICAL);
     if ((value == NULL) || crm_is_true(value)) {
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_critical);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_NOTIFY);
     if (crm_is_true(value)) {
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_notify);
     }
 
     if (xml_contains_remote_node((*rsc)->xml)) {
         (*rsc)->is_remote_node = TRUE;
         if (g_hash_table_lookup((*rsc)->meta, PCMK__META_CONTAINER)) {
             guest_node = true;
         } else {
             remote_node = true;
         }
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_ALLOW_MIGRATE);
     if (crm_is_true(value)) {
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_migratable);
     } else if ((value == NULL) && remote_node) {
         /* By default, we want remote nodes to be able
          * to float around the cluster without having to stop all the
          * resources within the remote-node before moving. Allowing
          * migration support enables this feature. If this ever causes
          * problems, migration support can be explicitly turned off with
          * PCMK_META_ALLOW_MIGRATE=false.
          */
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_migratable);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_IS_MANAGED);
     if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
         if (crm_is_true(value)) {
             pcmk__set_rsc_flags(*rsc, pcmk_rsc_managed);
         } else {
             pcmk__clear_rsc_flags(*rsc, pcmk_rsc_managed);
         }
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MAINTENANCE);
     if (crm_is_true(value)) {
         pcmk__clear_rsc_flags(*rsc, pcmk_rsc_managed);
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_maintenance);
     }
     if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
         pcmk__clear_rsc_flags(*rsc, pcmk_rsc_managed);
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_maintenance);
     }
 
     if (pe_rsc_is_clone(pe__const_top_resource(*rsc, false))) {
         value = g_hash_table_lookup((*rsc)->meta, PCMK_META_GLOBALLY_UNIQUE);
         if (crm_is_true(value)) {
             pcmk__set_rsc_flags(*rsc, pcmk_rsc_unique);
         }
         if (detect_promotable(*rsc)) {
             pcmk__set_rsc_flags(*rsc, pcmk_rsc_promotable);
         }
     } else {
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_unique);
     }
 
     // @COMPAT Deprecated meta-attribute
     value = g_hash_table_lookup((*rsc)->meta, PCMK__META_RESTART_TYPE);
     if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
         (*rsc)->restart_type = pe_restart_restart;
         pcmk__rsc_trace(*rsc, "%s dependency restart handling: restart",
                         (*rsc)->id);
         pcmk__warn_once(pcmk__wo_restart_type,
                         "Support for " PCMK__META_RESTART_TYPE " is deprecated "
                         "and will be removed in a future release");
 
     } else {
         (*rsc)->restart_type = pe_restart_ignore;
         pcmk__rsc_trace(*rsc, "%s dependency restart handling: ignore",
                         (*rsc)->id);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MULTIPLE_ACTIVE);
     if (pcmk__str_eq(value, "stop_only", pcmk__str_casei)) {
         (*rsc)->recovery_type = pcmk_multiply_active_stop;
         pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: stop only",
                         (*rsc)->id);
 
     } else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
         (*rsc)->recovery_type = pcmk_multiply_active_block;
         pcmk__rsc_trace(*rsc, "%s multiple running resource recovery: block",
                         (*rsc)->id);
 
     } else if (pcmk__str_eq(value, "stop_unexpected", pcmk__str_casei)) {
         (*rsc)->recovery_type = pcmk_multiply_active_unexpected;
         pcmk__rsc_trace(*rsc,
                         "%s multiple running resource recovery: "
                         "stop unexpected instances",
                         (*rsc)->id);
 
     } else { // "stop_start"
         if (!pcmk__str_eq(value, "stop_start",
                           pcmk__str_casei|pcmk__str_null_matches)) {
             pcmk__config_warn("%s is not a valid value for "
                               PCMK_META_MULTIPLE_ACTIVE
                               ", using default of \"stop_start\"", value);
         }
         (*rsc)->recovery_type = pcmk_multiply_active_restart;
         pcmk__rsc_trace(*rsc,
                         "%s multiple running resource recovery: stop/start",
                         (*rsc)->id);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_RESOURCE_STICKINESS);
     if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
         (*rsc)->stickiness = char2score(value);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_MIGRATION_THRESHOLD);
     if (value != NULL && !pcmk__str_eq("default", value, pcmk__str_casei)) {
         (*rsc)->migration_threshold = char2score(value);
         if ((*rsc)->migration_threshold < 0) {
             /* @TODO We use 1 here to preserve previous behavior, but this
              * should probably use the default (INFINITY) or 0 (to disable)
              * instead.
              */
             pcmk__warn_once(pcmk__wo_neg_threshold,
                             PCMK_META_MIGRATION_THRESHOLD
                             " must be non-negative, using 1 instead");
             (*rsc)->migration_threshold = 1;
         }
     }
 
     if (pcmk__str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
                      PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         pcmk__set_scheduler_flags(scheduler, pcmk_sched_have_fencing);
         pcmk__set_rsc_flags(*rsc, pcmk_rsc_fence_device);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_REQUIRES);
     unpack_requires(*rsc, value, false);
 
     value = g_hash_table_lookup((*rsc)->meta, PCMK_META_FAILURE_TIMEOUT);
     if (value != NULL) {
         guint interval_ms = 0U;
 
         // Stored as seconds
         pcmk_parse_interval_spec(value, &interval_ms);
         (*rsc)->failure_timeout = (int) (interval_ms / 1000);
     }
 
     if (remote_node) {
         GHashTable *params = pe_rsc_params(*rsc, NULL, scheduler);
 
         /* Grabbing the value now means that any rules based on node attributes
          * will evaluate to false, so such rules should not be used with
          * PCMK_REMOTE_RA_RECONNECT_INTERVAL.
          *
          * @TODO Evaluate per node before using
          */
         value = g_hash_table_lookup(params, PCMK_REMOTE_RA_RECONNECT_INTERVAL);
         if (value) {
             /* reconnect delay works by setting failure_timeout and preventing the
              * connection from starting until the failure is cleared. */
             pcmk_parse_interval_spec(value, &((*rsc)->remote_reconnect_ms));
 
             /* We want to override any default failure_timeout in use when remote
              * PCMK_REMOTE_RA_RECONNECT_INTERVAL is in use.
              */
             (*rsc)->failure_timeout = (*rsc)->remote_reconnect_ms / 1000;
         }
     }
 
     get_target_role(*rsc, &((*rsc)->next_role));
     pcmk__rsc_trace(*rsc, "%s desired next state: %s", (*rsc)->id,
                     (*rsc)->next_role != pcmk_role_unknown? role2text((*rsc)->next_role) : "default");
 
     if ((*rsc)->fns->unpack(*rsc, scheduler) == FALSE) {
         (*rsc)->fns->free(*rsc);
         *rsc = NULL;
         return pcmk_rc_unpack_error;
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
         // This tag must stay exactly the same because it is tested elsewhere
         resource_location(*rsc, NULL, 0, "symmetric_default", scheduler);
     } else if (guest_node) {
         /* remote resources tied to a container resource must always be allowed
          * to opt-in to the cluster. Whether the connection resource is actually
          * allowed to be placed on a node is dependent on the container resource */
         resource_location(*rsc, NULL, 0, "remote_connection_default",
                           scheduler);
     }
 
     pcmk__rsc_trace(*rsc, "%s action notification: %s", (*rsc)->id,
                     pcmk_is_set((*rsc)->flags, pcmk_rsc_notify)? "required" : "not required");
 
     (*rsc)->utilization = pcmk__strkey_table(free, free);
 
     pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, &rule_data,
                                (*rsc)->utilization, NULL, FALSE, scheduler);
 
     if (expanded_xml) {
         if (add_template_rsc(xml_obj, scheduler) == FALSE) {
             (*rsc)->fns->free(*rsc);
             *rsc = NULL;
             return pcmk_rc_unpack_error;
         }
     }
     return pcmk_rc_ok;
 }
 
 gboolean
 is_parent(pcmk_resource_t *child, pcmk_resource_t *rsc)
 {
     pcmk_resource_t *parent = child;
 
     if (parent == NULL || rsc == NULL) {
         return FALSE;
     }
     while (parent->parent != NULL) {
         if (parent->parent == rsc) {
             return TRUE;
         }
         parent = parent->parent;
     }
     return FALSE;
 }
 
 pcmk_resource_t *
 uber_parent(pcmk_resource_t *rsc)
 {
     pcmk_resource_t *parent = rsc;
 
     if (parent == NULL) {
         return NULL;
     }
     while ((parent->parent != NULL)
            && (parent->parent->variant != pcmk_rsc_variant_bundle)) {
         parent = parent->parent;
     }
     return parent;
 }
 
 /*!
  * \internal
  * \brief Get the topmost parent of a resource as a const pointer
  *
  * \param[in] rsc             Resource to check
  * \param[in] include_bundle  If true, go all the way to bundle
  *
  * \return \p NULL if \p rsc is NULL, \p rsc if \p rsc has no parent,
  *         the bundle if \p rsc is bundled and \p include_bundle is true,
  *         otherwise the topmost parent of \p rsc up to a clone
  */
 const pcmk_resource_t *
 pe__const_top_resource(const pcmk_resource_t *rsc, bool include_bundle)
 {
     const pcmk_resource_t *parent = rsc;
 
     if (parent == NULL) {
         return NULL;
     }
     while (parent->parent != NULL) {
         if (!include_bundle
             && (parent->parent->variant == pcmk_rsc_variant_bundle)) {
             break;
         }
         parent = parent->parent;
     }
     return parent;
 }
 
 void
 common_free(pcmk_resource_t * rsc)
 {
     if (rsc == NULL) {
         return;
     }
 
     pcmk__rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);
 
     g_list_free(rsc->rsc_cons);
     g_list_free(rsc->rsc_cons_lhs);
     g_list_free(rsc->rsc_tickets);
     g_list_free(rsc->dangling_migrations);
 
     if (rsc->parameter_cache != NULL) {
         g_hash_table_destroy(rsc->parameter_cache);
     }
     if (rsc->meta != NULL) {
         g_hash_table_destroy(rsc->meta);
     }
     if (rsc->utilization != NULL) {
         g_hash_table_destroy(rsc->utilization);
     }
 
     if ((rsc->parent == NULL)
         && pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
 
         free_xml(rsc->xml);
         rsc->xml = NULL;
         free_xml(rsc->orig_xml);
         rsc->orig_xml = NULL;
 
         /* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
     } else if (rsc->orig_xml) {
         free_xml(rsc->xml);
         rsc->xml = NULL;
     }
     if (rsc->running_on) {
         g_list_free(rsc->running_on);
         rsc->running_on = NULL;
     }
     if (rsc->known_on) {
         g_hash_table_destroy(rsc->known_on);
         rsc->known_on = NULL;
     }
     if (rsc->actions) {
         g_list_free(rsc->actions);
         rsc->actions = NULL;
     }
     if (rsc->allowed_nodes) {
         g_hash_table_destroy(rsc->allowed_nodes);
         rsc->allowed_nodes = NULL;
     }
     g_list_free(rsc->fillers);
     g_list_free(rsc->rsc_location);
     pcmk__rsc_trace(rsc, "Resource freed");
     free(rsc->id);
     free(rsc->clone_name);
     free(rsc->allocated_to);
     free(rsc->variant_opaque);
     free(rsc->pending_task);
     free(rsc);
 }
 
 /*!
  * \internal
  * \brief Count a node and update most preferred to it as appropriate
  *
  * \param[in]     rsc          An active resource
  * \param[in]     node         A node that \p rsc is active on
  * \param[in,out] active       This will be set to \p node if \p node is more
  *                             preferred than the current value
  * \param[in,out] count_all    If not NULL, this will be incremented
  * \param[in,out] count_clean  If not NULL, this will be incremented if \p node
  *                             is online and clean
  *
  * \return true if the count should continue, or false if sufficiently known
  */
 bool
 pe__count_active_node(const pcmk_resource_t *rsc, pcmk_node_t *node,
                       pcmk_node_t **active, unsigned int *count_all,
                       unsigned int *count_clean)
 {
     bool keep_looking = false;
     bool is_happy = false;
 
     CRM_CHECK((rsc != NULL) && (node != NULL) && (active != NULL),
               return false);
 
     is_happy = node->details->online && !node->details->unclean;
 
     if (count_all != NULL) {
         ++*count_all;
     }
     if ((count_clean != NULL) && is_happy) {
         ++*count_clean;
     }
     if ((count_all != NULL) || (count_clean != NULL)) {
         keep_looking = true; // We're counting, so go through entire list
     }
 
     if (rsc->partial_migration_source != NULL) {
         if (node->details == rsc->partial_migration_source->details) {
             *active = node; // This is the migration source
         } else {
             keep_looking = true;
         }
     } else if (!pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
         if (is_happy && ((*active == NULL) || !(*active)->details->online
                          || (*active)->details->unclean)) {
             *active = node; // This is the first clean node
         } else {
             keep_looking = true;
         }
     }
     if (*active == NULL) {
         *active = node; // This is the first node checked
     }
     return keep_looking;
 }
 
 // Shared implementation of pcmk_rsc_methods_t:active_node()
 static pcmk_node_t *
 active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
             unsigned int *count_clean)
 {
     pcmk_node_t *active = NULL;
 
     if (count_all != NULL) {
         *count_all = 0;
     }
     if (count_clean != NULL) {
         *count_clean = 0;
     }
     if (rsc == NULL) {
         return NULL;
     }
     for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
         if (!pe__count_active_node(rsc, (pcmk_node_t *) iter->data, &active,
                                    count_all, count_clean)) {
             break; // Don't waste time iterating if we don't have to
         }
     }
     return active;
 }
 
 /*!
  * \brief
  * \internal Find and count active nodes according to "requires"
  *
  * \param[in]  rsc    Resource to check
  * \param[out] count  If not NULL, will be set to count of active nodes
  *
  * \return An active node (or NULL if resource is not active anywhere)
  *
  * \note This is a convenience wrapper for active_node() where the count of all
  *       active nodes or only clean active nodes is desired according to the
  *       "requires" meta-attribute.
  */
 pcmk_node_t *
 pe__find_active_requires(const pcmk_resource_t *rsc, unsigned int *count)
 {
     if (rsc == NULL) {
         if (count != NULL) {
             *count = 0;
         }
         return NULL;
 
     } else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
         return rsc->fns->active_node(rsc, count, NULL);
 
     } else {
         return rsc->fns->active_node(rsc, NULL, count);
     }
 }
 
 void
 pe__count_common(pcmk_resource_t *rsc)
 {
     if (rsc->children != NULL) {
         for (GList *item = rsc->children; item != NULL; item = item->next) {
             ((pcmk_resource_t *) item->data)->fns->count(item->data);
         }
 
     } else if (!pcmk_is_set(rsc->flags, pcmk_rsc_removed)
                || (rsc->role > pcmk_role_stopped)) {
         rsc->cluster->ninstances++;
         if (pe__resource_is_disabled(rsc)) {
             rsc->cluster->disabled_resources++;
         }
         if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
             rsc->cluster->blocked_resources++;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Update a resource's next role
  *
  * \param[in,out] rsc   Resource to be updated
  * \param[in]     role  Resource's new next role
  * \param[in]     why   Human-friendly reason why role is changing (for logs)
  */
 void
 pe__set_next_role(pcmk_resource_t *rsc, enum rsc_role_e role, const char *why)
 {
     CRM_ASSERT((rsc != NULL) && (why != NULL));
     if (rsc->next_role != role) {
         pcmk__rsc_trace(rsc, "Resetting next role for %s from %s to %s (%s)",
                         rsc->id, role2text(rsc->next_role), role2text(role),
                         why);
         rsc->next_role = role;
     }
 }
diff --git a/lib/pengine/native.c b/lib/pengine/native.c
index 449294cd3b..8a3ac912d9 100644
--- a/lib/pengine/native.c
+++ b/lib/pengine/native.c
@@ -1,1468 +1,1468 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdint.h>
 
 #include <crm/common/output.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/complex.h>
 #include <crm/pengine/internal.h>
 #include <crm/msg_xml.h>
 #include <pe_status_private.h>
 
 #ifdef PCMK__COMPAT_2_0
 #define PROVIDER_SEP "::"
 #else
 #define PROVIDER_SEP ":"
 #endif
 
 /*!
  * \internal
  * \brief Check whether a resource is active on multiple nodes
  */
 static bool
 is_multiply_active(const pcmk_resource_t *rsc)
 {
     unsigned int count = 0;
 
     if (rsc->variant == pcmk_rsc_variant_primitive) {
         pe__find_active_requires(rsc, &count);
     }
     return count > 1;
 }
 
 static void
 native_priority_to_node(pcmk_resource_t *rsc, pcmk_node_t *node,
                         gboolean failed)
 {
     int priority = 0;
 
     if ((rsc->priority == 0) || (failed == TRUE)) {
         return;
     }
 
     if (rsc->role == pcmk_role_promoted) {
         // Promoted instance takes base priority + 1
         priority = rsc->priority + 1;
 
     } else {
         priority = rsc->priority;
     }
 
     node->details->priority += priority;
     pcmk__rsc_trace(rsc, "%s now has priority %d with %s'%s' (priority: %d%s)",
                     pe__node_name(node), node->details->priority,
                     (rsc->role == pcmk_role_promoted)? "promoted " : "",
                     rsc->id, rsc->priority,
                     (rsc->role == pcmk_role_promoted)? " + 1" : "");
 
     /* Priority of a resource running on a guest node is added to the cluster
      * node as well. */
     if (node->details->remote_rsc
         && node->details->remote_rsc->container) {
         GList *gIter = node->details->remote_rsc->container->running_on;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pcmk_node_t *a_node = gIter->data;
 
             a_node->details->priority += priority;
             pcmk__rsc_trace(rsc,
                             "%s now has priority %d with %s'%s' "
                             "(priority: %d%s) from guest node %s",
                             pe__node_name(a_node), a_node->details->priority,
                             (rsc->role == pcmk_role_promoted)? "promoted " : "",
                             rsc->id, rsc->priority,
                             (rsc->role == pcmk_role_promoted)? " + 1" : "",
                             pe__node_name(node));
         }
     }
 }
 
 void
 native_add_running(pcmk_resource_t *rsc, pcmk_node_t *node,
                    pcmk_scheduler_t *scheduler, gboolean failed)
 {
     GList *gIter = rsc->running_on;
 
     CRM_CHECK(node != NULL, return);
     for (; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *a_node = (pcmk_node_t *) gIter->data;
 
         CRM_CHECK(a_node != NULL, return);
         if (pcmk__str_eq(a_node->details->id, node->details->id, pcmk__str_casei)) {
             return;
         }
     }
 
     pcmk__rsc_trace(rsc, "Adding %s to %s %s", rsc->id, pe__node_name(node),
                     pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : "(unmanaged)");
 
     rsc->running_on = g_list_append(rsc->running_on, node);
     if (rsc->variant == pcmk_rsc_variant_primitive) {
         node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);
 
         native_priority_to_node(rsc, node, failed);
     }
 
     if ((rsc->variant == pcmk_rsc_variant_primitive)
         && node->details->maintenance) {
         pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
         pcmk__set_rsc_flags(rsc, pcmk_rsc_maintenance);
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
         pcmk_resource_t *p = rsc->parent;
 
         pcmk__rsc_info(rsc, "resource %s isn't managed", rsc->id);
         resource_location(rsc, node, INFINITY, "not_managed_default",
                           scheduler);
 
         while(p && node->details->online) {
             /* add without the additional location constraint */
             p->running_on = g_list_append(p->running_on, node);
             p = p->parent;
         }
         return;
     }
 
     if (is_multiply_active(rsc)) {
         switch (rsc->recovery_type) {
             case pcmk_multiply_active_stop:
                 {
                     GHashTableIter gIter;
                     pcmk_node_t *local_node = NULL;
 
                     /* make sure it doesn't come up again */
                     if (rsc->allowed_nodes != NULL) {
                         g_hash_table_destroy(rsc->allowed_nodes);
                     }
                     rsc->allowed_nodes = pe__node_list2table(scheduler->nodes);
                     g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
                     while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
                         local_node->weight = -INFINITY;
                     }
                 }
                 break;
             case pcmk_multiply_active_block:
                 pcmk__clear_rsc_flags(rsc, pcmk_rsc_managed);
                 pcmk__set_rsc_flags(rsc, pcmk_rsc_blocked);
 
                 /* If the resource belongs to a group or bundle configured with
                  * PCMK_META_MULTIPLE_ACTIVE=block, block the entire entity.
                  */
                 if (rsc->parent
                     && ((rsc->parent->variant == pcmk_rsc_variant_group)
                         || (rsc->parent->variant == pcmk_rsc_variant_bundle))
                     && (rsc->parent->recovery_type == pcmk_multiply_active_block)) {
                     GList *gIter = rsc->parent->children;
 
                     for (; gIter != NULL; gIter = gIter->next) {
                         pcmk_resource_t *child = gIter->data;
 
                         pcmk__clear_rsc_flags(child, pcmk_rsc_managed);
                         pcmk__set_rsc_flags(child, pcmk_rsc_blocked);
                     }
                 }
                 break;
 
             // pcmk_multiply_active_restart, pcmk_multiply_active_unexpected
             default:
                 /* The scheduler will do the right thing because the relevant
                  * variables and flags are set when unpacking the history.
                  */
                 break;
         }
         crm_debug("%s is active on multiple nodes including %s: %s",
                   rsc->id, pe__node_name(node),
                   pcmk_multiply_active_text(rsc->recovery_type));
 
     } else {
         pcmk__rsc_trace(rsc, "Resource %s is active on %s",
                         rsc->id, pe__node_name(node));
     }
 
     if (rsc->parent != NULL) {
         native_add_running(rsc->parent, node, scheduler, FALSE);
     }
 }
 
 static void
 recursive_clear_unique(pcmk_resource_t *rsc, gpointer user_data)
 {
     pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
     add_hash_param(rsc->meta, PCMK_META_GLOBALLY_UNIQUE, XML_BOOLEAN_FALSE);
     g_list_foreach(rsc->children, (GFunc) recursive_clear_unique, NULL);
 }
 
 gboolean
 native_unpack(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     pcmk_resource_t *parent = uber_parent(rsc);
     const char *standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     uint32_t ra_caps = pcmk_get_ra_caps(standard);
 
     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 
     // Only some agent standards support unique and promotable clones
     if (!pcmk_is_set(ra_caps, pcmk_ra_cap_unique)
         && pcmk_is_set(rsc->flags, pcmk_rsc_unique)
         && pe_rsc_is_clone(parent)) {
 
         /* @COMPAT We should probably reject this situation as an error (as we
          * do for promotable below) rather than warn and convert, but that would
          * be a backward-incompatible change that we should probably do with a
          * transform at a schema major version bump.
          */
         pe__force_anon(standard, parent, rsc->id, scheduler);
 
         /* Clear PCMK_META_GLOBALLY_UNIQUE on the parent and all its descendants
          * unpacked so far (clearing the parent should make any future children
          * unpacking correct). We have to clear this resource explicitly because
          * it isn't hooked into the parent's children yet.
          */
         recursive_clear_unique(parent, NULL);
         recursive_clear_unique(rsc, NULL);
     }
     if (!pcmk_is_set(ra_caps, pcmk_ra_cap_promotable)
         && pcmk_is_set(parent->flags, pcmk_rsc_promotable)) {
 
         pcmk__config_err("Resource %s is of type %s and therefore "
                          "cannot be used as a promotable clone resource",
                          rsc->id, standard);
         return FALSE;
     }
     return TRUE;
 }
 
 static bool
 rsc_is_on_node(pcmk_resource_t *rsc, const pcmk_node_t *node, int flags)
 {
     pcmk__rsc_trace(rsc, "Checking whether %s is on %s",
                     rsc->id, pe__node_name(node));
 
     if (pcmk_is_set(flags, pcmk_rsc_match_current_node)
         && (rsc->running_on != NULL)) {
 
         for (GList *iter = rsc->running_on; iter; iter = iter->next) {
             pcmk_node_t *loc = (pcmk_node_t *) iter->data;
 
             if (loc->details == node->details) {
                 return true;
             }
         }
 
     } else if (pcmk_is_set(flags, pe_find_inactive) // @COMPAT deprecated
                && (rsc->running_on == NULL)) {
         return true;
 
     } else if (!pcmk_is_set(flags, pcmk_rsc_match_current_node)
                && (rsc->allocated_to != NULL)
                && (rsc->allocated_to->details == node->details)) {
         return true;
     }
     return false;
 }
 
 pcmk_resource_t *
 native_find_rsc(pcmk_resource_t *rsc, const char *id,
                 const pcmk_node_t *on_node, int flags)
 {
     bool match = false;
     pcmk_resource_t *result = NULL;
 
     CRM_CHECK(id && rsc && rsc->id, return NULL);
 
     if (pcmk_is_set(flags, pcmk_rsc_match_clone_only)) {
         const char *rid = ID(rsc->xml);
 
         if (!pe_rsc_is_clone(pe__const_top_resource(rsc, false))) {
             match = false;
 
         } else if (!strcmp(id, rsc->id) || pcmk__str_eq(id, rid, pcmk__str_none)) {
             match = true;
         }
 
     } else if (!strcmp(id, rsc->id)) {
         match = true;
 
     } else if (pcmk_is_set(flags, pcmk_rsc_match_history)
                && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
         match = true;
 
     } else if (pcmk_is_set(flags, pcmk_rsc_match_basename)
                || (pcmk_is_set(flags, pcmk_rsc_match_anon_basename)
                    && !pcmk_is_set(rsc->flags, pcmk_rsc_unique))) {
         match = pe_base_name_eq(rsc, id);
     }
 
     if (match && on_node) {
         if (!rsc_is_on_node(rsc, on_node, flags)) {
             match = false;
         }
     }
 
     if (match) {
         return rsc;
     }
 
     for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pcmk_resource_t *child = (pcmk_resource_t *) gIter->data;
 
         result = rsc->fns->find_rsc(child, id, on_node, flags);
         if (result) {
             return result;
         }
     }
     return NULL;
 }
 
 // create is ignored
 char *
 native_parameter(pcmk_resource_t *rsc, pcmk_node_t *node, gboolean create,
                  const char *name, pcmk_scheduler_t *scheduler)
 {
     char *value_copy = NULL;
     const char *value = NULL;
     GHashTable *params = NULL;
 
     CRM_CHECK(rsc != NULL, return NULL);
     CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);
 
     pcmk__rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);
     params = pe_rsc_params(rsc, node, scheduler);
     value = g_hash_table_lookup(params, name);
     if (value == NULL) {
         /* try meta attributes instead */
         value = g_hash_table_lookup(rsc->meta, name);
     }
     pcmk__str_update(&value_copy, value);
     return value_copy;
 }
 
 gboolean
 native_active(pcmk_resource_t * rsc, gboolean all)
 {
     for (GList *gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *a_node = (pcmk_node_t *) gIter->data;
 
         if (a_node->details->unclean) {
             pcmk__rsc_trace(rsc, "Resource %s: %s is unclean",
                             rsc->id, pe__node_name(a_node));
             return TRUE;
         } else if (!a_node->details->online
                    && pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
             pcmk__rsc_trace(rsc, "Resource %s: %s is offline",
                             rsc->id, pe__node_name(a_node));
         } else {
             pcmk__rsc_trace(rsc, "Resource %s active on %s",
                             rsc->id, pe__node_name(a_node));
             return TRUE;
         }
     }
     return FALSE;
 }
 
 struct print_data_s {
     long options;
     void *print_data;
 };
 
 static const char *
 native_pending_state(const pcmk_resource_t *rsc)
 {
     const char *pending_state = NULL;
 
     if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_START, pcmk__str_casei)) {
         pending_state = "Starting";
 
     } else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_STOP,
                             pcmk__str_casei)) {
         pending_state = "Stopping";
 
     } else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_TO,
                             pcmk__str_casei)) {
         pending_state = "Migrating";
 
     } else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MIGRATE_FROM,
                             pcmk__str_casei)) {
        /* Work might be done in here. */
         pending_state = "Migrating";
 
     } else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_PROMOTE,
                             pcmk__str_casei)) {
         pending_state = "Promoting";
 
     } else if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_DEMOTE,
                             pcmk__str_casei)) {
         pending_state = "Demoting";
     }
 
     return pending_state;
 }
 
 static const char *
 native_pending_task(const pcmk_resource_t *rsc)
 {
     const char *pending_task = NULL;
 
     if (pcmk__str_eq(rsc->pending_task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
         pending_task = "Monitoring";
 
     /* Pending probes are not printed, even if pending
      * operations are requested. If someone ever requests that
      * behavior, uncomment this and the corresponding part of
      * unpack.c:unpack_rsc_op().
      */
     /*
     } else if (pcmk__str_eq(rsc->pending_task, "probe", pcmk__str_casei)) {
         pending_task = "Checking";
     */
     }
 
     return pending_task;
 }
 
 static enum rsc_role_e
 native_displayable_role(const pcmk_resource_t *rsc)
 {
     enum rsc_role_e role = rsc->role;
 
     if ((role == pcmk_role_started)
         && pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
                        pcmk_rsc_promotable)) {
 
         role = pcmk_role_unpromoted;
     }
     return role;
 }
 
 static const char *
 native_displayable_state(const pcmk_resource_t *rsc, bool print_pending)
 {
     const char *rsc_state = NULL;
 
     if (print_pending) {
         rsc_state = native_pending_state(rsc);
     }
     if (rsc_state == NULL) {
         rsc_state = role2text(native_displayable_role(rsc));
     }
     return rsc_state;
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 static void
 native_print_xml(pcmk_resource_t *rsc, const char *pre_text, long options,
                  void *print_data)
 {
     const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
-    const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+    const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
     const char *rsc_state = native_displayable_state(rsc, pcmk_is_set(options, pe_print_pending));
     const char *target_role = NULL;
 
     /* resource information. */
     status_print("%s<resource ", pre_text);
     status_print(PCMK_XA_ID "=\"%s\" ", rsc_printable_id(rsc));
     status_print("resource_agent=\"%s%s%s:%s\" ", class,
                  ((prov == NULL)? "" : PROVIDER_SEP),
                  ((prov == NULL)? "" : prov),
                  crm_element_value(rsc->xml, PCMK_XA_TYPE));
 
     status_print("role=\"%s\" ", rsc_state);
     if (rsc->meta) {
         target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
     }
     if (target_role) {
         status_print("target_role=\"%s\" ", target_role);
     }
     status_print("active=\"%s\" ", pcmk__btoa(rsc->fns->active(rsc, TRUE)));
     status_print("orphaned=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_removed));
     status_print("blocked=\"%s\" ",
                  pe__rsc_bool_str(rsc, pcmk_rsc_blocked));
     status_print("managed=\"%s\" ",
                  pe__rsc_bool_str(rsc, pcmk_rsc_managed));
     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pcmk_rsc_failed));
     status_print("failure_ignored=\"%s\" ",
                  pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure));
     status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));
 
     if (options & pe_print_pending) {
         const char *pending_task = native_pending_task(rsc);
 
         if (pending_task) {
             status_print("pending=\"%s\" ", pending_task);
         }
     }
 
     /* print out the nodes this resource is running on */
     if (options & pe_print_rsconly) {
         status_print("/>\n");
         /* do nothing */
     } else if (rsc->running_on != NULL) {
         GList *gIter = rsc->running_on;
 
         status_print(">\n");
         for (; gIter != NULL; gIter = gIter->next) {
             pcmk_node_t *node = (pcmk_node_t *) gIter->data;
 
             status_print("%s    <node name=\"%s\" " PCMK_XA_ID "=\"%s\" "
                          "cached=\"%s\"/>\n",
                          pre_text, pcmk__s(node->details->uname, ""),
                          node->details->id, pcmk__btoa(!node->details->online));
         }
         status_print("%s</resource>\n", pre_text);
     } else {
         status_print("/>\n");
     }
 }
 
 // Append a flag to resource description string's flags list
 static bool
 add_output_flag(GString *s, const char *flag_desc, bool have_flags)
 {
     g_string_append(s, (have_flags? ", " : " ("));
     g_string_append(s, flag_desc);
     return true;
 }
 
 // Append a node name to resource description string's node list
 static bool
 add_output_node(GString *s, const char *node, bool have_nodes)
 {
     g_string_append(s, (have_nodes? " " : " [ "));
     g_string_append(s, node);
     return true;
 }
 
 /*!
  * \internal
  * \brief Create a string description of a resource
  *
  * \param[in] rsc          Resource to describe
  * \param[in] name         Desired identifier for the resource
  * \param[in] node         If not NULL, node that resource is "on"
  * \param[in] show_opts    Bitmask of pcmk_show_opt_e.
  * \param[in] target_role  Resource's target role
  * \param[in] show_nodes   Whether to display nodes when multiply active
  *
  * \return Newly allocated string description of resource
  * \note Caller must free the result with g_free().
  */
 gchar *
 pcmk__native_output_string(const pcmk_resource_t *rsc, const char *name,
                            const pcmk_node_t *node, uint32_t show_opts,
                            const char *target_role, bool show_nodes)
 {
     const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     const char *provider = NULL;
     const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
     GString *outstr = NULL;
     bool have_flags = false;
 
     if (rsc->variant != pcmk_rsc_variant_primitive) {
         return NULL;
     }
 
     CRM_CHECK(name != NULL, name = "unknown");
     CRM_CHECK(kind != NULL, kind = "unknown");
     CRM_CHECK(class != NULL, class = "unknown");
 
     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
-        provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+        provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
     }
 
     if ((node == NULL) && (rsc->lock_node != NULL)) {
         node = rsc->lock_node;
     }
     if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only)
         || pcmk__list_of_multiple(rsc->running_on)) {
         node = NULL;
     }
 
     outstr = g_string_sized_new(128);
 
     // Resource name and agent
     pcmk__g_strcat(outstr,
                    name, "\t(", class, ((provider == NULL)? "" : PROVIDER_SEP),
                    pcmk__s(provider, ""), ":", kind, "):\t", NULL);
 
     // State on node
     if (pcmk_is_set(rsc->flags, pcmk_rsc_removed)) {
         g_string_append(outstr, " ORPHANED");
     }
     if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
         enum rsc_role_e role = native_displayable_role(rsc);
 
         g_string_append(outstr, " FAILED");
         if (role > pcmk_role_unpromoted) {
             pcmk__add_word(&outstr, 0, role2text(role));
         }
     } else {
         bool show_pending = pcmk_is_set(show_opts, pcmk_show_pending);
 
         pcmk__add_word(&outstr, 0, native_displayable_state(rsc, show_pending));
     }
     if (node) {
         pcmk__add_word(&outstr, 0, pe__node_name(node));
     }
 
     // Failed probe operation
     if (native_displayable_role(rsc) == pcmk_role_stopped) {
         xmlNode *probe_op = pe__failed_probe_for_rsc(rsc, node ? node->details->uname : NULL);
         if (probe_op != NULL) {
             int rc;
 
             pcmk__scan_min_int(crm_element_value(probe_op, XML_LRM_ATTR_RC), &rc, 0);
             pcmk__g_strcat(outstr, " (", services_ocf_exitcode_str(rc), ") ",
                            NULL);
         }
     }
 
     // Flags, as: (<flag> [...])
     if (node && !(node->details->online) && node->details->unclean) {
         have_flags = add_output_flag(outstr, "UNCLEAN", have_flags);
     }
     if (node && (node == rsc->lock_node)) {
         have_flags = add_output_flag(outstr, "LOCKED", have_flags);
     }
     if (pcmk_is_set(show_opts, pcmk_show_pending)) {
         const char *pending_task = native_pending_task(rsc);
 
         if (pending_task) {
             have_flags = add_output_flag(outstr, pending_task, have_flags);
         }
     }
     if (target_role != NULL) {
         switch (text2role(target_role)) {
             case pcmk_role_unknown:
                 pcmk__config_err("Invalid " PCMK_META_TARGET_ROLE
                                  " %s for resource %s", target_role, rsc->id);
                 break;
 
             case pcmk_role_stopped:
                 have_flags = add_output_flag(outstr, "disabled", have_flags);
                 break;
 
             case pcmk_role_unpromoted:
                 if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
                                 pcmk_rsc_promotable)) {
                     have_flags = add_output_flag(outstr,
                                                  PCMK_META_TARGET_ROLE ":",
                                                  have_flags);
                     g_string_append(outstr, target_role);
                 }
                 break;
 
             default:
                 /* Only show target role if it limits our abilities (i.e. ignore
                  * Started, as it is the default anyways, and doesn't prevent
                  * the resource from becoming promoted).
                  */
                 break;
         }
     }
 
     // Blocked or maintenance implies unmanaged
     if (pcmk_any_flags_set(rsc->flags,
                            pcmk_rsc_blocked|pcmk_rsc_maintenance)) {
         if (pcmk_is_set(rsc->flags, pcmk_rsc_blocked)) {
             have_flags = add_output_flag(outstr, "blocked", have_flags);
 
         } else if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
             have_flags = add_output_flag(outstr, "maintenance", have_flags);
         }
     } else if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
         have_flags = add_output_flag(outstr, "unmanaged", have_flags);
     }
 
     if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
         have_flags = add_output_flag(outstr, "failure ignored", have_flags);
     }
 
 
     if (have_flags) {
         g_string_append_c(outstr, ')');
     }
 
     // User-supplied description
     if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only|pcmk_show_description)
         || pcmk__list_of_multiple(rsc->running_on)) {
         const char *desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);
 
         if (desc) {
             g_string_append(outstr, " (");
             g_string_append(outstr, desc);
             g_string_append(outstr, ")");
 
         }
     }
 
     if (show_nodes && !pcmk_is_set(show_opts, pcmk_show_rsc_only)
         && pcmk__list_of_multiple(rsc->running_on)) {
         bool have_nodes = false;
 
         for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
             pcmk_node_t *n = (pcmk_node_t *) iter->data;
 
             have_nodes = add_output_node(outstr, n->details->uname, have_nodes);
         }
         if (have_nodes) {
             g_string_append(outstr, " ]");
         }
     }
 
     return g_string_free(outstr, FALSE);
 }
 
 int
 pe__common_output_html(pcmk__output_t *out, const pcmk_resource_t *rsc,
                        const char *name, const pcmk_node_t *node,
                        uint32_t show_opts)
 {
     const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
     const char *target_role = NULL;
 
     xmlNodePtr list_node = NULL;
     const char *cl = NULL;
 
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
     CRM_ASSERT(kind != NULL);
 
     if (rsc->meta) {
         const char *is_internal = g_hash_table_lookup(rsc->meta,
                                                       PCMK__META_INTERNAL_RSC);
 
         if (crm_is_true(is_internal)
             && !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {
 
             crm_trace("skipping print of internal resource %s", rsc->id);
             return pcmk_rc_no_output;
         }
         target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
         cl = "rsc-managed";
 
     } else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
         cl = "rsc-failed";
 
     } else if ((rsc->variant == pcmk_rsc_variant_primitive)
                && (rsc->running_on == NULL)) {
         cl = "rsc-failed";
 
     } else if (pcmk__list_of_multiple(rsc->running_on)) {
         cl = "rsc-multiple";
 
     } else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
         cl = "rsc-failure-ignored";
 
     } else {
         cl = "rsc-ok";
     }
 
     {
         gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
                                               target_role, true);
 
         list_node = pcmk__output_create_html_node(out, "li", NULL, NULL, NULL);
         pcmk_create_html_node(list_node, "span", NULL, cl, s);
         g_free(s);
     }
 
     return pcmk_rc_ok;
 }
 
 int
 pe__common_output_text(pcmk__output_t *out, const pcmk_resource_t *rsc,
                        const char *name, const pcmk_node_t *node,
                        uint32_t show_opts)
 {
     const char *target_role = NULL;
 
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
 
     if (rsc->meta) {
         const char *is_internal = g_hash_table_lookup(rsc->meta,
                                                       PCMK__META_INTERNAL_RSC);
 
         if (crm_is_true(is_internal)
             && !pcmk_is_set(show_opts, pcmk_show_implicit_rscs)) {
 
             crm_trace("skipping print of internal resource %s", rsc->id);
             return pcmk_rc_no_output;
         }
         target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
     }
 
     {
         gchar *s = pcmk__native_output_string(rsc, name, node, show_opts,
                                               target_role, true);
 
         out->list_item(out, NULL, "%s", s);
         g_free(s);
     }
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 void
 common_print(pcmk_resource_t *rsc, const char *pre_text, const char *name,
              const pcmk_node_t *node, long options, void *print_data)
 {
     const char *target_role = NULL;
 
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
 
     if (rsc->meta) {
         const char *is_internal = g_hash_table_lookup(rsc->meta,
                                                       PCMK__META_INTERNAL_RSC);
 
         if (crm_is_true(is_internal)
             && !pcmk_is_set(options, pe_print_implicit)) {
 
             crm_trace("skipping print of internal resource %s", rsc->id);
             return;
         }
         target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
     }
 
     if (options & pe_print_xml) {
         native_print_xml(rsc, pre_text, options, print_data);
         return;
     }
 
     if ((pre_text == NULL) && (options & pe_print_printf)) {
         pre_text = " ";
     }
 
     if (options & pe_print_html) {
         if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
             status_print("<font color=\"yellow\">");
 
         } else if (pcmk_is_set(rsc->flags, pcmk_rsc_failed)) {
             status_print("<font color=\"red\">");
 
         } else if (rsc->running_on == NULL) {
             status_print("<font color=\"red\">");
 
         } else if (pcmk__list_of_multiple(rsc->running_on)) {
             status_print("<font color=\"orange\">");
 
         } else if (pcmk_is_set(rsc->flags, pcmk_rsc_ignore_failure)) {
             status_print("<font color=\"yellow\">");
 
         } else {
             status_print("<font color=\"green\">");
         }
     }
 
     {
         gchar *resource_s = pcmk__native_output_string(rsc, name, node, options,
                                                        target_role, false);
         status_print("%s%s", (pre_text? pre_text : ""), resource_s);
         g_free(resource_s);
     }
 
     if (pcmk_is_set(options, pe_print_html)) {
         status_print(" </font> ");
     }
 
     if (!pcmk_is_set(options, pe_print_rsconly)
         && pcmk__list_of_multiple(rsc->running_on)) {
 
         GList *gIter = rsc->running_on;
         int counter = 0;
 
         if (options & pe_print_html) {
             status_print("<ul>\n");
         } else if ((options & pe_print_printf)
                    || (options & pe_print_ncurses)) {
             status_print("[");
         }
 
         for (; gIter != NULL; gIter = gIter->next) {
             pcmk_node_t *n = (pcmk_node_t *) gIter->data;
 
             counter++;
 
             if (options & pe_print_html) {
                 status_print("<li>\n%s", pe__node_name(n));
 
             } else if ((options & pe_print_printf)
                        || (options & pe_print_ncurses)) {
                 status_print(" %s", pe__node_name(n));
 
             } else if ((options & pe_print_log)) {
                 status_print("\t%d : %s", counter, pe__node_name(n));
 
             } else {
                 status_print("%s", pe__node_name(n));
             }
             if (options & pe_print_html) {
                 status_print("</li>\n");
 
             }
         }
 
         if (options & pe_print_html) {
             status_print("</ul>\n");
         } else if ((options & pe_print_printf)
                    || (options & pe_print_ncurses)) {
             status_print(" ]");
         }
     }
 
     if (options & pe_print_html) {
         status_print("<br/>\n");
     } else if (options & pe_print_suppres_nl) {
         /* nothing */
     } else if ((options & pe_print_printf) || (options & pe_print_ncurses)) {
         status_print("\n");
     }
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 void
 native_print(pcmk_resource_t *rsc, const char *pre_text, long options,
              void *print_data)
 {
     const pcmk_node_t *node = NULL;
 
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
     if (options & pe_print_xml) {
         native_print_xml(rsc, pre_text, options, print_data);
         return;
     }
 
     node = pe__current_node(rsc);
 
     if (node == NULL) {
         // This is set only if a non-probe action is pending on this node
         node = rsc->pending_node;
     }
 
     common_print(rsc, pre_text, rsc_printable_id(rsc), node, options, print_data);
 }
 
 PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__resource_xml(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node G_GNUC_UNUSED = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     bool print_pending = pcmk_is_set(show_opts, pcmk_show_pending);
     const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
-    const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+    const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
     const char *rsc_state = native_displayable_state(rsc, print_pending);
 
     const char *desc = NULL;
     char ra_name[LINE_MAX];
     char *nodes_running_on = NULL;
     const char *lock_node_name = NULL;
     int rc = pcmk_rc_no_output;
     const char *target_role = NULL;
 
     desc = pe__resource_description(rsc, show_opts);
 
     if (rsc->meta != NULL) {
        target_role = g_hash_table_lookup(rsc->meta, PCMK_META_TARGET_ROLE);
     }
 
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
 
     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return pcmk_rc_no_output;
     }
 
     /* resource information. */
     snprintf(ra_name, LINE_MAX, "%s%s%s:%s", class,
             ((prov == NULL)? "" : PROVIDER_SEP), ((prov == NULL)? "" : prov),
             crm_element_value(rsc->xml, PCMK_XA_TYPE));
 
     nodes_running_on = pcmk__itoa(g_list_length(rsc->running_on));
 
     if (rsc->lock_node != NULL) {
         lock_node_name = rsc->lock_node->details->uname;
     }
 
     rc = pe__name_and_nvpairs_xml(out, true, "resource", 15,
              PCMK_XA_ID, rsc_printable_id(rsc),
              "resource_agent", ra_name,
              "role", rsc_state,
              "target_role", target_role,
              "active", pcmk__btoa(rsc->fns->active(rsc, TRUE)),
              "orphaned", pe__rsc_bool_str(rsc, pcmk_rsc_removed),
              "blocked", pe__rsc_bool_str(rsc, pcmk_rsc_blocked),
              "maintenance", pe__rsc_bool_str(rsc, pcmk_rsc_maintenance),
              "managed", pe__rsc_bool_str(rsc, pcmk_rsc_managed),
              "failed", pe__rsc_bool_str(rsc, pcmk_rsc_failed),
              "failure_ignored", pe__rsc_bool_str(rsc, pcmk_rsc_ignore_failure),
              "nodes_running_on", nodes_running_on,
              "pending", (print_pending? native_pending_task(rsc) : NULL),
              "locked_to", lock_node_name,
              PCMK_XA_DESCRIPTION, desc);
     free(nodes_running_on);
 
     CRM_ASSERT(rc == pcmk_rc_ok);
 
     if (rsc->running_on != NULL) {
         GList *gIter = rsc->running_on;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pcmk_node_t *node = (pcmk_node_t *) gIter->data;
 
             rc = pe__name_and_nvpairs_xml(out, false, "node", 3,
                      PCMK_XA_NAME, node->details->uname,
                      PCMK_XA_ID, node->details->id,
                      "cached", pcmk__btoa(node->details->online));
             CRM_ASSERT(rc == pcmk_rc_ok);
         }
     }
 
     pcmk__output_xml_pop_parent(out);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__resource_html(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node G_GNUC_UNUSED = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     const pcmk_node_t *node = pe__current_node(rsc);
 
     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return pcmk_rc_no_output;
     }
 
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
 
     if (node == NULL) {
         // This is set only if a non-probe action is pending on this node
         node = rsc->pending_node;
     }
     return pe__common_output_html(out, rsc, rsc_printable_id(rsc), node, show_opts);
 }
 
 PCMK__OUTPUT_ARGS("primitive", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__resource_text(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node G_GNUC_UNUSED = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     const pcmk_node_t *node = pe__current_node(rsc);
 
     CRM_ASSERT(rsc->variant == pcmk_rsc_variant_primitive);
 
     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return pcmk_rc_no_output;
     }
 
     if (node == NULL) {
         // This is set only if a non-probe action is pending on this node
         node = rsc->pending_node;
     }
     return pe__common_output_text(out, rsc, rsc_printable_id(rsc), node, show_opts);
 }
 
 void
 native_free(pcmk_resource_t * rsc)
 {
     pcmk__rsc_trace(rsc, "Freeing resource action list (not the data)");
     common_free(rsc);
 }
 
 enum rsc_role_e
 native_resource_state(const pcmk_resource_t * rsc, gboolean current)
 {
     enum rsc_role_e role = rsc->next_role;
 
     if (current) {
         role = rsc->role;
     }
     pcmk__rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
     return role;
 }
 
 /*!
  * \internal
  * \brief List nodes where a resource (or any of its children) is
  *
  * \param[in]  rsc      Resource to check
  * \param[out] list     List to add result to
  * \param[in]  current  0 = where allocated, 1 = where running,
  *                      2 = where running or pending
  *
  * \return If list contains only one node, that node, or NULL otherwise
  */
 pcmk_node_t *
 native_location(const pcmk_resource_t *rsc, GList **list, int current)
 {
     // @COMPAT: Accept a pcmk__rsc_node argument instead of int current
     pcmk_node_t *one = NULL;
     GList *result = NULL;
 
     if (rsc->children) {
         GList *gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pcmk_resource_t *child = (pcmk_resource_t *) gIter->data;
 
             child->fns->location(child, &result, current);
         }
 
     } else if (current) {
 
         if (rsc->running_on) {
             result = g_list_copy(rsc->running_on);
         }
         if ((current == 2) && rsc->pending_node
             && !pe_find_node_id(result, rsc->pending_node->details->id)) {
                 result = g_list_append(result, rsc->pending_node);
         }
 
     } else if (current == FALSE && rsc->allocated_to) {
         result = g_list_append(NULL, rsc->allocated_to);
     }
 
     if (result && (result->next == NULL)) {
         one = result->data;
     }
 
     if (list) {
         GList *gIter = result;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pcmk_node_t *node = (pcmk_node_t *) gIter->data;
 
             if (*list == NULL || pe_find_node_id(*list, node->details->id) == NULL) {
                 *list = g_list_append(*list, node);
             }
         }
     }
 
     g_list_free(result);
     return one;
 }
 
 static void
 get_rscs_brief(GList *rsc_list, GHashTable * rsc_table, GHashTable * active_table)
 {
     GList *gIter = rsc_list;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) gIter->data;
 
         const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
         const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
 
         int offset = 0;
         char buffer[LINE_MAX];
 
         int *rsc_counter = NULL;
         int *active_counter = NULL;
 
         if (rsc->variant != pcmk_rsc_variant_primitive) {
             continue;
         }
 
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
         if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)) {
-            const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+            const char *prov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
 
             if (prov != NULL) {
                 offset += snprintf(buffer + offset, LINE_MAX - offset,
                                    PROVIDER_SEP "%s", prov);
             }
         }
         offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
         CRM_LOG_ASSERT(offset > 0);
 
         if (rsc_table) {
             rsc_counter = g_hash_table_lookup(rsc_table, buffer);
             if (rsc_counter == NULL) {
                 rsc_counter = calloc(1, sizeof(int));
                 *rsc_counter = 0;
                 g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
             }
             (*rsc_counter)++;
         }
 
         if (active_table) {
             GList *gIter2 = rsc->running_on;
 
             for (; gIter2 != NULL; gIter2 = gIter2->next) {
                 pcmk_node_t *node = (pcmk_node_t *) gIter2->data;
                 GHashTable *node_table = NULL;
 
                 if (node->details->unclean == FALSE && node->details->online == FALSE &&
                     pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
                     continue;
                 }
 
                 node_table = g_hash_table_lookup(active_table, node->details->uname);
                 if (node_table == NULL) {
                     node_table = pcmk__strkey_table(free, free);
                     g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
                 }
 
                 active_counter = g_hash_table_lookup(node_table, buffer);
                 if (active_counter == NULL) {
                     active_counter = calloc(1, sizeof(int));
                     *active_counter = 0;
                     g_hash_table_insert(node_table, strdup(buffer), active_counter);
                 }
                 (*active_counter)++;
             }
         }
     }
 }
 
 static void
 destroy_node_table(gpointer data)
 {
     GHashTable *node_table = data;
 
     if (node_table) {
         g_hash_table_destroy(node_table);
     }
 }
 
 /*!
  * \internal
  * \deprecated This function will be removed in a future release
  */
 void
 print_rscs_brief(GList *rsc_list, const char *pre_text, long options,
                  void *print_data, gboolean print_all)
 {
     GHashTable *rsc_table = pcmk__strkey_table(free, free);
     GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
     GHashTableIter hash_iter;
     char *type = NULL;
     int *rsc_counter = NULL;
 
     get_rscs_brief(rsc_list, rsc_table, active_table);
 
     g_hash_table_iter_init(&hash_iter, rsc_table);
     while (g_hash_table_iter_next(&hash_iter, (gpointer *)&type, (gpointer *)&rsc_counter)) {
         GHashTableIter hash_iter2;
         char *node_name = NULL;
         GHashTable *node_table = NULL;
         int active_counter_all = 0;
 
         g_hash_table_iter_init(&hash_iter2, active_table);
         while (g_hash_table_iter_next(&hash_iter2, (gpointer *)&node_name, (gpointer *)&node_table)) {
             int *active_counter = g_hash_table_lookup(node_table, type);
 
             if (active_counter == NULL || *active_counter == 0) {
                 continue;
 
             } else {
                 active_counter_all += *active_counter;
             }
 
             if (options & pe_print_rsconly) {
                 node_name = NULL;
             }
 
             if (options & pe_print_html) {
                 status_print("<li>\n");
             }
 
             if (print_all) {
                 status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
                              active_counter ? *active_counter : 0,
                              rsc_counter ? *rsc_counter : 0, type,
                              active_counter && (*active_counter > 0) && node_name ? node_name : "");
             } else {
                 status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
                              active_counter ? *active_counter : 0, type,
                              active_counter && (*active_counter > 0) && node_name ? node_name : "");
             }
 
             if (options & pe_print_html) {
                 status_print("</li>\n");
             }
         }
 
         if (print_all && active_counter_all == 0) {
             if (options & pe_print_html) {
                 status_print("<li>\n");
             }
 
             status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
                          active_counter_all,
                          rsc_counter ? *rsc_counter : 0, type);
 
             if (options & pe_print_html) {
                 status_print("</li>\n");
             }
         }
     }
 
     if (rsc_table) {
         g_hash_table_destroy(rsc_table);
         rsc_table = NULL;
     }
     if (active_table) {
         g_hash_table_destroy(active_table);
         active_table = NULL;
     }
 }
 
 int
 pe__rscs_brief_output(pcmk__output_t *out, GList *rsc_list, uint32_t show_opts)
 {
     GHashTable *rsc_table = pcmk__strkey_table(free, free);
     GHashTable *active_table = pcmk__strkey_table(free, destroy_node_table);
     GList *sorted_rscs;
     int rc = pcmk_rc_no_output;
 
     get_rscs_brief(rsc_list, rsc_table, active_table);
 
     /* Make a list of the rsc_table keys so that it can be sorted.  This is to make sure
      * output order stays consistent between systems.
      */
     sorted_rscs = g_hash_table_get_keys(rsc_table);
     sorted_rscs = g_list_sort(sorted_rscs, (GCompareFunc) strcmp);
 
     for (GList *gIter = sorted_rscs; gIter; gIter = gIter->next) {
         char *type = (char *) gIter->data;
         int *rsc_counter = g_hash_table_lookup(rsc_table, type);
 
         GList *sorted_nodes = NULL;
         int active_counter_all = 0;
 
         /* Also make a list of the active_table keys so it can be sorted.  If there's
          * more than one instance of a type of resource running, we need the nodes to
          * be sorted to make sure output order stays consistent between systems.
          */
         sorted_nodes = g_hash_table_get_keys(active_table);
         sorted_nodes = g_list_sort(sorted_nodes, (GCompareFunc) pcmk__numeric_strcasecmp);
 
         for (GList *gIter2 = sorted_nodes; gIter2; gIter2 = gIter2->next) {
             char *node_name = (char *) gIter2->data;
             GHashTable *node_table = g_hash_table_lookup(active_table, node_name);
             int *active_counter = NULL;
 
             if (node_table == NULL) {
                 continue;
             }
 
             active_counter = g_hash_table_lookup(node_table, type);
 
             if (active_counter == NULL || *active_counter == 0) {
                 continue;
 
             } else {
                 active_counter_all += *active_counter;
             }
 
             if (pcmk_is_set(show_opts, pcmk_show_rsc_only)) {
                 node_name = NULL;
             }
 
             if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
                 out->list_item(out, NULL, "%d/%d\t(%s):\tActive %s",
                                *active_counter,
                                rsc_counter ? *rsc_counter : 0, type,
                                (*active_counter > 0) && node_name ? node_name : "");
             } else {
                 out->list_item(out, NULL, "%d\t(%s):\tActive %s",
                                *active_counter, type,
                                (*active_counter > 0) && node_name ? node_name : "");
             }
 
             rc = pcmk_rc_ok;
         }
 
         if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs) && active_counter_all == 0) {
             out->list_item(out, NULL, "%d/%d\t(%s):\tActive",
                            active_counter_all,
                            rsc_counter ? *rsc_counter : 0, type);
             rc = pcmk_rc_ok;
         }
 
         if (sorted_nodes) {
             g_list_free(sorted_nodes);
         }
     }
 
     if (rsc_table) {
         g_hash_table_destroy(rsc_table);
         rsc_table = NULL;
     }
     if (active_table) {
         g_hash_table_destroy(active_table);
         active_table = NULL;
     }
     if (sorted_rscs) {
         g_list_free(sorted_rscs);
     }
 
     return rc;
 }
 
 gboolean
 pe__native_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                        gboolean check_parent)
 {
     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) ||
         pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches)) {
         return FALSE;
     } else if (check_parent && rsc->parent) {
         const pcmk_resource_t *up = pe__const_top_resource(rsc, true);
 
         return up->fns->is_filtered(up, only_rsc, FALSE);
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Get maximum primitive resource instances per node
  *
  * \param[in] rsc  Primitive resource to check
  *
  * \return Maximum number of \p rsc instances that can be active on one node
  */
 unsigned int
 pe__primitive_max_per_node(const pcmk_resource_t *rsc)
 {
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
     return 1U;
 }
diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
index 474c9cc224..694db9d537 100644
--- a/lib/pengine/pe_actions.c
+++ b/lib/pengine/pe_actions.c
@@ -1,1889 +1,1889 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 #include <stdbool.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/scheduler_internal.h>
 #include <crm/pengine/internal.h>
 #include <crm/common/xml_internal.h>
 #include "pe_status_private.h"
 
 static void unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj,
                              guint interval_ms);
 
 static void
 add_singleton(pcmk_scheduler_t *scheduler, pcmk_action_t *action)
 {
     if (scheduler->singletons == NULL) {
         scheduler->singletons = pcmk__strkey_table(NULL, NULL);
     }
     g_hash_table_insert(scheduler->singletons, action->uuid, action);
 }
 
 static pcmk_action_t *
 lookup_singleton(pcmk_scheduler_t *scheduler, const char *action_uuid)
 {
     if (scheduler->singletons == NULL) {
         return NULL;
     }
     return g_hash_table_lookup(scheduler->singletons, action_uuid);
 }
 
 /*!
  * \internal
  * \brief Find an existing action that matches arguments
  *
  * \param[in] key        Action key to match
  * \param[in] rsc        Resource to match (if any)
  * \param[in] node       Node to match (if any)
  * \param[in] scheduler  Scheduler data
  *
  * \return Existing action that matches arguments (or NULL if none)
  */
 static pcmk_action_t *
 find_existing_action(const char *key, const pcmk_resource_t *rsc,
                      const pcmk_node_t *node, const pcmk_scheduler_t *scheduler)
 {
     GList *matches = NULL;
     pcmk_action_t *action = NULL;
 
     /* When rsc is NULL, it would be quicker to check scheduler->singletons,
      * but checking all scheduler->actions takes the node into account.
      */
     matches = find_actions(((rsc == NULL)? scheduler->actions : rsc->actions),
                            key, node);
     if (matches == NULL) {
         return NULL;
     }
     CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));
 
     action = matches->data;
     g_list_free(matches);
     return action;
 }
 
 /*!
  * \internal
  * \brief Find the XML configuration corresponding to a specific action key
  *
  * \param[in] rsc               Resource to find action configuration for
  * \param[in] key               "RSC_ACTION_INTERVAL" of action to find
  * \param[in] include_disabled  If false, do not return disabled actions
  *
  * \return XML configuration of desired action if any, otherwise NULL
  */
 static xmlNode *
 find_exact_action_config(const pcmk_resource_t *rsc, const char *action_name,
                          guint interval_ms, bool include_disabled)
 {
     for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
          operation != NULL; operation = crm_next_same_xml(operation)) {
 
         bool enabled = false;
         const char *config_name = NULL;
         const char *interval_spec = NULL;
         guint tmp_ms = 0U;
 
         // @TODO This does not consider meta-attributes, rules, defaults, etc.
         if (!include_disabled
             && (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
                                        &enabled) == pcmk_rc_ok) && !enabled) {
             continue;
         }
 
         interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
         pcmk_parse_interval_spec(interval_spec, &tmp_ms);
         if (tmp_ms != interval_ms) {
             continue;
         }
 
         config_name = crm_element_value(operation, PCMK_XA_NAME);
         if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) {
             return operation;
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Find the XML configuration of a resource action
  *
  * \param[in] rsc               Resource to find action configuration for
  * \param[in] action_name       Action name to search for
  * \param[in] interval_ms       Action interval (in milliseconds) to search for
  * \param[in] include_disabled  If false, do not return disabled actions
  *
  * \return XML configuration of desired action if any, otherwise NULL
  */
 xmlNode *
 pcmk__find_action_config(const pcmk_resource_t *rsc, const char *action_name,
                          guint interval_ms, bool include_disabled)
 {
     xmlNode *action_config = NULL;
 
     // Try requested action first
     action_config = find_exact_action_config(rsc, action_name, interval_ms,
                                              include_disabled);
 
     // For migrate_to and migrate_from actions, retry with "migrate"
     // @TODO This should be either documented or deprecated
     if ((action_config == NULL)
         && pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO,
                             PCMK_ACTION_MIGRATE_FROM, NULL)) {
         action_config = find_exact_action_config(rsc, "migrate", 0,
                                                  include_disabled);
     }
 
     return action_config;
 }
 
 /*!
  * \internal
  * \brief Create a new action object
  *
  * \param[in]     key        Action key
  * \param[in]     task       Action name
  * \param[in,out] rsc        Resource that action is for (if any)
  * \param[in]     node       Node that action is on (if any)
  * \param[in]     optional   Whether action should be considered optional
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Newly allocated action
  * \note This function takes ownership of \p key. It is the caller's
  *       responsibility to free the return value with pe_free_action().
  */
 static pcmk_action_t *
 new_action(char *key, const char *task, pcmk_resource_t *rsc,
            const pcmk_node_t *node, bool optional, pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *action = calloc(1, sizeof(pcmk_action_t));
 
     CRM_ASSERT(action != NULL);
 
     action->rsc = rsc;
     action->task = strdup(task); CRM_ASSERT(action->task != NULL);
     action->uuid = key;
 
     if (node) {
         action->node = pe__copy_node(node);
     }
 
     if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
         // Resource history deletion for a node can be done on the DC
         pcmk__set_action_flags(action, pcmk_action_on_dc);
     }
 
     pcmk__set_action_flags(action, pcmk_action_runnable);
     if (optional) {
         pcmk__set_action_flags(action, pcmk_action_optional);
     } else {
         pcmk__clear_action_flags(action, pcmk_action_optional);
     }
 
     if (rsc == NULL) {
         action->meta = pcmk__strkey_table(free, free);
     } else {
         guint interval_ms = 0;
 
         parse_op_key(key, NULL, NULL, &interval_ms);
         action->op_entry = pcmk__find_action_config(rsc, task, interval_ms,
                                                     true);
 
         /* If the given key is for one of the many notification pseudo-actions
          * (pre_notify_promote, etc.), the actual action name is "notify"
          */
         if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) {
             action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY,
                                                         0, true);
         }
 
         unpack_operation(action, action->op_entry, interval_ms);
     }
 
     pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
                     (optional? "optional" : "required"),
                     scheduler->action_id, key, task,
                     ((rsc == NULL)? "no resource" : rsc->id),
                     pe__node_name(node));
     action->id = scheduler->action_id++;
 
     scheduler->actions = g_list_prepend(scheduler->actions, action);
     if (rsc == NULL) {
         add_singleton(scheduler, action);
     } else {
         rsc->actions = g_list_prepend(rsc->actions, action);
     }
     return action;
 }
 
 /*!
  * \internal
  * \brief Unpack a resource's action-specific instance parameters
  *
  * \param[in]     action_xml  XML of action's configuration in CIB (if any)
  * \param[in,out] node_attrs  Table of node attributes (for rule evaluation)
  * \param[in,out] scheduler   Cluster working set (for rule evaluation)
  *
  * \return Newly allocated hash table of action-specific instance parameters
  */
 GHashTable *
 pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
                                GHashTable *node_attrs,
                                pcmk_scheduler_t *scheduler)
 {
     GHashTable *params = pcmk__strkey_table(free, free);
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = node_attrs,
         .role = pcmk_role_unknown,
         .now = scheduler->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     pe__unpack_dataset_nvpairs(action_xml, XML_TAG_ATTR_SETS,
                                &rule_data, params, NULL,
                                FALSE, scheduler);
     return params;
 }
 
 /*!
  * \internal
  * \brief Update an action's optional flag
  *
  * \param[in,out] action    Action to update
  * \param[in]     optional  Requested optional status
  */
 static void
 update_action_optional(pcmk_action_t *action, gboolean optional)
 {
     // Force a non-recurring action to be optional if its resource is unmanaged
     if ((action->rsc != NULL) && (action->node != NULL)
         && !pcmk_is_set(action->flags, pcmk_action_pseudo)
         && !pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
         && (g_hash_table_lookup(action->meta,
                                 XML_LRM_ATTR_INTERVAL_MS) == NULL)) {
             pcmk__rsc_debug(action->rsc,
                             "%s on %s is optional (%s is unmanaged)",
                             action->uuid, pe__node_name(action->node),
                             action->rsc->id);
             pcmk__set_action_flags(action, pcmk_action_optional);
             // We shouldn't clear runnable here because ... something
 
     // Otherwise require the action if requested
     } else if (!optional) {
         pcmk__clear_action_flags(action, pcmk_action_optional);
     }
 }
 
 static enum pe_quorum_policy
 effective_quorum_policy(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     enum pe_quorum_policy policy = scheduler->no_quorum_policy;
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_quorate)) {
         policy = pcmk_no_quorum_ignore;
 
     } else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) {
         switch (rsc->role) {
             case pcmk_role_promoted:
             case pcmk_role_unpromoted:
                 if (rsc->next_role > pcmk_role_unpromoted) {
                     pe__set_next_role(rsc, pcmk_role_unpromoted,
                                       PCMK_OPT_NO_QUORUM_POLICY "=demote");
                 }
                 policy = pcmk_no_quorum_ignore;
                 break;
             default:
                 policy = pcmk_no_quorum_stop;
                 break;
         }
     }
     return policy;
 }
 
 /*!
  * \internal
  * \brief Update a resource action's runnable flag
  *
  * \param[in,out] action     Action to update
  * \param[in,out] scheduler  Scheduler data
  *
  * \note This may also schedule fencing if a stop is unrunnable.
  */
 static void
 update_resource_action_runnable(pcmk_action_t *action,
                                 pcmk_scheduler_t *scheduler)
 {
     if (pcmk_is_set(action->flags, pcmk_action_pseudo)) {
         return;
     }
 
     if (action->node == NULL) {
         pcmk__rsc_trace(action->rsc, "%s is unrunnable (unallocated)",
                         action->uuid);
         pcmk__clear_action_flags(action, pcmk_action_runnable);
 
     } else if (!pcmk_is_set(action->flags, pcmk_action_on_dc)
                && !(action->node->details->online)
                && (!pe__is_guest_node(action->node)
                    || action->node->details->remote_requires_reset)) {
         pcmk__clear_action_flags(action, pcmk_action_runnable);
         do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)",
                    action->uuid, pe__node_name(action->node));
         if (pcmk_is_set(action->rsc->flags, pcmk_rsc_managed)
             && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
             && !(action->node->details->unclean)) {
             pe_fence_node(scheduler, action->node, "stop is unrunnable", false);
         }
 
     } else if (!pcmk_is_set(action->flags, pcmk_action_on_dc)
                && action->node->details->pending) {
         pcmk__clear_action_flags(action, pcmk_action_runnable);
         do_crm_log(LOG_WARNING,
                    "Action %s on %s is unrunnable (node is pending)",
                    action->uuid, pe__node_name(action->node));
 
     } else if (action->needs == pcmk_requires_nothing) {
         pe_action_set_reason(action, NULL, TRUE);
         if (pe__is_guest_node(action->node)
             && !pe_can_fence(scheduler, action->node)) {
             /* An action that requires nothing usually does not require any
              * fencing in order to be runnable. However, there is an exception:
              * such an action cannot be completed if it is on a guest node whose
              * host is unclean and cannot be fenced.
              */
             pcmk__rsc_debug(action->rsc,
                             "%s on %s is unrunnable "
                             "(node's host cannot be fenced)",
                             action->uuid, pe__node_name(action->node));
             pcmk__clear_action_flags(action, pcmk_action_runnable);
         } else {
             pcmk__rsc_trace(action->rsc,
                             "%s on %s does not require fencing or quorum",
                             action->uuid, pe__node_name(action->node));
             pcmk__set_action_flags(action, pcmk_action_runnable);
         }
 
     } else {
         switch (effective_quorum_policy(action->rsc, scheduler)) {
             case pcmk_no_quorum_stop:
                 pcmk__rsc_debug(action->rsc,
                                 "%s on %s is unrunnable (no quorum)",
                                 action->uuid, pe__node_name(action->node));
                 pcmk__clear_action_flags(action, pcmk_action_runnable);
                 pe_action_set_reason(action, "no quorum", true);
                 break;
 
             case pcmk_no_quorum_freeze:
                 if (!action->rsc->fns->active(action->rsc, TRUE)
                     || (action->rsc->next_role > action->rsc->role)) {
                     pcmk__rsc_debug(action->rsc,
                                     "%s on %s is unrunnable (no quorum)",
                                     action->uuid, pe__node_name(action->node));
                     pcmk__clear_action_flags(action, pcmk_action_runnable);
                     pe_action_set_reason(action, "quorum freeze", true);
                 }
                 break;
 
             default:
                 //pe_action_set_reason(action, NULL, TRUE);
                 pcmk__set_action_flags(action, pcmk_action_runnable);
                 break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Update a resource object's flags for a new action on it
  *
  * \param[in,out] rsc     Resource that action is for (if any)
  * \param[in]     action  New action
  */
 static void
 update_resource_flags_for_action(pcmk_resource_t *rsc,
                                  const pcmk_action_t *action)
 {
     /* @COMPAT pcmk_rsc_starting and pcmk_rsc_stopping are deprecated and unused
      * within Pacemaker, and will eventually be removed
      */
     if (pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)) {
         pcmk__set_rsc_flags(rsc, pcmk_rsc_stopping);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)) {
         if (pcmk_is_set(action->flags, pcmk_action_runnable)) {
             pcmk__set_rsc_flags(rsc, pcmk_rsc_starting);
         } else {
             pcmk__clear_rsc_flags(rsc, pcmk_rsc_starting);
         }
     }
 }
 
 static bool
 valid_stop_on_fail(const char *value)
 {
     return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL);
 }
 
 /*!
  * \internal
  * \brief Validate (and possibly reset) resource action's on_fail meta-attribute
  *
  * \param[in]     rsc            Resource that action is for
  * \param[in]     action_name    Action name
  * \param[in]     action_config  Action configuration XML from CIB (if any)
  * \param[in,out] meta           Table of action meta-attributes
  */
 static void
 validate_on_fail(const pcmk_resource_t *rsc, const char *action_name,
                  const xmlNode *action_config, GHashTable *meta)
 {
     const char *name = NULL;
     const char *role = NULL;
     const char *interval_spec = NULL;
     const char *value = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);
     char *key = NULL;
     char *new_value = NULL;
     guint interval_ms = 0U;
 
     // Stop actions can only use certain on-fail values
     if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
         && !valid_stop_on_fail(value)) {
 
         pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
                          "action to default value because '%s' is not "
                          "allowed for stop", rsc->id, value);
         g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
         return;
     }
 
     /* Demote actions default on-fail to the on-fail value for the first
      * recurring monitor for the promoted role (if any).
      */
     if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
         && (value == NULL)) {
 
         /* @TODO This does not consider promote options set in a meta-attribute
          * block (which may have rules that need to be evaluated) rather than
          * XML properties.
          */
         for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
              operation != NULL; operation = crm_next_same_xml(operation)) {
             bool enabled = false;
             const char *promote_on_fail = NULL;
 
             /* We only care about explicit on-fail (if promote uses default, so
              * can demote)
              */
             promote_on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
             if (promote_on_fail == NULL) {
                 continue;
             }
 
             // We only care about recurring monitors for the promoted role
             name = crm_element_value(operation, PCMK_XA_NAME);
             role = crm_element_value(operation, "role");
             if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
                 || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
                                          PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
                 continue;
             }
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             pcmk_parse_interval_spec(interval_spec, &interval_ms);
             if (interval_ms == 0U) {
                 continue;
             }
 
             // We only care about enabled monitors
             if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
                                         &enabled) == pcmk_rc_ok) && !enabled) {
                 continue;
             }
 
             // Demote actions can't default to on-fail="demote"
             if (pcmk__str_eq(promote_on_fail, "demote", pcmk__str_casei)) {
                 continue;
             }
 
             // Use value from first applicable promote action found
             key = strdup(XML_OP_ATTR_ON_FAIL);
             new_value = strdup(promote_on_fail);
             CRM_ASSERT((key != NULL) && (new_value != NULL));
             g_hash_table_insert(meta, key, new_value);
         }
         return;
     }
 
     if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
         && !pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
         key = strdup(XML_OP_ATTR_ON_FAIL);
         new_value = strdup("ignore");
         CRM_ASSERT((key != NULL) && (new_value != NULL));
         g_hash_table_insert(meta, key, new_value);
         return;
     }
 
     // on-fail="demote" is allowed only for certain actions
     if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
         name = crm_element_value(action_config, PCMK_XA_NAME);
         role = crm_element_value(action_config, "role");
         interval_spec = crm_element_value(action_config,
                                           XML_LRM_ATTR_INTERVAL);
         pcmk_parse_interval_spec(interval_spec, &interval_ms);
 
         if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
             && ((interval_ms == 0U)
                 || !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
                 || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
                                          PCMK__ROLE_PROMOTED_LEGACY, NULL))) {
 
             pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
                              "action to default value because 'demote' is not "
                              "allowed for it", rsc->id, name);
             g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
             return;
         }
     }
 }
 
 static int
 unpack_timeout(const char *value)
 {
     int timeout_ms = crm_get_msec(value);
 
     if (timeout_ms < 0) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
     return timeout_ms;
 }
 
 // true if value contains valid, non-NULL interval origin for recurring op
 static bool
 unpack_interval_origin(const char *value, const xmlNode *xml_obj,
                        guint interval_ms, const crm_time_t *now,
                        long long *start_delay)
 {
     long long result = 0;
     guint interval_sec = interval_ms / 1000;
     crm_time_t *origin = NULL;
 
     // Ignore unspecified values and non-recurring operations
     if ((value == NULL) || (interval_ms == 0) || (now == NULL)) {
         return false;
     }
 
     // Parse interval origin from text
     origin = crm_time_new(value);
     if (origin == NULL) {
         pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
                          "'%s' because '%s' is not valid",
                          (ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
         return false;
     }
 
     // Get seconds since origin (negative if origin is in the future)
     result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
     crm_time_free(origin);
 
     // Calculate seconds from closest interval to now
     result = result % interval_sec;
 
     // Calculate seconds remaining until next interval
     result = ((result <= 0)? 0 : interval_sec) - result;
     crm_info("Calculated a start delay of %llds for operation '%s'",
              result,
              (ID(xml_obj)? ID(xml_obj) : "(unspecified)"));
 
     if (start_delay != NULL) {
         *start_delay = result * 1000; // milliseconds
     }
     return true;
 }
 
 static int
 unpack_start_delay(const char *value, GHashTable *meta)
 {
     int start_delay = 0;
 
     if (value != NULL) {
         start_delay = crm_get_msec(value);
 
         if (start_delay < 0) {
             start_delay = 0;
         }
 
         if (meta) {
             g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY),
                                  pcmk__itoa(start_delay));
         }
     }
 
     return start_delay;
 }
 
 /*!
  * \internal
  * \brief Find a resource's most frequent recurring monitor
  *
  * \param[in] rsc  Resource to check
  *
  * \return Operation XML configured for most frequent recurring monitor for
  *         \p rsc (if any)
  */
 static xmlNode *
 most_frequent_monitor(const pcmk_resource_t *rsc)
 {
     guint min_interval_ms = G_MAXUINT;
     xmlNode *op = NULL;
 
     for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
          operation != NULL; operation = crm_next_same_xml(operation)) {
         bool enabled = false;
         guint interval_ms = 0U;
         const char *interval_spec = crm_element_value(operation,
                                                       XML_LRM_ATTR_INTERVAL);
 
         // We only care about enabled recurring monitors
         if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME),
                           PCMK_ACTION_MONITOR, pcmk__str_none)) {
             continue;
         }
 
         pcmk_parse_interval_spec(interval_spec, &interval_ms);
         if (interval_ms == 0U) {
             continue;
         }
 
         // @TODO This does not consider meta-attributes, rules, defaults, etc.
         if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
                                     &enabled) == pcmk_rc_ok) && !enabled) {
             continue;
         }
 
         if (interval_ms < min_interval_ms) {
             min_interval_ms = interval_ms;
             op = operation;
         }
     }
     return op;
 }
 
 /*!
  * \internal
  * \brief Unpack action meta-attributes
  *
  * \param[in,out] rsc            Resource that action is for
  * \param[in]     node           Node that action is on
  * \param[in]     action_name    Action name
  * \param[in]     interval_ms    Action interval (in milliseconds)
  * \param[in]     action_config  Action XML configuration from CIB (if any)
  *
  * Unpack a resource action's meta-attributes (normalizing the interval,
  * timeout, and start delay values as integer milliseconds) from its CIB XML
  * configuration (including defaults).
  *
  * \return Newly allocated hash table with normalized action meta-attributes
  */
 GHashTable *
 pcmk__unpack_action_meta(pcmk_resource_t *rsc, const pcmk_node_t *node,
                          const char *action_name, guint interval_ms,
                          const xmlNode *action_config)
 {
     GHashTable *meta = NULL;
     char *name = NULL;
     char *value = NULL;
     const char *timeout_spec = NULL;
     const char *str = NULL;
 
     pe_rsc_eval_data_t rsc_rule_data = {
         .standard = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS),
-        .provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
+        .provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
         .agent = crm_element_value(rsc->xml, PCMK_XA_TYPE),
     };
 
     pe_op_eval_data_t op_rule_data = {
         .op_name = action_name,
         .interval = interval_ms,
     };
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = (node == NULL)? NULL : node->details->attrs,
         .role = pcmk_role_unknown,
         .now = rsc->cluster->now,
         .match_data = NULL,
         .rsc_data = &rsc_rule_data,
         .op_data = &op_rule_data,
     };
 
     meta = pcmk__strkey_table(free, free);
 
     // Cluster-wide <op_defaults> <meta_attributes>
     pe__unpack_dataset_nvpairs(rsc->cluster->op_defaults, XML_TAG_META_SETS,
                                &rule_data, meta, NULL, FALSE, rsc->cluster);
 
     // Derive default timeout for probes from recurring monitor timeouts
     if (pcmk_is_probe(action_name, interval_ms)) {
         xmlNode *min_interval_mon = most_frequent_monitor(rsc);
 
         if (min_interval_mon != NULL) {
             /* @TODO This does not consider timeouts set in meta_attributes
              * blocks (which may also have rules that need to be evaluated).
              */
             timeout_spec = crm_element_value(min_interval_mon,
                                              XML_ATTR_TIMEOUT);
             if (timeout_spec != NULL) {
                 pcmk__rsc_trace(rsc,
                                 "Setting default timeout for %s probe to "
                                 "most frequent monitor's timeout '%s'",
                                 rsc->id, timeout_spec);
                 name = strdup(XML_ATTR_TIMEOUT);
                 value = strdup(timeout_spec);
                 CRM_ASSERT((name != NULL) && (value != NULL));
                 g_hash_table_insert(meta, name, value);
             }
         }
     }
 
     if (action_config != NULL) {
         // <op> <meta_attributes> take precedence over defaults
         pe__unpack_dataset_nvpairs(action_config, XML_TAG_META_SETS, &rule_data,
                                    meta, NULL, TRUE, rsc->cluster);
 
         /* Anything set as an <op> XML property has highest precedence.
          * This ensures we use the name and interval from the <op> tag.
          * (See below for the only exception, fence device start/probe timeout.)
          */
         for (xmlAttrPtr attr = action_config->properties;
              attr != NULL; attr = attr->next) {
             name = strdup((const char *) attr->name);
             value = strdup(pcmk__xml_attr_value(attr));
 
             CRM_ASSERT((name != NULL) && (value != NULL));
             g_hash_table_insert(meta, name, value);
         }
     }
 
     g_hash_table_remove(meta, PCMK_XA_ID);
 
     // Normalize interval to milliseconds
     if (interval_ms > 0) {
         name = strdup(XML_LRM_ATTR_INTERVAL);
         CRM_ASSERT(name != NULL);
         value = crm_strdup_printf("%u", interval_ms);
         g_hash_table_insert(meta, name, value);
     } else {
         g_hash_table_remove(meta, XML_LRM_ATTR_INTERVAL);
     }
 
     /* Timeout order of precedence (highest to lowest):
      *   1. pcmk_monitor_timeout resource parameter (only for starts and probes
      *      when rsc has pcmk_ra_cap_fence_params; this gets used for recurring
      *      monitors via the executor instead)
      *   2. timeout configured in <op> (with <op timeout> taking precedence over
      *      <op> <meta_attributes>)
      *   3. timeout configured in <op_defaults> <meta_attributes>
      *   4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
      */
 
     // Check for pcmk_monitor_timeout
     if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
                     pcmk_ra_cap_fence_params)
         && (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)
             || pcmk_is_probe(action_name, interval_ms))) {
 
         GHashTable *params = pe_rsc_params(rsc, node, rsc->cluster);
 
         timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout");
         if (timeout_spec != NULL) {
             pcmk__rsc_trace(rsc,
                             "Setting timeout for %s %s to "
                             "pcmk_monitor_timeout (%s)",
                             rsc->id, action_name, timeout_spec);
             name = strdup(XML_ATTR_TIMEOUT);
             value = strdup(timeout_spec);
             CRM_ASSERT((name != NULL) && (value != NULL));
             g_hash_table_insert(meta, name, value);
         }
     }
 
     // Normalize timeout to positive milliseconds
     name = strdup(XML_ATTR_TIMEOUT);
     CRM_ASSERT(name != NULL);
     timeout_spec = g_hash_table_lookup(meta, XML_ATTR_TIMEOUT);
     g_hash_table_insert(meta, name, pcmk__itoa(unpack_timeout(timeout_spec)));
 
     // Ensure on-fail has a valid value
     validate_on_fail(rsc, action_name, action_config, meta);
 
     // Normalize start-delay
     str = g_hash_table_lookup(meta, XML_OP_ATTR_START_DELAY);
     if (str != NULL) {
         unpack_start_delay(str, meta);
     } else {
         long long start_delay = 0;
 
         str = g_hash_table_lookup(meta, XML_OP_ATTR_ORIGIN);
         if (unpack_interval_origin(str, action_config, interval_ms,
                                    rsc->cluster->now, &start_delay)) {
             name = strdup(XML_OP_ATTR_START_DELAY);
             CRM_ASSERT(name != NULL);
             g_hash_table_insert(meta, name,
                                 crm_strdup_printf("%lld", start_delay));
         }
     }
     return meta;
 }
 
 /*!
  * \internal
  * \brief Determine an action's quorum and fencing dependency
  *
  * \param[in] rsc          Resource that action is for
  * \param[in] action_name  Name of action being unpacked
  *
  * \return Quorum and fencing dependency appropriate to action
  */
 enum rsc_start_requirement
 pcmk__action_requires(const pcmk_resource_t *rsc, const char *action_name)
 {
     const char *value = NULL;
     enum rsc_start_requirement requires = pcmk_requires_nothing;
 
     CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires);
 
     if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START,
                               PCMK_ACTION_PROMOTE, NULL)) {
         value = "nothing (not start or promote)";
 
     } else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_fencing)) {
         requires = pcmk_requires_fencing;
         value = "fencing";
 
     } else if (pcmk_is_set(rsc->flags, pcmk_rsc_needs_quorum)) {
         requires = pcmk_requires_quorum;
         value = "quorum";
 
     } else {
         value = "nothing";
     }
     pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value);
     return requires;
 }
 
 /*!
  * \internal
  * \brief Parse action failure response from a user-provided string
  *
  * \param[in] rsc          Resource that action is for
  * \param[in] action_name  Name of action
  * \param[in] interval_ms  Action interval (in milliseconds)
  * \param[in] value        User-provided configuration value for on-fail
  *
  * \return Action failure response parsed from \p text
  */
 enum action_fail_response
 pcmk__parse_on_fail(const pcmk_resource_t *rsc, const char *action_name,
                     guint interval_ms, const char *value)
 {
     const char *desc = NULL;
     bool needs_remote_reset = false;
     enum action_fail_response on_fail = pcmk_on_fail_ignore;
 
     if (value == NULL) {
         // Use default
 
     } else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
         on_fail = pcmk_on_fail_block;
         desc = "block";
 
     } else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) {
         if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
             on_fail = pcmk_on_fail_fence_node;
             desc = "node fencing";
         } else {
             pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
                              "%s of %s to 'stop' because 'fence' is not "
                              "valid when fencing is disabled",
                              action_name, rsc->id);
             on_fail = pcmk_on_fail_stop;
             desc = "stop resource";
         }
 
     } else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) {
         on_fail = pcmk_on_fail_standby_node;
         desc = "node standby";
 
     } else if (pcmk__strcase_any_of(value, "ignore", PCMK__VALUE_NOTHING,
                                     NULL)) {
         desc = "ignore";
 
     } else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
         on_fail = pcmk_on_fail_ban;
         desc = "force migration";
 
     } else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) {
         on_fail = pcmk_on_fail_stop;
         desc = "stop resource";
 
     } else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
         on_fail = pcmk_on_fail_restart;
         desc = "restart (and possibly migrate)";
 
     } else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) {
         if (rsc->container == NULL) {
             pcmk__rsc_debug(rsc,
                             "Using default " XML_OP_ATTR_ON_FAIL " for %s "
                             "of %s because it does not have a container",
                             action_name, rsc->id);
         } else {
             on_fail = pcmk_on_fail_restart_container;
             desc = "restart container (and possibly migrate)";
         }
 
     } else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
         on_fail = pcmk_on_fail_demote;
         desc = "demote instance";
 
     } else {
         pcmk__config_err("Using default '" XML_OP_ATTR_ON_FAIL "' for "
                          "%s of %s because '%s' is not valid",
                          action_name, rsc->id, value);
     }
 
     /* Remote node connections are handled specially. Failures that result
      * in dropping an active connection must result in fencing. The only
      * failures that don't are probes and starts. The user can explicitly set
      * on-fail="fence" to fence after start failures.
      */
     if (pe__resource_is_remote_conn(rsc)
         && !pcmk_is_probe(action_name, interval_ms)
         && !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
         needs_remote_reset = true;
         if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
             desc = NULL; // Force default for unmanaged connections
         }
     }
 
     if (desc != NULL) {
         // Explicit value used, default not needed
 
     } else if (rsc->container != NULL) {
         on_fail = pcmk_on_fail_restart_container;
         desc = "restart container (and possibly migrate) (default)";
 
     } else if (needs_remote_reset) {
         if (pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
             if (pcmk_is_set(rsc->cluster->flags,
                             pcmk_sched_fencing_enabled)) {
                 desc = "fence remote node (default)";
             } else {
                 desc = "recover remote node connection (default)";
             }
             on_fail = pcmk_on_fail_reset_remote;
         } else {
             on_fail = pcmk_on_fail_stop;
             desc = "stop unmanaged remote node (enforcing default)";
         }
 
     } else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
         if (pcmk_is_set(rsc->cluster->flags, pcmk_sched_fencing_enabled)) {
             on_fail = pcmk_on_fail_fence_node;
             desc = "resource fence (default)";
         } else {
             on_fail = pcmk_on_fail_block;
             desc = "resource block (default)";
         }
 
     } else {
         on_fail = pcmk_on_fail_restart;
         desc = "restart (and possibly migrate) (default)";
     }
 
     pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s",
                     pcmk__readable_interval(interval_ms), action_name,
                     rsc->id, desc);
     return on_fail;
 }
 
 /*!
  * \internal
  * \brief Determine a resource's role after failure of an action
  *
  * \param[in] rsc          Resource that action is for
  * \param[in] action_name  Action name
  * \param[in] on_fail      Failure handling for action
  * \param[in] meta         Unpacked action meta-attributes
  *
  * \return Resource role that results from failure of action
  */
 enum rsc_role_e
 pcmk__role_after_failure(const pcmk_resource_t *rsc, const char *action_name,
                          enum action_fail_response on_fail, GHashTable *meta)
 {
     const char *value = NULL;
     enum rsc_role_e role = pcmk_role_unknown;
 
     // Set default for role after failure specially in certain circumstances
     switch (on_fail) {
         case pcmk_on_fail_stop:
             role = pcmk_role_stopped;
             break;
 
         case pcmk_on_fail_reset_remote:
             if (rsc->remote_reconnect_ms != 0) {
                 role = pcmk_role_stopped;
             }
             break;
 
         default:
             break;
     }
 
     // @COMPAT Check for explicitly configured role (deprecated)
     value = g_hash_table_lookup(meta, PCMK__META_ROLE_AFTER_FAILURE);
     if (value != NULL) {
         pcmk__warn_once(pcmk__wo_role_after,
                         "Support for " PCMK__META_ROLE_AFTER_FAILURE " is "
                         "deprecated and will be removed in a future release");
         if (role == pcmk_role_unknown) {
             role = text2role(value);
             if (role == pcmk_role_unknown) {
                 pcmk__config_err("Ignoring invalid value %s for "
                                  PCMK__META_ROLE_AFTER_FAILURE,
                                  value);
             }
         }
     }
 
     if (role == pcmk_role_unknown) {
         // Use default
         if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
             role = pcmk_role_unpromoted;
         } else {
             role = pcmk_role_started;
         }
     }
     pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s",
                     rsc->id, action_name, role2text(role));
     return role;
 }
 
 /*!
  * \internal
  * \brief Unpack action configuration
  *
  * Unpack a resource action's meta-attributes (normalizing the interval,
  * timeout, and start delay values as integer milliseconds), requirements, and
  * failure policy from its CIB XML configuration (including defaults).
  *
  * \param[in,out] action       Resource action to unpack into
  * \param[in]     xml_obj      Action configuration XML (NULL for defaults only)
  * \param[in]     interval_ms  How frequently to perform the operation
  */
 static void
 unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj,
                  guint interval_ms)
 {
     const char *value = NULL;
 
     action->meta = pcmk__unpack_action_meta(action->rsc, action->node,
                                             action->task, interval_ms, xml_obj);
     action->needs = pcmk__action_requires(action->rsc, action->task);
 
     value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
     action->on_fail = pcmk__parse_on_fail(action->rsc, action->task,
                                           interval_ms, value);
 
     action->fail_role = pcmk__role_after_failure(action->rsc, action->task,
                                                  action->on_fail, action->meta);
 }
 
 /*!
  * \brief Create or update an action object
  *
  * \param[in,out] rsc          Resource that action is for (if any)
  * \param[in,out] key          Action key (must be non-NULL)
  * \param[in]     task         Action name (must be non-NULL)
  * \param[in]     on_node      Node that action is on (if any)
  * \param[in]     optional     Whether action should be considered optional
  * \param[in,out] scheduler    Scheduler data
  *
  * \return Action object corresponding to arguments (guaranteed not to be
  *         \c NULL)
  * \note This function takes ownership of (and might free) \p key, and
  *       \p scheduler takes ownership of the returned action (the caller should
  *       not free it).
  */
 pcmk_action_t *
 custom_action(pcmk_resource_t *rsc, char *key, const char *task,
               const pcmk_node_t *on_node, gboolean optional,
               pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *action = NULL;
 
     CRM_ASSERT((key != NULL) && (task != NULL) && (scheduler != NULL));
 
     action = find_existing_action(key, rsc, on_node, scheduler);
     if (action == NULL) {
         action = new_action(key, task, rsc, on_node, optional, scheduler);
     } else {
         free(key);
     }
 
     update_action_optional(action, optional);
 
     if (rsc != NULL) {
         if ((action->node != NULL) && (action->op_entry != NULL)
             && !pcmk_is_set(action->flags, pcmk_action_attrs_evaluated)) {
 
             GHashTable *attrs = action->node->details->attrs;
 
             if (action->extra != NULL) {
                 g_hash_table_destroy(action->extra);
             }
             action->extra = pcmk__unpack_action_rsc_params(action->op_entry,
                                                            attrs, scheduler);
             pcmk__set_action_flags(action, pcmk_action_attrs_evaluated);
         }
 
         update_resource_action_runnable(action, scheduler);
         update_resource_flags_for_action(rsc, action);
     }
 
     if (action->extra == NULL) {
         action->extra = pcmk__strkey_table(free, free);
     }
 
     return action;
 }
 
 pcmk_action_t *
 get_pseudo_op(const char *name, pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *op = lookup_singleton(scheduler, name);
 
     if (op == NULL) {
         op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler);
         pcmk__set_action_flags(op, pcmk_action_pseudo|pcmk_action_runnable);
     }
     return op;
 }
 
 static GList *
 find_unfencing_devices(GList *candidates, GList *matches) 
 {
     for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
         pcmk_resource_t *candidate = gIter->data;
 
         if (candidate->children != NULL) {
             matches = find_unfencing_devices(candidate->children, matches);
 
         } else if (!pcmk_is_set(candidate->flags, pcmk_rsc_fence_device)) {
             continue;
 
         } else if (pcmk_is_set(candidate->flags, pcmk_rsc_needs_unfencing)) {
             matches = g_list_prepend(matches, candidate);
 
         } else if (pcmk__str_eq(g_hash_table_lookup(candidate->meta,
                                                     PCMK_STONITH_PROVIDES),
                                 PCMK__VALUE_UNFENCING,
                                 pcmk__str_casei)) {
             matches = g_list_prepend(matches, candidate);
         }
     }
     return matches;
 }
 
 static int
 node_priority_fencing_delay(const pcmk_node_t *node,
                             const pcmk_scheduler_t *scheduler)
 {
     int member_count = 0;
     int online_count = 0;
     int top_priority = 0;
     int lowest_priority = 0;
     GList *gIter = NULL;
 
     // PCMK_OPT_PRIORITY_FENCING_DELAY is disabled
     if (scheduler->priority_fencing_delay <= 0) {
         return 0;
     }
 
     /* No need to request a delay if the fencing target is not a normal cluster
      * member, for example if it's a remote node or a guest node. */
     if (node->details->type != pcmk_node_variant_cluster) {
         return 0;
     }
 
     // No need to request a delay if the fencing target is in our partition
     if (node->details->online) {
         return 0;
     }
 
     for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *n = gIter->data;
 
         if (n->details->type != pcmk_node_variant_cluster) {
             continue;
         }
 
         member_count ++;
 
         if (n->details->online) {
             online_count++;
         }
 
         if (member_count == 1
             || n->details->priority > top_priority) {
             top_priority = n->details->priority;
         }
 
         if (member_count == 1
             || n->details->priority < lowest_priority) {
             lowest_priority = n->details->priority;
         }
     }
 
     // No need to delay if we have more than half of the cluster members
     if (online_count > member_count / 2) {
         return 0;
     }
 
     /* All the nodes have equal priority.
      * Any configured corresponding `pcmk_delay_base/max` will be applied. */
     if (lowest_priority == top_priority) {
         return 0;
     }
 
     if (node->details->priority < top_priority) {
         return 0;
     }
 
     return scheduler->priority_fencing_delay;
 }
 
 pcmk_action_t *
 pe_fence_op(pcmk_node_t *node, const char *op, bool optional,
             const char *reason, bool priority_delay,
             pcmk_scheduler_t *scheduler)
 {
     char *op_key = NULL;
     pcmk_action_t *stonith_op = NULL;
 
     if(op == NULL) {
         op = scheduler->stonith_action;
     }
 
     op_key = crm_strdup_printf("%s-%s-%s",
                                PCMK_ACTION_STONITH, node->details->uname, op);
 
     stonith_op = lookup_singleton(scheduler, op_key);
     if(stonith_op == NULL) {
         stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
                                    TRUE, scheduler);
 
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
         add_hash_param(stonith_op->meta, "stonith_action", op);
 
         if (pcmk_is_set(scheduler->flags, pcmk_sched_enable_unfencing)) {
             /* Extra work to detect device changes
              */
             GString *digests_all = g_string_sized_new(1024);
             GString *digests_secure = g_string_sized_new(1024);
 
             GList *matches = find_unfencing_devices(scheduler->resources, NULL);
 
             char *key = NULL;
             char *value = NULL;
 
             for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
                 pcmk_resource_t *match = gIter->data;
                 const char *agent = g_hash_table_lookup(match->meta,
                                                         PCMK_XA_TYPE);
                 pcmk__op_digest_t *data = NULL;
 
                 data = pe__compare_fencing_digest(match, agent, node,
                                                   scheduler);
                 if (data->rc == pcmk__digest_mismatch) {
                     optional = FALSE;
                     crm_notice("Unfencing node %s because the definition of "
                                "%s changed", pe__node_name(node), match->id);
                     if (!pcmk__is_daemon && scheduler->priv != NULL) {
                         pcmk__output_t *out = scheduler->priv;
 
                         out->info(out,
                                   "notice: Unfencing node %s because the "
                                   "definition of %s changed",
                                   pe__node_name(node), match->id);
                     }
                 }
 
                 pcmk__g_strcat(digests_all,
                                match->id, ":", agent, ":",
                                data->digest_all_calc, ",", NULL);
                 pcmk__g_strcat(digests_secure,
                                match->id, ":", agent, ":",
                                data->digest_secure_calc, ",", NULL);
             }
             key = strdup(XML_OP_ATTR_DIGESTS_ALL);
             value = strdup((const char *) digests_all->str);
             CRM_ASSERT((key != NULL) && (value != NULL));
             g_hash_table_insert(stonith_op->meta, key, value);
             g_string_free(digests_all, TRUE);
 
             key = strdup(XML_OP_ATTR_DIGESTS_SECURE);
             value = strdup((const char *) digests_secure->str);
             CRM_ASSERT((key != NULL) && (value != NULL));
             g_hash_table_insert(stonith_op->meta, key, value);
             g_string_free(digests_secure, TRUE);
         }
 
     } else {
         free(op_key);
     }
 
     if (scheduler->priority_fencing_delay > 0
 
             /* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY
              * applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as
              * an indicator.
              */
         && (priority_delay
 
             /* The priority delay needs to be recalculated if this function has
              * been called by schedule_fencing_and_shutdowns() after node
              * priority has already been calculated by native_add_running().
              */
             || g_hash_table_lookup(stonith_op->meta,
                                    PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) {
 
             /* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if
              * it's 0 for the targeting node. So that it takes precedence over
              * any possible `pcmk_delay_base/max`.
              */
             char *delay_s = pcmk__itoa(node_priority_fencing_delay(node,
                                                                    scheduler));
 
             g_hash_table_insert(stonith_op->meta,
                                 strdup(PCMK_OPT_PRIORITY_FENCING_DELAY),
                                 delay_s);
     }
 
     if(optional == FALSE && pe_can_fence(scheduler, node)) {
         pcmk__clear_action_flags(stonith_op, pcmk_action_optional);
         pe_action_set_reason(stonith_op, reason, false);
 
     } else if(reason && stonith_op->reason == NULL) {
         stonith_op->reason = strdup(reason);
     }
 
     return stonith_op;
 }
 
 void
 pe_free_action(pcmk_action_t *action)
 {
     if (action == NULL) {
         return;
     }
     g_list_free_full(action->actions_before, free);
     g_list_free_full(action->actions_after, free);
     if (action->extra) {
         g_hash_table_destroy(action->extra);
     }
     if (action->meta) {
         g_hash_table_destroy(action->meta);
     }
     free(action->cancel_task);
     free(action->reason);
     free(action->task);
     free(action->uuid);
     free(action->node);
     free(action);
 }
 
 int
 pe_get_configured_timeout(pcmk_resource_t *rsc, const char *action,
                           pcmk_scheduler_t *scheduler)
 {
     xmlNode *child = NULL;
     GHashTable *action_meta = NULL;
     const char *timeout_spec = NULL;
     int timeout_ms = 0;
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = scheduler->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
          child != NULL; child = crm_next_same_xml(child)) {
         if (pcmk__str_eq(action, crm_element_value(child, PCMK_XA_NAME),
                 pcmk__str_casei)) {
             timeout_spec = crm_element_value(child, XML_ATTR_TIMEOUT);
             break;
         }
     }
 
     if (timeout_spec == NULL && scheduler->op_defaults) {
         action_meta = pcmk__strkey_table(free, free);
         pe__unpack_dataset_nvpairs(scheduler->op_defaults, XML_TAG_META_SETS,
                                    &rule_data, action_meta, NULL, FALSE,
                                    scheduler);
         timeout_spec = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
     }
 
     // @TODO check meta-attributes
     // @TODO maybe use min-interval monitor timeout as default for monitors
 
     timeout_ms = crm_get_msec(timeout_spec);
     if (timeout_ms < 0) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
 
     if (action_meta != NULL) {
         g_hash_table_destroy(action_meta);
     }
     return timeout_ms;
 }
 
 enum action_tasks
 get_complex_task(const pcmk_resource_t *rsc, const char *name)
 {
     enum action_tasks task = text2task(name);
 
     if ((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)) {
         switch (task) {
             case pcmk_action_stopped:
             case pcmk_action_started:
             case pcmk_action_demoted:
             case pcmk_action_promoted:
                 crm_trace("Folding %s back into its atomic counterpart for %s",
                           name, rsc->id);
                 --task;
                 break;
             default:
                 break;
         }
     }
     return task;
 }
 
 /*!
  * \internal
  * \brief Find first matching action in a list
  *
  * \param[in] input    List of actions to search
  * \param[in] uuid     If not NULL, action must have this UUID
  * \param[in] task     If not NULL, action must have this action name
  * \param[in] on_node  If not NULL, action must be on this node
  *
  * \return First action in list that matches criteria, or NULL if none
  */
 pcmk_action_t *
 find_first_action(const GList *input, const char *uuid, const char *task,
                   const pcmk_node_t *on_node)
 {
     CRM_CHECK(uuid || task, return NULL);
 
     for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
 
         if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
             continue;
 
         } else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
             continue;
 
         } else if (on_node == NULL) {
             return action;
 
         } else if (action->node == NULL) {
             continue;
 
         } else if (on_node->details == action->node->details) {
             return action;
         }
     }
 
     return NULL;
 }
 
 GList *
 find_actions(GList *input, const char *key, const pcmk_node_t *on_node)
 {
     GList *gIter = input;
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
 
         if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
             continue;
 
         } else if (on_node == NULL) {
             crm_trace("Action %s matches (ignoring node)", key);
             result = g_list_prepend(result, action);
 
         } else if (action->node == NULL) {
             crm_trace("Action %s matches (unallocated, assigning to %s)",
                       key, pe__node_name(on_node));
 
             action->node = pe__copy_node(on_node);
             result = g_list_prepend(result, action);
 
         } else if (on_node->details == action->node->details) {
             crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 GList *
 find_actions_exact(GList *input, const char *key, const pcmk_node_t *on_node)
 {
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     if (on_node == NULL) {
         return NULL;
     }
 
     for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
 
         if ((action->node != NULL)
             && pcmk__str_eq(key, action->uuid, pcmk__str_casei)
             && pcmk__str_eq(on_node->details->id, action->node->details->id,
                             pcmk__str_casei)) {
 
             crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 /*!
  * \brief Find all actions of given type for a resource
  *
  * \param[in] rsc           Resource to search
  * \param[in] node          Find only actions scheduled on this node
  * \param[in] task          Action name to search for
  * \param[in] require_node  If TRUE, NULL node or action node will not match
  *
  * \return List of actions found (or NULL if none)
  * \note If node is not NULL and require_node is FALSE, matching actions
  *       without a node will be assigned to node.
  */
 GList *
 pe__resource_actions(const pcmk_resource_t *rsc, const pcmk_node_t *node,
                      const char *task, bool require_node)
 {
     GList *result = NULL;
     char *key = pcmk__op_key(rsc->id, task, 0);
 
     if (require_node) {
         result = find_actions_exact(rsc->actions, key, node);
     } else {
         result = find_actions(rsc->actions, key, node);
     }
     free(key);
     return result;
 }
 
 /*!
  * \internal
  * \brief Create an action reason string based on the action itself
  *
  * \param[in] action  Action to create reason string for
  * \param[in] flag    Action flag that was cleared
  *
  * \return Newly allocated string suitable for use as action reason
  * \note It is the caller's responsibility to free() the result.
  */
 char *
 pe__action2reason(const pcmk_action_t *action, enum pe_action_flags flag)
 {
     const char *change = NULL;
 
     switch (flag) {
         case pcmk_action_runnable:
             change = "unrunnable";
             break;
         case pcmk_action_migratable:
             change = "unmigrateable";
             break;
         case pcmk_action_optional:
             change = "required";
             break;
         default:
             // Bug: caller passed unsupported flag
             CRM_CHECK(change != NULL, change = "");
             break;
     }
     return crm_strdup_printf("%s%s%s %s", change,
                              (action->rsc == NULL)? "" : " ",
                              (action->rsc == NULL)? "" : action->rsc->id,
                              action->task);
 }
 
 void pe_action_set_reason(pcmk_action_t *action, const char *reason,
                           bool overwrite)
 {
     if (action->reason != NULL && overwrite) {
         pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
                         action->uuid, action->reason,
                         pcmk__s(reason, "(none)"));
     } else if (action->reason == NULL) {
         pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'",
                         action->uuid, pcmk__s(reason, "(none)"));
     } else {
         // crm_assert(action->reason != NULL && !overwrite);
         return;
     }
 
     pcmk__str_update(&action->reason, reason);
 }
 
 /*!
  * \internal
  * \brief Create an action to clear a resource's history from CIB
  *
  * \param[in,out] rsc       Resource to clear
  * \param[in]     node      Node to clear history on
  */
 void
 pe__clear_resource_history(pcmk_resource_t *rsc, const pcmk_node_t *node)
 {
     CRM_ASSERT((rsc != NULL) && (node != NULL));
 
     custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
                   PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->cluster);
 }
 
 #define sort_return(an_int, why) do {					\
 	free(a_uuid);						\
 	free(b_uuid);						\
 	crm_trace("%s (%d) %c %s (%d) : %s",				\
 		  a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=',	\
 		  b_xml_id, b_call_id, why);				\
 	return an_int;							\
     } while(0)
 
 int
 pe__is_newer_op(const xmlNode *xml_a, const xmlNode *xml_b,
                 bool same_node_default)
 {
     int a_call_id = -1;
     int b_call_id = -1;
 
     char *a_uuid = NULL;
     char *b_uuid = NULL;
 
     const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID);
     const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID);
 
     const char *a_node = crm_element_value(xml_a, XML_LRM_ATTR_TARGET);
     const char *b_node = crm_element_value(xml_b, XML_LRM_ATTR_TARGET);
     bool same_node = true;
 
     /* @COMPAT The on_node attribute was added to last_failure as of 1.1.13 (via
      * 8b3ca1c) and the other entries as of 1.1.12 (via 0b07b5c).
      *
      * In case that any of the lrm_rsc_op entries doesn't have on_node
      * attribute, we need to explicitly tell whether the two operations are on
      * the same node.
      */
     if (a_node == NULL || b_node == NULL) {
         same_node = same_node_default;
 
     } else {
         same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);
     }
 
     if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
         /* We have duplicate lrm_rsc_op entries in the status
          * section which is unlikely to be a good thing
          *    - we can handle it easily enough, but we need to get
          *    to the bottom of why it's happening.
          */
         pcmk__config_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
         sort_return(0, "duplicate");
     }
 
     crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
     crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);
 
     if (a_call_id == -1 && b_call_id == -1) {
         /* both are pending ops so it doesn't matter since
          *   stops are never pending
          */
         sort_return(0, "pending");
 
     } else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
         sort_return(-1, "call id");
 
     } else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
         sort_return(1, "call id");
 
     } else if (a_call_id >= 0 && b_call_id >= 0
                && (!same_node || a_call_id == b_call_id)) {
         /*
          * The op and last_failed_op are the same
          * Order on last-rc-change
          */
         time_t last_a = -1;
         time_t last_b = -1;
 
         crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
         crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);
 
         crm_trace("rc-change: %lld vs %lld",
                   (long long) last_a, (long long) last_b);
         if (last_a >= 0 && last_a < last_b) {
             sort_return(-1, "rc-change");
 
         } else if (last_b >= 0 && last_a > last_b) {
             sort_return(1, "rc-change");
         }
         sort_return(0, "rc-change");
 
     } else {
         /* One of the inputs is a pending operation
          * Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
          */
 
         int a_id = -1;
         int b_id = -1;
 
         const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
         const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);
 
         CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
         if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic a");
         }
         if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic b");
         }
         /* try to determine the relative age of the operation...
          * some pending operations (e.g. a start) may have been superseded
          *   by a subsequent stop
          *
          * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last
          */
         if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) || a_id == b_id) {
             /*
              * some of the logic in here may be redundant...
              *
              * if the UUID from the TE doesn't match then one better
              *   be a pending operation.
              * pending operations don't survive between elections and joins
              *   because we query the LRM directly
              */
 
             if (b_call_id == -1) {
                 sort_return(-1, "transition + call");
 
             } else if (a_call_id == -1) {
                 sort_return(1, "transition + call");
             }
 
         } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
             sort_return(-1, "transition");
 
         } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
             sort_return(1, "transition");
         }
     }
 
     /* we should never end up here */
     CRM_CHECK(FALSE, sort_return(0, "default"));
 }
 
 gint
 sort_op_by_callid(gconstpointer a, gconstpointer b)
 {
     const xmlNode *xml_a = a;
     const xmlNode *xml_b = b;
 
     return pe__is_newer_op(xml_a, xml_b, true);
 }
 
 /*!
  * \internal
  * \brief Create a new pseudo-action for a resource
  *
  * \param[in,out] rsc       Resource to create action for
  * \param[in]     task      Action name
  * \param[in]     optional  Whether action should be considered optional
  * \param[in]     runnable  Whethe action should be considered runnable
  *
  * \return New action object corresponding to arguments
  */
 pcmk_action_t *
 pe__new_rsc_pseudo_action(pcmk_resource_t *rsc, const char *task, bool optional,
                           bool runnable)
 {
     pcmk_action_t *action = NULL;
 
     CRM_ASSERT((rsc != NULL) && (task != NULL));
 
     action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
                            optional, rsc->cluster);
     pcmk__set_action_flags(action, pcmk_action_pseudo);
     if (runnable) {
         pcmk__set_action_flags(action, pcmk_action_runnable);
     }
     return action;
 }
 
 /*!
  * \internal
  * \brief Add the expected result to an action
  *
  * \param[in,out] action           Action to add expected result to
  * \param[in]     expected_result  Expected result to add
  *
  * \note This is more efficient than calling add_hash_param().
  */
 void
 pe__add_action_expected_result(pcmk_action_t *action, int expected_result)
 {
     char *name = NULL;
 
     CRM_ASSERT((action != NULL) && (action->meta != NULL));
 
     name = strdup(XML_ATTR_TE_TARGET_RC);
     CRM_ASSERT (name != NULL);
 
     g_hash_table_insert(action->meta, name, pcmk__itoa(expected_result));
 }
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
index 73f494516b..1a52a5cf40 100644
--- a/lib/pengine/pe_output.c
+++ b/lib/pengine/pe_output.c
@@ -1,3176 +1,3179 @@
 /*
  * Copyright 2019-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdint.h>
 
 #include <crm/common/xml_internal.h>
 #include <crm/common/output.h>
 #include <crm/common/scheduler_internal.h>
 #include <crm/cib/util.h>
 #include <crm/msg_xml.h>
 #include <crm/pengine/internal.h>
 
 const char *
 pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
 {
     const char * desc = NULL;
     // User-supplied description
     if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only|pcmk_show_description)) {
         desc = crm_element_value(rsc->xml, PCMK_XA_DESCRIPTION);
     }
     return desc;
 }
 
 /* Never display node attributes whose name starts with one of these prefixes */
 #define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX,    \
                      "shutdown", PCMK_NODE_ATTR_TERMINATE, "standby", "#",  \
                      NULL }
 
 static int
 compare_attribute(gconstpointer a, gconstpointer b)
 {
     int rc;
 
     rc = strcmp((const char *)a, (const char *)b);
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Determine whether extended information about an attribute should be added.
  *
  * \param[in]     node            Node that ran this resource
  * \param[in,out] rsc_list        List of resources for this node
  * \param[in,out] scheduler       Scheduler data
  * \param[in]     attrname        Attribute to find
  * \param[out]    expected_score  Expected value for this attribute
  *
  * \return true if extended information should be printed, false otherwise
  * \note Currently, extended information is only supported for ping/pingd
  *       resources, for which a message will be printed if connectivity is lost
  *       or degraded.
  */
 static bool
 add_extra_info(const pcmk_node_t *node, GList *rsc_list,
                pcmk_scheduler_t *scheduler, const char *attrname,
                int *expected_score)
 {
     GList *gIter = NULL;
 
     for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) gIter->data;
         const char *type = g_hash_table_lookup(rsc->meta, PCMK_XA_TYPE);
         const char *name = NULL;
         GHashTable *params = NULL;
 
         if (rsc->children != NULL) {
             if (add_extra_info(node, rsc->children, scheduler, attrname,
                                expected_score)) {
                 return true;
             }
         }
 
         if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
             continue;
         }
 
         params = pe_rsc_params(rsc, node, scheduler);
         name = g_hash_table_lookup(params, PCMK_XA_NAME);
 
         if (name == NULL) {
             name = "pingd";
         }
 
         /* To identify the resource with the attribute name. */
         if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
             int host_list_num = 0;
             const char *hosts = g_hash_table_lookup(params, "host_list");
             const char *multiplier = g_hash_table_lookup(params, "multiplier");
             int multiplier_i;
 
             if (hosts) {
                 char **host_list = g_strsplit(hosts, " ", 0);
                 host_list_num = g_strv_length(host_list);
                 g_strfreev(host_list);
             }
 
             if ((multiplier == NULL)
                 || (pcmk__scan_min_int(multiplier, &multiplier_i,
                                        INT_MIN) != pcmk_rc_ok)) {
                 /* The ocf:pacemaker:ping resource agent defaults multiplier to
                  * 1. The agent currently does not handle invalid text, but it
                  * should, and this would be a reasonable choice ...
                  */
                 multiplier_i = 1;
             }
             *expected_score = host_list_num * multiplier_i;
 
             return true;
         }
     }
     return false;
 }
 
 static GList *
 filter_attr_list(GList *attr_list, char *name)
 {
     int i;
     const char *filt_str[] = FILTER_STR;
 
     CRM_CHECK(name != NULL, return attr_list);
 
     /* filtering automatic attributes */
     for (i = 0; filt_str[i] != NULL; i++) {
         if (g_str_has_prefix(name, filt_str[i])) {
             return attr_list;
         }
     }
 
     return g_list_insert_sorted(attr_list, name, compare_attribute);
 }
 
 static GList *
 get_operation_list(xmlNode *rsc_entry) {
     GList *op_list = NULL;
     xmlNode *rsc_op = NULL;
 
     for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
          rsc_op = pcmk__xe_next(rsc_op)) {
         const char *task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         const char *interval_ms_s = crm_element_value(rsc_op,
                                                       XML_LRM_ATTR_INTERVAL_MS);
         const char *op_rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
         int op_rc_i;
 
         pcmk__scan_min_int(op_rc, &op_rc_i, 0);
 
         /* Display 0-interval monitors as "probe" */
         if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
             && pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches | pcmk__str_casei)) {
             task = "probe";
         }
 
         /* Ignore notifies and some probes */
         if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)
             || (pcmk__str_eq(task, "probe", pcmk__str_none)
                 && (op_rc_i == CRM_EX_NOT_RUNNING))) {
             continue;
         }
 
         if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
             op_list = g_list_append(op_list, rsc_op);
         }
     }
 
     op_list = g_list_sort(op_list, sort_op_by_callid);
     return op_list;
 }
 
 static void
 add_dump_node(gpointer key, gpointer value, gpointer user_data)
 {
     xmlNodePtr node = user_data;
     pcmk_create_xml_text_node(node, (const char *) key, (const char *) value);
 }
 
 static void
 append_dump_text(gpointer key, gpointer value, gpointer user_data)
 {
     char **dump_text = user_data;
     char *new_text = crm_strdup_printf("%s %s=%s",
                                        *dump_text, (char *)key, (char *)value);
 
     free(*dump_text);
     *dump_text = new_text;
 }
 
 #define XPATH_STACK "//" XML_CIB_TAG_NVPAIR \
                     "[@" PCMK_XA_NAME "='"  \
                         PCMK_OPT_CLUSTER_INFRASTRUCTURE "']"
 
 static const char *
 get_cluster_stack(pcmk_scheduler_t *scheduler)
 {
     xmlNode *stack = get_xpath_object(XPATH_STACK, scheduler->input, LOG_DEBUG);
     return (stack != NULL)? crm_element_value(stack, PCMK_XA_VALUE) : "unknown";
 }
 
 static char *
 last_changed_string(const char *last_written, const char *user,
                     const char *client, const char *origin) {
     if (last_written != NULL || user != NULL || client != NULL || origin != NULL) {
         return crm_strdup_printf("%s%s%s%s%s%s%s",
                                  last_written ? last_written : "",
                                  user ? " by " : "",
                                  user ? user : "",
                                  client ? " via " : "",
                                  client ? client : "",
                                  origin ? " on " : "",
                                  origin ? origin : "");
     } else {
         return strdup("");
     }
 }
 
 static char *
 op_history_string(xmlNode *xml_op, const char *task, const char *interval_ms_s,
                   int rc, bool print_timing) {
     const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     char *interval_str = NULL;
     char *buf = NULL;
 
     if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
         char *pair = pcmk__format_nvpair("interval", interval_ms_s, "ms");
         interval_str = crm_strdup_printf(" %s", pair);
         free(pair);
     }
 
     if (print_timing) {
         char *last_change_str = NULL;
         char *exec_str = NULL;
         char *queue_str = NULL;
 
         const char *value = NULL;
 
         time_t epoch = 0;
 
         if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE, &epoch) == pcmk_ok)
             && (epoch > 0)) {
             char *epoch_str = pcmk__epoch2str(&epoch, 0);
 
             last_change_str = crm_strdup_printf(" %s=\"%s\"",
                                                 XML_RSC_OP_LAST_CHANGE,
                                                 pcmk__s(epoch_str, ""));
             free(epoch_str);
         }
 
         value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
         if (value) {
             char *pair = pcmk__format_nvpair(XML_RSC_OP_T_EXEC, value, "ms");
             exec_str = crm_strdup_printf(" %s", pair);
             free(pair);
         }
 
         value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
         if (value) {
             char *pair = pcmk__format_nvpair(XML_RSC_OP_T_QUEUE, value, "ms");
             queue_str = crm_strdup_printf(" %s", pair);
             free(pair);
         }
 
         buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
                                 interval_str ? interval_str : "",
                                 last_change_str ? last_change_str : "",
                                 exec_str ? exec_str : "",
                                 queue_str ? queue_str : "",
                                 rc, services_ocf_exitcode_str(rc));
 
         if (last_change_str) {
             free(last_change_str);
         }
 
         if (exec_str) {
             free(exec_str);
         }
 
         if (queue_str) {
             free(queue_str);
         }
     } else {
         buf = crm_strdup_printf("(%s) %s%s%s", call, task,
                                 interval_str ? ":" : "",
                                 interval_str ? interval_str : "");
     }
 
     if (interval_str) {
         free(interval_str);
     }
 
     return buf;
 }
 
 static char *
 resource_history_string(pcmk_resource_t *rsc, const char *rsc_id, bool all,
                         int failcount, time_t last_failure) {
     char *buf = NULL;
 
     if (rsc == NULL) {
         buf = crm_strdup_printf("%s: orphan", rsc_id);
     } else if (all || failcount || last_failure > 0) {
         char *failcount_s = NULL;
         char *lastfail_s = NULL;
 
         if (failcount > 0) {
             failcount_s = crm_strdup_printf(" %s=%d", PCMK__FAIL_COUNT_PREFIX,
                                             failcount);
         } else {
             failcount_s = strdup("");
         }
         if (last_failure > 0) {
             buf = pcmk__epoch2str(&last_failure, 0);
             lastfail_s = crm_strdup_printf(" %s='%s'",
                                            PCMK__LAST_FAILURE_PREFIX, buf);
             free(buf);
         }
 
         buf = crm_strdup_printf("%s: " PCMK_META_MIGRATION_THRESHOLD "=%d%s%s",
                                 rsc_id, rsc->migration_threshold, failcount_s,
                                 lastfail_s? lastfail_s : "");
         free(failcount_s);
         free(lastfail_s);
     } else {
         buf = crm_strdup_printf("%s:", rsc_id);
     }
 
     return buf;
 }
 
 /*!
  * \internal
  * \brief Get a node's feature set for status display purposes
  *
  * \param[in] node  Node to check
  *
  * \return String representation of feature set if the node is fully up (using
  *         "<3.15.1" for older nodes that don't set the #feature-set attribute),
  *         otherwise NULL
  */
 static const char *
 get_node_feature_set(const pcmk_node_t *node)
 {
     if (node->details->online && node->details->expected_up
         && !pe__is_guest_or_remote_node(node)) {
 
         const char *feature_set = g_hash_table_lookup(node->details->attrs,
                                                       CRM_ATTR_FEATURE_SET);
 
         /* The feature set attribute is present since 3.15.1. If it is missing,
          * then the node must be running an earlier version.
          */
         return pcmk__s(feature_set, "<3.15.1");
     }
     return NULL;
 }
 
 static bool
 is_mixed_version(pcmk_scheduler_t *scheduler)
 {
     const char *feature_set = NULL;
     for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = gIter->data;
         const char *node_feature_set = get_node_feature_set(node);
         if (node_feature_set != NULL) {
             if (feature_set == NULL) {
                 feature_set = node_feature_set;
             } else if (strcmp(feature_set, node_feature_set) != 0) {
                 return true;
             }
         }
     }
     return false;
 }
 
 static char *
 formatted_xml_buf(const pcmk_resource_t *rsc, bool raw)
 {
     if (raw) {
         return dump_xml_formatted(rsc->orig_xml ? rsc->orig_xml : rsc->xml);
     } else {
         return dump_xml_formatted(rsc->xml);
     }
 }
 
 #define XPATH_DC_VERSION "//" XML_CIB_TAG_NVPAIR    \
                          "[@" PCMK_XA_NAME "='" PCMK_OPT_DC_VERSION "']"
 
 PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
                   "enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
 static int
 cluster_summary(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     int rc = pcmk_rc_no_output;
     const char *stack_s = get_cluster_stack(scheduler);
 
     if (pcmk_is_set(section_opts, pcmk_section_stack)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-stack", stack_s, pcmkd_state);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_dc)) {
         xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
                                                scheduler->input, LOG_DEBUG);
         const char *dc_version_s = dc_version?
                                    crm_element_value(dc_version, PCMK_XA_VALUE)
                                    : NULL;
         const char *quorum = crm_element_value(scheduler->input,
                                                PCMK_XA_HAVE_QUORUM);
         char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
         bool mixed_version = is_mixed_version(scheduler);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-dc", scheduler->dc_node, quorum,
                      dc_version_s, dc_name, mixed_version);
         free(dc_name);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_times)) {
         const char *last_written = crm_element_value(scheduler->input,
                                                      PCMK_XA_CIB_LAST_WRITTEN);
         const char *user = crm_element_value(scheduler->input,
                                              XML_ATTR_UPDATE_USER);
         const char *client = crm_element_value(scheduler->input,
                                                XML_ATTR_UPDATE_CLIENT);
         const char *origin = crm_element_value(scheduler->input,
                                                XML_ATTR_UPDATE_ORIG);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-times",
                      scheduler->localhost, last_written, user, client, origin);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_counts)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
                      scheduler->ninstances, scheduler->disabled_resources,
                      scheduler->blocked_resources);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_options)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-options", scheduler);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
 
     if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
         if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
                   "enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
 static int
 cluster_summary_html(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     int rc = pcmk_rc_no_output;
     const char *stack_s = get_cluster_stack(scheduler);
 
     if (pcmk_is_set(section_opts, pcmk_section_stack)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-stack", stack_s, pcmkd_state);
     }
 
     /* Always print DC if none, even if not requested */
     if ((scheduler->dc_node == NULL)
         || pcmk_is_set(section_opts, pcmk_section_dc)) {
         xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
                                                scheduler->input, LOG_DEBUG);
         const char *dc_version_s = dc_version?
                                    crm_element_value(dc_version, PCMK_XA_VALUE)
                                    : NULL;
         const char *quorum = crm_element_value(scheduler->input,
                                                PCMK_XA_HAVE_QUORUM);
         char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
         bool mixed_version = is_mixed_version(scheduler);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-dc", scheduler->dc_node, quorum,
                      dc_version_s, dc_name, mixed_version);
         free(dc_name);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_times)) {
         const char *last_written = crm_element_value(scheduler->input,
                                                      PCMK_XA_CIB_LAST_WRITTEN);
         const char *user = crm_element_value(scheduler->input,
                                              XML_ATTR_UPDATE_USER);
         const char *client = crm_element_value(scheduler->input,
                                                XML_ATTR_UPDATE_CLIENT);
         const char *origin = crm_element_value(scheduler->input,
                                                XML_ATTR_UPDATE_ORIG);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-times",
                      scheduler->localhost, last_written, user, client, origin);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_counts)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
                      scheduler->ninstances, scheduler->disabled_resources,
                      scheduler->blocked_resources);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_options)) {
         /* Kind of a hack - close the list we may have opened earlier in this
          * function so we can put all the options into their own list.  We
          * only want to do this on HTML output, though.
          */
         PCMK__OUTPUT_LIST_FOOTER(out, rc);
 
         out->begin_list(out, NULL, NULL, "Config Options");
         out->message(out, "cluster-options", scheduler);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
 
     if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
         if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     return rc;
 }
 
 char *
 pe__node_display_name(pcmk_node_t *node, bool print_detail)
 {
     char *node_name;
     const char *node_host = NULL;
     const char *node_id = NULL;
     int name_len;
 
     CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));
 
     /* Host is displayed only if this is a guest node and detail is requested */
     if (print_detail && pe__is_guest_node(node)) {
         const pcmk_resource_t *container = node->details->remote_rsc->container;
         const pcmk_node_t *host_node = pe__current_node(container);
 
         if (host_node && host_node->details) {
             node_host = host_node->details->uname;
         }
         if (node_host == NULL) {
             node_host = ""; /* so we at least get "uname@" to indicate guest */
         }
     }
 
     /* Node ID is displayed if different from uname and detail is requested */
     if (print_detail && !pcmk__str_eq(node->details->uname, node->details->id, pcmk__str_casei)) {
         node_id = node->details->id;
     }
 
     /* Determine name length */
     name_len = strlen(node->details->uname) + 1;
     if (node_host) {
         name_len += strlen(node_host) + 1; /* "@node_host" */
     }
     if (node_id) {
         name_len += strlen(node_id) + 3; /* + " (node_id)" */
     }
 
     /* Allocate and populate display name */
     node_name = malloc(name_len);
     CRM_ASSERT(node_name != NULL);
     strcpy(node_name, node->details->uname);
     if (node_host) {
         strcat(node_name, "@");
         strcat(node_name, node_host);
     }
     if (node_id) {
         strcat(node_name, " (");
         strcat(node_name, node_id);
         strcat(node_name, ")");
     }
     return node_name;
 }
 
 int
 pe__name_and_nvpairs_xml(pcmk__output_t *out, bool is_list, const char *tag_name
                          , size_t pairs_count, ...)
 {
     xmlNodePtr xml_node = NULL;
     va_list args;
 
     CRM_ASSERT(tag_name != NULL);
 
     xml_node = pcmk__output_xml_peek_parent(out);
     CRM_ASSERT(xml_node != NULL);
     xml_node = create_xml_node(xml_node, tag_name);
 
     va_start(args, pairs_count);
     while(pairs_count--) {
         const char *param_name = va_arg(args, const char *);
         const char *param_value = va_arg(args, const char *);
         if (param_name && param_value) {
             crm_xml_add(xml_node, param_name, param_value);
         }
     };
     va_end(args);
 
     if (is_list) {
         pcmk__output_xml_push_parent(out, xml_node);
     }
     return pcmk_rc_ok;
 }
 
 static const char *
 role_desc(enum rsc_role_e role)
 {
     if (role == pcmk_role_promoted) {
 #ifdef PCMK__COMPAT_2_0
         return "as " PCMK__ROLE_PROMOTED_LEGACY " ";
 #else
         return "in " PCMK__ROLE_PROMOTED " role ";
 #endif
     }
     return "";
 }
 
 PCMK__OUTPUT_ARGS("ban", "pcmk_node_t *", "pcmk__location_t *", "uint32_t")
 static int
 ban_html(pcmk__output_t *out, va_list args) {
     pcmk_node_t *pe_node = va_arg(args, pcmk_node_t *);
     pcmk__location_t *location = va_arg(args, pcmk__location_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     char *node_name = pe__node_display_name(pe_node,
                                             pcmk_is_set(show_opts, pcmk_show_node_id));
     char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
                                   location->id, location->rsc->id,
                                   role_desc(location->role_filter), node_name);
 
     pcmk__output_create_html_node(out, "li", NULL, NULL, buf);
 
     free(node_name);
     free(buf);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ban", "pcmk_node_t *", "pcmk__location_t *", "uint32_t")
 static int
 ban_text(pcmk__output_t *out, va_list args) {
     pcmk_node_t *pe_node = va_arg(args, pcmk_node_t *);
     pcmk__location_t *location = va_arg(args, pcmk__location_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     char *node_name = pe__node_display_name(pe_node,
                                             pcmk_is_set(show_opts, pcmk_show_node_id));
     out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
                    location->id, location->rsc->id,
                    role_desc(location->role_filter), node_name);
 
     free(node_name);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ban", "pcmk_node_t *", "pcmk__location_t *", "uint32_t")
 static int
 ban_xml(pcmk__output_t *out, va_list args) {
     pcmk_node_t *pe_node = va_arg(args, pcmk_node_t *);
     pcmk__location_t *location = va_arg(args, pcmk__location_t *);
     uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
 
     const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
     char *weight_s = pcmk__itoa(pe_node->weight);
 
     pcmk__output_create_xml_node(out, "ban",
                                  PCMK_XA_ID, location->id,
                                  "resource", location->rsc->id,
                                  "node", pe_node->details->uname,
                                  "weight", weight_s,
                                  "promoted-only", promoted_only,
                                  /* This is a deprecated alias for
                                   * promoted_only. Removing it will break
                                   * backward compatibility of the API schema,
                                   * which will require an API schema major
                                   * version bump.
                                   */
                                  "master_only", promoted_only,
                                  NULL);
 
     free(weight_s);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t *", "const char *", "GList *",
                   "uint32_t", "bool")
 static int
 ban_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     const char *prefix = va_arg(args, const char *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     GList *gIter, *gIter2;
     int rc = pcmk_rc_no_output;
 
     /* Print each ban */
     for (gIter = scheduler->placement_constraints;
          gIter != NULL; gIter = gIter->next) {
         pcmk__location_t *location = gIter->data;
         const pcmk_resource_t *rsc = location->rsc;
 
         if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
             continue;
         }
 
         if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
                                pcmk__str_star_matches)
             && !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
                                   only_rsc, pcmk__str_star_matches)) {
             continue;
         }
 
         for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
             pcmk_node_t *node = (pcmk_node_t *) gIter2->data;
 
             if (node->weight < 0) {
                 PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
                 out->message(out, "ban", node, location, show_opts);
             }
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
 static int
 cluster_counts_html(pcmk__output_t *out, va_list args) {
     unsigned int nnodes = va_arg(args, unsigned int);
     int nresources = va_arg(args, int);
     int ndisabled = va_arg(args, int);
     int nblocked = va_arg(args, int);
 
     xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);
 
     char *nnodes_str = crm_strdup_printf("%d node%s configured",
                                          nnodes, pcmk__plural_s(nnodes));
 
     pcmk_create_html_node(nodes_node, "span", NULL, NULL, nnodes_str);
     free(nnodes_str);
 
     if (ndisabled && nblocked) {
         char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
                                     nresources, pcmk__plural_s(nresources),
                                     ndisabled);
         pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
         free(s);
 
         pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
 
         s = crm_strdup_printf(", %d ", nblocked);
         pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
         free(s);
 
         pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
         pcmk_create_html_node(resources_node, "span", NULL, NULL,
                               " from further action due to failure)");
     } else if (ndisabled && !nblocked) {
         char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
                                     nresources, pcmk__plural_s(nresources),
                                     ndisabled);
         pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
         free(s);
 
         pcmk_create_html_node(resources_node, "span", NULL, "bold", "DISABLED");
         pcmk_create_html_node(resources_node, "span", NULL, NULL, ")");
     } else if (!ndisabled && nblocked) {
         char *s = crm_strdup_printf("%d resource instance%s configured (%d ",
                                     nresources, pcmk__plural_s(nresources),
                                     nblocked);
         pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
         free(s);
 
         pcmk_create_html_node(resources_node, "span", NULL, "bold", "BLOCKED");
         pcmk_create_html_node(resources_node, "span", NULL, NULL,
                               " from further action due to failure)");
     } else {
         char *s = crm_strdup_printf("%d resource instance%s configured",
                                     nresources, pcmk__plural_s(nresources));
         pcmk_create_html_node(resources_node, "span", NULL, NULL, s);
         free(s);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
 static int
 cluster_counts_text(pcmk__output_t *out, va_list args) {
     unsigned int nnodes = va_arg(args, unsigned int);
     int nresources = va_arg(args, int);
     int ndisabled = va_arg(args, int);
     int nblocked = va_arg(args, int);
 
     out->list_item(out, NULL, "%d node%s configured",
                    nnodes, pcmk__plural_s(nnodes));
 
     if (ndisabled && nblocked) {
         out->list_item(out, NULL, "%d resource instance%s configured "
                                   "(%d DISABLED, %d BLOCKED from "
                                   "further action due to failure)",
                        nresources, pcmk__plural_s(nresources), ndisabled,
                        nblocked);
     } else if (ndisabled && !nblocked) {
         out->list_item(out, NULL, "%d resource instance%s configured "
                                   "(%d DISABLED)",
                        nresources, pcmk__plural_s(nresources), ndisabled);
     } else if (!ndisabled && nblocked) {
         out->list_item(out, NULL, "%d resource instance%s configured "
                                   "(%d BLOCKED from further action "
                                   "due to failure)",
                        nresources, pcmk__plural_s(nresources), nblocked);
     } else {
         out->list_item(out, NULL, "%d resource instance%s configured",
                        nresources, pcmk__plural_s(nresources));
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
 static int
 cluster_counts_xml(pcmk__output_t *out, va_list args) {
     unsigned int nnodes = va_arg(args, unsigned int);
     int nresources = va_arg(args, int);
     int ndisabled = va_arg(args, int);
     int nblocked = va_arg(args, int);
 
     xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "nodes_configured", NULL);
     xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "resources_configured", NULL);
 
     char *s = pcmk__itoa(nnodes);
     crm_xml_add(nodes_node, "number", s);
     free(s);
 
     s = pcmk__itoa(nresources);
     crm_xml_add(resources_node, "number", s);
     free(s);
 
     s = pcmk__itoa(ndisabled);
     crm_xml_add(resources_node, "disabled", s);
     free(s);
 
     s = pcmk__itoa(nblocked);
     crm_xml_add(resources_node, "blocked", s);
     free(s);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t *", "const char *", "const char *",
                   "char *", "int")
 static int
 cluster_dc_html(pcmk__output_t *out, va_list args) {
     pcmk_node_t *dc = va_arg(args, pcmk_node_t *);
     const char *quorum = va_arg(args, const char *);
     const char *dc_version_s = va_arg(args, const char *);
     char *dc_name = va_arg(args, char *);
     bool mixed_version = va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
 
     pcmk_create_html_node(node, "span", NULL, "bold", "Current DC: ");
 
     if (dc) {
         char *buf = crm_strdup_printf("%s (version %s) -", dc_name,
                                       dc_version_s ? dc_version_s : "unknown");
         pcmk_create_html_node(node, "span", NULL, NULL, buf);
         free(buf);
 
         if (mixed_version) {
             pcmk_create_html_node(node, "span", NULL, "warning",
                                   " MIXED-VERSION");
         }
         pcmk_create_html_node(node, "span", NULL, NULL, " partition");
         if (crm_is_true(quorum)) {
             pcmk_create_html_node(node, "span", NULL, NULL, " with");
         } else {
             pcmk_create_html_node(node, "span", NULL, "warning", " WITHOUT");
         }
         pcmk_create_html_node(node, "span", NULL, NULL, " quorum");
     } else {
         pcmk_create_html_node(node, "span", NULL, "warning", "NONE");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t *", "const char *", "const char *",
                   "char *", "int")
 static int
 cluster_dc_text(pcmk__output_t *out, va_list args) {
     pcmk_node_t *dc = va_arg(args, pcmk_node_t *);
     const char *quorum = va_arg(args, const char *);
     const char *dc_version_s = va_arg(args, const char *);
     char *dc_name = va_arg(args, char *);
     bool mixed_version = va_arg(args, int);
 
     if (dc) {
         out->list_item(out, "Current DC",
                        "%s (version %s) - %spartition %s quorum",
                        dc_name, dc_version_s ? dc_version_s : "unknown",
                        mixed_version ? "MIXED-VERSION " : "",
                        crm_is_true(quorum) ? "with" : "WITHOUT");
     } else {
         out->list_item(out, "Current DC", "NONE");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t *", "const char *", "const char *",
                   "char *", "int")
 static int
 cluster_dc_xml(pcmk__output_t *out, va_list args) {
     pcmk_node_t *dc = va_arg(args, pcmk_node_t *);
     const char *quorum = va_arg(args, const char *);
     const char *dc_version_s = va_arg(args, const char *);
     char *dc_name G_GNUC_UNUSED = va_arg(args, char *);
     bool mixed_version = va_arg(args, int);
 
     if (dc) {
         pcmk__output_create_xml_node(out, "current_dc",
                                      "present", "true",
                                      PCMK_XA_VERSION, pcmk__s(dc_version_s, ""),
                                      PCMK_XA_NAME, dc->details->uname,
                                      PCMK_XA_ID, dc->details->id,
                                      "with_quorum", pcmk__btoa(crm_is_true(quorum)),
                                      "mixed_version", pcmk__btoa(mixed_version),
                                      NULL);
     } else {
         pcmk__output_create_xml_node(out, "current_dc",
                                      "present", "false",
                                      NULL);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("maint-mode", "unsigned long long int")
 static int
 cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
     unsigned long long flags = va_arg(args, unsigned long long);
 
     if (pcmk_is_set(flags, pcmk_sched_in_maintenance)) {
         pcmk__formatted_printf(out, "\n              *** Resource management is DISABLED ***\n");
         pcmk__formatted_printf(out, "  The cluster will not attempt to start, stop or recover services\n");
         return pcmk_rc_ok;
     } else if (pcmk_is_set(flags, pcmk_sched_stop_all)) {
         pcmk__formatted_printf(out, "\n    *** Resource management is DISABLED ***\n");
         pcmk__formatted_printf(out, "  The cluster will keep all resources stopped\n");
         return pcmk_rc_ok;
     } else {
         return pcmk_rc_no_output;
     }
 }
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_html(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
         out->list_item(out, NULL, "STONITH of failed nodes enabled");
     } else {
         out->list_item(out, NULL, "STONITH of failed nodes disabled");
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
         out->list_item(out, NULL, "Cluster is symmetric");
     } else {
         out->list_item(out, NULL, "Cluster is asymmetric");
     }
 
     switch (scheduler->no_quorum_policy) {
         case pcmk_no_quorum_freeze:
             out->list_item(out, NULL, "No quorum policy: Freeze resources");
             break;
 
         case pcmk_no_quorum_stop:
             out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
             break;
 
         case pcmk_no_quorum_demote:
             out->list_item(out, NULL, "No quorum policy: Demote promotable "
                            "resources and stop all other resources");
             break;
 
         case pcmk_no_quorum_ignore:
             out->list_item(out, NULL, "No quorum policy: Ignore");
             break;
 
         case pcmk_no_quorum_fence:
             out->list_item(out, NULL, "No quorum policy: Suicide");
             break;
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
         xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
 
         pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
         pcmk_create_html_node(node, "span", NULL, "bold", "DISABLED");
         pcmk_create_html_node(node, "span", NULL, NULL,
                               " (the cluster will not attempt to start, stop, or recover services)");
     } else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
         xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
 
         pcmk_create_html_node(node, "span", NULL, NULL, "Resource management: ");
         pcmk_create_html_node(node, "span", NULL, "bold", "STOPPED");
         pcmk_create_html_node(node, "span", NULL, NULL,
                               " (the cluster will keep all resources stopped)");
     } else {
         out->list_item(out, NULL, "Resource management: enabled");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_log(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_in_maintenance)) {
         return out->info(out, "Resource management is DISABLED.  The cluster will not attempt to start, stop or recover services.");
     } else if (pcmk_is_set(scheduler->flags, pcmk_sched_stop_all)) {
         return out->info(out, "Resource management is DISABLED.  The cluster has stopped all resources.");
     } else {
         return pcmk_rc_no_output;
     }
 }
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_text(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_fencing_enabled)) {
         out->list_item(out, NULL, "STONITH of failed nodes enabled");
     } else {
         out->list_item(out, NULL, "STONITH of failed nodes disabled");
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk_sched_symmetric_cluster)) {
         out->list_item(out, NULL, "Cluster is symmetric");
     } else {
         out->list_item(out, NULL, "Cluster is asymmetric");
     }
 
     switch (scheduler->no_quorum_policy) {
         case pcmk_no_quorum_freeze:
             out->list_item(out, NULL, "No quorum policy: Freeze resources");
             break;
 
         case pcmk_no_quorum_stop:
             out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
             break;
 
         case pcmk_no_quorum_demote:
             out->list_item(out, NULL, "No quorum policy: Demote promotable "
                            "resources and stop all other resources");
             break;
 
         case pcmk_no_quorum_ignore:
             out->list_item(out, NULL, "No quorum policy: Ignore");
             break;
 
         case pcmk_no_quorum_fence:
             out->list_item(out, NULL, "No quorum policy: Suicide");
             break;
     }
 
     return pcmk_rc_ok;
 }
 
 #define bv(flag) pcmk__btoa(pcmk_is_set(scheduler->flags, (flag)))
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_xml(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     const char *no_quorum_policy = NULL;
     char *stonith_timeout_str = pcmk__itoa(scheduler->stonith_timeout);
     char *priority_fencing_delay_str = pcmk__itoa(scheduler->priority_fencing_delay * 1000);
 
     switch (scheduler->no_quorum_policy) {
         case pcmk_no_quorum_freeze:
             no_quorum_policy = "freeze";
             break;
 
         case pcmk_no_quorum_stop:
             no_quorum_policy = "stop";
             break;
 
         case pcmk_no_quorum_demote:
             no_quorum_policy = "demote";
             break;
 
         case pcmk_no_quorum_ignore:
             no_quorum_policy = "ignore";
             break;
 
         case pcmk_no_quorum_fence:
             no_quorum_policy = "suicide";
             break;
     }
 
     pcmk__output_create_xml_node(out, "cluster_options",
                                  PCMK_OPT_STONITH_ENABLED,
                                     bv(pcmk_sched_fencing_enabled),
                                  PCMK_OPT_SYMMETRIC_CLUSTER,
                                     bv(pcmk_sched_symmetric_cluster),
                                  PCMK_OPT_NO_QUORUM_POLICY, no_quorum_policy,
                                  PCMK_OPT_MAINTENANCE_MODE,
                                     bv(pcmk_sched_in_maintenance),
                                  PCMK_OPT_STOP_ALL_RESOURCES,
                                     bv(pcmk_sched_stop_all),
                                  PCMK_OPT_STONITH_TIMEOUT "-ms",
                                     stonith_timeout_str,
                                  PCMK_OPT_PRIORITY_FENCING_DELAY "-ms",
                                     priority_fencing_delay_str,
                                  NULL);
     free(stonith_timeout_str);
     free(priority_fencing_delay_str);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
 static int
 cluster_stack_html(pcmk__output_t *out, va_list args) {
     const char *stack_s = va_arg(args, const char *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
 
     pcmk_create_html_node(node, "span", NULL, "bold", "Stack: ");
     pcmk_create_html_node(node, "span", NULL, NULL, stack_s);
 
     if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
         pcmk_create_html_node(node, "span", NULL, NULL, " (");
         pcmk_create_html_node(node, "span", NULL, NULL,
                               pcmk__pcmkd_state_enum2friendly(pcmkd_state));
         pcmk_create_html_node(node, "span", NULL, NULL, ")");
     }
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
 static int
 cluster_stack_text(pcmk__output_t *out, va_list args) {
     const char *stack_s = va_arg(args, const char *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
 
     if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
         out->list_item(out, "Stack", "%s (%s)",
                        stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
     } else {
         out->list_item(out, "Stack", "%s", stack_s);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
 static int
 cluster_stack_xml(pcmk__output_t *out, va_list args) {
     const char *stack_s = va_arg(args, const char *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
 
     const char *state_s = NULL;
 
     if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
         state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
     }
 
     pcmk__output_create_xml_node(out, "stack",
                                  PCMK_XA_TYPE, stack_s,
                                  "pacemakerd-state", state_s,
                                  NULL);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *",
                   "const char *", "const char *", "const char *")
 static int
 cluster_times_html(pcmk__output_t *out, va_list args) {
     const char *our_nodename = va_arg(args, const char *);
     const char *last_written = va_arg(args, const char *);
     const char *user = va_arg(args, const char *);
     const char *client = va_arg(args, const char *);
     const char *origin = va_arg(args, const char *);
 
     xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);
 
     char *time_s = pcmk__epoch2str(NULL, 0);
 
     pcmk_create_html_node(updated_node, "span", NULL, "bold", "Last updated: ");
     pcmk_create_html_node(updated_node, "span", NULL, NULL, time_s);
 
     if (our_nodename != NULL) {
         pcmk_create_html_node(updated_node, "span", NULL, NULL, " on ");
         pcmk_create_html_node(updated_node, "span", NULL, NULL, our_nodename);
     }
 
     free(time_s);
     time_s = last_changed_string(last_written, user, client, origin);
 
     pcmk_create_html_node(changed_node, "span", NULL, "bold", "Last change: ");
     pcmk_create_html_node(changed_node, "span", NULL, NULL, time_s);
 
     free(time_s);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *",
                   "const char *", "const char *", "const char *")
 static int
 cluster_times_xml(pcmk__output_t *out, va_list args) {
     const char *our_nodename = va_arg(args, const char *);
     const char *last_written = va_arg(args, const char *);
     const char *user = va_arg(args, const char *);
     const char *client = va_arg(args, const char *);
     const char *origin = va_arg(args, const char *);
 
     char *time_s = pcmk__epoch2str(NULL, 0);
 
     pcmk__output_create_xml_node(out, "last_update",
                                  "time", time_s,
                                  "origin", our_nodename,
                                  NULL);
 
     pcmk__output_create_xml_node(out, "last_change",
                                  "time", last_written ? last_written : "",
                                  "user", user ? user : "",
                                  "client", client ? client : "",
                                  "origin", origin ? origin : "",
                                  NULL);
 
     free(time_s);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *",
                   "const char *", "const char *", "const char *")
 static int
 cluster_times_text(pcmk__output_t *out, va_list args) {
     const char *our_nodename = va_arg(args, const char *);
     const char *last_written = va_arg(args, const char *);
     const char *user = va_arg(args, const char *);
     const char *client = va_arg(args, const char *);
     const char *origin = va_arg(args, const char *);
 
     char *time_s = pcmk__epoch2str(NULL, 0);
 
     out->list_item(out, "Last updated", "%s%s%s",
                    time_s, (our_nodename != NULL)? " on " : "",
                    pcmk__s(our_nodename, ""));
 
     free(time_s);
     time_s = last_changed_string(last_written, user, client, origin);
 
     out->list_item(out, "Last change", " %s", time_s);
 
     free(time_s);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Display a failed action in less-technical natural language
  *
  * \param[in,out] out          Output object to use for display
  * \param[in]     xml_op       XML containing failed action
  * \param[in]     op_key       Operation key of failed action
  * \param[in]     node_name    Where failed action occurred
  * \param[in]     rc           OCF exit code of failed action
  * \param[in]     status       Execution status of failed action
  * \param[in]     exit_reason  Exit reason given for failed action
  * \param[in]     exec_time    String containing execution time in milliseconds
  */
 static void
 failed_action_friendly(pcmk__output_t *out, const xmlNode *xml_op,
                        const char *op_key, const char *node_name, int rc,
                        int status, const char *exit_reason,
                        const char *exec_time)
 {
     char *rsc_id = NULL;
     char *task = NULL;
     guint interval_ms = 0;
     time_t last_change_epoch = 0;
     GString *str = NULL;
 
     if (pcmk__str_empty(op_key)
         || !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {
         rsc_id = strdup("unknown resource");
         task = strdup("unknown action");
         interval_ms = 0;
     }
     CRM_ASSERT((rsc_id != NULL) && (task != NULL));
 
     str = g_string_sized_new(256); // Should be sufficient for most messages
 
     pcmk__g_strcat(str, rsc_id, " ", NULL);
 
     if (interval_ms != 0) {
         pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
                        NULL);
     }
     pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
                    node_name, NULL);
 
     if (status == PCMK_EXEC_DONE) {
         pcmk__g_strcat(str, " returned '", services_ocf_exitcode_str(rc), "'",
                        NULL);
         if (!pcmk__str_empty(exit_reason)) {
             pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
         }
 
     } else {
         pcmk__g_strcat(str, " could not be executed (",
                        pcmk_exec_status_str(status), NULL);
         if (!pcmk__str_empty(exit_reason)) {
             pcmk__g_strcat(str, ": ", exit_reason, NULL);
         }
         g_string_append_c(str, ')');
     }
 
 
     if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                 &last_change_epoch) == pcmk_ok) {
         char *s = pcmk__epoch2str(&last_change_epoch, 0);
 
         pcmk__g_strcat(str, " at ", s, NULL);
         free(s);
     }
     if (!pcmk__str_empty(exec_time)) {
         int exec_time_ms = 0;
 
         if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
             && (exec_time_ms > 0)) {
 
             pcmk__g_strcat(str, " after ",
                            pcmk__readable_interval(exec_time_ms), NULL);
         }
     }
 
     out->list_item(out, NULL, "%s", str->str);
     g_string_free(str, TRUE);
     free(rsc_id);
     free(task);
 }
 
 /*!
  * \internal
  * \brief Display a failed action with technical details
  *
  * \param[in,out] out          Output object to use for display
  * \param[in]     xml_op       XML containing failed action
  * \param[in]     op_key       Operation key of failed action
  * \param[in]     node_name    Where failed action occurred
  * \param[in]     rc           OCF exit code of failed action
  * \param[in]     status       Execution status of failed action
  * \param[in]     exit_reason  Exit reason given for failed action
  * \param[in]     exec_time    String containing execution time in milliseconds
  */
 static void
 failed_action_technical(pcmk__output_t *out, const xmlNode *xml_op,
                         const char *op_key, const char *node_name, int rc,
                         int status, const char *exit_reason,
                         const char *exec_time)
 {
     const char *call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     const char *queue_time = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
     const char *exit_status = services_ocf_exitcode_str(rc);
     const char *lrm_status = pcmk_exec_status_str(status);
     time_t last_change_epoch = 0;
     GString *str = NULL;
 
     if (pcmk__str_empty(op_key)) {
         op_key = "unknown operation";
     }
     if (pcmk__str_empty(exit_status)) {
         exit_status = "unknown exit status";
     }
     if (pcmk__str_empty(call_id)) {
         call_id = "unknown";
     }
 
     str = g_string_sized_new(256);
 
     g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
                            op_key, node_name, exit_status, rc, call_id,
                            lrm_status);
 
     if (!pcmk__str_empty(exit_reason)) {
         pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
     }
 
     if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                 &last_change_epoch) == pcmk_ok) {
         char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);
 
         pcmk__g_strcat(str,
                        ", " XML_RSC_OP_LAST_CHANGE "="
                        "'", last_change_str, "'", NULL);
         free(last_change_str);
     }
     if (!pcmk__str_empty(queue_time)) {
         pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
     }
     if (!pcmk__str_empty(exec_time)) {
         pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
     }
 
     out->list_item(out, NULL, "%s", str->str);
     g_string_free(str, TRUE);
 }
 
 PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
 static int
 failed_action_default(pcmk__output_t *out, va_list args)
 {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     const char *op_key = pe__xe_history_key(xml_op);
     const char *node_name = crm_element_value(xml_op, XML_ATTR_UNAME);
     const char *exit_reason = crm_element_value(xml_op,
                                                 XML_LRM_ATTR_EXIT_REASON);
     const char *exec_time = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
 
     int rc;
     int status;
 
     pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);
 
     pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
                        &status, 0);
 
     if (pcmk__str_empty(node_name)) {
         node_name = "unknown node";
     }
 
     if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
         failed_action_technical(out, xml_op, op_key, node_name, rc, status,
                                 exit_reason, exec_time);
     } else {
         failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
                                exit_reason, exec_time);
     }
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("failed-action", "xmlNodePtr", "uint32_t")
 static int
 failed_action_xml(pcmk__output_t *out, va_list args) {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
 
     const char *op_key = pe__xe_history_key(xml_op);
     const char *op_key_name = "op_key";
     int rc;
     int status;
     const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
 
     time_t epoch = 0;
     char *rc_s = NULL;
     char *reason_s = crm_xml_escape(exit_reason ? exit_reason : "none");
     xmlNodePtr node = NULL;
 
     pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), &rc, 0);
     pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
                        &status, 0);
 
     rc_s = pcmk__itoa(rc);
     if (crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY) == NULL) {
         op_key_name = PCMK_XA_ID;
     }
     node = pcmk__output_create_xml_node(out, "failure",
                                         op_key_name, op_key,
                                         "node", crm_element_value(xml_op, XML_ATTR_UNAME),
                                         "exitstatus", services_ocf_exitcode_str(rc),
                                         "exitreason", pcmk__s(reason_s, ""),
                                         "exitcode", rc_s,
                                         "call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
                                         "status", pcmk_exec_status_str(status),
                                         NULL);
     free(rc_s);
 
     if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                  &epoch) == pcmk_ok) && (epoch > 0)) {
         guint interval_ms = 0;
         char *interval_ms_s = NULL;
         char *rc_change = pcmk__epoch2str(&epoch,
                                           crm_time_log_date
                                           |crm_time_log_timeofday
                                           |crm_time_log_with_timezone);
 
         crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
         interval_ms_s = crm_strdup_printf("%u", interval_ms);
 
         pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE, rc_change,
                            "queued", crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                            "exec", crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
                            "interval", interval_ms_s,
                            "task", crm_element_value(xml_op, XML_LRM_ATTR_TASK),
                            NULL);
 
         free(interval_ms_s);
         free(rc_change);
     }
 
     free(reason_s);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t *", "GList *",
                   "GList *", "uint32_t", "bool")
 static int
 failed_action_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     xmlNode *xml_op = NULL;
     int rc = pcmk_rc_no_output;
 
     if (xmlChildElementCount(scheduler->failed) == 0) {
         return rc;
     }
 
     for (xml_op = pcmk__xml_first_child(scheduler->failed); xml_op != NULL;
          xml_op = pcmk__xml_next(xml_op)) {
         char *rsc = NULL;
 
         if (!pcmk__str_in_list(crm_element_value(xml_op, XML_ATTR_UNAME), only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         if (pcmk_xe_mask_probe_failure(xml_op)) {
             continue;
         }
 
         if (!parse_op_key(pe__xe_history_key(xml_op), &rsc, NULL, NULL)) {
             continue;
         }
 
         if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
             free(rsc);
             continue;
         }
 
         free(rsc);
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
         out->message(out, "failed-action", xml_op, show_opts);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 static void
 status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
 {
     int health = pe__node_health(node);
 
     // Cluster membership
     if (node->details->online) {
         pcmk_create_html_node(parent, "span", NULL, "online", " online");
     } else {
         pcmk_create_html_node(parent, "span", NULL, "offline", " OFFLINE");
     }
 
     // Standby mode
     if (node->details->standby_onfail && (node->details->running_rsc != NULL)) {
         pcmk_create_html_node(parent, "span", NULL, "standby",
                               " (in standby due to on-fail,"
                               " with active resources)");
     } else if (node->details->standby_onfail) {
         pcmk_create_html_node(parent, "span", NULL, "standby",
                               " (in standby due to on-fail)");
     } else if (node->details->standby && (node->details->running_rsc != NULL)) {
         pcmk_create_html_node(parent, "span", NULL, "standby",
                               " (in standby, with active resources)");
     } else if (node->details->standby) {
         pcmk_create_html_node(parent, "span", NULL, "standby", " (in standby)");
     }
 
     // Maintenance mode
     if (node->details->maintenance) {
         pcmk_create_html_node(parent, "span", NULL, "maint",
                               " (in maintenance mode)");
     }
 
     // Node health
     if (health < 0) {
         pcmk_create_html_node(parent, "span", NULL, "health_red",
                               " (health is RED)");
     } else if (health == 0) {
         pcmk_create_html_node(parent, "span", NULL, "health_yellow",
                               " (health is YELLOW)");
     }
 
     // Feature set
     if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
         const char *feature_set = get_node_feature_set(node);
         if (feature_set != NULL) {
             char *buf = crm_strdup_printf(", feature set %s", feature_set);
             pcmk_create_html_node(parent, "span", NULL, NULL, buf);
             free(buf);
         }
     }
 }
 
 PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
                   "GList *", "GList *")
 static int
 node_html(pcmk__output_t *out, va_list args) {
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool full = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
 
     if (full) {
         xmlNodePtr item_node;
 
         if (pcmk_all_flags_set(show_opts, pcmk_show_brief | pcmk_show_rscs_by_node)) {
             GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);
 
             out->begin_list(out, NULL, NULL, "%s:", node_name);
             item_node = pcmk__output_xml_create_parent(out, "li", NULL);
             pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
             status_node(node, item_node, show_opts);
 
             if (rscs != NULL) {
                 uint32_t new_show_opts = (show_opts | pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
                 out->begin_list(out, NULL, NULL, "Resources");
                 pe__rscs_brief_output(out, rscs, new_show_opts);
                 out->end_list(out);
             }
 
             pcmk__output_xml_pop_parent(out);
             out->end_list(out);
 
         } else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             GList *lpc2 = NULL;
             int rc = pcmk_rc_no_output;
 
             out->begin_list(out, NULL, NULL, "%s:", node_name);
             item_node = pcmk__output_xml_create_parent(out, "li", NULL);
             pcmk_create_html_node(item_node, "span", NULL, NULL, "Status:");
             status_node(node, item_node, show_opts);
 
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 pcmk_resource_t *rsc = (pcmk_resource_t *) lpc2->data;
                 PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");
 
                 show_opts |= pcmk_show_rsc_only;
                 out->message(out, crm_map_element_name(rsc->xml), show_opts,
                              rsc, only_node, only_rsc);
             }
 
             PCMK__OUTPUT_LIST_FOOTER(out, rc);
             pcmk__output_xml_pop_parent(out);
             out->end_list(out);
 
         } else {
             char *buf = crm_strdup_printf("%s:", node_name);
 
             item_node = pcmk__output_create_xml_node(out, "li", NULL);
             pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
             status_node(node, item_node, show_opts);
 
             free(buf);
         }
     } else {
         out->begin_list(out, NULL, NULL, "%s:", node_name);
     }
 
     free(node_name);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Get a human-friendly textual description of a node's status
  *
  * \param[in] node  Node to check
  *
  * \return String representation of node's status
  */
 static const char *
 node_text_status(const pcmk_node_t *node)
 {
     if (node->details->unclean) {
         if (node->details->online) {
             return "UNCLEAN (online)";
 
         } else if (node->details->pending) {
             return "UNCLEAN (pending)";
 
         } else {
             return "UNCLEAN (offline)";
         }
 
     } else if (node->details->pending) {
         return "pending";
 
     } else if (node->details->standby_onfail && node->details->online) {
         return "standby (on-fail)";
 
     } else if (node->details->standby) {
         if (node->details->online) {
             if (node->details->running_rsc) {
                 return "standby (with active resources)";
             } else {
                 return "standby";
             }
         } else {
             return "OFFLINE (standby)";
         }
 
     } else if (node->details->maintenance) {
         if (node->details->online) {
             return "maintenance";
         } else {
             return "OFFLINE (maintenance)";
         }
 
     } else if (node->details->online) {
         return "online";
     }
 
     return "OFFLINE";
 }
 
 PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool", "GList *",
                   "GList *")
 static int
 node_text(pcmk__output_t *out, va_list args) {
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool full = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     if (full) {
         char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
         GString *str = g_string_sized_new(64);
         int health = pe__node_health(node);
 
         // Create a summary line with node type, name, and status
         if (pe__is_guest_node(node)) {
             g_string_append(str, "GuestNode");
         } else if (pe__is_remote_node(node)) {
             g_string_append(str, "RemoteNode");
         } else {
             g_string_append(str, "Node");
         }
         pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);
 
         if (health < 0) {
             g_string_append(str, " (health is RED)");
         } else if (health == 0) {
             g_string_append(str, " (health is YELLOW)");
         }
         if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
             const char *feature_set = get_node_feature_set(node);
             if (feature_set != NULL) {
                 pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
             }
         }
 
         /* If we're grouping by node, print its resources */
         if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             if (pcmk_is_set(show_opts, pcmk_show_brief)) {
                 GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);
 
                 if (rscs != NULL) {
                     uint32_t new_show_opts = (show_opts | pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
                     out->begin_list(out, NULL, NULL, "%s", str->str);
                     out->begin_list(out, NULL, NULL, "Resources");
 
                     pe__rscs_brief_output(out, rscs, new_show_opts);
 
                     out->end_list(out);
                     out->end_list(out);
 
                     g_list_free(rscs);
                 }
 
             } else {
                 GList *gIter2 = NULL;
 
                 out->begin_list(out, NULL, NULL, "%s", str->str);
                 out->begin_list(out, NULL, NULL, "Resources");
 
                 for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
                     pcmk_resource_t *rsc = (pcmk_resource_t *) gIter2->data;
 
                     show_opts |= pcmk_show_rsc_only;
                     out->message(out, crm_map_element_name(rsc->xml), show_opts,
                                  rsc, only_node, only_rsc);
                 }
 
                 out->end_list(out);
                 out->end_list(out);
             }
         } else {
             out->list_item(out, NULL, "%s", str->str);
         }
 
         g_string_free(str, TRUE);
         free(node_name);
     } else {
         char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
         out->begin_list(out, NULL, NULL, "Node: %s", node_name);
         free(node_name);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool", "GList *",
                   "GList *")
 static int
 node_xml(pcmk__output_t *out, va_list args) {
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
     bool full = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     if (full) {
         const char *node_type = "unknown";
         char *length_s = pcmk__itoa(g_list_length(node->details->running_rsc));
         int health = pe__node_health(node);
         const char *health_s = NULL;
         const char *feature_set;
 
         switch (node->details->type) {
             case pcmk_node_variant_cluster:
                 node_type = "member";
                 break;
             case pcmk_node_variant_remote:
                 node_type = "remote";
                 break;
             case node_ping:
                 node_type = "ping";
                 break;
         }
 
         if (health < 0) {
             health_s = "red";
         } else if (health == 0) {
             health_s = "yellow";
         } else {
             health_s = "green";
         }
 
         feature_set = get_node_feature_set(node);
 
         pe__name_and_nvpairs_xml(out, true, "node", 15,
                                  PCMK_XA_NAME, node->details->uname,
                                  PCMK_XA_ID, node->details->id,
                                  "online", pcmk__btoa(node->details->online),
                                  "standby", pcmk__btoa(node->details->standby),
                                  "standby_onfail", pcmk__btoa(node->details->standby_onfail),
                                  "maintenance", pcmk__btoa(node->details->maintenance),
                                  "pending", pcmk__btoa(node->details->pending),
                                  "unclean", pcmk__btoa(node->details->unclean),
                                  "health", health_s,
                                  "feature_set", feature_set,
                                  "shutdown", pcmk__btoa(node->details->shutdown),
                                  "expected_up", pcmk__btoa(node->details->expected_up),
                                  "is_dc", pcmk__btoa(node->details->is_dc),
                                  "resources_running", length_s,
                                  PCMK_XA_TYPE, node_type);
 
         if (pe__is_guest_node(node)) {
             xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
             crm_xml_add(xml_node, "id_as_resource", node->details->remote_rsc->container->id);
         }
 
         if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             GList *lpc = NULL;
 
             for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
                 pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
 
                 show_opts |= pcmk_show_rsc_only;
                 out->message(out, crm_map_element_name(rsc->xml), show_opts,
                              rsc, only_node, only_rsc);
             }
         }
 
         free(length_s);
 
         out->end_list(out);
     } else {
         pcmk__output_xml_create_parent(out, "node",
                                        PCMK_XA_NAME, node->details->uname,
                                        NULL);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "bool", "int")
 static int
 node_attribute_text(pcmk__output_t *out, va_list args) {
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
     bool add_extra = va_arg(args, int);
     int expected_score = va_arg(args, int);
 
     if (add_extra) {
         int v;
 
         if (value == NULL) {
             v = 0;
         } else {
             pcmk__scan_min_int(value, &v, INT_MIN);
         }
         if (v <= 0) {
             out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
         } else if (v < expected_score) {
             out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
         } else {
             out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
         }
     } else {
         out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "bool", "int")
 static int
 node_attribute_html(pcmk__output_t *out, va_list args) {
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
     bool add_extra = va_arg(args, int);
     int expected_score = va_arg(args, int);
 
     if (add_extra) {
         int v;
         char *s = crm_strdup_printf("%s: %s", name, value);
         xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);
 
         if (value == NULL) {
             v = 0;
         } else {
             pcmk__scan_min_int(value, &v, INT_MIN);
         }
 
         pcmk_create_html_node(item_node, "span", NULL, NULL, s);
         free(s);
 
         if (v <= 0) {
             pcmk_create_html_node(item_node, "span", NULL, "bold", "(connectivity is lost)");
         } else if (v < expected_score) {
             char *buf = crm_strdup_printf("(connectivity is degraded -- expected %d", expected_score);
             pcmk_create_html_node(item_node, "span", NULL, "bold", buf);
             free(buf);
         }
     } else {
         out->list_item(out, NULL, "%s: %s", name, value);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
 static int
 node_and_op(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
 
     pcmk_resource_t *rsc = NULL;
     gchar *node_str = NULL;
     char *last_change_str = NULL;
 
     const char *op_rsc = crm_element_value(xml_op, "resource");
     int status;
     time_t last_change = 0;
 
     pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
                        &status, PCMK_EXEC_UNKNOWN);
 
     rsc = pe_find_resource(scheduler->resources, op_rsc);
 
     if (rsc) {
         const pcmk_node_t *node = pe__current_node(rsc);
         const char *target_role = g_hash_table_lookup(rsc->meta,
                                                       PCMK_META_TARGET_ROLE);
         uint32_t show_opts = pcmk_show_rsc_only | pcmk_show_pending;
 
         if (node == NULL) {
             node = rsc->pending_node;
         }
 
         node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
                                               show_opts, target_role, false);
     } else {
         node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
     }
 
     if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                 &last_change) == pcmk_ok) {
         last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
                                             XML_RSC_OP_LAST_CHANGE,
                                             pcmk__trim(ctime(&last_change)),
                                             crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
     }
 
     out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
                    node_str, pe__xe_history_key(xml_op),
                    crm_element_value(xml_op, XML_ATTR_UNAME),
                    crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
                    crm_element_value(xml_op, XML_LRM_ATTR_RC),
                    last_change_str ? last_change_str : "",
                    pcmk_exec_status_str(status));
 
     g_free(node_str);
     free(last_change_str);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNodePtr")
 static int
 node_and_op_xml(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
 
     pcmk_resource_t *rsc = NULL;
     const char *op_rsc = crm_element_value(xml_op, "resource");
     int status;
     time_t last_change = 0;
     xmlNode *node = NULL;
 
     pcmk__scan_min_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS),
                        &status, PCMK_EXEC_UNKNOWN);
     node = pcmk__output_create_xml_node(out, "operation",
                                         "op", pe__xe_history_key(xml_op),
                                         "node", crm_element_value(xml_op, XML_ATTR_UNAME),
                                         "call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
                                         "rc", crm_element_value(xml_op, XML_LRM_ATTR_RC),
                                         "status", pcmk_exec_status_str(status),
                                         NULL);
 
     rsc = pe_find_resource(scheduler->resources, op_rsc);
 
     if (rsc) {
         const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
+        const char *provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
         const char *kind = crm_element_value(rsc->xml, PCMK_XA_TYPE);
-        char *agent_tuple = NULL;
+        bool has_provider = pcmk_is_set(pcmk_get_ra_caps(class),
+                                        pcmk_ra_cap_provider);
 
-        agent_tuple = crm_strdup_printf("%s:%s:%s", class,
-                                        pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider) ? crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER) : "",
-                                        kind);
+        char *agent_tuple = crm_strdup_printf("%s:%s:%s",
+                                              class,
+                                              (has_provider? provider : ""),
+                                              kind);
 
         pcmk__xe_set_props(node, "rsc", rsc_printable_id(rsc),
                            "agent", agent_tuple,
                            NULL);
         free(agent_tuple);
     }
 
     if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                 &last_change) == pcmk_ok) {
         pcmk__xe_set_props(node, XML_RSC_OP_LAST_CHANGE,
                            pcmk__trim(ctime(&last_change)),
                            XML_RSC_OP_T_EXEC, crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
                            NULL);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "bool", "int")
 static int
 node_attribute_xml(pcmk__output_t *out, va_list args) {
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
     bool add_extra = va_arg(args, int);
     int expected_score = va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "attribute",
                                                    PCMK_XA_NAME, name,
                                                    PCMK_XA_VALUE, value,
                                                    NULL);
 
     if (add_extra) {
         char *buf = pcmk__itoa(expected_score);
         crm_xml_add(node, "expected", buf);
         free(buf);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
                   "bool", "GList *", "GList *")
 static int
 node_attribute_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     int rc = pcmk_rc_no_output;
 
     /* Display each node's attributes */
     for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = gIter->data;
 
         GList *attr_list = NULL;
         GHashTableIter iter;
         gpointer key;
 
         if (!node || !node->details || !node->details->online) {
             continue;
         }
 
         g_hash_table_iter_init(&iter, node->details->attrs);
         while (g_hash_table_iter_next (&iter, &key, NULL)) {
             attr_list = filter_attr_list(attr_list, key);
         }
 
         if (attr_list == NULL) {
             continue;
         }
 
         if (!pcmk__str_in_list(node->details->uname, only_node, pcmk__str_star_matches|pcmk__str_casei)) {
             g_list_free(attr_list);
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");
 
         out->message(out, "node", node, show_opts, false, only_node, only_rsc);
 
         for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
             const char *name = aIter->data;
             const char *value = NULL;
             int expected_score = 0;
             bool add_extra = false;
 
             value = pe_node_attribute_raw(node, name);
 
             add_extra = add_extra_info(node, node->details->running_rsc,
                                        scheduler, name, &expected_score);
 
             /* Print attribute name and value */
             out->message(out, "node-attribute", name, value, add_extra,
                          expected_score);
         }
 
         g_list_free(attr_list);
         out->end_list(out);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t *", "const char *")
 static int
 node_capacity(pcmk__output_t *out, va_list args)
 {
     const pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *comment = va_arg(args, const char *);
 
     char *dump_text = crm_strdup_printf("%s: %s capacity:",
                                         comment, pe__node_name(node));
 
     g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
     out->list_item(out, NULL, "%s", dump_text);
     free(dump_text);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t *", "const char *")
 static int
 node_capacity_xml(pcmk__output_t *out, va_list args)
 {
     const pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *comment = va_arg(args, const char *);
 
     xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "capacity",
                                                        "node", node->details->uname,
                                                        "comment", comment,
                                                        NULL);
     g_hash_table_foreach(node->details->utilization, add_dump_node, xml_node);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t *", "pcmk_node_t *",
                   "xmlNodePtr", "GList *", "GList *", "uint32_t", "uint32_t")
 static int
 node_history_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     xmlNode *node_state = va_arg(args, xmlNode *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     xmlNode *lrm_rsc = NULL;
     xmlNode *rsc_entry = NULL;
     int rc = pcmk_rc_no_output;
 
     lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
     lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
 
     /* Print history of each of the node's resources */
     for (rsc_entry = first_named_child(lrm_rsc, XML_LRM_TAG_RESOURCE);
          rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {
         const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
         pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
         const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
 
         /* We can't use is_filtered here to filter group resources.  For is_filtered,
          * we have to decide whether to check the parent or not.  If we check the
          * parent, all elements of a group will always be printed because that's how
          * is_filtered works for groups.  If we do not check the parent, sometimes
          * this will filter everything out.
          *
          * For other resource types, is_filtered is okay.
          */
         if (parent->variant == pcmk_rsc_variant_group) {
             if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
                                    pcmk__str_star_matches)
                 && !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
                                       pcmk__str_star_matches)) {
                 continue;
             }
         } else {
             if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
                 continue;
             }
         }
 
         if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
             time_t last_failure = 0;
             int failcount = pe_get_failcount(node, rsc, &last_failure,
                                              pcmk__fc_default, NULL);
 
             if (failcount <= 0) {
                 continue;
             }
 
             if (rc == pcmk_rc_no_output) {
                 rc = pcmk_rc_ok;
                 out->message(out, "node", node, show_opts, false, only_node,
                              only_rsc);
             }
 
             out->message(out, "resource-history", rsc, rsc_id, false,
                          failcount, last_failure, false);
         } else {
             GList *op_list = get_operation_list(rsc_entry);
             pcmk_resource_t *rsc = NULL;
 
             if (op_list == NULL) {
                 continue;
             }
 
             rsc = pe_find_resource(scheduler->resources,
                                    crm_element_value(rsc_entry, PCMK_XA_ID));
 
             if (rc == pcmk_rc_no_output) {
                 rc = pcmk_rc_ok;
                 out->message(out, "node", node, show_opts, false, only_node,
                              only_rsc);
             }
 
             out->message(out, "resource-operation-list", scheduler, rsc, node,
                          op_list, show_opts);
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-list", "GList *", "GList *", "GList *", "uint32_t", "bool")
 static int
 node_list_html(pcmk__output_t *out, va_list args) {
     GList *nodes = va_arg(args, GList *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer G_GNUC_UNUSED = va_arg(args, int);
 
     int rc = pcmk_rc_no_output;
 
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) gIter->data;
 
         if (!pcmk__str_in_list(node->details->uname, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");
 
         out->message(out, "node", node, show_opts, true, only_node, only_rsc);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-list", "GList *", "GList *", "GList *", "uint32_t", "bool")
 static int
 node_list_text(pcmk__output_t *out, va_list args) {
     GList *nodes = va_arg(args, GList *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     /* space-separated lists of node names */
     GString *online_nodes = NULL;
     GString *online_remote_nodes = NULL;
     GString *online_guest_nodes = NULL;
     GString *offline_nodes = NULL;
     GString *offline_remote_nodes = NULL;
 
     int rc = pcmk_rc_no_output;
 
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) gIter->data;
         char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
 
         if (!pcmk__str_in_list(node->details->uname, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             free(node_name);
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");
 
         // Determine whether to display node individually or in a list
         if (node->details->unclean || node->details->pending
             || (node->details->standby_onfail && node->details->online)
             || node->details->standby || node->details->maintenance
             || pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
             || pcmk_is_set(show_opts, pcmk_show_feature_set)
             || (pe__node_health(node) <= 0)) {
             // Display node individually
 
         } else if (node->details->online) {
             // Display online node in a list
             if (pe__is_guest_node(node)) {
                 pcmk__add_word(&online_guest_nodes, 1024, node_name);
 
             } else if (pe__is_remote_node(node)) {
                 pcmk__add_word(&online_remote_nodes, 1024, node_name);
 
             } else {
                 pcmk__add_word(&online_nodes, 1024, node_name);
             }
             free(node_name);
             continue;
 
         } else {
             // Display offline node in a list
             if (pe__is_remote_node(node)) {
                 pcmk__add_word(&offline_remote_nodes, 1024, node_name);
 
             } else if (pe__is_guest_node(node)) {
                 /* ignore offline guest nodes */
 
             } else {
                 pcmk__add_word(&offline_nodes, 1024, node_name);
             }
             free(node_name);
             continue;
         }
 
         /* If we get here, node is in bad state, or we're grouping by node */
         out->message(out, "node", node, show_opts, true, only_node, only_rsc);
         free(node_name);
     }
 
     /* If we're not grouping by node, summarize nodes by status */
     if (online_nodes != NULL) {
         out->list_item(out, "Online", "[ %s ]",
                        (const char *) online_nodes->str);
         g_string_free(online_nodes, TRUE);
     }
     if (offline_nodes != NULL) {
         out->list_item(out, "OFFLINE", "[ %s ]",
                        (const char *) offline_nodes->str);
         g_string_free(offline_nodes, TRUE);
     }
     if (online_remote_nodes) {
         out->list_item(out, "RemoteOnline", "[ %s ]",
                        (const char *) online_remote_nodes->str);
         g_string_free(online_remote_nodes, TRUE);
     }
     if (offline_remote_nodes) {
         out->list_item(out, "RemoteOFFLINE", "[ %s ]",
                        (const char *) offline_remote_nodes->str);
         g_string_free(offline_remote_nodes, TRUE);
     }
     if (online_guest_nodes != NULL) {
         out->list_item(out, "GuestOnline", "[ %s ]",
                        (const char *) online_guest_nodes->str);
         g_string_free(online_guest_nodes, TRUE);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-list", "GList *", "GList *", "GList *", "uint32_t", "bool")
 static int
 node_list_xml(pcmk__output_t *out, va_list args) {
     GList *nodes = va_arg(args, GList *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer G_GNUC_UNUSED = va_arg(args, int);
 
     out->begin_list(out, NULL, NULL, "nodes");
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) gIter->data;
 
         if (!pcmk__str_in_list(node->details->uname, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         out->message(out, "node", node, show_opts, true, only_node, only_rsc);
     }
     out->end_list(out);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t *", "GList *", "GList *",
                   "uint32_t", "uint32_t", "bool")
 static int
 node_summary(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     xmlNode *node_state = NULL;
     xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
                                                 XML_CIB_TAG_STATUS);
     int rc = pcmk_rc_no_output;
 
     if (xmlChildElementCount(cib_status) == 0) {
         return rc;
     }
 
     for (node_state = first_named_child(cib_status, XML_CIB_TAG_STATE);
          node_state != NULL; node_state = crm_next_same_xml(node_state)) {
         pcmk_node_t *node = pe_find_node_id(scheduler->nodes, ID(node_state));
 
         if (!node || !node->details || !node->details->online) {
             continue;
         }
 
         if (!pcmk__str_in_list(node->details->uname, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
                                  pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");
 
         out->message(out, "node-history-list", scheduler, node, node_state,
                      only_node, only_rsc, section_opts, show_opts);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t *", "const char *",
                   "const char *", "const char *")
 static int
 node_weight(pcmk__output_t *out, va_list args)
 {
     const pcmk_resource_t *rsc = va_arg(args, const pcmk_resource_t *);
     const char *prefix = va_arg(args, const char *);
     const char *uname = va_arg(args, const char *);
     const char *score = va_arg(args, const char *);
 
     if (rsc) {
         out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
                        prefix, rsc->id, uname, score);
     } else {
         out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t *", "const char *",
                   "const char *", "const char *")
 static int
 node_weight_xml(pcmk__output_t *out, va_list args)
 {
     const pcmk_resource_t *rsc = va_arg(args, const pcmk_resource_t *);
     const char *prefix = va_arg(args, const char *);
     const char *uname = va_arg(args, const char *);
     const char *score = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "node_weight",
                                                    "function", prefix,
                                                    "node", uname,
                                                    "score", score,
                                                    NULL);
 
     if (rsc) {
         crm_xml_add(node, PCMK_XA_ID, rsc->id);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char *", "const char *", "int", "uint32_t")
 static int
 op_history_text(pcmk__output_t *out, va_list args) {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     const char *task = va_arg(args, const char *);
     const char *interval_ms_s = va_arg(args, const char *);
     int rc = va_arg(args, int);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
                                   pcmk_is_set(show_opts, pcmk_show_timing));
 
     out->list_item(out, NULL, "%s", buf);
 
     free(buf);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("op-history", "xmlNodePtr", "const char *", "const char *", "int", "uint32_t")
 static int
 op_history_xml(pcmk__output_t *out, va_list args) {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     const char *task = va_arg(args, const char *);
     const char *interval_ms_s = va_arg(args, const char *);
     int rc = va_arg(args, int);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     char *rc_s = pcmk__itoa(rc);
     xmlNodePtr node = pcmk__output_create_xml_node(out, "operation_history",
                                                    "call", crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
                                                    "task", task,
                                                    "rc", rc_s,
                                                    "rc_text", services_ocf_exitcode_str(rc),
                                                    NULL);
     free(rc_s);
 
     if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
         char *s = crm_strdup_printf("%sms", interval_ms_s);
         crm_xml_add(node, "interval", s);
         free(s);
     }
 
     if (pcmk_is_set(show_opts, pcmk_show_timing)) {
         const char *value = NULL;
         time_t epoch = 0;
 
         if ((crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                      &epoch) == pcmk_ok) && (epoch > 0)) {
             char *s = pcmk__epoch2str(&epoch, 0);
             crm_xml_add(node, XML_RSC_OP_LAST_CHANGE, s);
             free(s);
         }
 
         value = crm_element_value(xml_op, XML_RSC_OP_T_EXEC);
         if (value) {
             char *s = crm_strdup_printf("%sms", value);
             crm_xml_add(node, XML_RSC_OP_T_EXEC, s);
             free(s);
         }
         value = crm_element_value(xml_op, XML_RSC_OP_T_QUEUE);
         if (value) {
             char *s = crm_strdup_printf("%sms", value);
             crm_xml_add(node, XML_RSC_OP_T_QUEUE, s);
             free(s);
         }
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 promotion_score(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *child_rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *chosen = va_arg(args, pcmk_node_t *);
     const char *score = va_arg(args, const char *);
 
     out->list_item(out, NULL, "%s promotion score on %s: %s",
                    child_rsc->id,
                    chosen? chosen->details->uname : "none",
                    score);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 promotion_score_xml(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *child_rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *chosen = va_arg(args, pcmk_node_t *);
     const char *score = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "promotion_score",
                                                    PCMK_XA_ID, child_rsc->id,
                                                    "score", score,
                                                    NULL);
 
     if (chosen) {
         crm_xml_add(node, "node", chosen->details->uname);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
 static int
 resource_config(pcmk__output_t *out, va_list args) {
     const pcmk_resource_t *rsc = va_arg(args, const pcmk_resource_t *);
     bool raw = va_arg(args, int);
 
     char *rsc_xml = formatted_xml_buf(rsc, raw);
 
     out->output_xml(out, "xml", rsc_xml);
 
     free(rsc_xml);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
 static int
 resource_config_text(pcmk__output_t *out, va_list args) {
     const pcmk_resource_t *rsc = va_arg(args, const pcmk_resource_t *);
     bool raw = va_arg(args, int);
 
     char *rsc_xml = formatted_xml_buf(rsc, raw);
 
     pcmk__formatted_printf(out, "Resource XML:\n");
     out->output_xml(out, "xml", rsc_xml);
 
     free(rsc_xml);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t *", "const char *",
                   "bool", "int", "time_t", "bool")
 static int
 resource_history_text(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *rsc_id = va_arg(args, const char *);
     bool all = va_arg(args, int);
     int failcount = va_arg(args, int);
     time_t last_failure = va_arg(args, time_t);
     bool as_header = va_arg(args, int);
 
     char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);
 
     if (as_header) {
         out->begin_list(out, NULL, NULL, "%s", buf);
     } else {
         out->list_item(out, NULL, "%s", buf);
     }
 
     free(buf);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t *", "const char *",
                   "bool", "int", "time_t", "bool")
 static int
 resource_history_xml(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *rsc_id = va_arg(args, const char *);
     bool all = va_arg(args, int);
     int failcount = va_arg(args, int);
     time_t last_failure = va_arg(args, time_t);
     bool as_header = va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource_history",
                                                      PCMK_XA_ID, rsc_id,
                                                      NULL);
 
     if (rsc == NULL) {
         pcmk__xe_set_bool_attr(node, "orphan", true);
     } else if (all || failcount || last_failure > 0) {
         char *migration_s = pcmk__itoa(rsc->migration_threshold);
 
         pcmk__xe_set_props(node, "orphan", "false",
                            PCMK_META_MIGRATION_THRESHOLD, migration_s,
                            NULL);
         free(migration_s);
 
         if (failcount > 0) {
             char *s = pcmk__itoa(failcount);
 
             crm_xml_add(node, PCMK__FAIL_COUNT_PREFIX, s);
             free(s);
         }
 
         if (last_failure > 0) {
             char *s = pcmk__epoch2str(&last_failure, 0);
 
             crm_xml_add(node, PCMK__LAST_FAILURE_PREFIX, s);
             free(s);
         }
     }
 
     if (!as_header) {
         pcmk__output_xml_pop_parent(out);
     }
 
     return pcmk_rc_ok;
 }
 
 static void
 print_resource_header(pcmk__output_t *out, uint32_t show_opts)
 {
     if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
         /* Active resources have already been printed by node */
         out->begin_list(out, NULL, NULL, "Inactive Resources");
     } else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
         out->begin_list(out, NULL, NULL, "Full List of Resources");
     } else {
         out->begin_list(out, NULL, NULL, "Active Resources");
     }
 }
 
 
 PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
                   "GList *", "GList *", "bool")
 static int
 resource_list(pcmk__output_t *out, va_list args)
 {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_summary = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     bool print_spacer = va_arg(args, int);
 
     GList *rsc_iter;
     int rc = pcmk_rc_no_output;
     bool printed_header = false;
 
     /* If we already showed active resources by node, and
      * we're not showing inactive resources, we have nothing to do
      */
     if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
         !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
         return rc;
     }
 
     /* If we haven't already printed resources grouped by node,
      * and brief output was requested, print resource summary */
     if (pcmk_is_set(show_opts, pcmk_show_brief)
         && !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
         GList *rscs = pe__filter_rsc_list(scheduler->resources, only_rsc);
 
         PCMK__OUTPUT_SPACER_IF(out, print_spacer);
         print_resource_header(out, show_opts);
         printed_header = true;
 
         rc = pe__rscs_brief_output(out, rscs, show_opts);
         g_list_free(rscs);
     }
 
     /* For each resource, display it if appropriate */
     for (rsc_iter = scheduler->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) rsc_iter->data;
         int x;
 
         /* Complex resources may have some sub-resources active and some inactive */
         gboolean is_active = rsc->fns->active(rsc, TRUE);
         gboolean partially_active = rsc->fns->active(rsc, FALSE);
 
         /* Skip inactive orphans (deleted but still in CIB) */
         if (pcmk_is_set(rsc->flags, pcmk_rsc_removed) && !is_active) {
             continue;
 
         /* Skip active resources if we already displayed them by node */
         } else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             if (is_active) {
                 continue;
             }
 
         /* Skip primitives already counted in a brief summary */
         } else if (pcmk_is_set(show_opts, pcmk_show_brief)
                    && (rsc->variant == pcmk_rsc_variant_primitive)) {
             continue;
 
         /* Skip resources that aren't at least partially active,
          * unless we're displaying inactive resources
          */
         } else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
             continue;
 
         } else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
             continue;
         }
 
         if (!printed_header) {
             PCMK__OUTPUT_SPACER_IF(out, print_spacer);
             print_resource_header(out, show_opts);
             printed_header = true;
         }
 
         /* Print this resource */
         x = out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc,
                          only_node, only_rsc);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     if (print_summary && rc != pcmk_rc_ok) {
         if (!printed_header) {
             PCMK__OUTPUT_SPACER_IF(out, print_spacer);
             print_resource_header(out, show_opts);
             printed_header = true;
         }
 
         if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             out->list_item(out, NULL, "No inactive resources");
         } else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
             out->list_item(out, NULL, "No resources");
         } else {
             out->list_item(out, NULL, "No active resources");
         }
     }
 
     if (printed_header) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
                   "pcmk_resource_t *", "pcmk_node_t *", "GList *", "uint32_t")
 static int
 resource_operation_list(pcmk__output_t *out, va_list args)
 {
     pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
                                                        pcmk_scheduler_t *);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     GList *op_list = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     GList *gIter = NULL;
     int rc = pcmk_rc_no_output;
 
     /* Print each operation */
     for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *xml_op = (xmlNode *) gIter->data;
         const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
         const char *interval_ms_s = crm_element_value(xml_op,
                                                       XML_LRM_ATTR_INTERVAL_MS);
         const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
         int op_rc_i;
 
         pcmk__scan_min_int(op_rc, &op_rc_i, 0);
 
         /* Display 0-interval monitors as "probe" */
         if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
             && pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches | pcmk__str_casei)) {
             task = "probe";
         }
 
         /* If this is the first printed operation, print heading for resource */
         if (rc == pcmk_rc_no_output) {
             time_t last_failure = 0;
             int failcount = pe_get_failcount(node, rsc, &last_failure,
                                              pcmk__fc_default, NULL);
 
             out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
                          failcount, last_failure, true);
             rc = pcmk_rc_ok;
         }
 
         /* Print the operation */
         out->message(out, "op-history", xml_op, task, interval_ms_s,
                      op_rc_i, show_opts);
     }
 
     /* Free the list we created (no need to free the individual items) */
     g_list_free(op_list);
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 resource_util(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *fn = va_arg(args, const char *);
 
     char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
                                         fn, rsc->id, pe__node_name(node));
 
     g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
     out->list_item(out, NULL, "%s", dump_text);
     free(dump_text);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 resource_util_xml(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *fn = va_arg(args, const char *);
 
     xmlNodePtr xml_node = pcmk__output_create_xml_node(out, "utilization",
                                                        "resource", rsc->id,
                                                        "node", node->details->uname,
                                                        "function", fn,
                                                        NULL);
     g_hash_table_foreach(rsc->utilization, add_dump_node, xml_node);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
 static int
 ticket_html(pcmk__output_t *out, va_list args) {
     pcmk_ticket_t *ticket = va_arg(args, pcmk_ticket_t *);
 
     if (ticket->last_granted > -1) {
         char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);
 
         out->list_item(out, NULL, "%s:\t%s%s %s=\"%s\"", ticket->id,
                        ticket->granted ? "granted" : "revoked",
                        ticket->standby ? " [standby]" : "",
                        "last-granted", pcmk__s(epoch_str, ""));
         free(epoch_str);
     } else {
         out->list_item(out, NULL, "%s:\t%s%s", ticket->id,
                        ticket->granted ? "granted" : "revoked",
                        ticket->standby ? " [standby]" : "");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
 static int
 ticket_text(pcmk__output_t *out, va_list args) {
     pcmk_ticket_t *ticket = va_arg(args, pcmk_ticket_t *);
 
     if (ticket->last_granted > -1) {
         char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);
 
         out->list_item(out, ticket->id, "%s%s %s=\"%s\"",
                        ticket->granted ? "granted" : "revoked",
                        ticket->standby ? " [standby]" : "",
                        "last-granted", pcmk__s(epoch_str, ""));
         free(epoch_str);
     } else {
         out->list_item(out, ticket->id, "%s%s",
                        ticket->granted ? "granted" : "revoked",
                        ticket->standby ? " [standby]" : "");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ticket", "pcmk_ticket_t *")
 static int
 ticket_xml(pcmk__output_t *out, va_list args) {
     pcmk_ticket_t *ticket = va_arg(args, pcmk_ticket_t *);
 
     xmlNodePtr node = NULL;
 
     node = pcmk__output_create_xml_node(out, "ticket",
                                         PCMK_XA_ID, ticket->id,
                                         "status", ticket->granted ? "granted" : "revoked",
                                         "standby", pcmk__btoa(ticket->standby),
                                         NULL);
 
     if (ticket->last_granted > -1) {
         char *buf = pcmk__epoch2str(&ticket->last_granted, 0);
 
         crm_xml_add(node, "last-granted", buf);
         free(buf);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ticket-list", "pcmk_scheduler_t *", "bool")
 static int
 ticket_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     bool print_spacer = va_arg(args, int);
 
     GHashTableIter iter;
     gpointer key, value;
 
     if (g_hash_table_size(scheduler->tickets) == 0) {
         return pcmk_rc_no_output;
     }
 
     PCMK__OUTPUT_SPACER_IF(out, print_spacer);
 
     /* Print section heading */
     out->begin_list(out, NULL, NULL, "Tickets");
 
     /* Print each ticket */
     g_hash_table_iter_init(&iter, scheduler->tickets);
     while (g_hash_table_iter_next(&iter, &key, &value)) {
         pcmk_ticket_t *ticket = (pcmk_ticket_t *) value;
         out->message(out, "ticket", ticket);
     }
 
     /* Close section */
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 static pcmk__message_entry_t fmt_functions[] = {
     { "ban", "default", ban_text },
     { "ban", "html", ban_html },
     { "ban", "xml", ban_xml },
     { "ban-list", "default", ban_list },
     { "bundle", "default", pe__bundle_text },
     { "bundle", "xml",  pe__bundle_xml },
     { "bundle", "html",  pe__bundle_html },
     { "clone", "default", pe__clone_default },
     { "clone", "xml",  pe__clone_xml },
     { "cluster-counts", "default", cluster_counts_text },
     { "cluster-counts", "html", cluster_counts_html },
     { "cluster-counts", "xml", cluster_counts_xml },
     { "cluster-dc", "default", cluster_dc_text },
     { "cluster-dc", "html", cluster_dc_html },
     { "cluster-dc", "xml", cluster_dc_xml },
     { "cluster-options", "default", cluster_options_text },
     { "cluster-options", "html", cluster_options_html },
     { "cluster-options", "log", cluster_options_log },
     { "cluster-options", "xml", cluster_options_xml },
     { "cluster-summary", "default", cluster_summary },
     { "cluster-summary", "html", cluster_summary_html },
     { "cluster-stack", "default", cluster_stack_text },
     { "cluster-stack", "html", cluster_stack_html },
     { "cluster-stack", "xml", cluster_stack_xml },
     { "cluster-times", "default", cluster_times_text },
     { "cluster-times", "html", cluster_times_html },
     { "cluster-times", "xml", cluster_times_xml },
     { "failed-action", "default", failed_action_default },
     { "failed-action", "xml", failed_action_xml },
     { "failed-action-list", "default", failed_action_list },
     { "group", "default",  pe__group_default},
     { "group", "xml",  pe__group_xml },
     { "maint-mode", "text", cluster_maint_mode_text },
     { "node", "default", node_text },
     { "node", "html", node_html },
     { "node", "xml", node_xml },
     { "node-and-op", "default", node_and_op },
     { "node-and-op", "xml", node_and_op_xml },
     { "node-capacity", "default", node_capacity },
     { "node-capacity", "xml", node_capacity_xml },
     { "node-history-list", "default", node_history_list },
     { "node-list", "default", node_list_text },
     { "node-list", "html", node_list_html },
     { "node-list", "xml", node_list_xml },
     { "node-weight", "default", node_weight },
     { "node-weight", "xml", node_weight_xml },
     { "node-attribute", "default", node_attribute_text },
     { "node-attribute", "html", node_attribute_html },
     { "node-attribute", "xml", node_attribute_xml },
     { "node-attribute-list", "default", node_attribute_list },
     { "node-summary", "default", node_summary },
     { "op-history", "default", op_history_text },
     { "op-history", "xml", op_history_xml },
     { "primitive", "default",  pe__resource_text },
     { "primitive", "xml",  pe__resource_xml },
     { "primitive", "html",  pe__resource_html },
     { "promotion-score", "default", promotion_score },
     { "promotion-score", "xml", promotion_score_xml },
     { "resource-config", "default", resource_config },
     { "resource-config", "text", resource_config_text },
     { "resource-history", "default", resource_history_text },
     { "resource-history", "xml", resource_history_xml },
     { "resource-list", "default", resource_list },
     { "resource-operation-list", "default", resource_operation_list },
     { "resource-util", "default", resource_util },
     { "resource-util", "xml", resource_util_xml },
     { "ticket", "default", ticket_text },
     { "ticket", "html", ticket_html },
     { "ticket", "xml", ticket_xml },
     { "ticket-list", "default", ticket_list },
 
     { NULL, NULL, NULL }
 };
 
 void
 pe__register_messages(pcmk__output_t *out) {
     pcmk__register_messages(out, fmt_functions);
 }
diff --git a/lib/pengine/remote.c b/lib/pengine/remote.c
index 41c6bd064e..0fcfdc0c9a 100644
--- a/lib/pengine/remote.c
+++ b/lib/pengine/remote.c
@@ -1,275 +1,275 @@
 /*
  * Copyright 2013-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/scheduler_internal.h>
 #include <crm/pengine/internal.h>
 #include <glib.h>
 
 bool
 pe__resource_is_remote_conn(const pcmk_resource_t *rsc)
 {
     return (rsc != NULL) && rsc->is_remote_node
            && pe__is_remote_node(pe_find_node(rsc->cluster->nodes, rsc->id));
 }
 
 bool
 pe__is_remote_node(const pcmk_node_t *node)
 {
     return (node != NULL) && (node->details->type == pcmk_node_variant_remote)
            && ((node->details->remote_rsc == NULL)
                || (node->details->remote_rsc->container == NULL));
 }
 
 bool
 pe__is_guest_node(const pcmk_node_t *node)
 {
     return (node != NULL) && (node->details->type == pcmk_node_variant_remote)
            && (node->details->remote_rsc != NULL)
            && (node->details->remote_rsc->container != NULL);
 }
 
 bool
 pe__is_guest_or_remote_node(const pcmk_node_t *node)
 {
     return (node != NULL) && (node->details->type == pcmk_node_variant_remote);
 }
 
 bool
 pe__is_bundle_node(const pcmk_node_t *node)
 {
     return pe__is_guest_node(node)
            && pe_rsc_is_bundled(node->details->remote_rsc);
 }
 
 /*!
  * \internal
  * \brief Check whether a resource creates a guest node
  *
  * If a given resource contains a filler resource that is a remote connection,
  * return that filler resource (or NULL if none is found).
  *
  * \param[in] scheduler  Scheduler data
  * \param[in] rsc        Resource to check
  *
  * \return Filler resource with remote connection, or NULL if none found
  */
 pcmk_resource_t *
 pe__resource_contains_guest_node(const pcmk_scheduler_t *scheduler,
                                  const pcmk_resource_t *rsc)
 {
     if ((rsc != NULL) && (scheduler != NULL)
         && pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
 
         for (GList *gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
             pcmk_resource_t *filler = gIter->data;
 
             if (filler->is_remote_node) {
                 return filler;
             }
         }
     }
     return NULL;
 }
 
 bool
 xml_contains_remote_node(xmlNode *xml)
 {
     const char *value = NULL;
 
     if (xml == NULL) {
         return false;
     }
 
     value = crm_element_value(xml, PCMK_XA_TYPE);
     if (!pcmk__str_eq(value, "remote", pcmk__str_casei)) {
         return false;
     }
 
     value = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
     if (!pcmk__str_eq(value, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
         return false;
     }
 
-    value = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
+    value = crm_element_value(xml, PCMK_XA_PROVIDER);
     if (!pcmk__str_eq(value, "pacemaker", pcmk__str_casei)) {
         return false;
     }
 
     return true;
 }
 
 /*!
  * \internal
  * \brief Execute a supplied function for each guest node running on a host
  *
  * \param[in]     scheduler  Scheduler data
  * \param[in]     host       Host node to check
  * \param[in]     helper     Function to call for each guest node
  * \param[in,out] user_data  Pointer to pass to helper function
  */
 void
 pe_foreach_guest_node(const pcmk_scheduler_t *scheduler,
                       const pcmk_node_t *host,
                       void (*helper)(const pcmk_node_t*, void*),
                       void *user_data)
 {
     GList *iter;
 
     CRM_CHECK(scheduler && host && host->details && helper, return);
     if (!pcmk_is_set(scheduler->flags, pcmk_sched_have_remote_nodes)) {
         return;
     }
     for (iter = host->details->running_rsc; iter != NULL; iter = iter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
         if (rsc->is_remote_node && (rsc->container != NULL)) {
             pcmk_node_t *guest_node = pe_find_node(scheduler->nodes, rsc->id);
 
             if (guest_node) {
                 (*helper)(guest_node, user_data);
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief Create CIB XML for an implicit remote connection
  *
  * \param[in,out] parent         If not \c NULL, use as parent XML element
  * \param[in]     uname          Name of Pacemaker Remote node
  * \param[in]     container_id   If not \c NULL, use this as connection container
  * \param[in]     migrateable    If not \c NULL, use as remote
  *                               \c PCMK_META_ALLOW_MIGRATE value
  * \param[in]     is_managed     If not \c NULL, use as remote
  *                               \c PCMK_META_IS_MANAGED value
  * \param[in]     start_timeout  If not \c NULL, use as remote connect timeout
  * \param[in]     server         If not \c NULL, use as \c PCMK_REMOTE_RA_ADDR
  * \param[in]     port           If not \c NULL, use as \c PCMK_REMOTE_RA_PORT
  *
  * \return Newly created XML
  */
 xmlNode *
 pe_create_remote_xml(xmlNode *parent, const char *uname,
                      const char *container_id, const char *migrateable,
                      const char *is_managed, const char *start_timeout,
                      const char *server, const char *port)
 {
     xmlNode *remote;
     xmlNode *xml_sub;
 
     remote = create_xml_node(parent, XML_CIB_TAG_RESOURCE);
 
     // Add identity
     crm_xml_add(remote, PCMK_XA_ID, uname);
     crm_xml_add(remote, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
-    crm_xml_add(remote, XML_AGENT_ATTR_PROVIDER, "pacemaker");
+    crm_xml_add(remote, PCMK_XA_PROVIDER, "pacemaker");
     crm_xml_add(remote, PCMK_XA_TYPE, "remote");
 
     // Add meta-attributes
     xml_sub = create_xml_node(remote, XML_TAG_META_SETS);
     crm_xml_set_id(xml_sub, "%s-%s", uname, XML_TAG_META_SETS);
     crm_create_nvpair_xml(xml_sub, NULL,
                           PCMK__META_INTERNAL_RSC, XML_BOOLEAN_TRUE);
     if (container_id) {
         crm_create_nvpair_xml(xml_sub, NULL,
                               PCMK__META_CONTAINER, container_id);
     }
     if (migrateable) {
         crm_create_nvpair_xml(xml_sub, NULL,
                               PCMK_META_ALLOW_MIGRATE, migrateable);
     }
     if (is_managed) {
         crm_create_nvpair_xml(xml_sub, NULL, PCMK_META_IS_MANAGED, is_managed);
     }
 
     // Add instance attributes
     if (port || server) {
         xml_sub = create_xml_node(remote, XML_TAG_ATTR_SETS);
         crm_xml_set_id(xml_sub, "%s-%s", uname, XML_TAG_ATTR_SETS);
         if (server) {
             crm_create_nvpair_xml(xml_sub, NULL, PCMK_REMOTE_RA_ADDR, server);
         }
         if (port) {
             crm_create_nvpair_xml(xml_sub, NULL, PCMK_REMOTE_RA_PORT, port);
         }
     }
 
     // Add operations
     xml_sub = create_xml_node(remote, "operations");
     crm_create_op_xml(xml_sub, uname, PCMK_ACTION_MONITOR, "30s", "30s");
     if (start_timeout) {
         crm_create_op_xml(xml_sub, uname, PCMK_ACTION_START, "0",
                           start_timeout);
     }
     return remote;
 }
 
 // History entry to be checked for fail count clearing
 struct check_op {
     const xmlNode *rsc_op;  // History entry XML
     pcmk_resource_t *rsc;   // Known resource corresponding to history entry
     pcmk_node_t *node;      // Known node corresponding to history entry
     enum pcmk__check_parameters check_type; // What needs checking
 };
 
 void
 pe__add_param_check(const xmlNode *rsc_op, pcmk_resource_t *rsc,
                     pcmk_node_t *node, enum pcmk__check_parameters flag,
                     pcmk_scheduler_t *scheduler)
 {
     struct check_op *check_op = NULL;
 
     CRM_CHECK(scheduler && rsc_op && rsc && node, return);
 
     check_op = calloc(1, sizeof(struct check_op));
     CRM_ASSERT(check_op != NULL);
 
     crm_trace("Deferring checks of %s until after allocation", ID(rsc_op));
     check_op->rsc_op = rsc_op;
     check_op->rsc = rsc;
     check_op->node = node;
     check_op->check_type = flag;
     scheduler->param_check = g_list_prepend(scheduler->param_check, check_op);
 }
 
 /*!
  * \internal
  * \brief Call a function for each action to be checked for addr substitution
  *
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     cb         Function to be called
  */
 void
 pe__foreach_param_check(pcmk_scheduler_t *scheduler,
                        void (*cb)(pcmk_resource_t*, pcmk_node_t*,
                                   const xmlNode*, enum pcmk__check_parameters))
 {
     CRM_CHECK(scheduler && cb, return);
 
     for (GList *item = scheduler->param_check;
          item != NULL; item = item->next) {
         struct check_op *check_op = item->data;
 
         cb(check_op->rsc, check_op->node, check_op->rsc_op,
            check_op->check_type);
     }
 }
 
 void
 pe__free_param_checks(pcmk_scheduler_t *scheduler)
 {
     if (scheduler && scheduler->param_check) {
         g_list_free_full(scheduler->param_check, free);
         scheduler->param_check = NULL;
     }
 }
diff --git a/lib/pengine/rules.c b/lib/pengine/rules.c
index 1c1640afdc..402734963a 100644
--- a/lib/pengine/rules.c
+++ b/lib/pengine/rules.c
@@ -1,1345 +1,1345 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <glib.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/rules_internal.h>
 #include <crm/pengine/internal.h>
 
 #include <sys/types.h>
 #include <regex.h>
 #include <ctype.h>
 
 CRM_TRACE_INIT_DATA(pe_rules);
 
 /*!
  * \brief Evaluate any rules contained by given XML element
  *
  * \param[in,out] xml          XML element to check for rules
  * \param[in]     node_hash    Node attributes to use to evaluate expressions
  * \param[in]     now          Time to use when evaluating expressions
  * \param[out]    next_change  If not NULL, set to when evaluation will change
  *
  * \return TRUE if no rules, or any of rules present is in effect, else FALSE
  */
 gboolean
 pe_evaluate_rules(xmlNode *ruleset, GHashTable *node_hash, crm_time_t *now,
                   crm_time_t *next_change)
 {
     pe_rule_eval_data_t rule_data = {
         .node_hash = node_hash,
         .role = pcmk_role_unknown,
         .now = now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     return pe_eval_rules(ruleset, &rule_data, next_change);
 }
 
 gboolean
 pe_test_rule(xmlNode *rule, GHashTable *node_hash, enum rsc_role_e role,
              crm_time_t *now, crm_time_t *next_change,
              pe_match_data_t *match_data)
 {
     pe_rule_eval_data_t rule_data = {
         .node_hash = node_hash,
         .role = role,
         .now = now,
         .match_data = match_data,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     return pe_eval_expr(rule, &rule_data, next_change);
 }
 
 /*!
  * \brief Evaluate one rule subelement (pass/fail)
  *
  * A rule element may contain another rule, a node attribute expression, or a
  * date expression. Given any one of those, evaluate it and return whether it
  * passed.
  *
  * \param[in,out] expr         Rule subelement XML
  * \param[in]     node_hash    Node attributes to use when evaluating expression
  * \param[in]     role         Resource role to use when evaluating expression
  * \param[in]     now          Time to use when evaluating expression
  * \param[out]    next_change  If not NULL, set to when evaluation will change
  * \param[in]     match_data   If not NULL, resource back-references and params
  *
  * \return TRUE if expression is in effect under given conditions, else FALSE
  */
 gboolean
 pe_test_expression(xmlNode *expr, GHashTable *node_hash, enum rsc_role_e role,
                    crm_time_t *now, crm_time_t *next_change,
                    pe_match_data_t *match_data)
 {
     pe_rule_eval_data_t rule_data = {
         .node_hash = node_hash,
         .role = role,
         .now = now,
         .match_data = match_data,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     return pe_eval_subexpr(expr, &rule_data, next_change);
 }
 
 enum expression_type
 find_expression_type(xmlNode * expr)
 {
     const char *attr = NULL;
 
     attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);
 
     if (pcmk__xe_is(expr, PCMK_XE_DATE_EXPRESSION)) {
         return pcmk__subexpr_datetime;
 
     } else if (pcmk__xe_is(expr, PCMK_XE_RSC_EXPRESSION)) {
         return pcmk__subexpr_resource;
 
     } else if (pcmk__xe_is(expr, PCMK_XE_OP_EXPRESSION)) {
         return pcmk__subexpr_operation;
 
     } else if (pcmk__xe_is(expr, XML_TAG_RULE)) {
         return pcmk__subexpr_rule;
 
     } else if (!pcmk__xe_is(expr, XML_TAG_EXPRESSION)) {
         return pcmk__subexpr_unknown;
 
     } else if (pcmk__str_any_of(attr, CRM_ATTR_UNAME, CRM_ATTR_KIND, CRM_ATTR_ID, NULL)) {
         return pcmk__subexpr_location;
 
     } else if (pcmk__str_eq(attr, CRM_ATTR_ROLE, pcmk__str_none)) {
         return pcmk__subexpr_role;
     }
 
     return pcmk__subexpr_attribute;
 }
 
 /* As per the nethack rules:
  *
  * moon period = 29.53058 days ~= 30, year = 365.2422 days
  * days moon phase advances on first day of year compared to preceding year
  *      = 365.2422 - 12*29.53058 ~= 11
  * years in Metonic cycle (time until same phases fall on the same days of
  *      the month) = 18.6 ~= 19
  * moon phase on first day of year (epact) ~= (11*(year%19) + 29) % 30
  *      (29 as initial condition)
  * current phase in days = first day phase + days elapsed in year
  * 6 moons ~= 177 days
  * 177 ~= 8 reported phases * 22
  * + 11/22 for rounding
  *
  * 0-7, with 0: new, 4: full
  */
 
 static int
 phase_of_the_moon(const crm_time_t *now)
 {
     uint32_t epact, diy, goldn;
     uint32_t y;
 
     crm_time_get_ordinal(now, &y, &diy);
 
     goldn = (y % 19) + 1;
     epact = (11 * goldn + 18) % 30;
     if ((epact == 25 && goldn > 11) || epact == 24)
         epact++;
 
     return ((((((diy + epact) * 6) + 11) % 177) / 22) & 7);
 }
 
 static int
 check_one(const xmlNode *cron_spec, const char *xml_field, uint32_t time_field)
 {
     int rc = pcmk_rc_undetermined;
     const char *value = crm_element_value(cron_spec, xml_field);
     long long low, high;
 
     if (value == NULL) {
         /* Return pe_date_result_undetermined if the field is missing. */
         goto bail;
     }
 
     if (pcmk__parse_ll_range(value, &low, &high) != pcmk_rc_ok) {
        goto bail;
     } else if (low == high) {
         /* A single number was given, not a range. */
         if (time_field < low) {
             rc = pcmk_rc_before_range;
         } else if (time_field > high) {
             rc = pcmk_rc_after_range;
         } else {
             rc = pcmk_rc_within_range;
         }
     } else if (low != -1 && high != -1) {
         /* This is a range with both bounds. */
         if (time_field < low) {
             rc = pcmk_rc_before_range;
         } else if (time_field > high) {
             rc = pcmk_rc_after_range;
         } else {
             rc = pcmk_rc_within_range;
         }
     } else if (low == -1) {
        /* This is a range with no starting value. */
         rc = time_field <= high ? pcmk_rc_within_range : pcmk_rc_after_range;
     } else if (high == -1) {
         /* This is a range with no ending value. */
         rc = time_field >= low ? pcmk_rc_within_range : pcmk_rc_before_range;
     }
 
 bail:
     if (rc == pcmk_rc_within_range) {
         crm_debug("Condition '%s' in %s: passed", value, xml_field);
     } else {
         crm_debug("Condition '%s' in %s: failed", value, xml_field);
     }
 
     return rc;
 }
 
 static gboolean
 check_passes(int rc) {
     /* _within_range is obvious.  _undetermined is a pass because
      * this is the return value if a field is not given.  In this
      * case, we just want to ignore it and check other fields to
      * see if they place some restriction on what can pass.
      */
     return rc == pcmk_rc_within_range || rc == pcmk_rc_undetermined;
 }
 
 #define CHECK_ONE(spec, name, var) do { \
     int subpart_rc = check_one(spec, name, var); \
     if (check_passes(subpart_rc) == FALSE) { \
         return subpart_rc; \
     } \
 } while (0)
 
 int
 pe_cron_range_satisfied(const crm_time_t *now, const xmlNode *cron_spec)
 {
     uint32_t h, m, s, y, d, w;
 
     CRM_CHECK(now != NULL, return pcmk_rc_op_unsatisfied);
 
     crm_time_get_gregorian(now, &y, &m, &d);
     CHECK_ONE(cron_spec, "years", y);
     CHECK_ONE(cron_spec, "months", m);
     CHECK_ONE(cron_spec, "monthdays", d);
 
     crm_time_get_timeofday(now, &h, &m, &s);
     CHECK_ONE(cron_spec, "hours", h);
     CHECK_ONE(cron_spec, "minutes", m);
     CHECK_ONE(cron_spec, "seconds", s);
 
     crm_time_get_ordinal(now, &y, &d);
     CHECK_ONE(cron_spec, "yeardays", d);
 
     crm_time_get_isoweek(now, &y, &w, &d);
     CHECK_ONE(cron_spec, "weekyears", y);
     CHECK_ONE(cron_spec, "weeks", w);
     CHECK_ONE(cron_spec, "weekdays", d);
 
     CHECK_ONE(cron_spec, "moon", phase_of_the_moon(now));
     if (crm_element_value(cron_spec, "moon") != NULL) {
         pcmk__config_warn("Support for 'moon' in date_spec elements "
                           "(such as %s) is deprecated and will be removed "
                           "in a future release of Pacemaker", ID(cron_spec));
     }
 
     /* If we get here, either no fields were specified (which is success), or all
      * the fields that were specified had their conditions met (which is also a
      * success).  Thus, the result is success.
      */
     return pcmk_rc_ok;
 }
 
 static void
 update_field(crm_time_t *t, const xmlNode *xml, const char *attr,
             void (*time_fn)(crm_time_t *, int))
 {
     long long value;
 
     if ((pcmk__scan_ll(crm_element_value(xml, attr), &value, 0LL) == pcmk_rc_ok)
         && (value != 0LL) && (value >= INT_MIN) && (value <= INT_MAX)) {
         time_fn(t, (int) value);
     }
 }
 
 static crm_time_t *
 parse_xml_duration(const crm_time_t *start, const xmlNode *duration_spec)
 {
     crm_time_t *end = pcmk_copy_time(start);
 
     update_field(end, duration_spec, "years", crm_time_add_years);
     update_field(end, duration_spec, "months", crm_time_add_months);
     update_field(end, duration_spec, "weeks", crm_time_add_weeks);
     update_field(end, duration_spec, "days", crm_time_add_days);
     update_field(end, duration_spec, "hours", crm_time_add_hours);
     update_field(end, duration_spec, "minutes", crm_time_add_minutes);
     update_field(end, duration_spec, "seconds", crm_time_add_seconds);
 
     return end;
 }
 
 // Set next_change to t if t is earlier
 static void
 crm_time_set_if_earlier(crm_time_t *next_change, crm_time_t *t)
 {
     if ((next_change != NULL) && (t != NULL)) {
         if (!crm_time_is_defined(next_change)
             || (crm_time_compare(t, next_change) < 0)) {
             crm_time_set(next_change, t);
         }
     }
 }
 
 // Information about a block of nvpair elements
 typedef struct sorted_set_s {
     int score;                  // This block's score for sorting
     const char *name;           // This block's ID
     const char *special_name;   // ID that should sort first
     xmlNode *attr_set;          // This block
     gboolean overwrite;         // Whether existing values will be overwritten
 } sorted_set_t;
 
 static gint
 sort_pairs(gconstpointer a, gconstpointer b)
 {
     const sorted_set_t *pair_a = a;
     const sorted_set_t *pair_b = b;
 
     if (a == NULL && b == NULL) {
         return 0;
     } else if (a == NULL) {
         return 1;
     } else if (b == NULL) {
         return -1;
     }
 
     if (pcmk__str_eq(pair_a->name, pair_a->special_name, pcmk__str_casei)) {
         return -1;
 
     } else if (pcmk__str_eq(pair_b->name, pair_a->special_name, pcmk__str_casei)) {
         return 1;
     }
 
     /* If we're overwriting values, we want lowest score first, so the highest
      * score is processed last; if we're not overwriting values, we want highest
      * score first, so nothing else overwrites it.
      */
     if (pair_a->score < pair_b->score) {
         return pair_a->overwrite? -1 : 1;
     } else if (pair_a->score > pair_b->score) {
         return pair_a->overwrite? 1 : -1;
     }
     return 0;
 }
 
 static void
 populate_hash(xmlNode * nvpair_list, GHashTable * hash, gboolean overwrite, xmlNode * top)
 {
     const char *name = NULL;
     const char *value = NULL;
     const char *old_value = NULL;
     xmlNode *list = nvpair_list;
     xmlNode *an_attr = NULL;
 
     if (pcmk__xe_is(list->children, XML_TAG_ATTRS)) {
         list = list->children;
     }
 
     for (an_attr = pcmk__xe_first_child(list); an_attr != NULL;
          an_attr = pcmk__xe_next(an_attr)) {
 
         if (pcmk__str_eq((const char *)an_attr->name, XML_CIB_TAG_NVPAIR, pcmk__str_none)) {
             xmlNode *ref_nvpair = expand_idref(an_attr, top);
 
             name = crm_element_value(an_attr, PCMK_XA_NAME);
             if ((name == NULL) && (ref_nvpair != NULL)) {
                 name = crm_element_value(ref_nvpair, PCMK_XA_NAME);
             }
 
             value = crm_element_value(an_attr, PCMK_XA_VALUE);
             if ((value == NULL) && (ref_nvpair != NULL)) {
                 value = crm_element_value(ref_nvpair, PCMK_XA_VALUE);
             }
 
             if (name == NULL || value == NULL) {
                 continue;
             }
 
             old_value = g_hash_table_lookup(hash, name);
 
             if (pcmk__str_eq(value, "#default", pcmk__str_casei)) {
                 if (old_value) {
                     crm_trace("Letting %s default (removing explicit value \"%s\")",
                               name, value);
                     g_hash_table_remove(hash, name);
                 }
                 continue;
 
             } else if (old_value == NULL) {
                 crm_trace("Setting %s=\"%s\"", name, value);
                 g_hash_table_insert(hash, strdup(name), strdup(value));
 
             } else if (overwrite) {
                 crm_trace("Setting %s=\"%s\" (overwriting old value \"%s\")",
                           name, value, old_value);
                 g_hash_table_replace(hash, strdup(name), strdup(value));
             }
         }
     }
 }
 
 typedef struct unpack_data_s {
     gboolean overwrite;
     void *hash;
     crm_time_t *next_change;
     const pe_rule_eval_data_t *rule_data;
     xmlNode *top;
 } unpack_data_t;
 
 static void
 unpack_attr_set(gpointer data, gpointer user_data)
 {
     sorted_set_t *pair = data;
     unpack_data_t *unpack_data = user_data;
 
     if (!pe_eval_rules(pair->attr_set, unpack_data->rule_data,
                        unpack_data->next_change)) {
         return;
     }
 
     crm_trace("Adding attributes from %s (score %d) %s overwrite",
               pair->name, pair->score,
               (unpack_data->overwrite? "with" : "without"));
     populate_hash(pair->attr_set, unpack_data->hash, unpack_data->overwrite, unpack_data->top);
 }
 
 /*!
  * \internal
  * \brief Create a sorted list of nvpair blocks
  *
  * \param[in,out] top           XML document root (used to expand id-ref's)
  * \param[in]     xml_obj       XML element containing blocks of nvpair elements
  * \param[in]     set_name      If not NULL, only get blocks of this element
  * \param[in]     always_first  If not NULL, sort block with this ID as first
  *
  * \return List of sorted_set_t entries for nvpair blocks
  */
 static GList *
 make_pairs(xmlNode *top, const xmlNode *xml_obj, const char *set_name,
            const char *always_first, gboolean overwrite)
 {
     GList *unsorted = NULL;
 
     if (xml_obj == NULL) {
         return NULL;
     }
     for (xmlNode *attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
          attr_set = pcmk__xe_next(attr_set)) {
 
         if (pcmk__str_eq(set_name, (const char *) attr_set->name,
                          pcmk__str_null_matches)) {
             const char *score = NULL;
             sorted_set_t *pair = NULL;
             xmlNode *expanded_attr_set = expand_idref(attr_set, top);
 
             if (expanded_attr_set == NULL) {
                 continue; // Not possible with schema validation enabled
             }
 
             pair = calloc(1, sizeof(sorted_set_t));
             pair->name = ID(expanded_attr_set);
             pair->special_name = always_first;
             pair->attr_set = expanded_attr_set;
             pair->overwrite = overwrite;
 
             score = crm_element_value(expanded_attr_set, XML_RULE_ATTR_SCORE);
             pair->score = char2score(score);
 
             unsorted = g_list_prepend(unsorted, pair);
         }
     }
     return g_list_sort(unsorted, sort_pairs);
 }
 
 /*!
  * \brief Extract nvpair blocks contained by an XML element into a hash table
  *
  * \param[in,out] top           XML document root (used to expand id-ref's)
  * \param[in]     xml_obj       XML element containing blocks of nvpair elements
  * \param[in]     set_name      If not NULL, only use blocks of this element
  * \param[in]     rule_data     Matching parameters to use when unpacking
  * \param[out]    hash          Where to store extracted name/value pairs
  * \param[in]     always_first  If not NULL, process block with this ID first
  * \param[in]     overwrite     Whether to replace existing values with same name
  * \param[out]    next_change   If not NULL, set to when evaluation will change
  */
 void
 pe_eval_nvpairs(xmlNode *top, const xmlNode *xml_obj, const char *set_name,
                 const pe_rule_eval_data_t *rule_data, GHashTable *hash,
                 const char *always_first, gboolean overwrite,
                 crm_time_t *next_change)
 {
     GList *pairs = make_pairs(top, xml_obj, set_name, always_first, overwrite);
 
     if (pairs) {
         unpack_data_t data = {
             .hash = hash,
             .overwrite = overwrite,
             .next_change = next_change,
             .top = top,
             .rule_data = rule_data
         };
 
         g_list_foreach(pairs, unpack_attr_set, &data);
         g_list_free_full(pairs, free);
     }
 }
 
 /*!
  * \brief Extract nvpair blocks contained by an XML element into a hash table
  *
  * \param[in,out] top           XML document root (used to expand id-ref's)
  * \param[in]     xml_obj       XML element containing blocks of nvpair elements
  * \param[in]     set_name      Element name to identify nvpair blocks
  * \param[in]     node_hash     Node attributes to use when evaluating rules
  * \param[out]    hash          Where to store extracted name/value pairs
  * \param[in]     always_first  If not NULL, process block with this ID first
  * \param[in]     overwrite     Whether to replace existing values with same name
  * \param[in]     now           Time to use when evaluating rules
  * \param[out]    next_change   If not NULL, set to when evaluation will change
  */
 void
 pe_unpack_nvpairs(xmlNode *top, const xmlNode *xml_obj, const char *set_name,
                   GHashTable *node_hash, GHashTable *hash,
                   const char *always_first, gboolean overwrite,
                   crm_time_t *now, crm_time_t *next_change)
 {
     pe_rule_eval_data_t rule_data = {
         .node_hash = node_hash,
         .role = pcmk_role_unknown,
         .now = now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash,
                     always_first, overwrite, next_change);
 }
 
 /*!
  * \brief Expand any regular expression submatches (%0-%9) in a string
  *
  * \param[in] string      String possibly containing submatch variables
  * \param[in] match_data  If not NULL, regular expression matches
  *
  * \return Newly allocated string identical to \p string with submatches
  *         expanded, or NULL if there were no matches
  */
 char *
 pe_expand_re_matches(const char *string, const pe_re_match_data_t *match_data)
 {
     size_t len = 0;
     int i;
     const char *p, *last_match_index;
     char *p_dst, *result = NULL;
 
     if (pcmk__str_empty(string) || !match_data) {
         return NULL;
     }
 
     p = last_match_index = string;
 
     while (*p) {
         if (*p == '%' && *(p + 1) && isdigit(*(p + 1))) {
             i = *(p + 1) - '0';
             if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
                 match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
                 len += p - last_match_index + (match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so);
                 last_match_index = p + 2;
             }
             p++;
         }
         p++;
     }
     len += p - last_match_index + 1;
 
     /* FIXME: Excessive? */
     if (len - 1 <= 0) {
         return NULL;
     }
 
     p_dst = result = calloc(1, len);
     p = string;
 
     while (*p) {
         if (*p == '%' && *(p + 1) && isdigit(*(p + 1))) {
             i = *(p + 1) - '0';
             if (match_data->nregs >= i && match_data->pmatch[i].rm_so != -1 &&
                 match_data->pmatch[i].rm_eo > match_data->pmatch[i].rm_so) {
                 /* rm_eo can be equal to rm_so, but then there is nothing to do */
                 int match_len = match_data->pmatch[i].rm_eo - match_data->pmatch[i].rm_so;
                 memcpy(p_dst, match_data->string + match_data->pmatch[i].rm_so, match_len);
                 p_dst += match_len;
             }
             p++;
         } else {
             *(p_dst) = *(p);
             p_dst++;
         }
         p++;
     }
 
     return result;
 }
 
 /*!
  * \brief Evaluate rules
  *
  * \param[in,out] ruleset      XML possibly containing rule sub-elements
  * \param[in]     rule_data
  * \param[out]    next_change  If not NULL, set to when evaluation will change
  *
  * \return TRUE if there are no rules or
  */
 gboolean
 pe_eval_rules(xmlNode *ruleset, const pe_rule_eval_data_t *rule_data,
               crm_time_t *next_change)
 {
     // If there are no rules, pass by default
     gboolean ruleset_default = TRUE;
 
     for (xmlNode *rule = first_named_child(ruleset, XML_TAG_RULE);
          rule != NULL; rule = crm_next_same_xml(rule)) {
 
         ruleset_default = FALSE;
         if (pe_eval_expr(rule, rule_data, next_change)) {
             /* Only the deprecated "lifetime" element of location constraints
              * may contain more than one rule at the top level -- the schema
              * limits a block of nvpairs to a single top-level rule. So, this
              * effectively means that a lifetime is active if any rule it
              * contains is active.
              */
             return TRUE;
         }
     }
 
     return ruleset_default;
 }
 
 /*!
  * \brief Evaluate all of a rule's expressions
  *
  * \param[in,out] rule         XML containing a rule definition or its id-ref
  * \param[in]     rule_data    Matching parameters to check against rule
  * \param[out]    next_change  If not NULL, set to when evaluation will change
  *
  * \return TRUE if \p rule_data passes \p rule, otherwise FALSE
  */
 gboolean
 pe_eval_expr(xmlNode *rule, const pe_rule_eval_data_t *rule_data,
              crm_time_t *next_change)
 {
     xmlNode *expr = NULL;
     gboolean test = TRUE;
     gboolean empty = TRUE;
     gboolean passed = TRUE;
     gboolean do_and = TRUE;
     const char *value = NULL;
 
     rule = expand_idref(rule, NULL);
     if (rule == NULL) {
         return FALSE; // Not possible with schema validation enabled
     }
 
     value = crm_element_value(rule, XML_RULE_ATTR_BOOLEAN_OP);
     if (pcmk__str_eq(value, "or", pcmk__str_casei)) {
         do_and = FALSE;
         passed = FALSE;
     }
 
     crm_trace("Testing rule %s", ID(rule));
     for (expr = pcmk__xe_first_child(rule); expr != NULL;
          expr = pcmk__xe_next(expr)) {
 
         test = pe_eval_subexpr(expr, rule_data, next_change);
         empty = FALSE;
 
         if (test && do_and == FALSE) {
             crm_trace("Expression %s/%s passed", ID(rule), ID(expr));
             return TRUE;
 
         } else if (test == FALSE && do_and) {
             crm_trace("Expression %s/%s failed", ID(rule), ID(expr));
             return FALSE;
         }
     }
 
     if (empty) {
         pcmk__config_err("Ignoring rule %s because it contains no expressions",
                          ID(rule));
     }
 
     crm_trace("Rule %s %s", ID(rule), passed ? "passed" : "failed");
     return passed;
 }
 
 /*!
  * \brief Evaluate a single rule expression, including any subexpressions
  *
  * \param[in,out] expr         XML containing a rule expression
  * \param[in]     rule_data    Matching parameters to check against expression
  * \param[out]    next_change  If not NULL, set to when evaluation will change
  *
  * \return TRUE if \p rule_data passes \p expr, otherwise FALSE
  */
 gboolean
 pe_eval_subexpr(xmlNode *expr, const pe_rule_eval_data_t *rule_data,
                 crm_time_t *next_change)
 {
     gboolean accept = FALSE;
     const char *uname = NULL;
 
     switch (find_expression_type(expr)) {
         case pcmk__subexpr_rule:
             accept = pe_eval_expr(expr, rule_data, next_change);
             break;
         case pcmk__subexpr_attribute:
         case pcmk__subexpr_location:
             /* these expressions can never succeed if there is
              * no node to compare with
              */
             if (rule_data->node_hash != NULL) {
                 accept = pe__eval_attr_expr(expr, rule_data);
             }
             break;
 
         case pcmk__subexpr_datetime:
             switch (pe__eval_date_expr(expr, rule_data, next_change)) {
                 case pcmk_rc_within_range:
                 case pcmk_rc_ok:
                     accept = TRUE;
                     break;
 
                 default:
                     accept = FALSE;
                     break;
             }
             break;
 
         case pcmk__subexpr_role:
             accept = pe__eval_role_expr(expr, rule_data);
             break;
 
         case pcmk__subexpr_resource:
             accept = pe__eval_rsc_expr(expr, rule_data);
             break;
 
         case pcmk__subexpr_operation:
             accept = pe__eval_op_expr(expr, rule_data);
             break;
 
         default:
             CRM_CHECK(FALSE /* bad type */ , return FALSE);
             accept = FALSE;
     }
     if (rule_data->node_hash) {
         uname = g_hash_table_lookup(rule_data->node_hash, CRM_ATTR_UNAME);
     }
 
     crm_trace("Expression %s %s on %s",
               ID(expr), accept ? "passed" : "failed", uname ? uname : "all nodes");
     return accept;
 }
 
 /*!
  * \internal
  * \brief   Compare two values in a rule's node attribute expression
  *
  * \param[in]   l_val   Value on left-hand side of comparison
  * \param[in]   r_val   Value on right-hand side of comparison
  * \param[in]   type    How to interpret the values (allowed values:
  *                      \c "string", \c "integer", \c "number",
  *                      \c "version", \c NULL)
  * \param[in]   op      Type of comparison
  *
  * \return  -1 if <tt>(l_val < r_val)</tt>,
  *           0 if <tt>(l_val == r_val)</tt>,
  *           1 if <tt>(l_val > r_val)</tt>
  */
 static int
 compare_attr_expr_vals(const char *l_val, const char *r_val, const char *type,
                        const char *op)
 {
     int cmp = 0;
 
     if (l_val != NULL && r_val != NULL) {
         if (type == NULL) {
             if (pcmk__strcase_any_of(op, "lt", "lte", "gt", "gte", NULL)) {
                 if (pcmk__char_in_any_str('.', l_val, r_val, NULL)) {
                     type = "number";
                 } else {
                     type = "integer";
                 }
 
             } else {
                 type = "string";
             }
             crm_trace("Defaulting to %s based comparison for '%s' op", type, op);
         }
 
         if (pcmk__str_eq(type, "string", pcmk__str_casei)) {
             cmp = strcasecmp(l_val, r_val);
 
         } else if (pcmk__str_eq(type, "integer", pcmk__str_casei)) {
             long long l_val_num;
             int rc1 = pcmk__scan_ll(l_val, &l_val_num, 0LL);
 
             long long r_val_num;
             int rc2 = pcmk__scan_ll(r_val, &r_val_num, 0LL);
 
             if ((rc1 == pcmk_rc_ok) && (rc2 == pcmk_rc_ok)) {
                 if (l_val_num < r_val_num) {
                     cmp = -1;
                 } else if (l_val_num > r_val_num) {
                     cmp = 1;
                 } else {
                     cmp = 0;
                 }
 
             } else {
                 crm_debug("Integer parse error. Comparing %s and %s as strings",
                           l_val, r_val);
                 cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
             }
 
         } else if (pcmk__str_eq(type, "number", pcmk__str_casei)) {
             double l_val_num;
             double r_val_num;
 
             int rc1 = pcmk__scan_double(l_val, &l_val_num, NULL, NULL);
             int rc2 = pcmk__scan_double(r_val, &r_val_num, NULL, NULL);
 
             if (rc1 == pcmk_rc_ok && rc2 == pcmk_rc_ok) {
                 if (l_val_num < r_val_num) {
                     cmp = -1;
                 } else if (l_val_num > r_val_num) {
                     cmp = 1;
                 } else {
                     cmp = 0;
                 }
 
             } else {
                 crm_debug("Floating-point parse error. Comparing %s and %s as "
                           "strings", l_val, r_val);
                 cmp = compare_attr_expr_vals(l_val, r_val, "string", op);
             }
 
         } else if (pcmk__str_eq(type, "version", pcmk__str_casei)) {
             cmp = compare_version(l_val, r_val);
 
         }
 
     } else if (l_val == NULL && r_val == NULL) {
         cmp = 0;
     } else if (r_val == NULL) {
         cmp = 1;
     } else {    // l_val == NULL && r_val != NULL
         cmp = -1;
     }
 
     return cmp;
 }
 
 /*!
  * \internal
  * \brief   Check whether an attribute expression evaluates to \c true
  *
  * \param[in]   l_val   Value on left-hand side of comparison
  * \param[in]   r_val   Value on right-hand side of comparison
  * \param[in]   type    How to interpret the values (allowed values:
  *                      \c "string", \c "integer", \c "number",
  *                      \c "version", \c NULL)
  * \param[in]   op      Type of comparison.
  *
  * \return  \c true if expression evaluates to \c true, \c false
  *          otherwise
  */
 static bool
 accept_attr_expr(const char *l_val, const char *r_val, const char *type,
                  const char *op)
 {
     int cmp;
 
     if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
         return (l_val != NULL);
 
     } else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
         return (l_val == NULL);
 
     }
 
     cmp = compare_attr_expr_vals(l_val, r_val, type, op);
 
     if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
         return (cmp == 0);
 
     } else if (pcmk__str_eq(op, "ne", pcmk__str_casei)) {
         return (cmp != 0);
 
     } else if (l_val == NULL || r_val == NULL) {
         // The comparison is meaningless from this point on
         return false;
 
     } else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
         return (cmp < 0);
 
     } else if (pcmk__str_eq(op, "lte", pcmk__str_casei)) {
         return (cmp <= 0);
 
     } else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
         return (cmp > 0);
 
     } else if (pcmk__str_eq(op, "gte", pcmk__str_casei)) {
         return (cmp >= 0);
     }
 
     return false;   // Should never reach this point
 }
 
 /*!
  * \internal
  * \brief Get correct value according to value-source
  *
  * \param[in] value         value given in rule expression
  * \param[in] value_source  value-source given in rule expressions
  * \param[in] match_data    If not NULL, resource back-references and params
  */
 static const char *
 expand_value_source(const char *value, const char *value_source,
                     const pe_match_data_t *match_data)
 {
     GHashTable *table = NULL;
 
     if (pcmk__str_empty(value)) {
         return NULL; // value_source is irrelevant
 
     } else if (pcmk__str_eq(value_source, "param", pcmk__str_casei)) {
         table = match_data->params;
 
     } else if (pcmk__str_eq(value_source, "meta", pcmk__str_casei)) {
         table = match_data->meta;
 
     } else { // literal
         return value;
     }
 
     if (table == NULL) {
         return NULL;
     }
     return (const char *) g_hash_table_lookup(table, value);
 }
 
 /*!
  * \internal
  * \brief Evaluate a node attribute expression based on #uname, #id, #kind,
  *        or a generic node attribute
  *
  * \param[in] expr       XML of rule expression
  * \param[in] rule_data  The match_data and node_hash members are used
  *
  * \return TRUE if rule_data satisfies the expression, FALSE otherwise
  */
 gboolean
 pe__eval_attr_expr(const xmlNode *expr, const pe_rule_eval_data_t *rule_data)
 {
     gboolean attr_allocated = FALSE;
     const char *h_val = NULL;
 
     const char *op = NULL;
     const char *type = NULL;
     const char *attr = NULL;
     const char *value = NULL;
     const char *value_source = NULL;
 
     attr = crm_element_value(expr, XML_EXPR_ATTR_ATTRIBUTE);
     op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);
     value = crm_element_value(expr, PCMK_XA_VALUE);
     type = crm_element_value(expr, PCMK_XA_TYPE);
     value_source = crm_element_value(expr, XML_EXPR_ATTR_VALUE_SOURCE);
 
     if (attr == NULL) {
         pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_ATTRIBUTE
                          " not specified", pcmk__s(ID(expr), "without ID"));
         return FALSE;
     } else if (op == NULL) {
         pcmk__config_err("Expression %s invalid: " XML_EXPR_ATTR_OPERATION
                          " not specified", pcmk__s(ID(expr), "without ID"));
         return FALSE;
     }
 
     if (rule_data->match_data != NULL) {
         // Expand any regular expression submatches (%0-%9) in attribute name
         if (rule_data->match_data->re != NULL) {
             char *resolved_attr = pe_expand_re_matches(attr, rule_data->match_data->re);
 
             if (resolved_attr != NULL) {
                 attr = (const char *) resolved_attr;
                 attr_allocated = TRUE;
             }
         }
 
         // Get value appropriate to value-source
         value = expand_value_source(value, value_source, rule_data->match_data);
     }
 
     if (rule_data->node_hash != NULL) {
         h_val = (const char *)g_hash_table_lookup(rule_data->node_hash, attr);
     }
 
     if (attr_allocated) {
         free((char *)attr);
         attr = NULL;
     }
 
     return accept_attr_expr(h_val, value, type, op);
 }
 
 /*!
  * \internal
  * \brief Evaluate a date_expression
  *
  * \param[in]  expr         XML of rule expression
  * \param[in]  rule_data    Only the now member is used
  * \param[out] next_change  If not NULL, set to when evaluation will change
  *
  * \return Standard Pacemaker return code
  */
 int
 pe__eval_date_expr(const xmlNode *expr, const pe_rule_eval_data_t *rule_data,
                    crm_time_t *next_change)
 {
     crm_time_t *start = NULL;
     crm_time_t *end = NULL;
     const char *value = NULL;
     const char *op = crm_element_value(expr, "operation");
 
     xmlNode *duration_spec = NULL;
     xmlNode *date_spec = NULL;
 
     // "undetermined" will also be returned for parsing errors
     int rc = pcmk_rc_undetermined;
 
     crm_trace("Testing expression: %s", ID(expr));
 
     duration_spec = first_named_child(expr, "duration");
     date_spec = first_named_child(expr, "date_spec");
 
     value = crm_element_value(expr, "start");
     if (value != NULL) {
         start = crm_time_new(value);
     }
     value = crm_element_value(expr, "end");
     if (value != NULL) {
         end = crm_time_new(value);
     }
 
     if (start != NULL && end == NULL && duration_spec != NULL) {
         end = parse_xml_duration(start, duration_spec);
     }
 
     if (pcmk__str_eq(op, "in_range", pcmk__str_null_matches | pcmk__str_casei)) {
         if ((start == NULL) && (end == NULL)) {
             // in_range requires at least one of start or end
         } else if ((start != NULL) && (crm_time_compare(rule_data->now, start) < 0)) {
             rc = pcmk_rc_before_range;
             crm_time_set_if_earlier(next_change, start);
         } else if ((end != NULL) && (crm_time_compare(rule_data->now, end) > 0)) {
             rc = pcmk_rc_after_range;
         } else {
             rc = pcmk_rc_within_range;
             if (end && next_change) {
                 // Evaluation doesn't change until second after end
                 crm_time_add_seconds(end, 1);
                 crm_time_set_if_earlier(next_change, end);
             }
         }
 
     } else if (pcmk__str_eq(op, "date_spec", pcmk__str_casei)) {
         rc = pe_cron_range_satisfied(rule_data->now, date_spec);
         // @TODO set next_change appropriately
 
     } else if (pcmk__str_eq(op, "gt", pcmk__str_casei)) {
         if (start == NULL) {
             // gt requires start
         } else if (crm_time_compare(rule_data->now, start) > 0) {
             rc = pcmk_rc_within_range;
         } else {
             rc = pcmk_rc_before_range;
 
             // Evaluation doesn't change until second after start
             crm_time_add_seconds(start, 1);
             crm_time_set_if_earlier(next_change, start);
         }
 
     } else if (pcmk__str_eq(op, "lt", pcmk__str_casei)) {
         if (end == NULL) {
             // lt requires end
         } else if (crm_time_compare(rule_data->now, end) < 0) {
             rc = pcmk_rc_within_range;
             crm_time_set_if_earlier(next_change, end);
         } else {
             rc = pcmk_rc_after_range;
         }
     }
 
     crm_time_free(start);
     crm_time_free(end);
     return rc;
 }
 
 gboolean
 pe__eval_op_expr(const xmlNode *expr, const pe_rule_eval_data_t *rule_data)
 {
     const char *name = crm_element_value(expr, PCMK_XA_NAME);
     const char *interval_s = crm_element_value(expr, XML_LRM_ATTR_INTERVAL);
     guint interval_ms = 0U;
 
     crm_trace("Testing op_defaults expression: %s", ID(expr));
 
     if (rule_data->op_data == NULL) {
         crm_trace("No operations data provided");
         return FALSE;
     }
 
     if (pcmk_parse_interval_spec(interval_s, &interval_ms) != pcmk_rc_ok) {
         crm_trace("Could not parse interval: %s", interval_s);
         return FALSE;
     }
 
     if ((interval_s != NULL) && (interval_ms != rule_data->op_data->interval)) {
         crm_trace("Interval doesn't match: %d != %d",
                   interval_ms, rule_data->op_data->interval);
         return FALSE;
     }
 
     if (!pcmk__str_eq(name, rule_data->op_data->op_name, pcmk__str_none)) {
         crm_trace("Name doesn't match: %s != %s", name, rule_data->op_data->op_name);
         return FALSE;
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Check whether a resource role matches a rule role
  *
  * \param[in] expr       XML of rule expression
  * \param[in] rule_data  Only the role member is used
  *
  * \return true if role matches, otherwise false
  */
 static bool
 role_matches(const xmlNode *expr, const pe_rule_eval_data_t *rule_data)
 {
     const char *value = crm_element_value(expr, PCMK_XA_VALUE);
     enum rsc_role_e role = text2role(value);
 
     if (role == pcmk_role_unknown) {
         pcmk__config_err("Invalid role %s in rule expression", value);
         return false;
     }
     return role == rule_data->role;
 }
 
 /*!
  * \internal
  * \brief Evaluate a node attribute expression based on #role
  *
  * \param[in] expr       XML of rule expression
  * \param[in] rule_data  Only the role member is used
  *
  * \return TRUE if rule_data->role satisfies the expression, FALSE otherwise
  * \todo Drop this whole code. The #role attribute was never implemented
  *       (rule_data->role is always pcmk_role_unknown), and it would be a poor
  *       design anyway, since a unique promotable clone could have multiple
  *       instances with different roles on a given node.
  */
 gboolean
 pe__eval_role_expr(const xmlNode *expr, const pe_rule_eval_data_t *rule_data)
 {
     const char *op = NULL;
 
     // A known role must be given to compare against
     if (rule_data->role == pcmk_role_unknown) {
         return FALSE;
     }
 
     op = crm_element_value(expr, XML_EXPR_ATTR_OPERATION);
 
     if (pcmk__str_eq(op, "defined", pcmk__str_casei)) {
         if (rule_data->role > pcmk_role_started) {
             return TRUE;
         }
 
     } else if (pcmk__str_eq(op, "not_defined", pcmk__str_casei)) {
         if ((rule_data->role > pcmk_role_unknown)
             && (rule_data->role < pcmk_role_unpromoted)) {
             return TRUE;
         }
 
     } else if (pcmk__str_eq(op, "eq", pcmk__str_casei)) {
         return role_matches(expr, rule_data)? TRUE : FALSE;
 
     } else if (pcmk__str_eq(op, "ne", pcmk__str_casei)
                // Test "ne" only with promotable clone roles
                && (rule_data->role >= pcmk_role_unpromoted)) {
         return role_matches(expr, rule_data)? FALSE : TRUE;
 
     } else {
         pcmk__config_err("Operation '%s' is not valid with " CRM_ATTR_ROLE
                          " comparisons", op);
     }
     return FALSE;
 }
 
 gboolean
 pe__eval_rsc_expr(const xmlNode *expr, const pe_rule_eval_data_t *rule_data)
 {
     const char *class = crm_element_value(expr, XML_AGENT_ATTR_CLASS);
-    const char *provider = crm_element_value(expr, XML_AGENT_ATTR_PROVIDER);
+    const char *provider = crm_element_value(expr, PCMK_XA_PROVIDER);
     const char *type = crm_element_value(expr, PCMK_XA_TYPE);
 
     crm_trace("Testing rsc_defaults expression: %s", ID(expr));
 
     if (rule_data->rsc_data == NULL) {
         crm_trace("No resource data provided");
         return FALSE;
     }
 
     if (class != NULL &&
         !pcmk__str_eq(class, rule_data->rsc_data->standard, pcmk__str_none)) {
         crm_trace("Class doesn't match: %s != %s", class, rule_data->rsc_data->standard);
         return FALSE;
     }
 
     if ((provider == NULL && rule_data->rsc_data->provider != NULL) ||
         (provider != NULL && rule_data->rsc_data->provider == NULL) ||
         !pcmk__str_eq(provider, rule_data->rsc_data->provider, pcmk__str_none)) {
         crm_trace("Provider doesn't match: %s != %s", provider, rule_data->rsc_data->provider);
         return FALSE;
     }
 
     if (type != NULL &&
         !pcmk__str_eq(type, rule_data->rsc_data->agent, pcmk__str_none)) {
         crm_trace("Agent doesn't match: %s != %s", type, rule_data->rsc_data->agent);
         return FALSE;
     }
 
     return TRUE;
 }
 
 // Deprecated functions kept only for backward API compatibility
 // LCOV_EXCL_START
 
 #include <crm/pengine/rules_compat.h>
 
 gboolean
 test_ruleset(xmlNode *ruleset, GHashTable *node_hash, crm_time_t *now)
 {
     return pe_evaluate_rules(ruleset, node_hash, now, NULL);
 }
 
 gboolean
 test_rule(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
 {
     return pe_test_rule(rule, node_hash, role, now, NULL, NULL);
 }
 
 gboolean
 pe_test_rule_re(xmlNode * rule, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
 {
     pe_match_data_t match_data = {
                                     .re = re_match_data,
                                     .params = NULL,
                                     .meta = NULL,
                                  };
     return pe_test_rule(rule, node_hash, role, now, NULL, &match_data);
 }
 
 gboolean
 pe_test_rule_full(xmlNode *rule, GHashTable *node_hash, enum rsc_role_e role,
                   crm_time_t *now, pe_match_data_t *match_data)
 {
     return pe_test_rule(rule, node_hash, role, now, NULL, match_data);
 }
 
 gboolean
 test_expression(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now)
 {
     return pe_test_expression(expr, node_hash, role, now, NULL, NULL);
 }
 
 gboolean
 pe_test_expression_re(xmlNode * expr, GHashTable * node_hash, enum rsc_role_e role, crm_time_t * now, pe_re_match_data_t * re_match_data)
 {
     pe_match_data_t match_data = {
                                     .re = re_match_data,
                                     .params = NULL,
                                     .meta = NULL,
                                  };
     return pe_test_expression(expr, node_hash, role, now, NULL, &match_data);
 }
 
 gboolean
 pe_test_expression_full(xmlNode *expr, GHashTable *node_hash,
                         enum rsc_role_e role, crm_time_t *now,
                         pe_match_data_t *match_data)
 {
     return pe_test_expression(expr, node_hash, role, now, NULL, match_data);
 }
 
 void
 unpack_instance_attributes(xmlNode *top, xmlNode *xml_obj, const char *set_name,
                            GHashTable *node_hash, GHashTable *hash,
                            const char *always_first, gboolean overwrite,
                            crm_time_t *now)
 {
     pe_rule_eval_data_t rule_data = {
         .node_hash = node_hash,
         .role = pcmk_role_unknown,
         .now = now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     pe_eval_nvpairs(top, xml_obj, set_name, &rule_data, hash, always_first,
                     overwrite, NULL);
 }
 
 // LCOV_EXCL_STOP
 // End deprecated API
diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
index 2315cb35f4..a3369c6939 100644
--- a/tools/crm_resource_print.c
+++ b/tools/crm_resource_print.c
@@ -1,824 +1,822 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdint.h>
 
 #include <crm_resource.h>
 #include <crm/common/lists_internal.h>
 #include <crm/common/output.h>
 #include <crm/common/results.h>
 
 #define cons_string(x) x?x:"NA"
 static int
 print_constraint(xmlNode *xml_obj, void *userdata)
 {
     pcmk_scheduler_t *scheduler = (pcmk_scheduler_t *) userdata;
     pcmk__output_t *out = scheduler->priv;
     xmlNode *lifetime = NULL;
     const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
 
     if (id == NULL) {
         return pcmk_rc_ok;
     }
 
     // @COMPAT lifetime is deprecated
     lifetime = first_named_child(xml_obj, "lifetime");
     if (pe_evaluate_rules(lifetime, NULL, scheduler->now, NULL) == FALSE) {
         return pcmk_rc_ok;
     }
 
     if (!pcmk__xe_is(xml_obj, XML_CONS_TAG_RSC_DEPEND)) {
         return pcmk_rc_ok;
     }
 
     out->info(out, "Constraint %s %s %s %s %s %s %s",
               xml_obj->name,
               cons_string(crm_element_value(xml_obj, PCMK_XA_ID)),
               cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
               cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
               cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
               cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
               cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));
 
     return pcmk_rc_ok;
 }
 
 void
 cli_resource_print_cts_constraints(pcmk_scheduler_t *scheduler)
 {
     pcmk__xe_foreach_child(pcmk_find_cib_element(scheduler->input,
                                                  XML_CIB_TAG_CONSTRAINTS),
                            NULL, print_constraint, scheduler);
 }
 
 void
 cli_resource_print_cts(pcmk_resource_t *rsc, pcmk__output_t *out)
 {
     const char *host = NULL;
     bool needs_quorum = TRUE;
     const char *rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
-    const char *rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+    const char *rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
     const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     pcmk_node_t *node = pe__current_node(rsc);
 
     if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         needs_quorum = FALSE;
     } else {
         // @TODO check requires in resource meta-data and rsc_defaults
     }
 
     if (node != NULL) {
         host = node->details->uname;
     }
 
     out->info(out, "Resource: %s %s %s %s %s %s %s %s %d %lld %#.16llx",
               rsc->xml->name, rsc->id,
               rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
               rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
               rsc->flags);
 
     g_list_foreach(rsc->children, (GFunc) cli_resource_print_cts, out);
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_print_operations(const char *rsc_id, const char *host_uname,
                               bool active, pcmk_scheduler_t *scheduler)
 {
     pcmk__output_t *out = scheduler->priv;
     int rc = pcmk_rc_no_output;
     GList *ops = find_operations(rsc_id, host_uname, active, scheduler);
 
     if (!ops) {
         return rc;
     }
 
     out->begin_list(out, NULL, NULL, "Resource Operations");
     rc = pcmk_rc_ok;
 
     for (GList *lpc = ops; lpc != NULL; lpc = lpc->next) {
         xmlNode *xml_op = (xmlNode *) lpc->data;
         out->message(out, "node-and-op", scheduler, xml_op);
     }
 
     out->end_list(out);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_print(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler,
                    bool expanded)
 {
     pcmk__output_t *out = scheduler->priv;
     uint32_t show_opts = pcmk_show_pending;
     GList *all = NULL;
 
     all = g_list_prepend(all, (gpointer) "*");
 
     out->begin_list(out, NULL, NULL, "Resource Config");
     out->message(out, crm_map_element_name(rsc->xml), show_opts, rsc, all, all);
     out->message(out, "resource-config", rsc, !expanded);
     out->end_list(out);
 
     g_list_free(all);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t *", "const char *",
                   "const char *")
 static int
 attribute_list_default(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *attr = va_arg(args, char *);
     const char *value = va_arg(args, const char *);
 
     if (value != NULL) {
         out->begin_list(out, NULL, NULL, "Attributes");
         out->list_item(out, attr, "%s", value);
         out->end_list(out);
         return pcmk_rc_ok;
     } else {
         out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
     }
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("agent-status", "int", "const char *", "const char *", "const char *",
                   "const char *", "const char *", "crm_exit_t", "const char *")
 static int
 agent_status_default(pcmk__output_t *out, va_list args) {
     int status = va_arg(args, int);
     const char *action = va_arg(args, const char *);
     const char *name = va_arg(args, const char *);
     const char *class = va_arg(args, const char *);
     const char *provider = va_arg(args, const char *);
     const char *type = va_arg(args, const char *);
     crm_exit_t rc = va_arg(args, crm_exit_t);
     const char *exit_reason = va_arg(args, const char *);
 
     if (status == PCMK_EXEC_DONE) {
         /* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
          * returned <exit-code> (<exit-description>[: <exit-reason>])
          */
         out->info(out, "Operation %s%s%s (%s%s%s:%s) returned %d (%s%s%s)",
                   action,
                   ((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
                   class,
                   ((provider == NULL)? "" : ":"),
                   ((provider == NULL)? "" : provider),
                   type, (int) rc, services_ocf_exitcode_str((int) rc),
                   ((exit_reason == NULL)? "" : ": "),
                   ((exit_reason == NULL)? "" : exit_reason));
     } else {
         /* Operation <action> [for <resource>] (<class>[:<provider>]:<agent>)
          * could not be executed (<execution-status>[: <exit-reason>])
          */
         out->err(out,
                  "Operation %s%s%s (%s%s%s:%s) could not be executed (%s%s%s)",
                  action,
                  ((name == NULL)? "" : " for "), ((name == NULL)? "" : name),
                  class,
                  ((provider == NULL)? "" : ":"),
                  ((provider == NULL)? "" : provider),
                  type, pcmk_exec_status_str(status),
                  ((exit_reason == NULL)? "" : ": "),
                  ((exit_reason == NULL)? "" : exit_reason));
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("agent-status", "int", "const char *", "const char *", "const char *",
                   "const char *", "const char *", "crm_exit_t", "const char *")
 static int
 agent_status_xml(pcmk__output_t *out, va_list args) {
     int status = va_arg(args, int);
     const char *action G_GNUC_UNUSED = va_arg(args, const char *);
     const char *name G_GNUC_UNUSED = va_arg(args, const char *);
     const char *class G_GNUC_UNUSED = va_arg(args, const char *);
     const char *provider G_GNUC_UNUSED = va_arg(args, const char *);
     const char *type G_GNUC_UNUSED = va_arg(args, const char *);
     crm_exit_t rc = va_arg(args, crm_exit_t);
     const char *exit_reason = va_arg(args, const char *);
 
     char *exit_str = pcmk__itoa(rc);
     char *status_str = pcmk__itoa(status);
 
     pcmk__output_create_xml_node(out, "agent-status",
                                  "code", exit_str,
                                  "message", services_ocf_exitcode_str((int) rc),
                                  "execution_code", status_str,
                                  "execution_message", pcmk_exec_status_str(status),
                                  "reason", exit_reason,
                                  NULL);
 
     free(exit_str);
     free(status_str);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("attribute-list", "pcmk_resource_t *", "const char *",
                   "const char *")
 static int
 attribute_list_text(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *attr = va_arg(args, char *);
     const char *value = va_arg(args, const char *);
 
     if (value != NULL) {
         pcmk__formatted_printf(out, "%s\n", value);
         return pcmk_rc_ok;
     } else {
         out->err(out, "Attribute '%s' not found for '%s'", attr, rsc->id);
     }
     return pcmk_rc_ok;
 }
 PCMK__OUTPUT_ARGS("override", "const char *", "const char *", "const char *")
 static int
 override_default(pcmk__output_t *out, va_list args) {
     const char *rsc_name = va_arg(args, const char *);
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
 
     if (rsc_name == NULL) {
         out->list_item(out, NULL, "Overriding the cluster configuration with '%s' = '%s'",
                        name, value);
     } else {
         out->list_item(out, NULL, "Overriding the cluster configuration for '%s' with '%s' = '%s'",
                        rsc_name, name, value);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("override", "const char *", "const char *", "const char *")
 static int
 override_xml(pcmk__output_t *out, va_list args) {
     const char *rsc_name = va_arg(args, const char *);
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "override",
                                                    PCMK_XA_NAME, name,
                                                    PCMK_XA_VALUE, value,
                                                    NULL);
 
     if (rsc_name != NULL) {
         crm_xml_add(node, "rsc", rsc_name);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t *", "const char *")
 static int
 property_list_default(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *attr = va_arg(args, char *);
 
     const char *value = crm_element_value(rsc->xml, attr);
 
     if (value != NULL) {
         out->begin_list(out, NULL, NULL, "Properties");
         out->list_item(out, attr, "%s", value);
         out->end_list(out);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("property-list", "pcmk_resource_t *", "const char *")
 static int
 property_list_text(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *attr = va_arg(args, const char *);
 
     const char *value = crm_element_value(rsc->xml, attr);
 
     if (value != NULL) {
         pcmk__formatted_printf(out, "%s\n", value);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char *", "const char *",
                   "const char *", "const char *", "const char *", "GHashTable *",
                   "crm_exit_t", "int", "const char *", "const char *", "const char *")
 static int
 resource_agent_action_default(pcmk__output_t *out, va_list args) {
     int verbose = va_arg(args, int);
 
     const char *class = va_arg(args, const char *);
     const char *provider = va_arg(args, const char *);
     const char *type = va_arg(args, const char *);
     const char *rsc_name = va_arg(args, const char *);
     const char *action = va_arg(args, const char *);
     GHashTable *overrides = va_arg(args, GHashTable *);
     crm_exit_t rc = va_arg(args, crm_exit_t);
     int status = va_arg(args, int);
     const char *exit_reason = va_arg(args, const char *);
     const char *stdout_data = va_arg(args, const char *);
     const char *stderr_data = va_arg(args, const char *);
 
     if (overrides) {
         GHashTableIter iter;
         const char *name = NULL;
         const char *value = NULL;
 
         out->begin_list(out, NULL, NULL, "overrides");
 
         g_hash_table_iter_init(&iter, overrides);
         while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *) &value)) {
             out->message(out, "override", rsc_name, name, value);
         }
 
         out->end_list(out);
     }
 
     out->message(out, "agent-status", status, action, rsc_name, class, provider,
                  type, rc, exit_reason);
 
     /* hide output for validate-all if not in verbose */
     if ((verbose == 0)
         && pcmk__str_eq(action, PCMK_ACTION_VALIDATE_ALL, pcmk__str_casei)) {
         return pcmk_rc_ok;
     }
 
     if (stdout_data || stderr_data) {
         xmlNodePtr doc = NULL;
 
         if (stdout_data != NULL) {
             doc = string2xml(stdout_data);
         }
         if (doc != NULL) {
             out->output_xml(out, "command", stdout_data);
             xmlFreeNode(doc);
         } else {
             out->subprocess_output(out, rc, stdout_data, stderr_data);
         }
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char *", "const char *",
                   "const char *", "const char *", "const char *", "GHashTable *",
                   "crm_exit_t", "int", "const char *", "const char *", "const char *")
 static int
 resource_agent_action_xml(pcmk__output_t *out, va_list args) {
     int verbose G_GNUC_UNUSED = va_arg(args, int);
 
     const char *class = va_arg(args, const char *);
     const char *provider = va_arg(args, const char *);
     const char *type = va_arg(args, const char *);
     const char *rsc_name = va_arg(args, const char *);
     const char *action = va_arg(args, const char *);
     GHashTable *overrides = va_arg(args, GHashTable *);
     crm_exit_t rc = va_arg(args, crm_exit_t);
     int status = va_arg(args, int);
     const char *exit_reason = va_arg(args, const char *);
     const char *stdout_data = va_arg(args, const char *);
     const char *stderr_data = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_xml_create_parent(out, "resource-agent-action",
                                                      "action", action,
                                                      "class", class,
                                                      PCMK_XA_TYPE, type,
                                                      NULL);
 
     if (rsc_name) {
         crm_xml_add(node, "rsc", rsc_name);
     }
 
-    if (provider) {
-        crm_xml_add(node, "provider", provider);
-    }
+    crm_xml_add(node, PCMK_XA_PROVIDER, provider);
 
     if (overrides) {
         GHashTableIter iter;
         const char *name = NULL;
         const char *value = NULL;
 
         out->begin_list(out, NULL, NULL, "overrides");
 
         g_hash_table_iter_init(&iter, overrides);
         while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *) &value)) {
             out->message(out, "override", rsc_name, name, value);
         }
 
         out->end_list(out);
     }
 
     out->message(out, "agent-status", status, action, rsc_name, class, provider,
                  type, rc, exit_reason);
 
     if (stdout_data || stderr_data) {
         xmlNodePtr doc = NULL;
 
         if (stdout_data != NULL) {
             doc = string2xml(stdout_data);
         }
         if (doc != NULL) {
             out->output_xml(out, "command", stdout_data);
             xmlFreeNode(doc);
         } else {
             out->subprocess_output(out, rc, stdout_data, stderr_data);
         }
     }
 
     pcmk__output_xml_pop_parent(out);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
 static int
 resource_check_list_default(pcmk__output_t *out, va_list args) {
     resource_checks_t *checks = va_arg(args, resource_checks_t *);
 
     const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);
 
     if (checks->flags == 0) {
         return pcmk_rc_no_output;
     }
 
     out->begin_list(out, NULL, NULL, "Resource Checks");
 
     if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
         out->list_item(out, "check", "Configuration specifies '%s' should remain stopped",
                        parent->id);
     }
 
     if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
         out->list_item(out, "check", "Configuration specifies '%s' should not be promoted",
                        parent->id);
     }
 
     if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
         out->list_item(out, "check", "Configuration prevents cluster from stopping or starting unmanaged '%s'",
                        parent->id);
     }
 
     if (pcmk_is_set(checks->flags, rsc_locked)) {
         out->list_item(out, "check", "'%s' is locked to node %s due to shutdown",
                        parent->id, checks->lock_node);
     }
 
     if (pcmk_is_set(checks->flags, rsc_node_health)) {
         out->list_item(out, "check",
                        "'%s' cannot run on unhealthy nodes due to "
                        PCMK_OPT_NODE_HEALTH_STRATEGY "='%s'",
                        parent->id,
                        pe_pref(checks->rsc->cluster->config_hash,
                                PCMK_OPT_NODE_HEALTH_STRATEGY));
     }
 
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
 static int
 resource_check_list_xml(pcmk__output_t *out, va_list args) {
     resource_checks_t *checks = va_arg(args, resource_checks_t *);
 
     const pcmk_resource_t *parent = pe__const_top_resource(checks->rsc, false);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "check",
                                                    PCMK_XA_ID, parent->id,
                                                    NULL);
 
     if (pcmk_is_set(checks->flags, rsc_remain_stopped)) {
         pcmk__xe_set_bool_attr(node, "remain_stopped", true);
     }
 
     if (pcmk_is_set(checks->flags, rsc_unpromotable)) {
         pcmk__xe_set_bool_attr(node, "promotable", false);
     }
 
     if (pcmk_is_set(checks->flags, rsc_unmanaged)) {
         pcmk__xe_set_bool_attr(node, "unmanaged", true);
     }
 
     if (pcmk_is_set(checks->flags, rsc_locked)) {
         crm_xml_add(node, "locked-to", checks->lock_node);
     }
 
     if (pcmk_is_set(checks->flags, rsc_node_health)) {
         pcmk__xe_set_bool_attr(node, "unhealthy", true);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-search-list", "GList *", "const char *")
 static int
 resource_search_list_default(pcmk__output_t *out, va_list args)
 {
     GList *nodes = va_arg(args, GList *);
     const char *requested_name = va_arg(args, const char *);
 
     bool printed = false;
     int rc = pcmk_rc_no_output;
 
     if (!out->is_quiet(out) && nodes == NULL) {
         out->err(out, "resource %s is NOT running", requested_name);
         return rc;
     }
 
     for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
         node_info_t *ni = (node_info_t *) lpc->data;
 
         if (!printed) {
             out->begin_list(out, NULL, NULL, "Nodes");
             printed = true;
             rc = pcmk_rc_ok;
         }
 
         if (out->is_quiet(out)) {
             out->list_item(out, "node", "%s", ni->node_name);
         } else {
             const char *role_text = "";
 
             if (ni->promoted) {
 #ifdef PCMK__COMPAT_2_0
                 role_text = " " PCMK__ROLE_PROMOTED_LEGACY;
 #else
                 role_text = " " PCMK__ROLE_PROMOTED;
 #endif
             }
             out->list_item(out, "node", "resource %s is running on: %s%s",
                            requested_name, ni->node_name, role_text);
         }
     }
 
     if (printed) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("resource-search-list", "GList *", "const char *")
 static int
 resource_search_list_xml(pcmk__output_t *out, va_list args)
 {
     GList *nodes = va_arg(args, GList *);
     const char *requested_name = va_arg(args, const char *);
 
     pcmk__output_xml_create_parent(out, "nodes",
                                    "resource", requested_name,
                                    NULL);
 
     for (GList *lpc = nodes; lpc != NULL; lpc = lpc->next) {
         node_info_t *ni = (node_info_t *) lpc->data;
         xmlNodePtr sub_node = pcmk__output_create_xml_text_node(out, "node", ni->node_name);
 
         if (ni->promoted) {
             crm_xml_add(sub_node, "state", "promoted");
         }
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-reasons-list", "GList *", "pcmk_resource_t *",
                   "pcmk_node_t *")
 static int
 resource_reasons_list_default(pcmk__output_t *out, va_list args)
 {
     GList *resources = va_arg(args, GList *);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
 
     const char *host_uname = (node == NULL)? NULL : node->details->uname;
 
     out->begin_list(out, NULL, NULL, "Resource Reasons");
 
     if ((rsc == NULL) && (host_uname == NULL)) {
         GList *lpc = NULL;
         GList *hosts = NULL;
 
         for (lpc = resources; lpc != NULL; lpc = lpc->next) {
             pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
             rsc->fns->location(rsc, &hosts, TRUE);
 
             if (hosts == NULL) {
                 out->list_item(out, "reason", "Resource %s is not running", rsc->id);
             } else {
                 out->list_item(out, "reason", "Resource %s is running", rsc->id);
             }
 
             cli_resource_check(out, rsc, NULL);
             g_list_free(hosts);
             hosts = NULL;
         }
 
     } else if ((rsc != NULL) && (host_uname != NULL)) {
         if (resource_is_running_on(rsc, host_uname)) {
             out->list_item(out, "reason", "Resource %s is running on host %s",
                            rsc->id, host_uname);
         } else {
             out->list_item(out, "reason", "Resource %s is not running on host %s",
                            rsc->id, host_uname);
         }
 
         cli_resource_check(out, rsc, node);
 
     } else if ((rsc == NULL) && (host_uname != NULL)) {
         const char* host_uname =  node->details->uname;
         GList *allResources = node->details->allocated_rsc;
         GList *activeResources = node->details->running_rsc;
         GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
         GList *lpc = NULL;
 
         for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
             pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
             out->list_item(out, "reason", "Resource %s is running on host %s",
                            rsc->id, host_uname);
             cli_resource_check(out, rsc, node);
         }
 
         for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
             pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
             out->list_item(out, "reason", "Resource %s is assigned to host %s but not running",
                            rsc->id, host_uname);
             cli_resource_check(out, rsc, node);
         }
 
         g_list_free(allResources);
         g_list_free(activeResources);
         g_list_free(unactiveResources);
 
     } else if ((rsc != NULL) && (host_uname == NULL)) {
         GList *hosts = NULL;
 
         rsc->fns->location(rsc, &hosts, TRUE);
         out->list_item(out, "reason", "Resource %s is %srunning",
                        rsc->id, (hosts? "" : "not "));
         cli_resource_check(out, rsc, NULL);
         g_list_free(hosts);
     }
 
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-reasons-list", "GList *", "pcmk_resource_t *",
                   "pcmk_node_t *")
 static int
 resource_reasons_list_xml(pcmk__output_t *out, va_list args)
 {
     GList *resources = va_arg(args, GList *);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
 
     const char *host_uname = (node == NULL)? NULL : node->details->uname;
 
     xmlNodePtr xml_node = pcmk__output_xml_create_parent(out, "reason", NULL);
 
     if ((rsc == NULL) && (host_uname == NULL)) {
         GList *lpc = NULL;
         GList *hosts = NULL;
 
         pcmk__output_xml_create_parent(out, "resources", NULL);
 
         for (lpc = resources; lpc != NULL; lpc = lpc->next) {
             pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
 
             rsc->fns->location(rsc, &hosts, TRUE);
 
             pcmk__output_xml_create_parent(out, "resource",
                                            PCMK_XA_ID, rsc->id,
                                            "running", pcmk__btoa(hosts != NULL),
                                            NULL);
 
             cli_resource_check(out, rsc, NULL);
             pcmk__output_xml_pop_parent(out);
             g_list_free(hosts);
             hosts = NULL;
         }
 
         pcmk__output_xml_pop_parent(out);
 
     } else if ((rsc != NULL) && (host_uname != NULL)) {
         if (resource_is_running_on(rsc, host_uname)) {
             crm_xml_add(xml_node, "running_on", host_uname);
         }
 
         cli_resource_check(out, rsc, node);
 
     } else if ((rsc == NULL) && (host_uname != NULL)) {
         const char* host_uname =  node->details->uname;
         GList *allResources = node->details->allocated_rsc;
         GList *activeResources = node->details->running_rsc;
         GList *unactiveResources = pcmk__subtract_lists(allResources, activeResources, (GCompareFunc) strcmp);
         GList *lpc = NULL;
 
         pcmk__output_xml_create_parent(out, "resources", NULL);
 
         for (lpc = activeResources; lpc != NULL; lpc = lpc->next) {
             pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
 
             pcmk__output_xml_create_parent(out, "resource",
                                            PCMK_XA_ID, rsc->id,
                                            "running", "true",
                                            "host", host_uname,
                                            NULL);
 
             cli_resource_check(out, rsc, node);
             pcmk__output_xml_pop_parent(out);
         }
 
         for(lpc = unactiveResources; lpc != NULL; lpc = lpc->next) {
             pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
 
             pcmk__output_xml_create_parent(out, "resource",
                                            PCMK_XA_ID, rsc->id,
                                            "running", "false",
                                            "host", host_uname,
                                            NULL);
 
             cli_resource_check(out, rsc, node);
             pcmk__output_xml_pop_parent(out);
         }
 
         pcmk__output_xml_pop_parent(out);
         g_list_free(allResources);
         g_list_free(activeResources);
         g_list_free(unactiveResources);
 
     } else if ((rsc != NULL) && (host_uname == NULL)) {
         GList *hosts = NULL;
 
         rsc->fns->location(rsc, &hosts, TRUE);
         crm_xml_add(xml_node, "running", pcmk__btoa(hosts != NULL));
         cli_resource_check(out, rsc, NULL);
         g_list_free(hosts);
     }
 
     return pcmk_rc_ok;
 }
 
 static void
 add_resource_name(pcmk_resource_t *rsc, pcmk__output_t *out)
 {
     if (rsc->children == NULL) {
         out->list_item(out, "resource", "%s", rsc->id);
     } else {
         g_list_foreach(rsc->children, (GFunc) add_resource_name, out);
     }
 }
 
 PCMK__OUTPUT_ARGS("resource-names-list", "GList *")
 static int
 resource_names(pcmk__output_t *out, va_list args) {
     GList *resources = va_arg(args, GList *);
 
     if (resources == NULL) {
         out->err(out, "NO resources configured\n");
         return pcmk_rc_no_output;
     }
 
     out->begin_list(out, NULL, NULL, "Resource Names");
     g_list_foreach(resources, (GFunc) add_resource_name, out);
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 static pcmk__message_entry_t fmt_functions[] = {
     { "agent-status", "default", agent_status_default },
     { "agent-status", "xml", agent_status_xml },
     { "attribute-list", "default", attribute_list_default },
     { "attribute-list", "text", attribute_list_text },
     { "override", "default", override_default },
     { "override", "xml", override_xml },
     { "property-list", "default", property_list_default },
     { "property-list", "text", property_list_text },
     { "resource-agent-action", "default", resource_agent_action_default },
     { "resource-agent-action", "xml", resource_agent_action_xml },
     { "resource-check-list", "default", resource_check_list_default },
     { "resource-check-list", "xml", resource_check_list_xml },
     { "resource-search-list", "default", resource_search_list_default },
     { "resource-search-list", "xml", resource_search_list_xml },
     { "resource-reasons-list", "default", resource_reasons_list_default },
     { "resource-reasons-list", "xml", resource_reasons_list_xml },
     { "resource-names-list", "default", resource_names },
 
     { NULL, NULL, NULL }
 };
 
 void
 crm_resource_register_messages(pcmk__output_t *out) {
     pcmk__register_messages(out, fmt_functions);
 }
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 52e8e54066..1084e9f2a9 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,2254 +1,2254 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm_resource.h>
 #include <crm/common/ipc_attrd_internal.h>
 #include <crm/common/ipc_controld.h>
 #include <crm/common/lists_internal.h>
 #include <crm/services_internal.h>
 
 static GList *
 build_node_info_list(const pcmk_resource_t *rsc)
 {
     GList *retval = NULL;
 
     for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
 
         for (const GList *iter2 = child->running_on;
              iter2 != NULL; iter2 = iter2->next) {
 
             const pcmk_node_t *node = (const pcmk_node_t *) iter2->data;
             node_info_t *ni = calloc(1, sizeof(node_info_t));
 
             ni->node_name = node->details->uname;
             ni->promoted = pcmk_is_set(rsc->flags, pcmk_rsc_promotable) &&
                            child->fns->state(child, TRUE) == pcmk_role_promoted;
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 GList *
 cli_resource_search(pcmk_resource_t *rsc, const char *requested_name,
                     pcmk_scheduler_t *scheduler)
 {
     GList *retval = NULL;
     const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
 
     if (pe_rsc_is_clone(rsc)) {
         retval = build_node_info_list(rsc);
 
     /* The anonymous clone children's common ID is supplied */
     } else if (pe_rsc_is_clone(parent)
                && !pcmk_is_set(rsc->flags, pcmk_rsc_unique)
                && rsc->clone_name
                && pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
 
         retval = build_node_info_list(parent);
 
     } else if (rsc->running_on != NULL) {
         for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
             pcmk_node_t *node = (pcmk_node_t *) iter->data;
             node_info_t *ni = calloc(1, sizeof(node_info_t));
             ni->node_name = node->details->uname;
             ni->promoted = (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted);
 
             retval = g_list_prepend(retval, ni);
         }
     }
 
     return retval;
 }
 
 // \return Standard Pacemaker return code
 static int
 find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
                    const char *rsc, const char *attr_set_type, const char *set_name,
                    const char *attr_id, const char *attr_name, char **value)
 {
     int rc = pcmk_rc_ok;
     xmlNode *xml_search = NULL;
     GString *xpath = NULL;
     const char *xpath_base = NULL;
 
     if(value) {
         *value = NULL;
     }
 
     if(the_cib == NULL) {
         return ENOTCONN;
     }
 
     xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
     if (xpath_base == NULL) {
         crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
         return ENOMSG;
     }
 
     xpath = g_string_sized_new(1024);
     pcmk__g_strcat(xpath,
                    xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);
 
     if (attr_set_type != NULL) {
         pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
         if (set_name != NULL) {
             pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
                            NULL);
         }
     }
 
     g_string_append(xpath, "//" XML_CIB_TAG_NVPAIR "[");
     if (attr_id != NULL) {
         pcmk__g_strcat(xpath, "@" PCMK_XA_ID "=\"", attr_id, "\"", NULL);
     }
 
     if (attr_name != NULL) {
         if (attr_id != NULL) {
             g_string_append(xpath, " and ");
         }
         pcmk__g_strcat(xpath, "@" PCMK_XA_NAME "=\"", attr_name, "\"", NULL);
     }
     g_string_append_c(xpath, ']');
 
     rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
     rc = pcmk_legacy2rc(rc);
 
     if (rc != pcmk_rc_ok) {
         goto done;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_search->children != NULL) {
         xmlNode *child = NULL;
 
         rc = ENOTUNIQ;
         out->info(out, "Multiple attributes match name=%s", attr_name);
 
         for (child = pcmk__xml_first_child(xml_search); child != NULL;
              child = pcmk__xml_next(child)) {
             out->info(out, "  Value: %s \t(id=%s)",
                       crm_element_value(child, PCMK_XA_VALUE), ID(child));
         }
 
         out->spacer(out);
 
     } else if(value) {
         pcmk__str_update(value, crm_element_value(xml_search, attr));
     }
 
   done:
     g_string_free(xpath, TRUE);
     free_xml(xml_search);
     return rc;
 }
 
 /* PRIVATE. Use the find_matching_attr_resources instead. */
 static void
 find_matching_attr_resources_recursive(pcmk__output_t *out,
                                        GList /* <pcmk_resource_t*> */ **result,
                                        pcmk_resource_t *rsc, const char *rsc_id,
                                        const char * attr_set, const char * attr_set_type,
                                        const char * attr_id, const char * attr_name,
                                        cib_t * cib, const char * cmd, int depth)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = clone_strip(rsc->id);
     char *local_attr_id = NULL;
 
     /* visit the children */
     for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
         find_matching_attr_resources_recursive(out, result,
                                                (pcmk_resource_t *) gIter->data,
                                                rsc_id, attr_set, attr_set_type,
                                                attr_id, attr_name, cib, cmd, depth+1);
         /* do it only once for clones */
         if (rsc->variant == pcmk_rsc_variant_clone) {
             break;
         }
     }
 
     rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
                             attr_set, attr_id, attr_name, &local_attr_id);
     /* Post-order traversal.
      * The root is always on the list and it is the last item. */
     if((0 == depth) || (pcmk_rc_ok == rc)) {
         /* push the head */
         *result = g_list_append(*result, rsc);
     }
 
     free(local_attr_id);
     free(lookup_id);
 }
 
 
 /* The result is a linearized pre-ordered tree of resources. */
 static GList/*<pcmk_resource_t*>*/ *
 find_matching_attr_resources(pcmk__output_t *out, pcmk_resource_t *rsc,
                              const char * rsc_id, const char * attr_set,
                              const char * attr_set_type, const char * attr_id,
                              const char * attr_name, cib_t * cib, const char * cmd,
                              gboolean force)
 {
     int rc = pcmk_rc_ok;
     char *lookup_id = NULL;
     char *local_attr_id = NULL;
     GList * result = NULL;
     /* If --force is used, update only the requested resource (clone or primitive).
      * Otherwise, if the primitive has the attribute, use that.
      * Otherwise use the clone. */
     if(force == TRUE) {
         return g_list_append(result, rsc);
     }
     if ((rsc->parent != NULL)
         && (rsc->parent->variant == pcmk_rsc_variant_clone)) {
         int rc = pcmk_rc_ok;
         char *local_attr_id = NULL;
         rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &local_attr_id);
         free(local_attr_id);
 
         if(rc != pcmk_rc_ok) {
             rsc = rsc->parent;
             out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
                       cmd, attr_name, rsc->id, rsc_id);
         }
         return g_list_append(result, rsc);
 
     } else if ((rsc->parent == NULL) && (rsc->children != NULL)
                && (rsc->variant == pcmk_rsc_variant_clone)) {
         pcmk_resource_t *child = rsc->children->data;
 
         if (child->variant == pcmk_rsc_variant_primitive) {
             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
             rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
                                     attr_set_type, attr_set, attr_id, attr_name,
                                     &local_attr_id);
 
             if(rc == pcmk_rc_ok) {
                 rsc = child;
                 out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
                           attr_name, lookup_id, cmd, rsc_id);
             }
 
             free(local_attr_id);
             free(lookup_id);
         }
         return g_list_append(result, rsc);
     }
     /* If the resource is a group ==> children inherit the attribute if defined. */
     find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
                                            attr_set_type, attr_id, attr_name,
                                            cib, cmd, 0);
     return result;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_update_attribute(pcmk_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
                               const char *attr_value, gboolean recursive,
                               cib_t *cib, int cib_options, gboolean force)
 {
     pcmk__output_t *out = rsc->cluster->priv;
     int rc = pcmk_rc_ok;
 
     char *found_attr_id = NULL;
 
     GList/*<pcmk_resource_t*>*/ *resources = NULL;
     const char *top_id = pe__const_top_resource(rsc, false)->id;
 
     if ((attr_id == NULL) && !force) {
         find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
                            attr_name, NULL);
     }
 
     if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
         if (!force) {
             rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &found_attr_id);
             if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
                 out->err(out,
                          "WARNING: There is already a meta attribute "
                          "for '%s' called '%s' (id=%s)",
                          top_id, attr_name, found_attr_id);
                 out->err(out,
                          "         Delete '%s' first or use the force option "
                          "to override", found_attr_id);
             }
             free(found_attr_id);
             if (rc == pcmk_rc_ok) {
                 return ENOTUNIQ;
             }
         }
         resources = g_list_append(resources, rsc);
 
     } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
         crm_xml_add(rsc->xml, attr_name, attr_value);
         CRM_ASSERT(cib != NULL);
         rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
                                 cib_options);
         rc = pcmk_legacy2rc(rc);
         if (rc == pcmk_rc_ok) {
             out->info(out, "Set attribute: name=%s value=%s",
                       attr_name, attr_value);
         }
         return rc;
 
     } else {
         resources = find_matching_attr_resources(out, rsc, requested_name,
                                                  attr_set, attr_set_type,
                                                  attr_id, attr_name, cib,
                                                  "update", force);
     }
 
     /* If the user specified attr_set or attr_id, the intent is to modify a
      * single resource, which will be the last item in the list.
      */
     if ((attr_set != NULL) || (attr_id != NULL)) {
         GList *last = g_list_last(resources);
 
         resources = g_list_remove_link(resources, last);
         g_list_free(resources);
         resources = last;
     }
 
     for (GList *iter = resources; iter != NULL; iter = iter->next) {
         char *lookup_id = NULL;
         char *local_attr_set = NULL;
         const char *rsc_attr_id = attr_id;
         const char *rsc_attr_set = attr_set;
 
         xmlNode *xml_top = NULL;
         xmlNode *xml_obj = NULL;
         found_attr_id = NULL;
 
         rsc = (pcmk_resource_t *) iter->data;
 
         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &found_attr_id);
 
         switch (rc) {
             case pcmk_rc_ok:
                 crm_debug("Found a match for name=%s: id=%s",
                           attr_name, found_attr_id);
                 rsc_attr_id = found_attr_id;
                 break;
 
             case ENXIO:
                 if (rsc_attr_set == NULL) {
                     local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
                                                        attr_set_type);
                     rsc_attr_set = local_attr_set;
                 }
                 if (rsc_attr_id == NULL) {
                     found_attr_id = crm_strdup_printf("%s-%s",
                                                       rsc_attr_set, attr_name);
                     rsc_attr_id = found_attr_id;
                 }
 
                 xml_top = create_xml_node(NULL, (const char *) rsc->xml->name);
                 crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);
 
                 xml_obj = create_xml_node(xml_top, attr_set_type);
                 crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
                 break;
 
             default:
                 free(lookup_id);
                 free(found_attr_id);
                 g_list_free(resources);
                 return rc;
         }
 
         xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
                                         attr_value);
         if (xml_top == NULL) {
             xml_top = xml_obj;
         }
 
         crm_log_xml_debug(xml_top, "Update");
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top,
                                cib_options);
         rc = pcmk_legacy2rc(rc);
         if (rc == pcmk_rc_ok) {
             out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s",
                       lookup_id, found_attr_id,
                       ((rsc_attr_set == NULL)? "" : " set="),
                       pcmk__s(rsc_attr_set, ""),
                       ((attr_name == NULL)? "" : " name="),
                       pcmk__s(attr_name, ""), attr_value);
         }
 
         free_xml(xml_top);
 
         free(lookup_id);
         free(found_attr_id);
         free(local_attr_set);
 
         if (recursive
             && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS,
                             pcmk__str_casei)) {
             GList *lpc = NULL;
             static bool need_init = true;
 
             if (need_init) {
                 need_init = false;
                 pcmk__unpack_constraints(rsc->cluster);
                 pe__clear_resource_flags_on_all(rsc->cluster,
                                                 pcmk_rsc_detect_loop);
             }
 
             /* We want to set the attribute only on resources explicitly
              * colocated with this one, so we use rsc->rsc_cons_lhs directly
              * rather than the with_this_colocations() method.
              */
             pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
             for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
                 pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
 
                 crm_debug("Checking %s %d", cons->id, cons->score);
                 if (!pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)
                     && (cons->score > 0)) {
                     crm_debug("Setting %s=%s for dependent resource %s",
                               attr_name, attr_value, cons->dependent->id);
                     cli_resource_update_attribute(cons->dependent,
                                                   cons->dependent->id, NULL,
                                                   attr_set_type, NULL,
                                                   attr_name, attr_value,
                                                   recursive, cib, cib_options,
                                                   force);
                 }
             }
         }
     }
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete_attribute(pcmk_resource_t *rsc, const char *requested_name,
                               const char *attr_set, const char *attr_set_type,
                               const char *attr_id, const char *attr_name,
                               cib_t *cib, int cib_options, gboolean force)
 {
     pcmk__output_t *out = rsc->cluster->priv;
     int rc = pcmk_rc_ok;
     GList/*<pcmk_resource_t*>*/ *resources = NULL;
 
     if ((attr_id == NULL) && !force) {
         find_resource_attr(out, cib, PCMK_XA_ID,
                            pe__const_top_resource(rsc, false)->id, NULL,
                            NULL, NULL, attr_name, NULL);
     }
 
     if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
         resources = find_matching_attr_resources(out, rsc, requested_name,
                                                  attr_set, attr_set_type,
                                                  attr_id, attr_name, cib,
                                                  "delete", force);
 
     } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
         xml_remove_prop(rsc->xml, attr_name);
         CRM_ASSERT(cib != NULL);
         rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
                                 cib_options);
         rc = pcmk_legacy2rc(rc);
         if (rc == pcmk_rc_ok) {
             out->info(out, "Deleted attribute: %s", attr_name);
         }
         return rc;
 
     } else {
         resources = g_list_append(resources, rsc);
     }
 
     for (GList *iter = resources; iter != NULL; iter = iter->next) {
         char *lookup_id = NULL;
         xmlNode *xml_obj = NULL;
         char *found_attr_id = NULL;
         const char *rsc_attr_id = attr_id;
 
         rsc = (pcmk_resource_t *) iter->data;
 
         lookup_id = clone_strip(rsc->id);
         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
                                 attr_set, attr_id, attr_name, &found_attr_id);
         switch (rc) {
             case pcmk_rc_ok:
                 break;
 
             case ENXIO:
                 free(lookup_id);
                 rc = pcmk_rc_ok;
                 continue;
 
             default:
                 free(lookup_id);
                 g_list_free(resources);
                 return rc;
         }
 
         if (rsc_attr_id == NULL) {
             rsc_attr_id = found_attr_id;
         }
 
         xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
         crm_log_xml_debug(xml_obj, "Delete");
 
         CRM_ASSERT(cib);
         rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj,
                                cib_options);
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok) {
             out->info(out, "Deleted '%s' option: id=%s%s%s%s%s",
                       lookup_id, found_attr_id,
                       ((attr_set == NULL)? "" : " set="),
                       pcmk__s(attr_set, ""),
                       ((attr_name == NULL)? "" : " name="),
                       pcmk__s(attr_name, ""));
         }
 
         free(lookup_id);
         free_xml(xml_obj);
         free(found_attr_id);
     }
     g_list_free(resources);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
                 const char *host_uname, const char *rsc_id,
                 pcmk_scheduler_t *scheduler)
 {
     pcmk__output_t *out = scheduler->priv;
     const char *router_node = host_uname;
     const char *rsc_api_id = NULL;
     const char *rsc_long_id = NULL;
     const char *rsc_class = NULL;
     const char *rsc_provider = NULL;
     const char *rsc_type = NULL;
     bool cib_only = false;
     pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
 
     if (rsc == NULL) {
         out->err(out, "Resource %s not found", rsc_id);
         return ENXIO;
 
     } else if (rsc->variant != pcmk_rsc_variant_primitive) {
         out->err(out, "We can only process primitive resources, not %s", rsc_id);
         return EINVAL;
     }
 
     rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
-    rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
+    rsc_provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
     rsc_type = crm_element_value(rsc->xml, PCMK_XA_TYPE);
     if ((rsc_class == NULL) || (rsc_type == NULL)) {
         out->err(out, "Resource %s does not have a class and type", rsc_id);
         return EINVAL;
     }
 
     {
         pcmk_node_t *node = pe_find_node(scheduler->nodes, host_uname);
 
         if (node == NULL) {
             out->err(out, "Node %s not found", host_uname);
             return pcmk_rc_node_unknown;
         }
 
         if (!(node->details->online)) {
             if (do_fail_resource) {
                 out->err(out, "Node %s is not online", host_uname);
                 return ENOTCONN;
             } else {
                 cib_only = true;
             }
         }
         if (!cib_only && pe__is_guest_or_remote_node(node)) {
             node = pe__current_node(node->details->remote_rsc);
             if (node == NULL) {
                 out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
                          host_uname);
                 return ENOTCONN;
             }
             router_node = node->details->uname;
         }
     }
 
     if (rsc->clone_name) {
         rsc_api_id = rsc->clone_name;
         rsc_long_id = rsc->id;
     } else {
         rsc_api_id = rsc->id;
     }
     if (do_fail_resource) {
         return pcmk_controld_api_fail(controld_api, host_uname, router_node,
                                       rsc_api_id, rsc_long_id,
                                       rsc_class, rsc_provider, rsc_type);
     } else {
         return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
                                          rsc_api_id, rsc_long_id, rsc_class,
                                          rsc_provider, rsc_type, cib_only);
     }
 }
 
 /*!
  * \internal
  * \brief Get resource name as used in failure-related node attributes
  *
  * \param[in] rsc  Resource to check
  *
  * \return Newly allocated string containing resource's fail name
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 rsc_fail_name(const pcmk_resource_t *rsc)
 {
     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 
     if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
         return strdup(name);
     }
     return clone_strip(name);
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pcmk_scheduler_t *scheduler)
 {
     int rc = pcmk_rc_ok;
 
     /* Erase the resource's entire LRM history in the CIB, even if we're only
      * clearing a single operation's fail count. If we erased only entries for a
      * single operation, we might wind up with a wrong idea of the current
      * resource state, and we might not re-probe the resource.
      */
     rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     crm_trace("Processing %d mainloop inputs",
               pcmk_controld_api_replies_expected(controld_api));
     while (g_main_context_iteration(NULL, FALSE)) {
         crm_trace("Processed mainloop input, %d still remaining",
                   pcmk_controld_api_replies_expected(controld_api));
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
                    const char *node_name, const char *rsc_id, const char *operation,
                    const char *interval_spec, pcmk_scheduler_t *scheduler)
 {
     int rc = pcmk_rc_ok;
     const char *failed_value = NULL;
     const char *failed_id = NULL;
     char *interval_ms_s = NULL;
     GHashTable *rscs = NULL;
     GHashTableIter iter;
 
     /* Create a hash table to use as a set of resources to clean. This lets us
      * clean each resource only once (per node) regardless of how many failed
      * operations it has.
      */
     rscs = pcmk__strkey_table(NULL, NULL);
 
     // Normalize interval to milliseconds for comparison to history entry
     if (operation) {
         guint interval_ms = 0U;
 
         pcmk_parse_interval_spec(interval_spec, &interval_ms);
         interval_ms_s = crm_strdup_printf("%u", interval_ms);
     }
 
     for (xmlNode *xml_op = pcmk__xml_first_child(scheduler->failed);
          xml_op != NULL;
          xml_op = pcmk__xml_next(xml_op)) {
 
         failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
         if (failed_id == NULL) {
             // Malformed history entry, should never happen
             continue;
         }
 
         // No resource specified means all resources match
         if (rsc_id) {
             pcmk_resource_t *fail_rsc = NULL;
 
             fail_rsc = pe_find_resource_with_flags(scheduler->resources,
                                                    failed_id,
                                                    pcmk_rsc_match_history
                                                    |pcmk_rsc_match_anon_basename);
             if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
                 continue;
             }
         }
 
         // Host name should always have been provided by this point
         failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
         if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
             continue;
         }
 
         // No operation specified means all operations match
         if (operation) {
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
             if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
                 continue;
             }
 
             // Interval (if operation was specified) defaults to 0 (not all)
             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
             if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
                 continue;
             }
         }
 
         g_hash_table_add(rscs, (gpointer) failed_id);
     }
 
     free(interval_ms_s);
 
     g_hash_table_iter_init(&iter, rscs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
         rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
         if (rc != pcmk_rc_ok) {
             return rc;
         }
     }
     g_hash_table_destroy(rscs);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 clear_rsc_fail_attrs(const pcmk_resource_t *rsc, const char *operation,
                      const char *interval_spec, const pcmk_node_t *node)
 {
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     char *rsc_name = rsc_fail_name(rsc);
 
     if (pe__is_guest_or_remote_node(node)) {
         attr_options |= pcmk__node_attr_remote;
     }
 
     rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
                                         operation, interval_spec, NULL,
                                         attr_options);
     free(rsc_name);
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
                     const pcmk_resource_t *rsc, const char *operation,
                     const char *interval_spec, bool just_failures,
                     pcmk_scheduler_t *scheduler, gboolean force)
 {
     pcmk__output_t *out = scheduler->priv;
     int rc = pcmk_rc_ok;
     pcmk_node_t *node = NULL;
 
     if (rsc == NULL) {
         return ENXIO;
 
     } else if (rsc->children) {
 
         for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
             const pcmk_resource_t *child = (const pcmk_resource_t *) lpc->data;
 
             rc = cli_resource_delete(controld_api, host_uname, child, operation,
                                      interval_spec, just_failures, scheduler,
                                      force);
             if (rc != pcmk_rc_ok) {
                 return rc;
             }
         }
         return pcmk_rc_ok;
 
     } else if (host_uname == NULL) {
         GList *lpc = NULL;
         GList *nodes = g_hash_table_get_values(rsc->known_on);
 
         if(nodes == NULL && force) {
             nodes = pcmk__copy_node_list(scheduler->nodes, false);
 
         } else if(nodes == NULL && rsc->exclusive_discover) {
             GHashTableIter iter;
             pcmk_node_t *node = NULL;
 
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
                 if(node->weight >= 0) {
                     nodes = g_list_prepend(nodes, node);
                 }
             }
 
         } else if(nodes == NULL) {
             nodes = g_hash_table_get_values(rsc->allowed_nodes);
         }
 
         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
             node = (pcmk_node_t *) lpc->data;
 
             if (node->details->online) {
                 rc = cli_resource_delete(controld_api, node->details->uname, rsc,
                                          operation, interval_spec, just_failures,
                                          scheduler, force);
             }
             if (rc != pcmk_rc_ok) {
                 g_list_free(nodes);
                 return rc;
             }
         }
 
         g_list_free(nodes);
         return pcmk_rc_ok;
     }
 
     node = pe_find_node(scheduler->nodes, host_uname);
 
     if (node == NULL) {
         out->err(out, "Unable to clean up %s because node %s not found",
                  rsc->id, host_uname);
         return ENODEV;
     }
 
     if (!node->details->rsc_discovery_enabled) {
         out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
                  rsc->id, host_uname);
         return EOPNOTSUPP;
     }
 
     if (controld_api == NULL) {
         out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
                  rsc->id, host_uname);
         return pcmk_rc_ok;
     }
 
     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up %s failures on %s: %s",
                  rsc->id, host_uname, pcmk_rc_str(rc));
         return rc;
     }
 
     if (just_failures) {
         rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
                                 interval_spec, scheduler);
     } else {
         rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
     }
     if (rc != pcmk_rc_ok) {
         out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
                  rsc->id, host_uname, pcmk_rc_str(rc));
     } else {
         out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
     }
     return rc;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
                 const char *operation, const char *interval_spec,
                 pcmk_scheduler_t *scheduler)
 {
     pcmk__output_t *out = scheduler->priv;
     int rc = pcmk_rc_ok;
     int attr_options = pcmk__node_attr_none;
     const char *display_name = node_name? node_name : "all nodes";
 
     if (controld_api == NULL) {
         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
                   display_name);
         return rc;
     }
 
     if (node_name) {
         pcmk_node_t *node = pe_find_node(scheduler->nodes, node_name);
 
         if (node == NULL) {
             out->err(out, "Unknown node: %s", node_name);
             return ENXIO;
         }
         if (pe__is_guest_or_remote_node(node)) {
             attr_options |= pcmk__node_attr_remote;
         }
     }
 
     rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
                                         interval_spec, NULL, attr_options);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Unable to clean up all failures on %s: %s",
                  display_name, pcmk_rc_str(rc));
         return rc;
     }
 
     if (node_name) {
         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
                                 operation, interval_spec, scheduler);
         if (rc != pcmk_rc_ok) {
             out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
                      node_name, pcmk_rc_str(rc));
             return rc;
         }
     } else {
         for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
             pcmk_node_t *node = (pcmk_node_t *) iter->data;
 
             rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
                                     operation, interval_spec, scheduler);
             if (rc != pcmk_rc_ok) {
                 out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
                          pcmk_rc_str(rc));
                 return rc;
             }
         }
     }
 
     out->info(out, "Cleaned up all resources on %s", display_name);
     return rc;
 }
 
 static void
 check_role(resource_checks_t *checks)
 {
     const char *role_s = g_hash_table_lookup(checks->rsc->meta,
                                              PCMK_META_TARGET_ROLE);
 
     if (role_s == NULL) {
         return;
     }
     switch (text2role(role_s)) {
         case pcmk_role_stopped:
             checks->flags |= rsc_remain_stopped;
             break;
 
         case pcmk_role_unpromoted:
             if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
                             pcmk_rsc_promotable)) {
                 checks->flags |= rsc_unpromotable;
             }
             break;
 
         default:
             break;
     }
 }
 
 static void
 check_managed(resource_checks_t *checks)
 {
     const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
                                                 PCMK_META_IS_MANAGED);
 
     if ((managed_s != NULL) && !crm_is_true(managed_s)) {
         checks->flags |= rsc_unmanaged;
     }
 }
 
 static void
 check_locked(resource_checks_t *checks)
 {
     if (checks->rsc->lock_node != NULL) {
         checks->flags |= rsc_locked;
         checks->lock_node = checks->rsc->lock_node->details->uname;
     }
 }
 
 static bool
 node_is_unhealthy(pcmk_node_t *node)
 {
     switch (pe__health_strategy(node->details->data_set)) {
         case pcmk__health_strategy_none:
             break;
 
         case pcmk__health_strategy_no_red:
             if (pe__node_health(node) < 0) {
                 return true;
             }
             break;
 
         case pcmk__health_strategy_only_green:
             if (pe__node_health(node) <= 0) {
                 return true;
             }
             break;
 
         case pcmk__health_strategy_progressive:
         case pcmk__health_strategy_custom:
             /* @TODO These are finite scores, possibly with rules, and possibly
              * combining with other scores, so attributing these as a cause is
              * nontrivial.
              */
             break;
     }
     return false;
 }
 
 static void
 check_node_health(resource_checks_t *checks, pcmk_node_t *node)
 {
     if (node == NULL) {
         GHashTableIter iter;
         bool allowed = false;
         bool all_nodes_unhealthy = true;
 
         g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
             allowed = true;
             if (!node_is_unhealthy(node)) {
                 all_nodes_unhealthy = false;
                 break;
             }
         }
         if (allowed && all_nodes_unhealthy) {
             checks->flags |= rsc_node_health;
         }
 
     } else if (node_is_unhealthy(node)) {
         checks->flags |= rsc_node_health;
     }
 }
 
 int
 cli_resource_check(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
 {
     resource_checks_t checks = { .rsc = rsc };
 
     check_role(&checks);
     check_managed(&checks);
     check_locked(&checks);
     check_node_health(&checks, node);
 
     return out->message(out, "resource-check-list", &checks);
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
                   const char *rsc_id, pcmk_scheduler_t *scheduler)
 {
     crm_notice("Failing %s on %s", rsc_id, host_uname);
     return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
 }
 
 static GHashTable *
 generate_resource_params(pcmk_resource_t *rsc, pcmk_node_t *node,
                          pcmk_scheduler_t *scheduler)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
     char *key = NULL;
     char *value = NULL;
 
     combined = pcmk__strkey_table(free, free);
 
     params = pe_rsc_params(rsc, node, scheduler);
     if (params != NULL) {
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             g_hash_table_insert(combined, strdup(key), strdup(value));
         }
     }
 
     meta = pcmk__strkey_table(free, free);
     get_meta_attributes(meta, rsc, node, scheduler);
     if (meta != NULL) {
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
 
             g_hash_table_insert(combined, crm_name, strdup(value));
         }
         g_hash_table_destroy(meta);
     }
 
     return combined;
 }
 
 bool resource_is_running_on(pcmk_resource_t *rsc, const char *host)
 {
     bool found = true;
     GList *hIter = NULL;
     GList *hosts = NULL;
 
     if (rsc == NULL) {
         return false;
     }
 
     rsc->fns->location(rsc, &hosts, TRUE);
     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) hIter->data;
 
         if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         }
     }
 
     if (host != NULL) {
         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
         found = false;
 
     } else if(host == NULL && hosts == NULL) {
         crm_trace("Resource %s is not running\n", rsc->id);
         found = false;
     }
 
   done:
     g_list_free(hosts);
     return found;
 }
 
 /*!
  * \internal
  * \brief Create a list of all resources active on host from a given list
  *
  * \param[in] host      Name of host to check whether resources are active
  * \param[in] rsc_list  List of resources to check
  *
  * \return New list of resources from list that are active on host
  */
 static GList *
 get_active_resources(const char *host, GList *rsc_list)
 {
     GList *rIter = NULL;
     GList *active = NULL;
 
     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) rIter->data;
 
         /* Expand groups to their members, because if we're restarting a member
          * other than the first, we can't otherwise tell which resources are
          * stopping and starting.
          */
         if (rsc->variant == pcmk_rsc_variant_group) {
             active = g_list_concat(active,
                                    get_active_resources(host, rsc->children));
         } else if (resource_is_running_on(rsc, host)) {
             active = g_list_append(active, strdup(rsc->id));
         }
     }
     return active;
 }
 
 static void dump_list(GList *items, const char *tag)
 {
     int lpc = 0;
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
         lpc++;
     }
 }
 
 static void display_list(pcmk__output_t *out, GList *items, const char *tag)
 {
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         out->info(out, "%s%s", tag, (const char *)item->data);
     }
 }
 
 /*!
  * \internal
  * \brief Upgrade XML to latest schema version and use it as scheduler input
  *
  * This also updates the scheduler timestamp to the current time.
  *
  * \param[in,out] scheduler  Scheduler data to update
  * \param[in,out] xml        XML to use as input
  *
  * \return Standard Pacemaker return code
  * \note On success, caller is responsible for freeing memory allocated for
  *       scheduler->now.
  * \todo This follows the example of other callers of cli_config_update()
  *       and returns ENOKEY ("Required key not available") if that fails,
  *       but perhaps pcmk_rc_schema_validation would be better in that case.
  */
 int
 update_scheduler_input(pcmk_scheduler_t *scheduler, xmlNode **xml)
 {
     if (cli_config_update(xml, NULL, FALSE) == FALSE) {
         return ENOKEY;
     }
     scheduler->input = *xml;
     scheduler->now = crm_time_new(NULL);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Update scheduler XML input based on a CIB query
  *
  * \param[in] scheduler  Scheduler data to initialize
  * \param[in] cib        Connection to the CIB manager
  *
  * \return Standard Pacemaker return code
  * \note On success, caller is responsible for freeing memory allocated for
  *       scheduler->input and scheduler->now.
  */
 static int
 update_scheduler_input_to_cib(pcmk__output_t *out, pcmk_scheduler_t *scheduler,
                               cib_t *cib)
 {
     xmlNode *cib_xml_copy = NULL;
     int rc = pcmk_rc_ok;
 
     rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
     rc = pcmk_legacy2rc(rc);
 
     if (rc != pcmk_rc_ok) {
         out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
         return rc;
     }
     rc = update_scheduler_input(scheduler, &cib_xml_copy);
     if (rc != pcmk_rc_ok) {
         out->err(out, "Could not upgrade the current CIB XML");
         free_xml(cib_xml_copy);
         return rc;
     }
 
     return rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 update_dataset(cib_t *cib, pcmk_scheduler_t *scheduler, bool simulate)
 {
     char *pid = NULL;
     char *shadow_file = NULL;
     cib_t *shadow_cib = NULL;
     int rc = pcmk_rc_ok;
 
     pcmk__output_t *out = scheduler->priv;
 
     pe_reset_working_set(scheduler);
     pcmk__set_scheduler_flags(scheduler,
                               pcmk_sched_no_counts|pcmk_sched_no_compat);
     rc = update_scheduler_input_to_cib(out, scheduler, cib);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     if(simulate) {
         bool prev_quiet = false;
 
         pid = pcmk__getpid_s();
         shadow_cib = cib_shadow_new(pid);
         shadow_file = get_shadow_file(pid);
 
         if (shadow_cib == NULL) {
             out->err(out, "Could not create shadow cib: '%s'", pid);
             rc = ENXIO;
             goto done;
         }
 
         rc = write_xml_file(scheduler->input, shadow_file, FALSE);
 
         if (rc < 0) {
             out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
             goto done;
         }
 
         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
         rc = pcmk_legacy2rc(rc);
 
         if (rc != pcmk_rc_ok) {
             out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_rc_str(rc), rc);
             goto done;
         }
 
         pcmk__schedule_actions(scheduler->input,
                                pcmk_sched_no_counts|pcmk_sched_no_compat,
                                scheduler);
 
         prev_quiet = out->is_quiet(out);
         out->quiet = true;
         pcmk__simulate_transition(scheduler, shadow_cib, NULL);
         out->quiet = prev_quiet;
 
         rc = update_dataset(shadow_cib, scheduler, false);
 
     } else {
         cluster_status(scheduler);
     }
 
   done:
     // Do not free scheduler->input here, we need rsc->xml to be valid later on
     cib_delete(shadow_cib);
     free(pid);
 
     if(shadow_file) {
         unlink(shadow_file);
         free(shadow_file);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Find the maximum stop timeout of a resource and its children (if any)
  *
  * \param[in,out] rsc  Resource to get timeout for
  *
  * \return Maximum stop timeout for \p rsc (in milliseconds)
  */
 static int
 max_rsc_stop_timeout(pcmk_resource_t *rsc)
 {
     long long result_ll;
     int max_delay = 0;
     xmlNode *config = NULL;
     GHashTable *meta = NULL;
 
     if (rsc == NULL) {
         return 0;
     }
 
     // If resource is collective, use maximum of its children's stop timeouts
     if (rsc->children != NULL) {
         for (GList *iter = rsc->children; iter; iter = iter->next) {
             pcmk_resource_t *child = iter->data;
             int delay = max_rsc_stop_timeout(child);
 
             if (delay > max_delay) {
                 pcmk__rsc_trace(rsc,
                                 "Maximum stop timeout for %s is now %s "
                                 "due to %s", rsc->id,
                                 pcmk__readable_interval(delay), child->id);
                 max_delay = delay;
             }
         }
         return max_delay;
     }
 
     // Get resource's stop action configuration from CIB
     config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);
 
     /* Get configured timeout for stop action (fully evaluated for rules,
      * defaults, etc.).
      *
      * @TODO This currently ignores node (which might matter for rules)
      */
     meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
     if ((pcmk__scan_ll(g_hash_table_lookup(meta, XML_ATTR_TIMEOUT),
                        &result_ll, -1LL) == pcmk_rc_ok)
         && (result_ll >= 0) && (result_ll <= INT_MAX)) {
         max_delay = (int) result_ll;
     }
     g_hash_table_destroy(meta);
 
     return max_delay;
 }
 
 /*!
  * \internal
  * \brief Find a reasonable waiting time for stopping any one resource in a list
  *
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     resources  List of names of resources that will be stopped
  *
  * \return Rough estimate of a reasonable time to wait (in seconds) to stop any
  *         one resource in \p resources
  * \note This estimate is very rough, simply the maximum stop timeout of all
  *       given resources and their children, plus a small fudge factor. It does
  *       not account for children that must be stopped in sequence, action
  *       throttling, or any demotions needed. It checks the stop timeout, even
  *       if the resources in question are actually being started.
  */
 static int
 wait_time_estimate(pcmk_scheduler_t *scheduler, const GList *resources)
 {
     int max_delay = 0;
 
     // Find maximum stop timeout in milliseconds
     for (const GList *item = resources; item != NULL; item = item->next) {
         pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
                                                 (const char *) item->data);
         int delay = max_rsc_stop_timeout(rsc);
 
         if (delay > max_delay) {
             pcmk__rsc_trace(rsc,
                             "Wait time is now %s due to %s",
                             pcmk__readable_interval(delay), rsc->id);
             max_delay = delay;
         }
     }
 
     return (max_delay / 1000) + 5;
 }
 
 #define waiting_for_starts(d, r, h) ((d != NULL) || \
                                     (!resource_is_running_on((r), (h))))
 
 /*!
  * \internal
  * \brief Restart a resource (on a particular host if requested).
  *
  * \param[in,out] out                 Output object
  * \param[in,out] rsc                 The resource to restart
  * \param[in]     node                Node to restart resource on (NULL for all)
  * \param[in]     move_lifetime       If not NULL, how long constraint should
  *                                    remain in effect (as ISO 8601 string)
  * \param[in]     timeout_ms          Consider failed if actions do not complete
  *                                    in this time (specified in milliseconds,
  *                                    but a two-second granularity is actually
  *                                    used; if 0, it will be calculated based on
  *                                    the resource timeout)
  * \param[in,out] cib                 Connection to the CIB manager
  * \param[in]     cib_options         Group of enum cib_call_options flags to
  *                                    use with CIB calls
  * \param[in]     promoted_role_only  If true, limit to promoted instances
  * \param[in]     force               If true, apply only to requested instance
  *                                    if part of a collective resource
  *
  * \return Standard Pacemaker return code (exits on certain failures)
  */
 int
 cli_resource_restart(pcmk__output_t *out, pcmk_resource_t *rsc,
                      const pcmk_node_t *node, const char *move_lifetime,
                      int timeout_ms, cib_t *cib, int cib_options,
                      gboolean promoted_role_only, gboolean force)
 {
     int rc = pcmk_rc_ok;
     int lpc = 0;
     int before = 0;
     int step_timeout_s = 0;
     int sleep_interval = 2;
     int timeout = timeout_ms / 1000;
 
     bool stop_via_ban = false;
     char *rsc_id = NULL;
     char *lookup_id = NULL;
     char *orig_target_role = NULL;
 
     GList *list_delta = NULL;
     GList *target_active = NULL;
     GList *current_active = NULL;
     GList *restart_target_active = NULL;
 
     pcmk_scheduler_t *scheduler = NULL;
     pcmk_resource_t *parent = uber_parent(rsc);
 
     bool running = false;
     const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
     const char *host = node ? node->details->uname : NULL;
 
     /* If the implicit resource or primitive resource of a bundle is given, operate on the
      * bundle itself instead.
      */
     if (pe_rsc_is_bundled(rsc)) {
         rsc = parent->parent;
     }
 
     running = resource_is_running_on(rsc, host);
 
     if (pe_rsc_is_clone(parent) && !running) {
         if (pe_rsc_is_unique_clone(parent)) {
             lookup_id = strdup(rsc->id);
         } else {
             lookup_id = clone_strip(rsc->id);
         }
 
         rsc = parent->fns->find_rsc(parent, lookup_id, node,
                                     pcmk_rsc_match_basename
                                     |pcmk_rsc_match_current_node);
         free(lookup_id);
         running = resource_is_running_on(rsc, host);
     }
 
     if (!running) {
         if (host) {
             out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
         } else {
             out->err(out, "%s is not running anywhere and so cannot be restarted", id);
         }
         return ENXIO;
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
         out->err(out, "Unmanaged resources cannot be restarted.");
         return EAGAIN;
     }
 
     rsc_id = strdup(rsc->id);
 
     if (pe_rsc_is_unique_clone(parent)) {
         lookup_id = strdup(rsc->id);
     } else {
         lookup_id = clone_strip(rsc->id);
     }
 
     if (host) {
         if (pe_rsc_is_clone(rsc) || pe_bundle_replicas(rsc)) {
             stop_via_ban = true;
         } else if (pe_rsc_is_clone(parent)) {
             stop_via_ban = true;
             free(lookup_id);
             lookup_id = strdup(parent->id);
         }
     }
 
     /*
       grab full cib
       determine originally active resources
       disable or ban
       poll cib and watch for affected resources to get stopped
       without --timeout, calculate the stop timeout for each step and wait for that
       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
       if everything stopped, re-enable or un-ban
       poll cib and watch for affected resources to get started
       without --timeout, calculate the start timeout for each step and wait for that
       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
       report success
 
       Optimizations:
       - use constraints to determine ordered list of affected resources
       - Allow a --no-deps option (aka. --force-restart)
     */
 
     scheduler = pe_new_working_set();
     if (scheduler == NULL) {
         rc = errno;
         out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
         goto done;
     }
 
     scheduler->priv = out;
     rc = update_dataset(cib, scheduler, false);
 
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
         goto done;
     }
 
     restart_target_active = get_active_resources(host, scheduler->resources);
     current_active = get_active_resources(host, scheduler->resources);
 
     dump_list(current_active, "Origin");
 
     if (stop_via_ban) {
         /* Stop the clone or bundle instance by banning it from the host */
         out->quiet = true;
         rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
                               cib_options, promoted_role_only,
                               PCMK__ROLE_PROMOTED);
     } else {
         /* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
          * Remember any existing PCMK_META_TARGET_ROLE so we can restore it
          * later (though it only makes any difference if it's Unpromoted).
          */
 
         find_resource_attr(out, cib, PCMK_XA_VALUE, lookup_id, NULL, NULL, NULL,
                            PCMK_META_TARGET_ROLE, &orig_target_role);
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, PCMK_META_TARGET_ROLE,
                                            PCMK_ACTION_STOPPED, FALSE, cib,
                                            cib_options, force);
     }
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
                  rsc_id, pcmk_rc_str(rc), rc);
         if (current_active != NULL) {
             g_list_free_full(current_active, free);
             current_active = NULL;
         }
         if (restart_target_active != NULL) {
             g_list_free_full(restart_target_active, free);
             restart_target_active = NULL;
         }
         goto done;
     }
 
     rc = update_dataset(cib, scheduler, true);
     if(rc != pcmk_rc_ok) {
         out->err(out, "Could not determine which resources would be stopped");
         goto failure;
     }
 
     target_active = get_active_resources(host, scheduler->resources);
     dump_list(target_active, "Target");
 
     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (list_delta != NULL) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = wait_time_estimate(scheduler, list_delta)
                              / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
             rc = update_dataset(cib, scheduler, FALSE);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were stopped");
                 goto failure;
             }
 
             if (current_active != NULL) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(host, scheduler->resources);
 
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
 
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
             out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     if (stop_via_ban) {
         rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
 
     } else if (orig_target_role) {
         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, PCMK_META_TARGET_ROLE,
                                            orig_target_role, FALSE, cib,
                                            cib_options, force);
         free(orig_target_role);
         orig_target_role = NULL;
     } else {
         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                            NULL, PCMK_META_TARGET_ROLE, cib,
                                            cib_options, force);
     }
 
     if(rc != pcmk_rc_ok) {
         out->err(out,
                  "Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
                  rsc_id, pcmk_rc_str(rc), rc);
         goto done;
     }
 
     if (target_active != NULL) {
         g_list_free_full(target_active, free);
     }
     target_active = restart_target_active;
 
     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
     out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
     display_list(out, list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (waiting_for_starts(list_delta, rsc, host)) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = wait_time_estimate(scheduler, list_delta)
                              / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
 
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
 
             rc = update_dataset(cib, scheduler, false);
             if(rc != pcmk_rc_ok) {
                 out->err(out, "Could not determine which resources were started");
                 goto failure;
             }
 
             /* It's OK if dependent resources moved to a different node,
              * so we check active resources on all nodes.
              */
             if (current_active != NULL) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(NULL, scheduler->resources);
 
             g_list_free(list_delta);
             list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
             out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
             display_list(out, list_delta, " * ");
             rc = ETIME;
             goto failure;
         }
 
     }
 
     rc = pcmk_rc_ok;
     goto done;
 
   failure:
     if (stop_via_ban) {
         cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
     } else if (orig_target_role) {
         cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
                                       PCMK_META_TARGET_ROLE, orig_target_role,
                                       FALSE, cib, cib_options, force);
         free(orig_target_role);
     } else {
         cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
                                       NULL, PCMK_META_TARGET_ROLE, cib,
                                       cib_options, force);
     }
 
 done:
     if (list_delta != NULL) {
         g_list_free(list_delta);
     }
     if (current_active != NULL) {
         g_list_free_full(current_active, free);
     }
     if (target_active != NULL && (target_active != restart_target_active)) {
         g_list_free_full(target_active, free);
     }
     if (restart_target_active != NULL) {
         g_list_free_full(restart_target_active, free);
     }
     free(rsc_id);
     free(lookup_id);
     pe_free_working_set(scheduler);
     return rc;
 }
 
 static inline bool
 action_is_pending(const pcmk_action_t *action)
 {
     if (pcmk_any_flags_set(action->flags,
                            pcmk_action_optional|pcmk_action_pseudo)
         || !pcmk_is_set(action->flags, pcmk_action_runnable)
         || pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
         return false;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether any actions in a list are pending
  *
  * \param[in] actions   List of actions to check
  *
  * \return true if any actions in the list are pending, otherwise false
  */
 static bool
 actions_are_pending(const GList *actions)
 {
     for (const GList *action = actions; action != NULL; action = action->next) {
         const pcmk_action_t *a = (const pcmk_action_t *) action->data;
 
         if (action_is_pending(a)) {
             crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
             return true;
         }
     }
     return false;
 }
 
 static void
 print_pending_actions(pcmk__output_t *out, GList *actions)
 {
     GList *action;
 
     out->info(out, "Pending actions:");
     for (action = actions; action != NULL; action = action->next) {
         pcmk_action_t *a = (pcmk_action_t *) action->data;
 
         if (!action_is_pending(a)) {
             continue;
         }
 
         if (a->node) {
             out->info(out, "\tAction %d: %s\ton %s",
                       a->id, a->uuid, pe__node_name(a->node));
         } else {
             out->info(out, "\tAction %d: %s", a->id, a->uuid);
         }
     }
 }
 
 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
 
 /* For --wait, how long to sleep between cluster state checks */
 #define WAIT_SLEEP_S (2)
 
 /*!
  * \internal
  * \brief Wait until all pending cluster actions are complete
  *
  * This waits until either the CIB's transition graph is idle or a timeout is
  * reached.
  *
  * \param[in,out] out          Output object
  * \param[in]     timeout_ms   Consider failed if actions do not complete in
  *                             this time (specified in milliseconds, but
  *                             one-second granularity is actually used; if 0, a
  *                             default will be used)
  * \param[in,out] cib          Connection to the CIB manager
  *
  * \return Standard Pacemaker return code
  */
 int
 wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
 {
     pcmk_scheduler_t *scheduler = NULL;
     xmlXPathObjectPtr search;
     int rc = pcmk_rc_ok;
     bool pending_unknown_state_resources;
     int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
     time_t expire_time = time(NULL) + timeout_s;
     time_t time_diff;
     bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
 
     scheduler = pe_new_working_set();
     if (scheduler == NULL) {
         return ENOMEM;
     }
 
     do {
         /* Abort if timeout is reached */
         time_diff = expire_time - time(NULL);
         if (time_diff > 0) {
             crm_info("Waiting up to %lld seconds for cluster actions to complete", (long long) time_diff);
         } else {
             print_pending_actions(out, scheduler->actions);
             pe_free_working_set(scheduler);
             return ETIME;
         }
         if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
             sleep(WAIT_SLEEP_S);
         }
 
         /* Get latest transition graph */
         pe_reset_working_set(scheduler);
         rc = update_scheduler_input_to_cib(out, scheduler, cib);
         if (rc != pcmk_rc_ok) {
             pe_free_working_set(scheduler);
             return rc;
         }
         pcmk__schedule_actions(scheduler->input,
                                pcmk_sched_no_counts|pcmk_sched_no_compat,
                                scheduler);
 
         if (!printed_version_warning) {
             /* If the DC has a different version than the local node, the two
              * could come to different conclusions about what actions need to be
              * done. Warn the user in this case.
              *
              * @TODO A possible long-term solution would be to reimplement the
              * wait as a new controller operation that would be forwarded to the
              * DC. However, that would have potential problems of its own.
              */
             const char *dc_version = g_hash_table_lookup(scheduler->config_hash,
                                                          PCMK_OPT_DC_VERSION);
 
             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
                 out->info(out, "warning: wait option may not work properly in "
                           "mixed-version cluster");
                 printed_version_warning = true;
             }
         }
 
         search = xpath_search(scheduler->input, "/cib/status/node_state/lrm/lrm_resources/lrm_resource/"
                                                 XML_LRM_TAG_RSC_OP "[@" XML_LRM_ATTR_RC "='193']");
         pending_unknown_state_resources = (numXpathResults(search) > 0);
         freeXpathObject(search);
     } while (actions_are_pending(scheduler->actions) || pending_unknown_state_resources);
 
     pe_free_working_set(scheduler);
     return rc;
 }
 
 static const char *
 get_action(const char *rsc_action) {
     const char *action = NULL;
 
     if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
         action = PCMK_ACTION_VALIDATE_ALL;
 
     } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
         action = PCMK_ACTION_MONITOR;
 
     } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
                                     "force-demote", "force-promote", NULL)) {
         action = rsc_action+6;
     } else {
         action = rsc_action;
     }
 
     return action;
 }
 
 /*!
  * \brief Set up environment variables as expected by resource agents
  *
  * When the cluster executes resource agents, it adds certain environment
  * variables (directly or via resource meta-attributes) expected by some
  * resource agents. Add the essential ones that many resource agents expect, so
  * the behavior is the same for command-line execution.
  *
  * \param[in,out] params       Resource parameters that will be passed to agent
  * \param[in]     timeout_ms   Action timeout (in milliseconds)
  * \param[in]     check_level  OCF check level
  * \param[in]     verbosity    Verbosity level
  */
 static void
 set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
                       int verbosity)
 {
     g_hash_table_insert(params, strdup("CRM_meta_timeout"),
                         crm_strdup_printf("%d", timeout_ms));
 
     g_hash_table_insert(params, strdup(PCMK_XA_CRM_FEATURE_SET),
                         strdup(CRM_FEATURE_SET));
 
     if (check_level >= 0) {
         char *level = crm_strdup_printf("%d", check_level);
 
         setenv("OCF_CHECK_LEVEL", level, 1);
         free(level);
     }
 
     pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
     if (verbosity > 1) {
         setenv("OCF_TRACE_RA", "1", 1);
     }
 
     /* A resource agent using the standard ocf-shellfuncs library will not print
      * messages to stderr if it doesn't have a controlling terminal (e.g. if
      * crm_resource is called via script or ssh). This forces it to do so.
      */
     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
 }
 
 /*!
  * \internal
  * \brief Apply command-line overrides to resource parameters
  *
  * \param[in,out] params     Parameters to be passed to agent
  * \param[in]     overrides  Parameters to override (or NULL if none)
  */
 static void
 apply_overrides(GHashTable *params, GHashTable *overrides)
 {
     if (overrides != NULL) {
         GHashTableIter iter;
         char *name = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, overrides);
         while (g_hash_table_iter_next(&iter, (gpointer *) &name,
                                       (gpointer *) &value)) {
             g_hash_table_replace(params, strdup(name), strdup(value));
         }
     }
 }
 
 crm_exit_t
 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
                                  const char *rsc_class, const char *rsc_prov,
                                  const char *rsc_type, const char *rsc_action,
                                  GHashTable *params, GHashTable *override_hash,
                                  int timeout_ms, int resource_verbose, gboolean force,
                                  int check_level)
 {
     const char *class = rsc_class;
     const char *action = get_action(rsc_action);
     crm_exit_t exit_code = CRM_EX_OK;
     svc_action_t *op = NULL;
 
     // If no timeout was provided, use the same default as the cluster
     if (timeout_ms == 0) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
 
     set_agent_environment(params, timeout_ms, check_level, resource_verbose);
     apply_overrides(params, override_hash);
 
     op = services__create_resource_action(rsc_name? rsc_name : "test",
                                           rsc_class, rsc_prov, rsc_type, action,
                                           0, timeout_ms, params, 0);
     if (op == NULL) {
         out->err(out, "Could not execute %s using %s%s%s:%s: %s",
                  action, rsc_class, (rsc_prov? ":" : ""),
                  (rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
         g_hash_table_destroy(params);
         return CRM_EX_OSERR;
     }
 
     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
         class = resources_find_service_class(rsc_type);
     }
     if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
                               PCMK_RESOURCE_CLASS_LSB, NULL)) {
         services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
                                 "Manual execution of the %s standard is "
                                 "unsupported", pcmk__s(class, "unspecified"));
     }
 
     if (op->rc != PCMK_OCF_UNKNOWN) {
         exit_code = op->rc;
         goto done;
     }
 
     services_action_sync(op);
 
     // Map results to OCF codes for consistent reporting to user
     {
         enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
 
         // Cast variable instead of function return to keep compilers happy
         exit_code = (crm_exit_t) ocf_code;
     }
 
 done:
     out->message(out, "resource-agent-action", resource_verbose, rsc_class,
                  rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
                  exit_code, op->status, services__exit_reason(op),
                  op->stdout_data, op->stderr_data);
     services_action_free(op);
     return exit_code;
 }
 
 crm_exit_t
 cli_resource_execute(pcmk_resource_t *rsc, const char *requested_name,
                      const char *rsc_action, GHashTable *override_hash,
                      int timeout_ms, cib_t *cib, pcmk_scheduler_t *scheduler,
                      int resource_verbose, gboolean force, int check_level)
 {
     pcmk__output_t *out = scheduler->priv;
     crm_exit_t exit_code = CRM_EX_OK;
     const char *rid = NULL;
     const char *rtype = NULL;
     const char *rprov = NULL;
     const char *rclass = NULL;
     GHashTable *params = NULL;
 
     if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
                                     "force-promote", NULL)) {
         if(pe_rsc_is_clone(rsc)) {
             GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
             if(nodes != NULL && force == FALSE) {
                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
                          rsc_action, rsc->id);
                 out->err(out,
                          "Try setting "
                          PCMK_META_TARGET_ROLE "=" PCMK__ROLE_STOPPED
                          " first or specifying the force option");
                 return CRM_EX_UNSAFE;
             }
 
             g_list_free_full(nodes, free);
         }
     }
 
     if(pe_rsc_is_clone(rsc)) {
         /* Grab the first child resource in the hope it's not a group */
         rsc = rsc->children->data;
     }
 
     if (rsc->variant == pcmk_rsc_variant_group) {
         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     } else if (pe_rsc_is_bundled(rsc)) {
         out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
         return CRM_EX_UNIMPLEMENT_FEATURE;
     }
 
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
-    rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
+    rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
     rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
 
     params = generate_resource_params(rsc, NULL /* @TODO use local node */,
                                       scheduler);
 
     if (timeout_ms == 0) {
         timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action),
                                                scheduler);
     }
 
     rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
 
     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
                                                  params, override_hash, timeout_ms,
                                                  resource_verbose, force, check_level);
     return exit_code;
 }
 
 // \return Standard Pacemaker return code
 int
 cli_resource_move(const pcmk_resource_t *rsc, const char *rsc_id,
                   const char *host_name, const char *move_lifetime, cib_t *cib,
                   int cib_options, pcmk_scheduler_t *scheduler,
                   gboolean promoted_role_only, gboolean force)
 {
     pcmk__output_t *out = scheduler->priv;
     int rc = pcmk_rc_ok;
     unsigned int count = 0;
     pcmk_node_t *current = NULL;
     pcmk_node_t *dest = pe_find_node(scheduler->nodes, host_name);
     bool cur_is_dest = false;
 
     if (dest == NULL) {
         return pcmk_rc_node_unknown;
     }
 
     if (promoted_role_only
         && !pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
 
         const pcmk_resource_t *p = pe__const_top_resource(rsc, false);
 
         if (pcmk_is_set(p->flags, pcmk_rsc_promotable)) {
             out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
             rsc_id = p->id;
             rsc = p;
 
         } else {
             out->info(out, "Ignoring --promoted option: %s is not promotable",
                       rsc_id);
             promoted_role_only = FALSE;
         }
     }
 
     current = pe__find_active_requires(rsc, &count);
 
     if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
         unsigned int promoted_count = 0;
         pcmk_node_t *promoted_node = NULL;
 
         for (const GList *iter = rsc->children; iter; iter = iter->next) {
             const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
             enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
             if (child_role == pcmk_role_promoted) {
                 rsc = child;
                 promoted_node = pe__current_node(child);
                 promoted_count++;
             }
         }
         if (promoted_role_only || (promoted_count != 0)) {
             count = promoted_count;
             current = promoted_node;
         }
 
     }
 
     if (count > 1) {
         if (pe_rsc_is_clone(rsc)) {
             current = NULL;
         } else {
             return pcmk_rc_multiple;
         }
     }
 
     if (current && (current->details == dest->details)) {
         cur_is_dest = true;
         if (force) {
             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
                      rsc_id, promoted_role_only?"promoted":"active",
                      pe__node_name(dest));
         } else {
             return pcmk_rc_already;
         }
     }
 
     /* Clear any previous prefer constraints across all nodes. */
     cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, cib_options, false,
                        force);
 
     /* Clear any previous ban constraints on 'dest'. */
     cli_resource_clear(rsc_id, dest->details->uname, scheduler->nodes, cib,
                        cib_options, TRUE, force);
 
     /* Record an explicit preference for 'dest' */
     rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
                              cib, cib_options, promoted_role_only,
                              PCMK__ROLE_PROMOTED);
 
     crm_trace("%s%s now prefers %s%s",
               rsc->id, (promoted_role_only? " (promoted)" : ""),
               pe__node_name(dest), force?"(forced)":"");
 
     /* only ban the previous location if current location != destination location.
      * it is possible to use -M to enforce a location without regard of where the
      * resource is currently located */
     if (force && !cur_is_dest) {
         /* Ban the original location if possible */
         if(current) {
             (void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
                                    cib, cib_options, promoted_role_only,
                                    PCMK__ROLE_PROMOTED);
         } else if(count > 1) {
             out->info(out, "Resource '%s' is currently %s in %d locations. "
                       "One may now move to %s",
                       rsc_id, (promoted_role_only? "promoted" : "active"),
                       count, pe__node_name(dest));
             out->info(out, "To prevent '%s' from being %s at a specific location, "
                       "specify a node.",
                       rsc_id, (promoted_role_only? "promoted" : "active"));
 
         } else {
             crm_trace("Not banning %s from its current location: not active", rsc_id);
         }
     }
 
     return rc;
 }