diff --git a/include/crm/common/scheduler_internal.h b/include/crm/common/scheduler_internal.h
index 39e806d92c..c2ebaf329b 100644
--- a/include/crm/common/scheduler_internal.h
+++ b/include/crm/common/scheduler_internal.h
@@ -1,286 +1,289 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H
 #define PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H
 
 #include <crm/common/action_relation_internal.h>
 #include <crm/common/actions_internal.h>
 #include <crm/common/attrs_internal.h>
 #include <crm/common/bundles_internal.h>
 #include <crm/common/clone_internal.h>
 #include <crm/common/digest_internal.h>
 #include <crm/common/failcounts_internal.h>
 #include <crm/common/group_internal.h>
 #include <crm/common/history_internal.h>
 #include <crm/common/location_internal.h>
 #include <crm/common/nodes_internal.h>
 #include <crm/common/primitive_internal.h>
 #include <crm/common/remote_internal.h>
 #include <crm/common/resources_internal.h>
 #include <crm/common/roles_internal.h>
 #include <crm/common/rules_internal.h>
 #include <crm/common/tickets_internal.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 enum pcmk__check_parameters {
     /* Clear fail count if parameters changed for un-expired start or monitor
      * last_failure.
      */
     pcmk__check_last_failure,
 
     /* Clear fail count if parameters changed for start, monitor, promote, or
      * migrate_from actions for active resources.
      */
     pcmk__check_active,
 };
 
 // Scheduling options and conditions
 enum pcmk__scheduler_flags {
     // No scheduler flags set (compare with equality rather than bit set)
     pcmk__sched_none                    = 0ULL,
 
     /* These flags are dynamically determined conditions */
 
     // Whether partition has quorum (via \c PCMK_XA_HAVE_QUORUM attribute)
     //! \deprecated Call pcmk_has_quorum() to check quorum instead
     pcmk__sched_quorate                 = (1ULL << 0),
 
     // Whether cluster is symmetric (via symmetric-cluster property)
     pcmk__sched_symmetric_cluster       = (1ULL << 1),
 
     // Whether scheduling encountered a non-configuration error
     pcmk__sched_processing_error        = (1ULL << 2),
 
     // Whether cluster is in maintenance mode (via maintenance-mode property)
     pcmk__sched_in_maintenance          = (1ULL << 3),
 
     // Whether fencing is enabled (via stonith-enabled property)
     pcmk__sched_fencing_enabled         = (1ULL << 4),
 
     // Whether cluster has a fencing resource (via CIB resources)
     /*! \deprecated To indicate the cluster has a fencing resource, add either a
      * fencing resource configuration or the have-watchdog cluster option to the
      * input CIB
      */
     pcmk__sched_have_fencing            = (1ULL << 5),
 
     // Whether any resource provides or requires unfencing (via CIB resources)
     pcmk__sched_enable_unfencing        = (1ULL << 6),
 
     // Whether concurrent fencing is allowed (via concurrent-fencing property)
     pcmk__sched_concurrent_fencing      = (1ULL << 7),
 
     /*
      * Whether resources removed from the configuration should be stopped (via
      * stop-orphan-resources property)
      */
     pcmk__sched_stop_removed_resources  = (1ULL << 8),
 
     /*
      * Whether recurring actions removed from the configuration should be
      * cancelled (via stop-orphan-actions property)
      */
     pcmk__sched_cancel_removed_actions  = (1ULL << 9),
 
     // Whether to stop all resources (via stop-all-resources property)
     pcmk__sched_stop_all                = (1ULL << 10),
 
     // Whether scheduler processing encountered a warning
     pcmk__sched_processing_warning      = (1ULL << 11),
 
     /*
      * Whether start failure should be treated as if
      * \c PCMK_META_MIGRATION_THRESHOLD is 1 (via
      * \c PCMK_OPT_START_FAILURE_IS_FATAL property)
      */
     pcmk__sched_start_failure_fatal     = (1ULL << 12),
 
     // Whether unseen nodes should be fenced (via startup-fencing property)
     pcmk__sched_startup_fencing         = (1ULL << 14),
 
     /*
      * Whether resources should be left stopped when their node shuts down
      * cleanly (via shutdown-lock property)
      */
     pcmk__sched_shutdown_lock           = (1ULL << 15),
 
     /*
      * Whether resources' current state should be probed (when unknown) before
      * scheduling any other actions (via the enable-startup-probes property)
      */
     pcmk__sched_probe_resources         = (1ULL << 16),
 
     // Whether the CIB status section has been parsed yet
     pcmk__sched_have_status             = (1ULL << 17),
 
     // Whether the cluster includes any Pacemaker Remote nodes (via CIB)
     pcmk__sched_have_remote_nodes       = (1ULL << 18),
 
 
     /* The remaining flags are scheduling options that must be set explicitly */
 
     /*
      * Whether to skip unpacking the CIB status section and stop the scheduling
      * sequence after applying node-specific location criteria (skipping
      * assignment, ordering, actions, etc.).
      */
     pcmk__sched_location_only           = (1ULL << 20),
 
     // Whether sensitive resource attributes have been masked
     pcmk__sched_sanitized               = (1ULL << 21),
 
     // Skip counting of total, disabled, and blocked resource instances
     pcmk__sched_no_counts               = (1ULL << 23),
 
     // Whether node scores should be output instead of logged
     pcmk__sched_output_scores           = (1ULL << 25),
 
     // Whether to show node and resource utilization (in log or output)
     pcmk__sched_show_utilization        = (1ULL << 26),
 
     /*
      * Whether to stop the scheduling sequence after unpacking the CIB,
      * calculating cluster status, and applying node health (skipping
      * applying node-specific location criteria, assignment, etc.)
      */
     pcmk__sched_validate_only           = (1ULL << 27),
 };
 
 // Implementation of pcmk__scheduler_private_t
 struct pcmk__scheduler_private {
     // Be careful about when each piece of information is available and final
 
     char *local_node_name;          // Name of node running scheduler (if known)
     crm_time_t *now;                // Time to use when evaluating rules
     pcmk__output_t *out;            // Output object for displaying messages
     GHashTable *options;            // Cluster options
     const char *fence_action;       // Default fencing action
     guint fence_timeout_ms;         // Default fencing action timeout (in ms)
     guint priority_fencing_ms;      // Priority-based fencing delay (in ms)
     guint shutdown_lock_ms;         // How long to lock resources (in ms)
     guint node_pending_ms;          // Pending join times out after this (in ms)
+
+    // @TODO convert to enum
     const char *placement_strategy; // Value of placement-strategy property
+
     xmlNode *rsc_defaults;          // Configured resource defaults
     xmlNode *op_defaults;           // Configured operation defaults
     GList *resources;               // Resources in cluster
     GHashTable *templates;          // Key = template ID, value = resource list
     GHashTable *tags;               // Key = tag ID, value = element list
     GList *actions;                 // All scheduled actions
     GHashTable *singletons;         // Scheduled non-resource actions
     int next_action_id;             // Counter used as ID for actions
     xmlNode *failed;                // History entries of failed actions
     GList *param_check;             // History entries that need to be checked
     GList *stop_needed;             // Containers that need stop actions
     GList *location_constraints;    // Location constraints
     GList *colocation_constraints;  // Colocation constraints
     GList *ordering_constraints;    // Ordering constraints
     GHashTable *ticket_constraints; // Key = ticket ID, value = pcmk__ticket_t
     int next_ordering_id;           // Counter used as ID for orderings
     int ninstances;                 // Total number of resource instances
     int blocked_resources;          // Number of blocked resources in cluster
     int disabled_resources;         // Number of disabled resources in cluster
     time_t recheck_by;              // Hint to controller when to reschedule
     xmlNode *graph;                 // Transition graph
     int synapse_count;              // Number of transition graph synapses
 };
 
 // Group of enum pcmk__warnings flags for warnings we want to log once
 extern uint32_t pcmk__warnings;
 
 /*!
  * \internal
  * \brief Log a resource-tagged message at info severity
  *
  * \param[in] rsc       Tag message with this resource's ID
  * \param[in] fmt...    printf(3)-style format and arguments
  */
 #define pcmk__rsc_info(rsc, fmt, args...)   \
     crm_log_tag(LOG_INFO, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)
 
 /*!
  * \internal
  * \brief Log a resource-tagged message at debug severity
  *
  * \param[in] rsc       Tag message with this resource's ID
  * \param[in] fmt...    printf(3)-style format and arguments
  */
 #define pcmk__rsc_debug(rsc, fmt, args...)  \
     crm_log_tag(LOG_DEBUG, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)
 
 /*!
  * \internal
  * \brief Log a resource-tagged message at trace severity
  *
  * \param[in] rsc       Tag message with this resource's ID
  * \param[in] fmt...    printf(3)-style format and arguments
  */
 #define pcmk__rsc_trace(rsc, fmt, args...)  \
     crm_log_tag(LOG_TRACE, ((rsc) == NULL)? "<NULL>" : (rsc)->id, (fmt), ##args)
 
 /*!
  * \internal
  * \brief Log an error and remember that current scheduler input has errors
  *
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     fmt...     printf(3)-style format and arguments
  */
 #define pcmk__sched_err(scheduler, fmt...) do {                     \
         pcmk__set_scheduler_flags((scheduler),                      \
                                   pcmk__sched_processing_error);    \
         crm_err(fmt);                                               \
     } while (0)
 
 /*!
  * \internal
  * \brief Log a warning and remember that current scheduler input has warnings
  *
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     fmt...     printf(3)-style format and arguments
  */
 #define pcmk__sched_warn(scheduler, fmt...) do {                    \
         pcmk__set_scheduler_flags((scheduler),                      \
                                   pcmk__sched_processing_warning);  \
         crm_warn(fmt);                                              \
     } while (0)
 
 /*!
  * \internal
  * \brief Set scheduler flags
  *
  * \param[in,out] scheduler     Scheduler data
  * \param[in]     flags_to_set  Group of enum pcmk__scheduler_flags to set
  */
 #define pcmk__set_scheduler_flags(scheduler, flags_to_set) do {             \
         (scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__,         \
             LOG_TRACE, "Scheduler", crm_system_name,                        \
             (scheduler)->flags, (flags_to_set), #flags_to_set);             \
     } while (0)
 
 /*!
  * \internal
  * \brief Clear scheduler flags
  *
  * \param[in,out] scheduler       Scheduler data
  * \param[in]     flags_to_clear  Group of enum pcmk__scheduler_flags to clear
  */
 #define pcmk__clear_scheduler_flags(scheduler, flags_to_clear) do {         \
         (scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__,       \
             LOG_TRACE, "Scheduler", crm_system_name,                        \
             (scheduler)->flags, (flags_to_clear), #flags_to_clear);         \
     } while (0)
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif // PCMK__CRM_COMMON_SCHEDULER_INTERNAL__H
diff --git a/include/pcmki/pcmki_transition.h b/include/pcmki/pcmki_transition.h
index d66d7e43ce..63ac639281 100644
--- a/include/pcmki/pcmki_transition.h
+++ b/include/pcmki/pcmki_transition.h
@@ -1,181 +1,185 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__PCMKI_PCMKI_TRANSITION__H
 #define PCMK__PCMKI_PCMKI_TRANSITION__H
 
 #include <stdbool.h>                    // bool
 #include <stdint.h>                     // uint32_t
 #include <sys/types.h>                  // time_t
 #include <glib.h>                       // guint, GList, GHashTable
 #include <libxml/tree.h>                // xmlNode
 
 #include <crm/common/scheduler_types.h> // pcmk_scheduler_t
 #include <crm/lrmd_events.h>            // lrmd_event_data_t
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 enum pcmk__graph_action_type {
     pcmk__pseudo_graph_action,
     pcmk__rsc_graph_action,
     pcmk__cluster_graph_action,
+    /* @TODO maybe separate a new pcmk__fencing_graph_action from
+     * pcmk__cluster_graph_action to make code cleaner (for example, see
+     * initiate_action())
+     */
 };
 
 enum pcmk__synapse_flags {
     pcmk__synapse_ready       = (1 << 0),
     pcmk__synapse_failed      = (1 << 1),
     pcmk__synapse_executed    = (1 << 2),
     pcmk__synapse_confirmed   = (1 << 3),
 };
 
 typedef struct {
     int id;
     int priority;
 
     uint32_t flags; // Group of pcmk__synapse_flags
 
     GList *actions;           /* pcmk__graph_action_t* */
     GList *inputs;            /* pcmk__graph_action_t* */
 } pcmk__graph_synapse_t;
 
 #define pcmk__set_synapse_flags(synapse, flags_to_set) do {             \
         (synapse)->flags = pcmk__set_flags_as(__func__, __LINE__,       \
             LOG_TRACE,                                                  \
             "Synapse", "synapse",                       \
             (synapse)->flags, (flags_to_set), #flags_to_set);           \
     } while (0)
 
 #define pcmk__clear_synapse_flags(synapse, flags_to_clear) do {         \
         (synapse)->flags = pcmk__clear_flags_as(__func__, __LINE__,     \
             LOG_TRACE,                                                  \
             "Synapse", "synapse",                      \
             (synapse)->flags, (flags_to_clear), #flags_to_clear);       \
     } while (0)
 
 enum pcmk__graph_action_flags {
     pcmk__graph_action_sent_update   = (1 << 0),     /* sent to the CIB */
     pcmk__graph_action_executed      = (1 << 1),     /* sent to the CRM */
     pcmk__graph_action_confirmed     = (1 << 2),
     pcmk__graph_action_failed        = (1 << 3),
 };
 
 typedef struct {
     int id;
     int timeout;
     int timer;
     guint interval_ms;
     GHashTable *params;
     enum pcmk__graph_action_type type;
     pcmk__graph_synapse_t *synapse;
 
     uint32_t flags; // Group of pcmk__graph_action_flags
 
     xmlNode *xml;
 
 } pcmk__graph_action_t;
 
 #define pcmk__set_graph_action_flags(action, flags_to_set) do {       \
         (action)->flags = pcmk__set_flags_as(__func__, __LINE__,      \
             LOG_TRACE,                                                \
             "Action", "action",                                       \
             (action)->flags, (flags_to_set), #flags_to_set);          \
     } while (0)
 
 #define pcmk__clear_graph_action_flags(action, flags_to_clear) do {   \
         (action)->flags = pcmk__clear_flags_as(__func__, __LINE__,    \
             LOG_TRACE,                                                \
             "Action", "action",                                       \
             (action)->flags, (flags_to_clear), #flags_to_clear);      \
     } while (0)
 
 // What to do after finished processing a transition graph
 enum pcmk__graph_next {
     // Order matters: lowest priority to highest
     pcmk__graph_done,       // Transition complete, nothing further needed
     pcmk__graph_wait,       // Transition interrupted, wait for further changes
     pcmk__graph_restart,    // Transition interrupted, start a new one
     pcmk__graph_shutdown,   // Transition interrupted, local shutdown needed
 };
 
 typedef struct {
     int id;
     char *source;
     int abort_priority;
 
     bool complete;
     const char *abort_reason;
     enum pcmk__graph_next completion_action;
 
     int num_actions;
     int num_synapses;
 
     int batch_limit;
     guint network_delay;
     guint stonith_timeout;
 
     int fired;
     int pending;
     int skipped;
     int completed;
     int incomplete;
 
     GList *synapses;          /* pcmk__graph_synapse_t* */
 
     int migration_limit;
 
     //! Failcount after one failed stop action
     char *failed_stop_offset;
 
     //! Failcount after one failed start action
     char *failed_start_offset;
 
     //! Time (from epoch) by which the controller should re-run the scheduler
     time_t recheck_by;
 } pcmk__graph_t;
 
 
 typedef struct {
     int (*pseudo) (pcmk__graph_t *graph, pcmk__graph_action_t *action);
     int (*rsc) (pcmk__graph_t *graph, pcmk__graph_action_t *action);
     int (*cluster) (pcmk__graph_t *graph, pcmk__graph_action_t *action);
     int (*fence) (pcmk__graph_t *graph, pcmk__graph_action_t *action);
     bool (*allowed) (pcmk__graph_t *graph, pcmk__graph_action_t *action);
 } pcmk__graph_functions_t;
 
 enum pcmk__graph_status {
     pcmk__graph_active,     // Some actions have been performed
     pcmk__graph_pending,    // No actions performed yet
     pcmk__graph_complete,
     pcmk__graph_terminated,
 };
 
 void pcmk__set_graph_functions(pcmk__graph_functions_t *fns);
 pcmk__graph_t *pcmk__unpack_graph(const xmlNode *xml_graph,
                                   const char *reference);
 enum pcmk__graph_status pcmk__execute_graph(pcmk__graph_t *graph);
 void pcmk__update_graph(pcmk__graph_t *graph,
                         const pcmk__graph_action_t *action);
 void pcmk__free_graph(pcmk__graph_t *graph);
 const char *pcmk__graph_status2text(enum pcmk__graph_status state);
 void pcmk__log_graph(unsigned int log_level, pcmk__graph_t *graph);
 void pcmk__log_graph_action(int log_level, pcmk__graph_action_t *action);
 void pcmk__log_transition_summary(const pcmk_scheduler_t *scheduler,
                                   const char *filename);
 lrmd_event_data_t *pcmk__event_from_graph_action(const xmlNode *resource,
                                                  const pcmk__graph_action_t *action,
                                                  int status, int rc,
                                                  const char *exit_reason);
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif // PCMK__PCMKI_PCMKI_TRANSITION__H
diff --git a/lib/common/ipc_client.c b/lib/common/ipc_client.c
index c644b9c94c..6f2d65be2f 100644
--- a/lib/common/ipc_client.c
+++ b/lib/common/ipc_client.c
@@ -1,1678 +1,1682 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #if defined(HAVE_UCRED) || defined(HAVE_SOCKPEERCRED)
 #include <sys/socket.h>
 #elif defined(HAVE_GETPEERUCRED)
 #include <ucred.h>
 #endif
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <errno.h>
 #include <bzlib.h>
 
 #include <crm/crm.h>   /* indirectly: pcmk_err_generic */
 #include <crm/common/xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include "crmcommon_private.h"
 
 static int is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
                                     uid_t refuid, gid_t refgid, pid_t *gotpid,
                                     uid_t *gotuid, gid_t *gotgid);
 
 /*!
  * \brief Create a new object for using Pacemaker daemon IPC
  *
  * \param[out] api     Where to store new IPC object
  * \param[in]  server  Which Pacemaker daemon the object is for
  *
  * \return Standard Pacemaker result code
  *
  * \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
  * \note This is intended to supersede crm_ipc_new() but currently only
  *       supports the controller, pacemakerd, and schedulerd IPC API.
  */
 int
 pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
 {
     if (api == NULL) {
         return EINVAL;
     }
 
     *api = calloc(1, sizeof(pcmk_ipc_api_t));
     if (*api == NULL) {
         return errno;
     }
 
     (*api)->server = server;
     if (pcmk_ipc_name(*api, false) == NULL) {
         pcmk_free_ipc_api(*api);
         *api = NULL;
         return EOPNOTSUPP;
     }
 
     (*api)->ipc_size_max = 0;
 
     // Set server methods and max_size (if not default)
     switch (server) {
         case pcmk_ipc_attrd:
             (*api)->cmds = pcmk__attrd_api_methods();
             break;
 
         case pcmk_ipc_based:
             (*api)->ipc_size_max = 512 * 1024; // 512KB
             break;
 
         case pcmk_ipc_controld:
             (*api)->cmds = pcmk__controld_api_methods();
             break;
 
         case pcmk_ipc_execd:
             break;
 
         case pcmk_ipc_fenced:
             break;
 
         case pcmk_ipc_pacemakerd:
             (*api)->cmds = pcmk__pacemakerd_api_methods();
             break;
 
         case pcmk_ipc_schedulerd:
             (*api)->cmds = pcmk__schedulerd_api_methods();
             // @TODO max_size could vary by client, maybe take as argument?
             (*api)->ipc_size_max = 5 * 1024 * 1024; // 5MB
             break;
 
         default: // pcmk_ipc_unknown
             pcmk_free_ipc_api(*api);
             *api = NULL;
             return EINVAL;
     }
     if ((*api)->cmds == NULL) {
         pcmk_free_ipc_api(*api);
         *api = NULL;
         return ENOMEM;
     }
 
     (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false),
                               (*api)->ipc_size_max);
     if ((*api)->ipc == NULL) {
         pcmk_free_ipc_api(*api);
         *api = NULL;
         return ENOMEM;
     }
 
     // If daemon API has its own data to track, allocate it
     if ((*api)->cmds->new_data != NULL) {
         if ((*api)->cmds->new_data(*api) != pcmk_rc_ok) {
             pcmk_free_ipc_api(*api);
             *api = NULL;
             return ENOMEM;
         }
     }
     crm_trace("Created %s API IPC object", pcmk_ipc_name(*api, true));
     return pcmk_rc_ok;
 }
 
 static void
 free_daemon_specific_data(pcmk_ipc_api_t *api)
 {
     if ((api != NULL) && (api->cmds != NULL)) {
         if ((api->cmds->free_data != NULL) && (api->api_data != NULL)) {
             api->cmds->free_data(api->api_data);
             api->api_data = NULL;
         }
         free(api->cmds);
         api->cmds = NULL;
     }
 }
 
 /*!
  * \internal
  * \brief Call an IPC API event callback, if one is registed
  *
  * \param[in,out] api         IPC API connection
  * \param[in]     event_type  The type of event that occurred
  * \param[in]     status      Event status
  * \param[in,out] event_data  Event-specific data
  */
 void
 pcmk__call_ipc_callback(pcmk_ipc_api_t *api, enum pcmk_ipc_event event_type,
                         crm_exit_t status, void *event_data)
 {
     if ((api != NULL) && (api->cb != NULL)) {
         api->cb(api, event_type, status, event_data, api->user_data);
     }
 }
 
 /*!
  * \internal
  * \brief Clean up after an IPC disconnect
  *
  * \param[in,out] user_data  IPC API connection that disconnected
  *
  * \note This function can be used as a main loop IPC destroy callback.
  */
 static void
 ipc_post_disconnect(gpointer user_data)
 {
     pcmk_ipc_api_t *api = user_data;
 
     crm_info("Disconnected from %s", pcmk_ipc_name(api, true));
 
     // Perform any daemon-specific handling needed
     if ((api->cmds != NULL) && (api->cmds->post_disconnect != NULL)) {
         api->cmds->post_disconnect(api);
     }
 
     // Call client's registered event callback
     pcmk__call_ipc_callback(api, pcmk_ipc_event_disconnect, CRM_EX_DISCONNECT,
                             NULL);
 
     /* If this is being called from a running main loop, mainloop_gio_destroy()
      * will free ipc and mainloop_io immediately after calling this function.
      * If this is called from a stopped main loop, these will leak, so the best
      * practice is to close the connection before stopping the main loop.
      */
     api->ipc = NULL;
     api->mainloop_io = NULL;
 
     if (api->free_on_disconnect) {
         /* pcmk_free_ipc_api() has already been called, but did not free api
          * or api->cmds because this function needed them. Do that now.
          */
         free_daemon_specific_data(api);
         crm_trace("Freeing IPC API object after disconnect");
         free(api);
     }
 }
 
 /*!
  * \brief Free the contents of an IPC API object
  *
  * \param[in,out] api  IPC API object to free
  */
 void
 pcmk_free_ipc_api(pcmk_ipc_api_t *api)
 {
     bool free_on_disconnect = false;
 
     if (api == NULL) {
         return;
     }
     crm_debug("Releasing %s IPC API", pcmk_ipc_name(api, true));
 
     if (api->ipc != NULL) {
         if (api->mainloop_io != NULL) {
             /* We need to keep the api pointer itself around, because it is the
              * user data for the IPC client destroy callback. That will be
              * triggered by the pcmk_disconnect_ipc() call below, but it might
              * happen later in the main loop (if still running).
              *
              * This flag tells the destroy callback to free the object. It can't
              * do that unconditionally, because the application might call this
              * function after a disconnect that happened by other means.
              */
             free_on_disconnect = api->free_on_disconnect = true;
         }
         pcmk_disconnect_ipc(api); // Frees api if free_on_disconnect is true
     }
     if (!free_on_disconnect) {
         free_daemon_specific_data(api);
         crm_trace("Freeing IPC API object");
         free(api);
     }
 }
 
 /*!
  * \brief Get the IPC name used with an IPC API connection
  *
  * \param[in] api      IPC API connection
  * \param[in] for_log  If true, return human-friendly name instead of IPC name
  *
  * \return IPC API's human-friendly or connection name, or if none is available,
  *         "Pacemaker" if for_log is true and NULL if for_log is false
  */
 const char *
 pcmk_ipc_name(const pcmk_ipc_api_t *api, bool for_log)
 {
     if (api == NULL) {
         return for_log? "Pacemaker" : NULL;
     }
     if (for_log) {
         const char *name = pcmk__server_log_name(api->server);
 
         return pcmk__s(name, "Pacemaker");
     }
     switch (api->server) {
         // These servers do not have pcmk_ipc_api_t implementations yet
         case pcmk_ipc_based:
         case pcmk_ipc_execd:
         case pcmk_ipc_fenced:
             return NULL;
 
         default:
             return pcmk__server_ipc_name(api->server);
     }
 }
 
 /*!
  * \brief Check whether an IPC API connection is active
  *
  * \param[in,out] api  IPC API connection
  *
  * \return true if IPC is connected, false otherwise
  */
 bool
 pcmk_ipc_is_connected(pcmk_ipc_api_t *api)
 {
     return (api != NULL) && crm_ipc_connected(api->ipc);
 }
 
 /*!
  * \internal
  * \brief Call the daemon-specific API's dispatch function
  *
  * Perform daemon-specific handling of IPC reply dispatch. It is the daemon
  * method's responsibility to call the client's registered event callback, as
  * well as allocate and free any event data.
  *
  * \param[in,out] api      IPC API connection
  * \param[in,out] message  IPC reply XML to dispatch
  */
 static bool
 call_api_dispatch(pcmk_ipc_api_t *api, xmlNode *message)
 {
     crm_log_xml_trace(message, "ipc-received");
     if ((api->cmds != NULL) && (api->cmds->dispatch != NULL)) {
         return api->cmds->dispatch(api, message);
     }
 
     return false;
 }
 
 /*!
  * \internal
  * \brief Dispatch previously read IPC data
  *
  * \param[in]     buffer  Data read from IPC
  * \param[in,out] api     IPC object
  *
  * \return Standard Pacemaker return code.  In particular:
  *
  * pcmk_rc_ok: There are no more messages expected from the server.  Quit
  *             reading.
  * EINPROGRESS: There are more messages expected from the server.  Keep reading.
  *
  * All other values indicate an error.
  */
 static int
 dispatch_ipc_data(const char *buffer, pcmk_ipc_api_t *api)
 {
     bool more = false;
     xmlNode *msg;
 
     if (buffer == NULL) {
         crm_warn("Empty message received from %s IPC",
                  pcmk_ipc_name(api, true));
         return ENOMSG;
     }
 
     msg = pcmk__xml_parse(buffer);
     if (msg == NULL) {
         crm_warn("Malformed message received from %s IPC",
                  pcmk_ipc_name(api, true));
         return EPROTO;
     }
 
     more = call_api_dispatch(api, msg);
     pcmk__xml_free(msg);
 
     if (more) {
         return EINPROGRESS;
     } else {
         return pcmk_rc_ok;
     }
 }
 
 /*!
  * \internal
  * \brief Dispatch data read from IPC source
  *
  * \param[in]     buffer     Data read from IPC
  * \param[in]     length     Number of bytes of data in buffer (ignored)
  * \param[in,out] user_data  IPC object
  *
  * \return Always 0 (meaning connection is still required)
  *
  * \note This function can be used as a main loop IPC dispatch callback.
  */
 static int
 dispatch_ipc_source_data(const char *buffer, ssize_t length, gpointer user_data)
 {
     pcmk_ipc_api_t *api = user_data;
 
     CRM_CHECK(api != NULL, return 0);
     dispatch_ipc_data(buffer, api);
     return 0;
 }
 
 /*!
  * \brief Check whether an IPC connection has data available (without main loop)
  *
  * \param[in]  api         IPC API connection
  * \param[in]  timeout_ms  If less than 0, poll indefinitely; if 0, poll once
  *                         and return immediately; otherwise, poll for up to
  *                         this many milliseconds
  *
  * \return Standard Pacemaker return code
  *
  * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
  *       this function to check whether IPC data is available. Return values of
  *       interest include pcmk_rc_ok meaning data is available, and EAGAIN
  *       meaning no data is available; all other values indicate errors.
  * \todo This does not allow the caller to poll multiple file descriptors at
  *       once. If there is demand for that, we could add a wrapper for
  *       pcmk__ipc_fd(api->ipc), so the caller can call poll() themselves.
  */
 int
 pcmk_poll_ipc(const pcmk_ipc_api_t *api, int timeout_ms)
 {
     int rc;
     struct pollfd pollfd = { 0, };
 
     if ((api == NULL) || (api->dispatch_type != pcmk_ipc_dispatch_poll)) {
         return EINVAL;
     }
 
     rc = pcmk__ipc_fd(api->ipc, &(pollfd.fd));
     if (rc != pcmk_rc_ok) {
         crm_debug("Could not obtain file descriptor for %s IPC: %s",
                   pcmk_ipc_name(api, true), pcmk_rc_str(rc));
         return rc;
     }
 
     pollfd.events = POLLIN;
     rc = poll(&pollfd, 1, timeout_ms);
     if (rc < 0) {
         /* Some UNIX systems return negative and set EAGAIN for failure to
          * allocate memory; standardize the return code in that case
          */
         return (errno == EAGAIN)? ENOMEM : errno;
     } else if (rc == 0) {
         return EAGAIN;
     }
     return pcmk_rc_ok;
 }
 
 /*!
  * \brief Dispatch available messages on an IPC connection (without main loop)
  *
  * \param[in,out] api  IPC API connection
  *
  * \return Standard Pacemaker return code
  *
  * \note Callers of pcmk_connect_ipc() using pcmk_ipc_dispatch_poll should call
  *       this function when IPC data is available.
  */
 void
 pcmk_dispatch_ipc(pcmk_ipc_api_t *api)
 {
     if (api == NULL) {
         return;
     }
     while (crm_ipc_ready(api->ipc) > 0) {
         if (crm_ipc_read(api->ipc) > 0) {
             dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);
         }
     }
 }
 
 // \return Standard Pacemaker return code
 static int
 connect_with_main_loop(pcmk_ipc_api_t *api)
 {
     int rc;
 
     struct ipc_client_callbacks callbacks = {
         .dispatch = dispatch_ipc_source_data,
         .destroy = ipc_post_disconnect,
     };
 
     rc = pcmk__add_mainloop_ipc(api->ipc, G_PRIORITY_DEFAULT, api,
                                 &callbacks, &(api->mainloop_io));
     if (rc != pcmk_rc_ok) {
         return rc;
     }
     crm_debug("Connected to %s IPC (attached to main loop)",
               pcmk_ipc_name(api, true));
     /* After this point, api->mainloop_io owns api->ipc, so api->ipc
      * should not be explicitly freed.
      */
     return pcmk_rc_ok;
 }
 
 // \return Standard Pacemaker return code
 static int
 connect_without_main_loop(pcmk_ipc_api_t *api)
 {
     int rc = pcmk__connect_generic_ipc(api->ipc);
 
     if (rc != pcmk_rc_ok) {
         crm_ipc_close(api->ipc);
     } else {
         crm_debug("Connected to %s IPC (without main loop)",
                   pcmk_ipc_name(api, true));
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Connect to a Pacemaker daemon via IPC (retrying after soft errors)
  *
  * \param[in,out] api            IPC API instance
  * \param[in]     dispatch_type  How IPC replies should be dispatched
  * \param[in]     attempts       How many times to try (in case of soft error)
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type,
                   int attempts)
 {
     int rc = pcmk_rc_ok;
 
     if ((api == NULL) || (attempts < 1)) {
         return EINVAL;
     }
 
     if (api->ipc == NULL) {
         api->ipc = crm_ipc_new(pcmk_ipc_name(api, false), api->ipc_size_max);
         if (api->ipc == NULL) {
             return ENOMEM;
         }
     }
 
     if (crm_ipc_connected(api->ipc)) {
         crm_trace("Already connected to %s", pcmk_ipc_name(api, true));
         return pcmk_rc_ok;
     }
 
     api->dispatch_type = dispatch_type;
 
     crm_debug("Attempting connection to %s (up to %d time%s)",
               pcmk_ipc_name(api, true), attempts, pcmk__plural_s(attempts));
     for (int remaining = attempts - 1; remaining >= 0; --remaining) {
         switch (dispatch_type) {
             case pcmk_ipc_dispatch_main:
                 rc = connect_with_main_loop(api);
                 break;
 
             case pcmk_ipc_dispatch_sync:
             case pcmk_ipc_dispatch_poll:
                 rc = connect_without_main_loop(api);
                 break;
         }
 
         if ((remaining == 0) || ((rc != EAGAIN) && (rc != EALREADY))) {
             break; // Result is final
         }
 
         // Retry after soft error (interrupted by signal, etc.)
         pcmk__sleep_ms((attempts - remaining) * 500);
         crm_debug("Re-attempting connection to %s (%d attempt%s remaining)",
                   pcmk_ipc_name(api, true), remaining,
                   pcmk__plural_s(remaining));
     }
 
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     if ((api->cmds != NULL) && (api->cmds->post_connect != NULL)) {
         rc = api->cmds->post_connect(api);
         if (rc != pcmk_rc_ok) {
             crm_ipc_close(api->ipc);
         }
     }
     return rc;
 }
 
 /*!
  * \brief Connect to a Pacemaker daemon via IPC
  *
  * \param[in,out] api            IPC API instance
  * \param[in]     dispatch_type  How IPC replies should be dispatched
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type)
 {
     int rc = pcmk__connect_ipc(api, dispatch_type, 2);
 
     if (rc != pcmk_rc_ok) {
         crm_err("Connection to %s failed: %s",
                 pcmk_ipc_name(api, true), pcmk_rc_str(rc));
     }
     return rc;
 }
 
 /*!
  * \brief Disconnect an IPC API instance
  *
  * \param[in,out] api  IPC API connection
  *
  * \return Standard Pacemaker return code
  *
  * \note If the connection is attached to a main loop, this function should be
  *       called before quitting the main loop, to ensure that all memory is
  *       freed.
  */
 void
 pcmk_disconnect_ipc(pcmk_ipc_api_t *api)
 {
     if ((api == NULL) || (api->ipc == NULL)) {
         return;
     }
     switch (api->dispatch_type) {
         case pcmk_ipc_dispatch_main:
             {
                 mainloop_io_t *mainloop_io = api->mainloop_io;
 
                 // Make sure no code with access to api can use these again
                 api->mainloop_io = NULL;
                 api->ipc = NULL;
 
                 mainloop_del_ipc_client(mainloop_io);
                 // After this point api might have already been freed
             }
             break;
 
         case pcmk_ipc_dispatch_poll:
         case pcmk_ipc_dispatch_sync:
             {
                 crm_ipc_t *ipc = api->ipc;
 
                 // Make sure no code with access to api can use ipc again
                 api->ipc = NULL;
 
                 // This should always be the case already, but to be safe
                 api->free_on_disconnect = false;
 
                 crm_ipc_close(ipc);
                 crm_ipc_destroy(ipc);
                 ipc_post_disconnect(api);
             }
             break;
     }
 }
 
 /*!
  * \brief Register a callback for IPC API events
  *
  * \param[in,out] api       IPC API connection
  * \param[in]     callback  Callback to register
  * \param[in]     userdata  Caller data to pass to callback
  *
  * \note This function may be called multiple times to update the callback
  *       and/or user data. The caller remains responsible for freeing
  *       userdata in any case (after the IPC is disconnected, if the
  *       user data is still registered with the IPC).
  */
 void
 pcmk_register_ipc_callback(pcmk_ipc_api_t *api, pcmk_ipc_callback_t cb,
                            void *user_data)
 {
     if (api == NULL) {
         return;
     }
     api->cb = cb;
     api->user_data = user_data;
 }
 
 /*!
  * \internal
  * \brief Send an XML request across an IPC API connection
  *
  * \param[in,out] api      IPC API connection
  * \param[in]     request  XML request to send
  *
  * \return Standard Pacemaker return code
  *
  * \note Daemon-specific IPC API functions should call this function to send
  *       requests, because it handles different dispatch types appropriately.
  */
 int
 pcmk__send_ipc_request(pcmk_ipc_api_t *api, const xmlNode *request)
 {
     int rc;
     xmlNode *reply = NULL;
     enum crm_ipc_flags flags = crm_ipc_flags_none;
 
     if ((api == NULL) || (api->ipc == NULL) || (request == NULL)) {
         return EINVAL;
     }
     crm_log_xml_trace(request, "ipc-sent");
 
     // Synchronous dispatch requires waiting for a reply
     if ((api->dispatch_type == pcmk_ipc_dispatch_sync)
         && (api->cmds != NULL)
         && (api->cmds->reply_expected != NULL)
         && (api->cmds->reply_expected(api, request))) {
         flags = crm_ipc_client_response;
     }
 
-    // The 0 here means a default timeout of 5 seconds
+    /* The 0 here means a default timeout of 5 seconds
+     *
+     * @TODO Maybe add a timeout_ms member to pcmk_ipc_api_t and a
+     * pcmk_set_ipc_timeout() setter for it, then use it here.
+     */
     rc = crm_ipc_send(api->ipc, request, flags, 0, &reply);
 
     if (rc < 0) {
         return pcmk_legacy2rc(rc);
     } else if (rc == 0) {
         return ENODATA;
     }
 
     // With synchronous dispatch, we dispatch any reply now
     if (reply != NULL) {
         bool more = call_api_dispatch(api, reply);
 
         pcmk__xml_free(reply);
 
         while (more) {
             rc = crm_ipc_read(api->ipc);
 
             if (rc == -EAGAIN) {
                 continue;
             } else if (rc == -ENOMSG || rc == pcmk_ok) {
                 return pcmk_rc_ok;
             } else if (rc < 0) {
                 return -rc;
             }
 
             rc = dispatch_ipc_data(crm_ipc_buffer(api->ipc), api);
 
             if (rc == pcmk_rc_ok) {
                 more = false;
             } else if (rc == EINPROGRESS) {
                 more = true;
             } else {
                 continue;
             }
         }
     }
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Create the XML for an IPC request to purge a node from the peer cache
  *
  * \param[in]  api        IPC API connection
  * \param[in]  node_name  If not NULL, name of node to purge
  * \param[in]  nodeid     If not 0, node ID of node to purge
  *
  * \return Newly allocated IPC request XML
  *
  * \note The controller, fencer, and pacemakerd use the same request syntax, but
  *       the attribute manager uses a different one. The CIB manager doesn't
  *       have any syntax for it. The executor and scheduler don't connect to the
  *       cluster layer and thus don't have or need any syntax for it.
  *
  * \todo Modify the attribute manager to accept the common syntax (as well
  *       as its current one, for compatibility with older clients). Modify
  *       the CIB manager to accept and honor the common syntax. Modify the
  *       executor and scheduler to accept the syntax (immediately returning
  *       success), just for consistency. Modify this function to use the
  *       common syntax with all daemons if their version supports it.
  */
 static xmlNode *
 create_purge_node_request(const pcmk_ipc_api_t *api, const char *node_name,
                           uint32_t nodeid)
 {
     xmlNode *request = NULL;
     const char *client = crm_system_name? crm_system_name : "client";
 
     switch (api->server) {
         case pcmk_ipc_attrd:
             request = pcmk__xe_create(NULL, __func__);
             crm_xml_add(request, PCMK__XA_T, PCMK__VALUE_ATTRD);
             crm_xml_add(request, PCMK__XA_SRC, crm_system_name);
             crm_xml_add(request, PCMK_XA_TASK, PCMK__ATTRD_CMD_PEER_REMOVE);
             pcmk__xe_set_bool_attr(request, PCMK__XA_REAP, true);
             pcmk__xe_add_node(request, node_name, nodeid);
             break;
 
         case pcmk_ipc_controld:
         case pcmk_ipc_fenced:
         case pcmk_ipc_pacemakerd:
             request = pcmk__new_request(api->server, client, NULL,
                                         pcmk_ipc_name(api, false),
                                         CRM_OP_RM_NODE_CACHE, NULL);
             if (nodeid > 0) {
                 crm_xml_add_ll(request, PCMK_XA_ID, (long long) nodeid);
             }
             crm_xml_add(request, PCMK_XA_UNAME, node_name);
             break;
 
         case pcmk_ipc_based:
         case pcmk_ipc_execd:
         case pcmk_ipc_schedulerd:
             break;
 
         default: // pcmk_ipc_unknown (shouldn't be possible)
             return NULL;
     }
     return request;
 }
 
 /*!
  * \brief Ask a Pacemaker daemon to purge a node from its peer cache
  *
  * \param[in,out] api        IPC API connection
  * \param[in]     node_name  If not NULL, name of node to purge
  * \param[in]     nodeid     If not 0, node ID of node to purge
  *
  * \return Standard Pacemaker return code
  *
  * \note At least one of node_name or nodeid must be specified.
  */
 int
 pcmk_ipc_purge_node(pcmk_ipc_api_t *api, const char *node_name, uint32_t nodeid)
 {
     int rc = 0;
     xmlNode *request = NULL;
 
     if (api == NULL) {
         return EINVAL;
     }
     if ((node_name == NULL) && (nodeid == 0)) {
         return EINVAL;
     }
 
     request = create_purge_node_request(api, node_name, nodeid);
     if (request == NULL) {
         return EOPNOTSUPP;
     }
     rc = pcmk__send_ipc_request(api, request);
     pcmk__xml_free(request);
 
     crm_debug("%s peer cache purge of node %s[%lu]: rc=%d",
               pcmk_ipc_name(api, true), node_name, (unsigned long) nodeid, rc);
     return rc;
 }
 
 /*
  * Generic IPC API (to eventually be deprecated as public API and made internal)
  */
 
 struct crm_ipc_s {
     struct pollfd pfd;
     unsigned int max_buf_size; // maximum bytes we can send or receive over IPC
     unsigned int buf_size;     // size of allocated buffer
     int msg_size;
     int need_reply;
     char *buffer;
     char *server_name;          // server IPC name being connected to
     qb_ipcc_connection_t *ipc;
 };
 
 /*!
  * \brief Create a new (legacy) object for using Pacemaker daemon IPC
  *
  * \param[in] name      IPC system name to connect to
  * \param[in] max_size  Use a maximum IPC buffer size of at least this size
  *
  * \return Newly allocated IPC object on success, NULL otherwise
  *
  * \note The caller is responsible for freeing the result using
  *       crm_ipc_destroy().
  * \note This should be considered deprecated for use with daemons supported by
  *       pcmk_new_ipc_api().
  */
 crm_ipc_t *
 crm_ipc_new(const char *name, size_t max_size)
 {
     crm_ipc_t *client = NULL;
 
     client = calloc(1, sizeof(crm_ipc_t));
     if (client == NULL) {
         crm_err("Could not create IPC connection: %s", strerror(errno));
         return NULL;
     }
 
     client->server_name = strdup(name);
     if (client->server_name == NULL) {
         crm_err("Could not create %s IPC connection: %s",
                 name, strerror(errno));
         free(client);
         return NULL;
     }
     client->buf_size = pcmk__ipc_buffer_size(max_size);
     client->buffer = malloc(client->buf_size);
     if (client->buffer == NULL) {
         crm_err("Could not create %s IPC connection: %s",
                 name, strerror(errno));
         free(client->server_name);
         free(client);
         return NULL;
     }
 
     /* Clients initiating connection pick the max buf size */
     client->max_buf_size = client->buf_size;
 
     client->pfd.fd = -1;
     client->pfd.events = POLLIN;
     client->pfd.revents = 0;
 
     return client;
 }
 
 /*!
  * \internal
  * \brief Connect a generic (not daemon-specific) IPC object
  *
  * \param[in,out] ipc  Generic IPC object to connect
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__connect_generic_ipc(crm_ipc_t *ipc)
 {
     uid_t cl_uid = 0;
     gid_t cl_gid = 0;
     pid_t found_pid = 0;
     uid_t found_uid = 0;
     gid_t found_gid = 0;
     int rc = pcmk_rc_ok;
 
     if (ipc == NULL) {
         return EINVAL;
     }
 
     ipc->need_reply = FALSE;
     ipc->ipc = qb_ipcc_connect(ipc->server_name, ipc->buf_size);
     if (ipc->ipc == NULL) {
         return errno;
     }
 
     rc = qb_ipcc_fd_get(ipc->ipc, &ipc->pfd.fd);
     if (rc < 0) { // -errno
         crm_ipc_close(ipc);
         return -rc;
     }
 
     rc = pcmk_daemon_user(&cl_uid, &cl_gid);
     rc = pcmk_legacy2rc(rc);
     if (rc != pcmk_rc_ok) {
         crm_ipc_close(ipc);
         return rc;
     }
 
     rc = is_ipc_provider_expected(ipc->ipc, ipc->pfd.fd, cl_uid, cl_gid,
                                   &found_pid, &found_uid, &found_gid);
     if (rc != pcmk_rc_ok) {
         if (rc == pcmk_rc_ipc_unauthorized) {
             crm_info("%s IPC provider authentication failed: process %lld has "
                      "uid %lld (expected %lld) and gid %lld (expected %lld)",
                      ipc->server_name,
                      (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
                      (long long) found_uid, (long long) cl_uid,
                      (long long) found_gid, (long long) cl_gid);
         }
         crm_ipc_close(ipc);
         return rc;
     }
 
     ipc->max_buf_size = qb_ipcc_get_buffer_size(ipc->ipc);
     if (ipc->max_buf_size > ipc->buf_size) {
         free(ipc->buffer);
         ipc->buffer = calloc(ipc->max_buf_size, sizeof(char));
         if (ipc->buffer == NULL) {
             rc = errno;
             crm_ipc_close(ipc);
             return rc;
         }
         ipc->buf_size = ipc->max_buf_size;
     }
 
     return pcmk_rc_ok;
 }
 
 void
 crm_ipc_close(crm_ipc_t * client)
 {
     if (client) {
         if (client->ipc) {
             qb_ipcc_connection_t *ipc = client->ipc;
 
             client->ipc = NULL;
             qb_ipcc_disconnect(ipc);
         }
     }
 }
 
 void
 crm_ipc_destroy(crm_ipc_t * client)
 {
     if (client) {
         if (client->ipc && qb_ipcc_is_connected(client->ipc)) {
             crm_notice("Destroying active %s IPC connection",
                        client->server_name);
             /* The next line is basically unsafe
              *
              * If this connection was attached to mainloop and mainloop is active,
              *   the 'disconnected' callback will end up back here and we'll end
              *   up free'ing the memory twice - something that can still happen
              *   even without this if we destroy a connection and it closes before
              *   we call exit
              */
             /* crm_ipc_close(client); */
         } else {
             crm_trace("Destroying inactive %s IPC connection",
                       client->server_name);
         }
         free(client->buffer);
         free(client->server_name);
         free(client);
     }
 }
 
 /*!
  * \internal
  * \brief Get the file descriptor for a generic IPC object
  *
  * \param[in,out] ipc  Generic IPC object to get file descriptor for
  * \param[out]    fd   Where to store file descriptor
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__ipc_fd(crm_ipc_t *ipc, int *fd)
 {
     if ((ipc == NULL) || (fd == NULL)) {
         return EINVAL;
     }
     if ((ipc->ipc == NULL) || (ipc->pfd.fd < 0)) {
         return ENOTCONN;
     }
     *fd = ipc->pfd.fd;
     return pcmk_rc_ok;
 }
 
 int
 crm_ipc_get_fd(crm_ipc_t * client)
 {
     int fd = -1;
 
     if (pcmk__ipc_fd(client, &fd) != pcmk_rc_ok) {
         crm_err("Could not obtain file descriptor for %s IPC",
                 ((client == NULL)? "unspecified" : client->server_name));
         errno = EINVAL;
         return -EINVAL;
     }
     return fd;
 }
 
 bool
 crm_ipc_connected(crm_ipc_t * client)
 {
     bool rc = FALSE;
 
     if (client == NULL) {
         crm_trace("No client");
         return FALSE;
 
     } else if (client->ipc == NULL) {
         crm_trace("No connection");
         return FALSE;
 
     } else if (client->pfd.fd < 0) {
         crm_trace("Bad descriptor");
         return FALSE;
     }
 
     rc = qb_ipcc_is_connected(client->ipc);
     if (rc == FALSE) {
         client->pfd.fd = -EINVAL;
     }
     return rc;
 }
 
 /*!
  * \brief Check whether an IPC connection is ready to be read
  *
  * \param[in,out] client  Connection to check
  *
  * \return Positive value if ready to be read, 0 if not ready, -errno on error
  */
 int
 crm_ipc_ready(crm_ipc_t *client)
 {
     int rc;
 
     pcmk__assert(client != NULL);
 
     if (!crm_ipc_connected(client)) {
         return -ENOTCONN;
     }
 
     client->pfd.revents = 0;
     rc = poll(&(client->pfd), 1, 0);
     return (rc < 0)? -errno : rc;
 }
 
 // \return Standard Pacemaker return code
 static int
 crm_ipc_decompress(crm_ipc_t * client)
 {
     pcmk__ipc_header_t *header = (pcmk__ipc_header_t *)(void*)client->buffer;
 
     if (header->size_compressed) {
         int rc = 0;
         unsigned int size_u = 1 + header->size_uncompressed;
         /* never let buf size fall below our max size required for ipc reads. */
         unsigned int new_buf_size = QB_MAX((sizeof(pcmk__ipc_header_t) + size_u), client->max_buf_size);
         char *uncompressed = pcmk__assert_alloc(1, new_buf_size);
 
         crm_trace("Decompressing message data %u bytes into %u bytes",
                  header->size_compressed, size_u);
 
         rc = BZ2_bzBuffToBuffDecompress(uncompressed + sizeof(pcmk__ipc_header_t), &size_u,
                                         client->buffer + sizeof(pcmk__ipc_header_t), header->size_compressed, 1, 0);
         rc = pcmk__bzlib2rc(rc);
 
         if (rc != pcmk_rc_ok) {
             crm_err("Decompression failed: %s " QB_XS " rc=%d",
                     pcmk_rc_str(rc), rc);
             free(uncompressed);
             return rc;
         }
 
         pcmk__assert(size_u == header->size_uncompressed);
 
         memcpy(uncompressed, client->buffer, sizeof(pcmk__ipc_header_t));       /* Preserve the header */
         header = (pcmk__ipc_header_t *)(void*)uncompressed;
 
         free(client->buffer);
         client->buf_size = new_buf_size;
         client->buffer = uncompressed;
     }
 
     pcmk__assert(client->buffer[sizeof(pcmk__ipc_header_t)
                                 + header->size_uncompressed - 1] == 0);
     return pcmk_rc_ok;
 }
 
 long
 crm_ipc_read(crm_ipc_t * client)
 {
     pcmk__ipc_header_t *header = NULL;
 
     pcmk__assert((client != NULL) && (client->ipc != NULL)
                  && (client->buffer != NULL));
 
     client->buffer[0] = 0;
     client->msg_size = qb_ipcc_event_recv(client->ipc, client->buffer,
                                           client->buf_size, 0);
     if (client->msg_size >= 0) {
         int rc = crm_ipc_decompress(client);
 
         if (rc != pcmk_rc_ok) {
             return pcmk_rc2legacy(rc);
         }
 
         header = (pcmk__ipc_header_t *)(void*)client->buffer;
         if (!pcmk__valid_ipc_header(header)) {
             return -EBADMSG;
         }
 
         crm_trace("Received %s IPC event %d size=%u rc=%d text='%.100s'",
                   client->server_name, header->qb.id, header->qb.size,
                   client->msg_size,
                   client->buffer + sizeof(pcmk__ipc_header_t));
 
     } else {
         crm_trace("No message received from %s IPC: %s",
                   client->server_name, pcmk_strerror(client->msg_size));
 
         if (client->msg_size == -EAGAIN) {
             return -EAGAIN;
         }
     }
 
     if (!crm_ipc_connected(client) || client->msg_size == -ENOTCONN) {
         crm_err("Connection to %s IPC failed", client->server_name);
     }
 
     if (header) {
         /* Data excluding the header */
         return header->size_uncompressed;
     }
     return -ENOMSG;
 }
 
 const char *
 crm_ipc_buffer(crm_ipc_t * client)
 {
     pcmk__assert(client != NULL);
     return client->buffer + sizeof(pcmk__ipc_header_t);
 }
 
 uint32_t
 crm_ipc_buffer_flags(crm_ipc_t * client)
 {
     pcmk__ipc_header_t *header = NULL;
 
     pcmk__assert(client != NULL);
     if (client->buffer == NULL) {
         return 0;
     }
 
     header = (pcmk__ipc_header_t *)(void*)client->buffer;
     return header->flags;
 }
 
 const char *
 crm_ipc_name(crm_ipc_t * client)
 {
     pcmk__assert(client != NULL);
     return client->server_name;
 }
 
 // \return Standard Pacemaker return code
 static int
 internal_ipc_get_reply(crm_ipc_t *client, int request_id, int ms_timeout,
                        ssize_t *bytes)
 {
     time_t timeout = time(NULL) + 1 + pcmk__timeout_ms2s(ms_timeout);
     int rc = pcmk_rc_ok;
 
     /* get the reply */
     crm_trace("Waiting on reply to %s IPC message %d",
               client->server_name, request_id);
     do {
 
         *bytes = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, 1000);
         if (*bytes > 0) {
             pcmk__ipc_header_t *hdr = NULL;
 
             rc = crm_ipc_decompress(client);
             if (rc != pcmk_rc_ok) {
                 return rc;
             }
 
             hdr = (pcmk__ipc_header_t *)(void*)client->buffer;
             if (hdr->qb.id == request_id) {
                 /* Got it */
                 break;
             } else if (hdr->qb.id < request_id) {
                 xmlNode *bad = pcmk__xml_parse(crm_ipc_buffer(client));
 
                 crm_err("Discarding old reply %d (need %d)", hdr->qb.id, request_id);
                 crm_log_xml_notice(bad, "OldIpcReply");
 
             } else {
                 xmlNode *bad = pcmk__xml_parse(crm_ipc_buffer(client));
 
                 crm_err("Discarding newer reply %d (need %d)", hdr->qb.id, request_id);
                 crm_log_xml_notice(bad, "ImpossibleReply");
                 pcmk__assert(hdr->qb.id <= request_id);
             }
         } else if (!crm_ipc_connected(client)) {
             crm_err("%s IPC provider disconnected while waiting for message %d",
                     client->server_name, request_id);
             break;
         }
 
     } while (time(NULL) < timeout);
 
     if (*bytes < 0) {
         rc = (int) -*bytes; // System errno
     }
     return rc;
 }
 
 /*!
  * \brief Send an IPC XML message
  *
  * \param[in,out] client      Connection to IPC server
  * \param[in]     message     XML message to send
  * \param[in]     flags       Bitmask of crm_ipc_flags
  * \param[in]     ms_timeout  Give up if not sent within this much time
  *                            (5 seconds if 0, or no timeout if negative)
  * \param[out]    reply       Reply from server (or NULL if none)
  *
  * \return Negative errno on error, otherwise size of reply received in bytes
  *         if reply was needed, otherwise number of bytes sent
  */
 int
 crm_ipc_send(crm_ipc_t *client, const xmlNode *message,
              enum crm_ipc_flags flags, int32_t ms_timeout, xmlNode **reply)
 {
     int rc = 0;
     ssize_t qb_rc = 0;
     ssize_t bytes = 0;
     struct iovec *iov;
     static uint32_t id = 0;
     static int factor = 8;
     pcmk__ipc_header_t *header;
 
     if (client == NULL) {
         crm_notice("Can't send IPC request without connection (bug?): %.100s",
                    message);
         return -ENOTCONN;
 
     } else if (!crm_ipc_connected(client)) {
         /* Don't even bother */
         crm_notice("Can't send %s IPC requests: Connection closed",
                    client->server_name);
         return -ENOTCONN;
     }
 
     if (ms_timeout == 0) {
         ms_timeout = 5000;
     }
 
     if (client->need_reply) {
         qb_rc = qb_ipcc_recv(client->ipc, client->buffer, client->buf_size, ms_timeout);
         if (qb_rc < 0) {
             crm_warn("Sending %s IPC disabled until pending reply received",
                      client->server_name);
             return -EALREADY;
 
         } else {
             crm_notice("Sending %s IPC re-enabled after pending reply received",
                        client->server_name);
             client->need_reply = FALSE;
         }
     }
 
     id++;
     CRM_LOG_ASSERT(id != 0); /* Crude wrap-around detection */
     rc = pcmk__ipc_prepare_iov(id, message, client->max_buf_size, &iov, &bytes);
     if (rc != pcmk_rc_ok) {
         crm_warn("Couldn't prepare %s IPC request: %s " QB_XS " rc=%d",
                  client->server_name, pcmk_rc_str(rc), rc);
         return pcmk_rc2legacy(rc);
     }
 
     header = iov[0].iov_base;
     pcmk__set_ipc_flags(header->flags, client->server_name, flags);
 
     if (pcmk_is_set(flags, crm_ipc_proxied)) {
         /* Don't look for a synchronous response */
         pcmk__clear_ipc_flags(flags, "client", crm_ipc_client_response);
     }
 
     if(header->size_compressed) {
         if(factor < 10 && (client->max_buf_size / 10) < (bytes / factor)) {
             crm_notice("Compressed message exceeds %d0%% of configured IPC "
                        "limit (%u bytes); consider setting PCMK_ipc_buffer to "
                        "%u or higher",
                        factor, client->max_buf_size, 2 * client->max_buf_size);
             factor++;
         }
     }
 
     crm_trace("Sending %s IPC request %d of %u bytes using %dms timeout",
               client->server_name, header->qb.id, header->qb.size, ms_timeout);
 
     if ((ms_timeout > 0) || !pcmk_is_set(flags, crm_ipc_client_response)) {
 
         time_t timeout = time(NULL) + 1 + pcmk__timeout_ms2s(ms_timeout);
 
         do {
             /* @TODO Is this check really needed? Won't qb_ipcc_sendv() return
              * an error if it's not connected?
              */
             if (!crm_ipc_connected(client)) {
                 goto send_cleanup;
             }
 
             qb_rc = qb_ipcc_sendv(client->ipc, iov, 2);
         } while ((qb_rc == -EAGAIN) && (time(NULL) < timeout));
 
         rc = (int) qb_rc; // Negative of system errno, or bytes sent
         if (qb_rc <= 0) {
             goto send_cleanup;
 
         } else if (!pcmk_is_set(flags, crm_ipc_client_response)) {
             crm_trace("Not waiting for reply to %s IPC request %d",
                       client->server_name, header->qb.id);
             goto send_cleanup;
         }
 
         rc = internal_ipc_get_reply(client, header->qb.id, ms_timeout, &bytes);
         if (rc != pcmk_rc_ok) {
             /* We didn't get the reply in time, so disable future sends for now.
              * The only alternative would be to close the connection since we
              * don't know how to detect and discard out-of-sequence replies.
              *
              * @TODO Implement out-of-sequence detection
              */
             client->need_reply = TRUE;
         }
         rc = (int) bytes; // Negative system errno, or size of reply received
 
     } else {
         // No timeout, and client response needed
         do {
             qb_rc = qb_ipcc_sendv_recv(client->ipc, iov, 2, client->buffer,
                                        client->buf_size, -1);
         } while ((qb_rc == -EAGAIN) && crm_ipc_connected(client));
         rc = (int) qb_rc; // Negative system errno, or size of reply received
     }
 
     if (rc > 0) {
         pcmk__ipc_header_t *hdr = (pcmk__ipc_header_t *)(void*)client->buffer;
 
         crm_trace("Received %d-byte reply %d to %s IPC %d: %.100s",
                   rc, hdr->qb.id, client->server_name, header->qb.id,
                   crm_ipc_buffer(client));
 
         if (reply) {
             *reply = pcmk__xml_parse(crm_ipc_buffer(client));
         }
 
     } else {
         crm_trace("No reply to %s IPC %d: rc=%d",
                   client->server_name, header->qb.id, rc);
     }
 
   send_cleanup:
     if (!crm_ipc_connected(client)) {
         crm_notice("Couldn't send %s IPC request %d: Connection closed "
                    QB_XS " rc=%d", client->server_name, header->qb.id, rc);
 
     } else if (rc == -ETIMEDOUT) {
         crm_warn("%s IPC request %d failed: %s after %dms " QB_XS " rc=%d",
                  client->server_name, header->qb.id, pcmk_strerror(rc),
                  ms_timeout, rc);
         crm_write_blackbox(0, NULL);
 
     } else if (rc <= 0) {
         crm_warn("%s IPC request %d failed: %s " QB_XS " rc=%d",
                  client->server_name, header->qb.id,
                  ((rc == 0)? "No bytes sent" : pcmk_strerror(rc)), rc);
     }
 
     pcmk_free_ipc_event(iov);
     return rc;
 }
 
 /*!
  * \brief Ensure an IPC provider has expected user or group
  *
  * \param[in]  qb_ipc  libqb client connection if available
  * \param[in]  sock    Connected Unix socket for IPC
  * \param[in]  refuid  Expected user ID
  * \param[in]  refgid  Expected group ID
  * \param[out] gotpid  If not NULL, where to store provider's actual process ID
  *                     (or 1 on platforms where ID is not available)
  * \param[out] gotuid  If not NULL, where to store provider's actual user ID
  * \param[out] gotgid  If not NULL, where to store provider's actual group ID
  *
  * \return Standard Pacemaker return code
  * \note An actual user ID of 0 (root) will always be considered authorized,
  *       regardless of the expected values provided. The caller can use the
  *       output arguments to be stricter than this function.
  */
 static int
 is_ipc_provider_expected(qb_ipcc_connection_t *qb_ipc, int sock,
                          uid_t refuid, gid_t refgid,
                          pid_t *gotpid, uid_t *gotuid, gid_t *gotgid)
 {
     int rc = EOPNOTSUPP;
     pid_t found_pid = 0;
     uid_t found_uid = 0;
     gid_t found_gid = 0;
 
 #ifdef HAVE_QB_IPCC_AUTH_GET
     if (qb_ipc != NULL) {
         rc = qb_ipcc_auth_get(qb_ipc, &found_pid, &found_uid, &found_gid);
         rc = -rc; // libqb returns 0 or -errno
         if (rc == pcmk_rc_ok) {
             goto found;
         }
     }
 #endif
 
 #ifdef HAVE_UCRED
     {
         struct ucred ucred;
         socklen_t ucred_len = sizeof(ucred);
 
         if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED, &ucred, &ucred_len) < 0) {
             rc = errno;
         } else if (ucred_len != sizeof(ucred)) {
             rc = EOPNOTSUPP;
         } else {
             found_pid = ucred.pid;
             found_uid = ucred.uid;
             found_gid = ucred.gid;
             goto found;
         }
     }
 #endif
 
 #ifdef HAVE_SOCKPEERCRED
     {
         struct sockpeercred sockpeercred;
         socklen_t sockpeercred_len = sizeof(sockpeercred);
 
         if (getsockopt(sock, SOL_SOCKET, SO_PEERCRED,
                        &sockpeercred, &sockpeercred_len) < 0) {
             rc = errno;
         } else if (sockpeercred_len != sizeof(sockpeercred)) {
             rc = EOPNOTSUPP;
         } else {
             found_pid = sockpeercred.pid;
             found_uid = sockpeercred.uid;
             found_gid = sockpeercred.gid;
             goto found;
         }
     }
 #endif
 
 #ifdef HAVE_GETPEEREID // For example, FreeBSD
     if (getpeereid(sock, &found_uid, &found_gid) < 0) {
         rc = errno;
     } else {
         found_pid = PCMK__SPECIAL_PID;
         goto found;
     }
 #endif
 
 #ifdef HAVE_GETPEERUCRED
     {
         ucred_t *ucred = NULL;
 
         if (getpeerucred(sock, &ucred) < 0) {
             rc = errno;
         } else {
             found_pid = ucred_getpid(ucred);
             found_uid = ucred_geteuid(ucred);
             found_gid = ucred_getegid(ucred);
             ucred_free(ucred);
             goto found;
         }
     }
 #endif
 
     return rc; // If we get here, nothing succeeded
 
 found:
     if (gotpid != NULL) {
         *gotpid = found_pid;
     }
     if (gotuid != NULL) {
         *gotuid = found_uid;
     }
     if (gotgid != NULL) {
         *gotgid = found_gid;
     }
     if ((found_uid != 0) && (found_uid != refuid) && (found_gid != refgid)) {
         return pcmk_rc_ipc_unauthorized;
     }
     return pcmk_rc_ok;
 }
 
 int
 crm_ipc_is_authentic_process(int sock, uid_t refuid, gid_t refgid,
                              pid_t *gotpid, uid_t *gotuid, gid_t *gotgid)
 {
     int ret = is_ipc_provider_expected(NULL, sock, refuid, refgid,
                                        gotpid, gotuid, gotgid);
 
     /* The old function had some very odd return codes*/
     if (ret == 0) {
         return 1;
     } else if (ret == pcmk_rc_ipc_unauthorized) {
         return 0;
     } else {
         return pcmk_rc2legacy(ret);
     }
 }
 
 int
 pcmk__ipc_is_authentic_process_active(const char *name, uid_t refuid,
                                       gid_t refgid, pid_t *gotpid)
 {
     static char last_asked_name[PATH_MAX / 2] = "";  /* log spam prevention */
     int fd;
     int rc = pcmk_rc_ipc_unresponsive;
     int auth_rc = 0;
     int32_t qb_rc;
     pid_t found_pid = 0; uid_t found_uid = 0; gid_t found_gid = 0;
     qb_ipcc_connection_t *c;
 #ifdef HAVE_QB_IPCC_CONNECT_ASYNC
     struct pollfd pollfd = { 0, };
     int poll_rc;
 
     c = qb_ipcc_connect_async(name, 0,
                               &(pollfd.fd));
 #else
     c = qb_ipcc_connect(name, 0);
 #endif
     if (c == NULL) {
         crm_info("Could not connect to %s IPC: %s", name, strerror(errno));
         rc = pcmk_rc_ipc_unresponsive;
         goto bail;
     }
 #ifdef HAVE_QB_IPCC_CONNECT_ASYNC
     pollfd.events = POLLIN;
     do {
         poll_rc = poll(&pollfd, 1, 2000);
     } while ((poll_rc == -1) && (errno == EINTR));
 
     /* If poll() failed, given that disconnect function is not registered yet,
      * qb_ipcc_disconnect() won't clean up the socket. In any case, call
      * qb_ipcc_connect_continue() here so that it may fail and do the cleanup
      * for us.
      */
     if (qb_ipcc_connect_continue(c) != 0) {
         crm_info("Could not connect to %s IPC: %s", name,
                  (poll_rc == 0)?"timeout":strerror(errno));
         rc = pcmk_rc_ipc_unresponsive;
         c = NULL; // qb_ipcc_connect_continue cleaned up for us
         goto bail;
     }
 #endif
 
     qb_rc = qb_ipcc_fd_get(c, &fd);
     if (qb_rc != 0) {
         rc = (int) -qb_rc; // System errno
         crm_err("Could not get fd from %s IPC: %s " QB_XS " rc=%d",
                 name, pcmk_rc_str(rc), rc);
         goto bail;
     }
 
     auth_rc = is_ipc_provider_expected(c, fd, refuid, refgid,
                                        &found_pid, &found_uid, &found_gid);
     if (auth_rc == pcmk_rc_ipc_unauthorized) {
         crm_err("Daemon (IPC %s) effectively blocked with unauthorized"
                 " process %lld (uid: %lld, gid: %lld)",
                 name, (long long) PCMK__SPECIAL_PID_AS_0(found_pid),
                 (long long) found_uid, (long long) found_gid);
         rc = pcmk_rc_ipc_unauthorized;
         goto bail;
     }
 
     if (auth_rc != pcmk_rc_ok) {
         rc = auth_rc;
         crm_err("Could not get peer credentials from %s IPC: %s "
                 QB_XS " rc=%d", name, pcmk_rc_str(rc), rc);
         goto bail;
     }
 
     if (gotpid != NULL) {
         *gotpid = found_pid;
     }
 
     rc = pcmk_rc_ok;
     if ((found_uid != refuid || found_gid != refgid)
             && strncmp(last_asked_name, name, sizeof(last_asked_name))) {
         if ((found_uid == 0) && (refuid != 0)) {
             crm_warn("Daemon (IPC %s) runs as root, whereas the expected"
                      " credentials are %lld:%lld, hazard of violating"
                      " the least privilege principle",
                      name, (long long) refuid, (long long) refgid);
         } else {
             crm_notice("Daemon (IPC %s) runs as %lld:%lld, whereas the"
                        " expected credentials are %lld:%lld, which may"
                        " mean a different set of privileges than expected",
                        name, (long long) found_uid, (long long) found_gid,
                        (long long) refuid, (long long) refgid);
         }
         memccpy(last_asked_name, name, '\0', sizeof(last_asked_name));
     }
 
 bail:
     if (c != NULL) {
         qb_ipcc_disconnect(c);
     }
     return rc;
 }
 
 // Deprecated functions kept only for backward API compatibility
 // LCOV_EXCL_START
 
 #include <crm/common/ipc_client_compat.h>
 
 bool
 crm_ipc_connect(crm_ipc_t *client)
 {
     int rc = pcmk__connect_generic_ipc(client);
 
     if (rc == pcmk_rc_ok) {
         return true;
     }
     if ((client != NULL) && (client->ipc == NULL)) {
         errno = (rc > 0)? rc : ENOTCONN;
         crm_debug("Could not establish %s IPC connection: %s (%d)",
                   client->server_name, pcmk_rc_str(errno), errno);
     } else if (rc == pcmk_rc_ipc_unauthorized) {
         crm_err("%s IPC provider authentication failed",
                 (client == NULL)? "Pacemaker" : client->server_name);
         errno = ECONNABORTED;
     } else {
         crm_err("Could not verify authenticity of %s IPC provider",
                 (client == NULL)? "Pacemaker" : client->server_name);
         errno = ENOTCONN;
     }
     return false;
 }
 
 // LCOV_EXCL_STOP
 // End deprecated API
diff --git a/lib/common/iso8601.c b/lib/common/iso8601.c
index dd2e75de6c..007a64fff5 100644
--- a/lib/common/iso8601.c
+++ b/lib/common/iso8601.c
@@ -1,2251 +1,2251 @@
 /*
  * Copyright 2005-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 /*
  * References:
  *	https://en.wikipedia.org/wiki/ISO_8601
  *	http://www.staff.science.uu.nl/~gent0113/calendar/isocalendar.htm
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <time.h>
 #include <ctype.h>
 #include <inttypes.h>
 #include <limits.h>         // INT_MIN, INT_MAX
 #include <string.h>
 #include <stdbool.h>
 #include <crm/common/iso8601.h>
 #include <crm/common/iso8601_internal.h>
 #include "crmcommon_private.h"
 
 /*
  * Andrew's code was originally written for OSes whose "struct tm" contains:
  *	long tm_gmtoff;		:: Seconds east of UTC
  *	const char *tm_zone;	:: Timezone abbreviation
  * Some OSes lack these, instead having:
  *	time_t (or long) timezone;
 		:: "difference between UTC and local standard time"
  *	char *tzname[2] = { "...", "..." };
  * I (David Lee) confess to not understanding the details.  So my attempted
  * generalisations for where their use is necessary may be flawed.
  *
  * 1. Does "difference between ..." subtract the same or opposite way?
  * 2. Should it use "altzone" instead of "timezone"?
  * 3. Should it use tzname[0] or tzname[1]?  Interaction with timezone/altzone?
  */
 #if defined(HAVE_STRUCT_TM_TM_GMTOFF)
 #  define GMTOFF(tm) ((tm)->tm_gmtoff)
 #else
 /* Note: extern variable; macro argument not actually used.  */
 #  define GMTOFF(tm) (-timezone+daylight)
 #endif
 
 #define HOUR_SECONDS    (60 * 60)
 #define DAY_SECONDS     (HOUR_SECONDS * 24)
 
 /*!
  * \internal
  * \brief Validate a seconds/microseconds tuple
  *
  * The microseconds value must be in the correct range, and if both are nonzero
  * they must have the same sign.
  *
  * \param[in] sec   Seconds
  * \param[in] usec  Microseconds
  *
  * \return true if the seconds/microseconds tuple is valid, or false otherwise
  */
 #define valid_sec_usec(sec, usec)               \
         ((QB_ABS(usec) < QB_TIME_US_IN_SEC)     \
          && (((sec) == 0) || ((usec) == 0) || (((sec) < 0) == ((usec) < 0))))
 
 // A date/time or duration
 struct crm_time_s {
     int years;      // Calendar year (date/time) or number of years (duration)
     int months;     // Number of months (duration only)
     int days;       // Ordinal day of year (date/time) or number of days (duration)
     int seconds;    // Seconds of day (date/time) or number of seconds (duration)
     int offset;     // Seconds offset from UTC (date/time only)
     bool duration;  // True if duration
 };
 
 static crm_time_t *parse_date(const char *date_str);
 
 static crm_time_t *
 crm_get_utc_time(const crm_time_t *dt)
 {
     crm_time_t *utc = NULL;
 
     if (dt == NULL) {
         errno = EINVAL;
         return NULL;
     }
 
     utc = crm_time_new_undefined();
     utc->years = dt->years;
     utc->days = dt->days;
     utc->seconds = dt->seconds;
     utc->offset = 0;
 
     if (dt->offset) {
         crm_time_add_seconds(utc, -dt->offset);
     } else {
         /* Durations (which are the only things that can include months, never have a timezone */
         utc->months = dt->months;
     }
 
     crm_time_log(LOG_TRACE, "utc-source", dt,
                  crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
     crm_time_log(LOG_TRACE, "utc-target", utc,
                  crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
     return utc;
 }
 
 crm_time_t *
 crm_time_new(const char *date_time)
 {
     tzset();
     if (date_time == NULL) {
         return pcmk__copy_timet(time(NULL));
     }
     return parse_date(date_time);
 }
 
 /*!
  * \brief Allocate memory for an uninitialized time object
  *
  * \return Newly allocated time object
  * \note The caller is responsible for freeing the return value using
  *       crm_time_free().
  */
 crm_time_t *
 crm_time_new_undefined(void)
 {
     return (crm_time_t *) pcmk__assert_alloc(1, sizeof(crm_time_t));
 }
 
 /*!
  * \brief Check whether a time object has been initialized yet
  *
  * \param[in] t  Time object to check
  *
  * \return TRUE if time object has been initialized, FALSE otherwise
  */
 bool
 crm_time_is_defined(const crm_time_t *t)
 {
     // Any nonzero member indicates something has been done to t
     return (t != NULL) && (t->years || t->months || t->days || t->seconds
                            || t->offset || t->duration);
 }
 
 void
 crm_time_free(crm_time_t * dt)
 {
     if (dt == NULL) {
         return;
     }
     free(dt);
 }
 
 static int
 year_days(int year)
 {
     int d = 365;
 
     if (crm_time_leapyear(year)) {
         d++;
     }
     return d;
 }
 
 /* From http://myweb.ecu.edu/mccartyr/ISOwdALG.txt :
  *
  * 5. Find the Jan1Weekday for Y (Monday=1, Sunday=7)
  *  YY = (Y-1) % 100
  *  C = (Y-1) - YY
  *  G = YY + YY/4
  *  Jan1Weekday = 1 + (((((C / 100) % 4) x 5) + G) % 7)
  */
 int
 crm_time_january1_weekday(int year)
 {
     int YY = (year - 1) % 100;
     int C = (year - 1) - YY;
     int G = YY + YY / 4;
     int jan1 = 1 + (((((C / 100) % 4) * 5) + G) % 7);
 
     crm_trace("YY=%d, C=%d, G=%d", YY, C, G);
     crm_trace("January 1 %.4d: %d", year, jan1);
     return jan1;
 }
 
 int
 crm_time_weeks_in_year(int year)
 {
     int weeks = 52;
     int jan1 = crm_time_january1_weekday(year);
 
     /* if jan1 == thursday */
     if (jan1 == 4) {
         weeks++;
     } else {
         jan1 = crm_time_january1_weekday(year + 1);
         /* if dec31 == thursday aka. jan1 of next year is a friday */
         if (jan1 == 5) {
             weeks++;
         }
 
     }
     return weeks;
 }
 
 // Jan-Dec plus Feb of leap years
 static int month_days[13] = {
     31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 29
 };
 
 /*!
  * \brief Return number of days in given month of given year
  *
  * \param[in] month  Ordinal month (1-12)
  * \param[in] year   Gregorian year
  *
  * \return Number of days in given month (0 if given month or year is invalid)
  */
 int
 crm_time_days_in_month(int month, int year)
 {
     if ((month < 1) || (month > 12) || (year < 1)) {
         return 0;
     }
     if ((month == 2) && crm_time_leapyear(year)) {
         month = 13;
     }
     return month_days[month - 1];
 }
 
 bool
 crm_time_leapyear(int year)
 {
     gboolean is_leap = FALSE;
 
     if (year % 4 == 0) {
         is_leap = TRUE;
     }
     if (year % 100 == 0 && year % 400 != 0) {
         is_leap = FALSE;
     }
     return is_leap;
 }
 
 /*!
  * \internal
  * \brief Get ordinal day number of year corresponding to given date
  *
  * \param[in] y   Year
  * \param[in] m   Month (1-12)
  * \param[in] d   Day of month (1-31)
  *
  * \return Day number of year \p y corresponding to month \p m and day \p d,
  *         or 0 for invalid arguments
  */
 static int
 get_ordinal_days(uint32_t y, uint32_t m, uint32_t d)
 {
     int result = 0;
 
     CRM_CHECK((y > 0) && (y <= INT_MAX) && (m >= 1) && (m <= 12)
               && (d >= 1) && (d <= 31), return 0);
 
     result = d;
     for (int lpc = 1; lpc < m; lpc++) {
         result += crm_time_days_in_month(lpc, y);
     }
     return result;
 }
 
 void
 crm_time_log_alias(int log_level, const char *file, const char *function,
                    int line, const char *prefix, const crm_time_t *date_time,
                    int flags)
 {
     char *date_s = crm_time_as_string(date_time, flags);
 
     if (log_level == LOG_STDOUT) {
         printf("%s%s%s\n",
                (prefix? prefix : ""), (prefix? ": " : ""), date_s);
     } else {
         do_crm_log_alias(log_level, file, function, line, "%s%s%s",
                          (prefix? prefix : ""), (prefix? ": " : ""), date_s);
     }
     free(date_s);
 }
 
 static void
 crm_time_get_sec(int sec, uint32_t *h, uint32_t *m, uint32_t *s)
 {
     uint32_t hours, minutes, seconds;
 
     seconds = QB_ABS(sec);
 
     hours = seconds / HOUR_SECONDS;
     seconds -= HOUR_SECONDS * hours;
 
     minutes = seconds / 60;
     seconds -= 60 * minutes;
 
     crm_trace("%d == %.2" PRIu32 ":%.2" PRIu32 ":%.2" PRIu32,
               sec, hours, minutes, seconds);
 
     *h = hours;
     *m = minutes;
     *s = seconds;
 }
 
 int
 crm_time_get_timeofday(const crm_time_t *dt, uint32_t *h, uint32_t *m,
                        uint32_t *s)
 {
     crm_time_get_sec(dt->seconds, h, m, s);
     return TRUE;
 }
 
 int
 crm_time_get_timezone(const crm_time_t *dt, uint32_t *h, uint32_t *m)
 {
     uint32_t s;
 
     crm_time_get_sec(dt->seconds, h, m, &s);
     return TRUE;
 }
 
 long long
 crm_time_get_seconds(const crm_time_t *dt)
 {
     int lpc;
     crm_time_t *utc = NULL;
     long long in_seconds = 0;
 
     if (dt == NULL) {
         return 0;
     }
 
+    // @TODO This is inefficient if dt is already in UTC
     utc = crm_get_utc_time(dt);
     if (utc == NULL) {
         return 0;
     }
 
+    // @TODO We should probably use <= if dt is a duration
     for (lpc = 1; lpc < utc->years; lpc++) {
         long long dmax = year_days(lpc);
 
         in_seconds += DAY_SECONDS * dmax;
     }
 
-    /* utc->months is an offset that can only be set for a duration.
-     * By definition, the value is variable depending on the date to
-     * which it is applied.
-     *
-     * Force 30-day months so that something vaguely sane happens
-     * for anyone that tries to use a month in this way.
+    /* utc->months can be set only for durations. By definition, the value
+     * varies depending on the (unknown) start date to which the duration will
+     * be applied. Assume 30-day months so that something vaguely sane happens
+     * in this case.
      */
     if (utc->months > 0) {
         in_seconds += DAY_SECONDS * 30 * (long long) (utc->months);
     }
 
     if (utc->days > 0) {
         in_seconds += DAY_SECONDS * (long long) (utc->days - 1);
     }
     in_seconds += utc->seconds;
 
     crm_time_free(utc);
     return in_seconds;
 }
 
 #define EPOCH_SECONDS 62135596800ULL    /* Calculated using crm_time_get_seconds() */
 long long
 crm_time_get_seconds_since_epoch(const crm_time_t *dt)
 {
     return (dt == NULL)? 0 : (crm_time_get_seconds(dt) - EPOCH_SECONDS);
 }
 
 int
 crm_time_get_gregorian(const crm_time_t *dt, uint32_t *y, uint32_t *m,
                        uint32_t *d)
 {
     int months = 0;
     int days = dt->days;
 
     if(dt->years != 0) {
         for (months = 1; months <= 12 && days > 0; months++) {
             int mdays = crm_time_days_in_month(months, dt->years);
 
             if (mdays >= days) {
                 break;
             } else {
                 days -= mdays;
             }
         }
 
     } else if (dt->months) {
         /* This is a duration including months, don't convert the days field */
         months = dt->months;
 
     } else {
         /* This is a duration not including months, still don't convert the days field */
     }
 
     *y = dt->years;
     *m = months;
     *d = days;
     crm_trace("%.4d-%.3d -> %.4d-%.2d-%.2d", dt->years, dt->days, dt->years, months, days);
     return TRUE;
 }
 
 int
 crm_time_get_ordinal(const crm_time_t *dt, uint32_t *y, uint32_t *d)
 {
     *y = dt->years;
     *d = dt->days;
     return TRUE;
 }
 
 int
 crm_time_get_isoweek(const crm_time_t *dt, uint32_t *y, uint32_t *w,
                      uint32_t *d)
 {
     /*
      * Monday 29 December 2008 is written "2009-W01-1"
      * Sunday 3 January 2010 is written "2009-W53-7"
      */
     int year_num = 0;
     int jan1 = crm_time_january1_weekday(dt->years);
     int h = -1;
 
     CRM_CHECK(dt->days > 0, return FALSE);
 
 /* 6. Find the Weekday for Y M D */
     h = dt->days + jan1 - 1;
     *d = 1 + ((h - 1) % 7);
 
 /* 7. Find if Y M D falls in YearNumber Y-1, WeekNumber 52 or 53 */
     if (dt->days <= (8 - jan1) && jan1 > 4) {
         crm_trace("year--, jan1=%d", jan1);
         year_num = dt->years - 1;
         *w = crm_time_weeks_in_year(year_num);
 
     } else {
         year_num = dt->years;
     }
 
 /* 8. Find if Y M D falls in YearNumber Y+1, WeekNumber 1 */
     if (year_num == dt->years) {
         int dmax = year_days(year_num);
         int correction = 4 - *d;
 
         if ((dmax - dt->days) < correction) {
             crm_trace("year++, jan1=%d, i=%d vs. %d", jan1, dmax - dt->days, correction);
             year_num = dt->years + 1;
             *w = 1;
         }
     }
 
 /* 9. Find if Y M D falls in YearNumber Y, WeekNumber 1 through 53 */
     if (year_num == dt->years) {
         int j = dt->days + (7 - *d) + (jan1 - 1);
 
         *w = j / 7;
         if (jan1 > 4) {
             *w -= 1;
         }
     }
 
     *y = year_num;
     crm_trace("Converted %.4d-%.3d to %.4" PRIu32 "-W%.2" PRIu32 "-%" PRIu32,
               dt->years, dt->days, *y, *w, *d);
     return TRUE;
 }
 
 #define DATE_MAX 128
 
 /*!
  * \internal
  * \brief Print "<seconds>.<microseconds>" to a buffer
  *
  * \param[in]     sec     Seconds
  * \param[in]     usec    Microseconds (must be of same sign as \p sec and of
  *                        absolute value less than \p QB_TIME_US_IN_SEC)
  * \param[in,out] buf     Result buffer
  * \param[in,out] offset  Current offset within \p buf
  */
 static inline void
 sec_usec_as_string(long long sec, int usec, char *buf, size_t *offset)
 {
     *offset += snprintf(buf + *offset, DATE_MAX - *offset, "%s%lld.%06d",
                         ((sec == 0) && (usec < 0))? "-" : "",
                         sec, QB_ABS(usec));
 }
 
 /*!
  * \internal
  * \brief Get a string representation of a duration
  *
  * \param[in]  dt         Time object to interpret as a duration
  * \param[in]  usec       Microseconds to add to \p dt
  * \param[in]  show_usec  Whether to include microseconds in \p result
  * \param[out] result     Where to store the result string
  */
 static void
 crm_duration_as_string(const crm_time_t *dt, int usec, bool show_usec,
                        char *result)
 {
     size_t offset = 0;
 
     pcmk__assert(valid_sec_usec(dt->seconds, usec));
 
     if (dt->years) {
         offset += snprintf(result + offset, DATE_MAX - offset, "%4d year%s ",
                            dt->years, pcmk__plural_s(dt->years));
     }
     if (dt->months) {
         offset += snprintf(result + offset, DATE_MAX - offset, "%2d month%s ",
                            dt->months, pcmk__plural_s(dt->months));
     }
     if (dt->days) {
         offset += snprintf(result + offset, DATE_MAX - offset, "%2d day%s ",
                            dt->days, pcmk__plural_s(dt->days));
     }
 
     // At least print seconds (and optionally usecs)
     if ((offset == 0) || (dt->seconds != 0) || (show_usec && (usec != 0))) {
         if (show_usec) {
             sec_usec_as_string(dt->seconds, usec, result, &offset);
         } else {
             offset += snprintf(result + offset, DATE_MAX - offset, "%d",
                                dt->seconds);
         }
         offset += snprintf(result + offset, DATE_MAX - offset, " second%s",
                            pcmk__plural_s(dt->seconds));
     }
 
     // More than one minute, so provide a more readable breakdown into units
     if (QB_ABS(dt->seconds) >= 60) {
         uint32_t h = 0;
         uint32_t m = 0;
         uint32_t s = 0;
         uint32_t u = QB_ABS(usec);
         bool print_sec_component = false;
 
         crm_time_get_sec(dt->seconds, &h, &m, &s);
         print_sec_component = ((s != 0) || (show_usec && (u != 0)));
 
         offset += snprintf(result + offset, DATE_MAX - offset, " (");
 
         if (h) {
             offset += snprintf(result + offset, DATE_MAX - offset,
                                "%" PRIu32 " hour%s%s", h, pcmk__plural_s(h),
                                ((m != 0) || print_sec_component)? " " : "");
         }
 
         if (m) {
             offset += snprintf(result + offset, DATE_MAX - offset,
                                "%" PRIu32 " minute%s%s", m, pcmk__plural_s(m),
                                print_sec_component? " " : "");
         }
 
         if (print_sec_component) {
             if (show_usec) {
                 sec_usec_as_string(s, u, result, &offset);
             } else {
                 offset += snprintf(result + offset, DATE_MAX - offset,
                                    "%" PRIu32, s);
             }
             offset += snprintf(result + offset, DATE_MAX - offset, " second%s",
                                pcmk__plural_s(dt->seconds));
         }
 
         offset += snprintf(result + offset, DATE_MAX - offset, ")");
     }
 }
 
 /*!
  * \internal
  * \brief Get a string representation of a time object
  *
  * \param[in]  dt      Time to convert to string
  * \param[in]  usec    Microseconds to add to \p dt
  * \param[in]  flags   Group of \p crm_time_* string format options
  * \param[out] result  Where to store the result string
  *
  * \note \p result must be of size \p DATE_MAX or larger.
  */
 static void
 time_as_string_common(const crm_time_t *dt, int usec, uint32_t flags,
                       char *result)
 {
     crm_time_t *utc = NULL;
     size_t offset = 0;
 
     if (!crm_time_is_defined(dt)) {
         strcpy(result, "<undefined time>");
         return;
     }
 
     pcmk__assert(valid_sec_usec(dt->seconds, usec));
 
     /* Simple cases: as duration, seconds, or seconds since epoch.
      * These never depend on time zone.
      */
 
     if (pcmk_is_set(flags, crm_time_log_duration)) {
         crm_duration_as_string(dt, usec, pcmk_is_set(flags, crm_time_usecs),
                                result);
         return;
     }
 
     if (pcmk_any_flags_set(flags, crm_time_seconds|crm_time_epoch)) {
         long long seconds = 0;
 
         if (pcmk_is_set(flags, crm_time_seconds)) {
             seconds = crm_time_get_seconds(dt);
         } else {
             seconds = crm_time_get_seconds_since_epoch(dt);
         }
 
         if (pcmk_is_set(flags, crm_time_usecs)) {
             sec_usec_as_string(seconds, usec, result, &offset);
         } else {
             snprintf(result, DATE_MAX, "%lld", seconds);
         }
         return;
     }
 
     // Convert to UTC if local timezone was not requested
     if ((dt->offset != 0) && !pcmk_is_set(flags, crm_time_log_with_timezone)) {
         crm_trace("UTC conversion");
         utc = crm_get_utc_time(dt);
         dt = utc;
     }
 
     // As readable string
 
     if (pcmk_is_set(flags, crm_time_log_date)) {
         if (pcmk_is_set(flags, crm_time_weeks)) { // YYYY-WW-D
             uint32_t y = 0;
             uint32_t w = 0;
             uint32_t d = 0;
 
             if (crm_time_get_isoweek(dt, &y, &w, &d)) {
                 offset += snprintf(result + offset, DATE_MAX - offset,
                                    "%" PRIu32 "-W%.2" PRIu32 "-%" PRIu32,
                                    y, w, d);
             }
 
         } else if (pcmk_is_set(flags, crm_time_ordinal)) { // YYYY-DDD
             uint32_t y = 0;
             uint32_t d = 0;
 
             if (crm_time_get_ordinal(dt, &y, &d)) {
                 offset += snprintf(result + offset, DATE_MAX - offset,
                                    "%" PRIu32 "-%.3" PRIu32, y, d);
             }
 
         } else { // YYYY-MM-DD
             uint32_t y = 0;
             uint32_t m = 0;
             uint32_t d = 0;
 
             if (crm_time_get_gregorian(dt, &y, &m, &d)) {
                 offset += snprintf(result + offset, DATE_MAX - offset,
                                    "%.4" PRIu32 "-%.2" PRIu32 "-%.2" PRIu32,
                                    y, m, d);
             }
         }
     }
 
     if (pcmk_is_set(flags, crm_time_log_timeofday)) {
         uint32_t h = 0, m = 0, s = 0;
 
         if (offset > 0) {
             offset += snprintf(result + offset, DATE_MAX - offset, " ");
         }
 
         if (crm_time_get_timeofday(dt, &h, &m, &s)) {
             offset += snprintf(result + offset, DATE_MAX - offset,
                                "%.2" PRIu32 ":%.2" PRIu32 ":%.2" PRIu32,
                                h, m, s);
 
             if (pcmk_is_set(flags, crm_time_usecs)) {
                 offset += snprintf(result + offset, DATE_MAX - offset,
                                    ".%06" PRIu32, QB_ABS(usec));
             }
         }
 
         if (pcmk_is_set(flags, crm_time_log_with_timezone)
             && (dt->offset != 0)) {
             crm_time_get_sec(dt->offset, &h, &m, &s);
             offset += snprintf(result + offset, DATE_MAX - offset,
                                " %c%.2" PRIu32 ":%.2" PRIu32,
                                ((dt->offset < 0)? '-' : '+'), h, m);
         } else {
             offset += snprintf(result + offset, DATE_MAX - offset, "Z");
         }
     }
 
     crm_time_free(utc);
 }
 
 /*!
  * \brief Get a string representation of a \p crm_time_t object
  *
  * \param[in]  dt      Time to convert to string
  * \param[in]  flags   Group of \p crm_time_* string format options
  *
  * \note The caller is responsible for freeing the return value using \p free().
  */
 char *
 crm_time_as_string(const crm_time_t *dt, int flags)
 {
     char result[DATE_MAX] = { '\0', };
 
     time_as_string_common(dt, 0, flags, result);
     return pcmk__str_copy(result);
 }
 
 /*!
  * \internal
  * \brief Determine number of seconds from an hour:minute:second string
  *
  * \param[in]  time_str  Time specification string
  * \param[out] result    Number of seconds equivalent to time_str
  *
  * \return TRUE if specification was valid, FALSE (and set errno) otherwise
  * \note This may return the number of seconds in a day (which is out of bounds
  *       for a time object) if given 24:00:00.
  */
 static bool
 crm_time_parse_sec(const char *time_str, int *result)
 {
     int rc;
     uint32_t hour = 0;
     uint32_t minute = 0;
     uint32_t second = 0;
 
     *result = 0;
 
     // Must have at least hour, but minutes and seconds are optional
     rc = sscanf(time_str, "%" SCNu32 ":%" SCNu32 ":%" SCNu32,
                 &hour, &minute, &second);
     if (rc == 1) {
         rc = sscanf(time_str, "%2" SCNu32 "%2" SCNu32 "%2" SCNu32,
                     &hour, &minute, &second);
     }
     if (rc == 0) {
         crm_err("%s is not a valid ISO 8601 time specification", time_str);
         errno = EINVAL;
         return FALSE;
     }
 
     crm_trace("Got valid time: %.2" PRIu32 ":%.2" PRIu32 ":%.2" PRIu32,
               hour, minute, second);
 
     if ((hour == 24) && (minute == 0) && (second == 0)) {
         // Equivalent to 00:00:00 of next day, return number of seconds in day
     } else if (hour >= 24) {
         crm_err("%s is not a valid ISO 8601 time specification "
                 "because %" PRIu32 " is not a valid hour", time_str, hour);
         errno = EINVAL;
         return FALSE;
     }
     if (minute >= 60) {
         crm_err("%s is not a valid ISO 8601 time specification "
                 "because %" PRIu32 " is not a valid minute", time_str, minute);
         errno = EINVAL;
         return FALSE;
     }
     if (second >= 60) {
         crm_err("%s is not a valid ISO 8601 time specification "
                 "because %" PRIu32 " is not a valid second", time_str, second);
         errno = EINVAL;
         return FALSE;
     }
 
     *result = (hour * HOUR_SECONDS) + (minute * 60) + second;
     return TRUE;
 }
 
 static bool
 crm_time_parse_offset(const char *offset_str, int *offset)
 {
     tzset();
 
     if (offset_str == NULL) {
         // Use local offset
 #if defined(HAVE_STRUCT_TM_TM_GMTOFF)
         time_t now = time(NULL);
         struct tm *now_tm = localtime(&now);
 #endif
         int h_offset = GMTOFF(now_tm) / HOUR_SECONDS;
         int m_offset = (GMTOFF(now_tm) - (HOUR_SECONDS * h_offset)) / 60;
 
         if (h_offset < 0 && m_offset < 0) {
             m_offset = 0 - m_offset;
         }
         *offset = (HOUR_SECONDS * h_offset) + (60 * m_offset);
         return TRUE;
     }
 
     if (offset_str[0] == 'Z') { // @TODO invalid if anything after?
         *offset = 0;
         return TRUE;
     }
 
     *offset = 0;
     if ((offset_str[0] == '+') || (offset_str[0] == '-')
         || isdigit((int)offset_str[0])) {
 
         gboolean negate = FALSE;
 
         if (offset_str[0] == '+') {
             offset_str++;
         } else if (offset_str[0] == '-') {
             negate = TRUE;
             offset_str++;
         }
         if (crm_time_parse_sec(offset_str, offset) == FALSE) {
             return FALSE;
         }
         if (negate) {
             *offset = 0 - *offset;
         }
     } // @TODO else invalid?
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Parse the time portion of an ISO 8601 date/time string
  *
  * \param[in]     time_str  Time portion of specification (after any 'T')
  * \param[in,out] a_time    Time object to parse into
  *
  * \return TRUE if valid time was parsed, FALSE (and set errno) otherwise
  * \note This may add a day to a_time (if the time is 24:00:00).
  */
 static bool
 crm_time_parse(const char *time_str, crm_time_t *a_time)
 {
     uint32_t h, m, s;
     char *offset_s = NULL;
 
     tzset();
 
     if (time_str) {
         if (crm_time_parse_sec(time_str, &(a_time->seconds)) == FALSE) {
             return FALSE;
         }
         offset_s = strstr(time_str, "Z");
         if (offset_s == NULL) {
             offset_s = strstr(time_str, " ");
             if (offset_s) {
                 while (isspace(offset_s[0])) {
                     offset_s++;
                 }
             }
         }
     }
 
     if (crm_time_parse_offset(offset_s, &(a_time->offset)) == FALSE) {
         return FALSE;
     }
     crm_time_get_sec(a_time->offset, &h, &m, &s);
     crm_trace("Got tz: %c%2." PRIu32 ":%.2" PRIu32,
               (a_time->offset < 0)? '-' : '+', h, m);
 
     if (a_time->seconds == DAY_SECONDS) {
         // 24:00:00 == 00:00:00 of next day
         a_time->seconds = 0;
         crm_time_add_days(a_time, 1);
     }
     return TRUE;
 }
 
 /*
  * \internal
  * \brief Parse a time object from an ISO 8601 date/time specification
  *
  * \param[in] date_str  ISO 8601 date/time specification (or
  *                      \c PCMK__VALUE_EPOCH)
  *
  * \return New time object on success, NULL (and set errno) otherwise
  */
 static crm_time_t *
 parse_date(const char *date_str)
 {
     const char *time_s = NULL;
     crm_time_t *dt = NULL;
 
     uint32_t year = 0U;
     uint32_t month = 0U;
     uint32_t day = 0U;
     uint32_t week = 0U;
 
     int rc = 0;
 
     if (pcmk__str_empty(date_str)) {
         crm_err("No ISO 8601 date/time specification given");
         goto invalid;
     }
 
     if ((date_str[0] == 'T')
         || ((strlen(date_str) > 2) && (date_str[2] == ':'))) {
         /* Just a time supplied - Infer current date */
         dt = crm_time_new(NULL);
         if (date_str[0] == 'T') {
             time_s = date_str + 1;
         } else {
             time_s = date_str;
         }
         goto parse_time;
     }
 
     dt = crm_time_new_undefined();
 
     if ((strncasecmp(PCMK__VALUE_EPOCH, date_str, 5) == 0)
         && ((date_str[5] == '\0')
             || (date_str[5] == '/')
             || isspace(date_str[5]))) {
         dt->days = 1;
         dt->years = 1970;
         crm_time_log(LOG_TRACE, "Unpacked", dt, crm_time_log_date | crm_time_log_timeofday);
         return dt;
     }
 
     /* YYYY-MM-DD */
     rc = sscanf(date_str, "%" SCNu32 "-%" SCNu32 "-%" SCNu32 "",
                 &year, &month, &day);
     if (rc == 1) {
         /* YYYYMMDD */
         rc = sscanf(date_str, "%4" SCNu32 "%2" SCNu32 "%2" SCNu32 "",
                     &year, &month, &day);
     }
     if (rc == 3) {
         if ((month < 1U) || (month > 12U)) {
             crm_err("'%s' is not a valid ISO 8601 date/time specification "
                     "because '%" PRIu32 "' is not a valid month",
                     date_str, month);
             goto invalid;
         } else if ((year < 1U) || (year > INT_MAX)) {
             crm_err("'%s' is not a valid ISO 8601 date/time specification "
                     "because '%" PRIu32 "' is not a valid year",
                     date_str, year);
             goto invalid;
         } else if ((day < 1) || (day > INT_MAX)
                    || (day > crm_time_days_in_month(month, year))) {
             crm_err("'%s' is not a valid ISO 8601 date/time specification "
                     "because '%" PRIu32 "' is not a valid day of the month",
                     date_str, day);
             goto invalid;
         } else {
             dt->years = year;
             dt->days = get_ordinal_days(year, month, day);
             crm_trace("Parsed Gregorian date '%.4" PRIu32 "-%.3d' "
                       "from date string '%s'", year, dt->days, date_str);
         }
         goto parse_time;
     }
 
     /* YYYY-DDD */
     rc = sscanf(date_str, "%" SCNu32 "-%" SCNu32, &year, &day);
     if (rc == 2) {
         if ((year < 1U) || (year > INT_MAX)) {
             crm_err("'%s' is not a valid ISO 8601 date/time specification "
                     "because '%" PRIu32 "' is not a valid year",
                     date_str, year);
             goto invalid;
         } else if ((day < 1U) || (day > INT_MAX) || (day > year_days(year))) {
             crm_err("'%s' is not a valid ISO 8601 date/time specification "
                     "because '%" PRIu32 "' is not a valid day of year %"
                     PRIu32 " (1-%d)",
                     date_str, day, year, year_days(year));
             goto invalid;
         }
         crm_trace("Parsed ordinal year %d and days %d from date string '%s'",
                   year, day, date_str);
         dt->days = day;
         dt->years = year;
         goto parse_time;
     }
 
     /* YYYY-Www-D */
     rc = sscanf(date_str, "%" SCNu32 "-W%" SCNu32 "-%" SCNu32,
                 &year, &week, &day);
     if (rc == 3) {
         if ((week < 1U) || (week > crm_time_weeks_in_year(year))) {
             crm_err("'%s' is not a valid ISO 8601 date/time specification "
                     "because '%" PRIu32 "' is not a valid week of year %"
                     PRIu32 " (1-%d)",
                     date_str, week, year, crm_time_weeks_in_year(year));
             goto invalid;
         } else if ((day < 1U) || (day > 7U)) {
             crm_err("'%s' is not a valid ISO 8601 date/time specification "
                     "because '%" PRIu32 "' is not a valid day of the week",
                     date_str, day);
             goto invalid;
         } else {
             /*
              * See https://en.wikipedia.org/wiki/ISO_week_date
              *
              * Monday 29 December 2008 is written "2009-W01-1"
              * Sunday 3 January 2010 is written "2009-W53-7"
              * Saturday 27 September 2008 is written "2008-W37-6"
              *
              * If 1 January is on a Monday, Tuesday, Wednesday or Thursday, it
              * is in week 1. If 1 January is on a Friday, Saturday or Sunday,
              * it is in week 52 or 53 of the previous year.
              */
             int jan1 = crm_time_january1_weekday(year);
 
             crm_trace("Parsed year %" PRIu32 " (Jan 1 = %d), week %" PRIu32
                       ", and day %" PRIu32 " from date string '%s'",
                       year, jan1, week, day, date_str);
 
             dt->years = year;
             crm_time_add_days(dt, (week - 1) * 7);
 
             if (jan1 <= 4) {
                 crm_time_add_days(dt, 1 - jan1);
             } else {
                 crm_time_add_days(dt, 8 - jan1);
             }
 
             crm_time_add_days(dt, day);
         }
         goto parse_time;
     }
 
     crm_err("'%s' is not a valid ISO 8601 date/time specification", date_str);
     goto invalid;
 
   parse_time:
 
     if (time_s == NULL) {
         time_s = date_str + strspn(date_str, "0123456789-W");
         if ((time_s[0] == ' ') || (time_s[0] == 'T')) {
             ++time_s;
         } else {
             time_s = NULL;
         }
     }
     if ((time_s != NULL) && (crm_time_parse(time_s, dt) == FALSE)) {
         goto invalid;
     }
 
     crm_time_log(LOG_TRACE, "Unpacked", dt, crm_time_log_date | crm_time_log_timeofday);
     if (crm_time_check(dt) == FALSE) {
         crm_err("'%s' is not a valid ISO 8601 date/time specification",
                 date_str);
         goto invalid;
     }
     return dt;
 
 invalid:
     crm_time_free(dt);
     errno = EINVAL;
     return NULL;
 }
 
 // Parse an ISO 8601 numeric value and return number of characters consumed
 static int
 parse_int(const char *str, int *result)
 {
     unsigned int lpc;
     int offset = (str[0] == 'T')? 1 : 0;
     bool negate = false;
 
     *result = 0;
 
     // @TODO This cannot handle combinations of these characters
     switch (str[offset]) {
         case '.':
         case ',':
             return 0; // Fractions are not supported
 
         case '-':
             negate = true;
             offset++;
             break;
 
         case '+':
         case ':':
             offset++;
             break;
 
         default:
             break;
     }
 
     for (lpc = 0; (lpc < 10) && isdigit(str[offset]); lpc++) {
         const int digit = str[offset++] - '0';
 
         if ((*result * 10LL + digit) > INT_MAX) {
             return 0; // Overflow
         }
         *result = *result * 10 + digit;
     }
     if (negate) {
         *result = 0 - *result;
     }
     return (lpc > 0)? offset : 0;
 }
 
 /*!
  * \brief Parse a time duration from an ISO 8601 duration specification
  *
  * \param[in] period_s  ISO 8601 duration specification (optionally followed by
  *                      whitespace, after which the rest of the string will be
  *                      ignored)
  *
  * \return New time object on success, NULL (and set errno) otherwise
  * \note It is the caller's responsibility to return the result using
  *       crm_time_free().
  */
 crm_time_t *
 crm_time_parse_duration(const char *period_s)
 {
     gboolean is_time = FALSE;
     crm_time_t *diff = NULL;
 
     if (pcmk__str_empty(period_s)) {
         crm_err("No ISO 8601 time duration given");
         goto invalid;
     }
     if (period_s[0] != 'P') {
         crm_err("'%s' is not a valid ISO 8601 time duration "
                 "because it does not start with a 'P'", period_s);
         goto invalid;
     }
     if ((period_s[1] == '\0') || isspace(period_s[1])) {
         crm_err("'%s' is not a valid ISO 8601 time duration "
                 "because nothing follows 'P'", period_s);
         goto invalid;
     }
 
     diff = crm_time_new_undefined();
 
     for (const char *current = period_s + 1;
          current[0] && (current[0] != '/') && !isspace(current[0]);
          ++current) {
 
         int an_int = 0, rc;
         long long result = 0LL;
 
         if (current[0] == 'T') {
             /* A 'T' separates year/month/day from hour/minute/seconds. We don't
              * require it strictly, but just use it to differentiate month from
              * minutes.
              */
             is_time = TRUE;
             continue;
         }
 
         // An integer must be next
         rc = parse_int(current, &an_int);
         if (rc == 0) {
             crm_err("'%s' is not a valid ISO 8601 time duration "
                     "because no valid integer at '%s'", period_s, current);
             goto invalid;
         }
         current += rc;
 
         // A time unit must be next (we're not strict about the order)
         switch (current[0]) {
             case 'Y':
                 diff->years = an_int;
                 break;
 
             case 'M':
                 if (!is_time) { // Months
                     diff->months = an_int;
                 } else { // Minutes
                     result = diff->seconds + an_int * 60LL;
                     if ((result < INT_MIN) || (result > INT_MAX)) {
                         crm_err("'%s' is not a valid ISO 8601 time duration "
                                 "because integer at '%s' is too %s",
                                 period_s, current - rc,
                                 ((result > 0)? "large" : "small"));
                         goto invalid;
                     } else {
                         diff->seconds = (int) result;
                     }
                 }
 
                 break;
 
             case 'W':
                 result = diff->days + an_int * 7LL;
                 if ((result < INT_MIN) || (result > INT_MAX)) {
                     crm_err("'%s' is not a valid ISO 8601 time duration "
                             "because integer at '%s' is too %s",
                             period_s, current - rc,
                             ((result > 0)? "large" : "small"));
                     goto invalid;
                 } else {
                     diff->days = (int) result;
                 }
                 break;
 
             case 'D':
                 result = diff->days + (long long) an_int;
                 if ((result < INT_MIN) || (result > INT_MAX)) {
                     crm_err("'%s' is not a valid ISO 8601 time duration "
                             "because integer at '%s' is too %s",
                             period_s, current - rc,
                             ((result > 0)? "large" : "small"));
                     goto invalid;
                 } else {
                     diff->days = (int) result;
                 }
                 break;
 
             case 'H':
                 result = diff->seconds + (long long) an_int * HOUR_SECONDS;
                 if ((result < INT_MIN) || (result > INT_MAX)) {
                     crm_err("'%s' is not a valid ISO 8601 time duration "
                             "because integer at '%s' is too %s",
                             period_s, current - rc,
                             ((result > 0)? "large" : "small"));
                     goto invalid;
                 } else {
                     diff->seconds = (int) result;
                 }
                 break;
 
             case 'S':
                 result = diff->seconds + (long long) an_int;
                 if ((result < INT_MIN) || (result > INT_MAX)) {
                     crm_err("'%s' is not a valid ISO 8601 time duration "
                             "because integer at '%s' is too %s",
                             period_s, current - rc,
                             ((result > 0)? "large" : "small"));
                     goto invalid;
                 } else {
                     diff->seconds = (int) result;
                 }
                 break;
 
             case '\0':
                 crm_err("'%s' is not a valid ISO 8601 time duration "
                         "because no units after %d", period_s, an_int);
                 goto invalid;
 
             default:
                 crm_err("'%s' is not a valid ISO 8601 time duration "
                         "because '%c' is not a valid time unit",
                         period_s, current[0]);
                 goto invalid;
         }
     }
 
     if (!crm_time_is_defined(diff)) {
         crm_err("'%s' is not a valid ISO 8601 time duration "
                 "because no amounts and units given", period_s);
         goto invalid;
     }
 
     diff->duration = TRUE;
     return diff;
 
 invalid:
     crm_time_free(diff);
     errno = EINVAL;
     return NULL;
 }
 
 /*!
  * \brief Parse a time period from an ISO 8601 interval specification
  *
  * \param[in] period_str  ISO 8601 interval specification (start/end,
  *                        start/duration, or duration/end)
  *
  * \return New time period object on success, NULL (and set errno) otherwise
  * \note The caller is responsible for freeing the result using
  *       crm_time_free_period().
  */
 crm_time_period_t *
 crm_time_parse_period(const char *period_str)
 {
     const char *original = period_str;
     crm_time_period_t *period = NULL;
 
     if (pcmk__str_empty(period_str)) {
         crm_err("No ISO 8601 time period given");
         goto invalid;
     }
 
     tzset();
     period = pcmk__assert_alloc(1, sizeof(crm_time_period_t));
 
     if (period_str[0] == 'P') {
         period->diff = crm_time_parse_duration(period_str);
         if (period->diff == NULL) {
             goto error;
         }
     } else {
         period->start = parse_date(period_str);
         if (period->start == NULL) {
             goto error;
         }
     }
 
     period_str = strstr(original, "/");
     if (period_str) {
         ++period_str;
         if (period_str[0] == 'P') {
             if (period->diff != NULL) {
                 crm_err("'%s' is not a valid ISO 8601 time period "
                         "because it has two durations",
                         original);
                 goto invalid;
             }
             period->diff = crm_time_parse_duration(period_str);
             if (period->diff == NULL) {
                 goto error;
             }
         } else {
             period->end = parse_date(period_str);
             if (period->end == NULL) {
                 goto error;
             }
         }
 
     } else if (period->diff != NULL) {
         // Only duration given, assume start is now
         period->start = crm_time_new(NULL);
 
     } else {
         // Only start given
         crm_err("'%s' is not a valid ISO 8601 time period "
                 "because it has no duration or ending time",
                 original);
         goto invalid;
     }
 
     if (period->start == NULL) {
         period->start = crm_time_subtract(period->end, period->diff);
 
     } else if (period->end == NULL) {
         period->end = crm_time_add(period->start, period->diff);
     }
 
     if (crm_time_check(period->start) == FALSE) {
         crm_err("'%s' is not a valid ISO 8601 time period "
                 "because the start is invalid", period_str);
         goto invalid;
     }
     if (crm_time_check(period->end) == FALSE) {
         crm_err("'%s' is not a valid ISO 8601 time period "
                 "because the end is invalid", period_str);
         goto invalid;
     }
     return period;
 
 invalid:
     errno = EINVAL;
 error:
     crm_time_free_period(period);
     return NULL;
 }
 
 /*!
  * \brief Free a dynamically allocated time period object
  *
  * \param[in,out] period  Time period to free
  */
 void
 crm_time_free_period(crm_time_period_t *period)
 {
     if (period) {
         crm_time_free(period->start);
         crm_time_free(period->end);
         crm_time_free(period->diff);
         free(period);
     }
 }
 
 void
 crm_time_set(crm_time_t *target, const crm_time_t *source)
 {
     crm_trace("target=%p, source=%p", target, source);
 
     CRM_CHECK(target != NULL && source != NULL, return);
 
     target->years = source->years;
     target->days = source->days;
     target->months = source->months;    /* Only for durations */
     target->seconds = source->seconds;
     target->offset = source->offset;
 
     crm_time_log(LOG_TRACE, "source", source,
                  crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
     crm_time_log(LOG_TRACE, "target", target,
                  crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
 }
 
 static void
 ha_set_tm_time(crm_time_t *target, const struct tm *source)
 {
     int h_offset = 0;
     int m_offset = 0;
 
     /* Ensure target is fully initialized */
     target->years = 0;
     target->months = 0;
     target->days = 0;
     target->seconds = 0;
     target->offset = 0;
     target->duration = FALSE;
 
     if (source->tm_year > 0) {
         /* years since 1900 */
         target->years = 1900;
         crm_time_add_years(target, source->tm_year);
     }
 
     if (source->tm_yday >= 0) {
         /* days since January 1 [0-365] */
         target->days = 1 + source->tm_yday;
     }
 
     if (source->tm_hour >= 0) {
         target->seconds += HOUR_SECONDS * source->tm_hour;
     }
     if (source->tm_min >= 0) {
         target->seconds += 60 * source->tm_min;
     }
     if (source->tm_sec >= 0) {
         target->seconds += source->tm_sec;
     }
 
     /* tm_gmtoff == offset from UTC in seconds */
     h_offset = GMTOFF(source) / HOUR_SECONDS;
     m_offset = (GMTOFF(source) - (HOUR_SECONDS * h_offset)) / 60;
     crm_trace("Time offset is %lds (%.2d:%.2d)",
               GMTOFF(source), h_offset, m_offset);
 
     target->offset += HOUR_SECONDS * h_offset;
     target->offset += 60 * m_offset;
 }
 
 void
 crm_time_set_timet(crm_time_t *target, const time_t *source)
 {
     ha_set_tm_time(target, localtime(source));
 }
 
 /*!
  * \internal
  * \brief Set one time object to another if the other is earlier
  *
  * \param[in,out] target  Time object to set
  * \param[in]     source  Time object to use if earlier
  */
 void
 pcmk__set_time_if_earlier(crm_time_t *target, const crm_time_t *source)
 {
     if ((target != NULL) && (source != NULL)
         && (!crm_time_is_defined(target)
             || (crm_time_compare(source, target) < 0))) {
         crm_time_set(target, source);
     }
 }
 
 crm_time_t *
 pcmk_copy_time(const crm_time_t *source)
 {
     crm_time_t *target = crm_time_new_undefined();
 
     crm_time_set(target, source);
     return target;
 }
 
 /*!
  * \internal
  * \brief Convert a \p time_t time to a \p crm_time_t time
  *
  * \param[in] source  Time to convert
  *
  * \return A \p crm_time_t object representing \p source
  */
 crm_time_t *
 pcmk__copy_timet(time_t source)
 {
     crm_time_t *target = crm_time_new_undefined();
 
     crm_time_set_timet(target, &source);
     return target;
 }
 
 crm_time_t *
 crm_time_add(const crm_time_t *dt, const crm_time_t *value)
 {
     crm_time_t *utc = NULL;
     crm_time_t *answer = NULL;
 
     if ((dt == NULL) || (value == NULL)) {
         errno = EINVAL;
         return NULL;
     }
 
     answer = pcmk_copy_time(dt);
 
     utc = crm_get_utc_time(value);
     if (utc == NULL) {
         crm_time_free(answer);
         return NULL;
     }
 
     crm_time_add_years(answer, utc->years);
     crm_time_add_months(answer, utc->months);
     crm_time_add_days(answer, utc->days);
     crm_time_add_seconds(answer, utc->seconds);
 
     crm_time_free(utc);
     return answer;
 }
 
 /*!
  * \internal
  * \brief Return the XML attribute name corresponding to a time component
  *
  * \param[in] component  Component to check
  *
  * \return XML attribute name corresponding to \p component, or NULL if
  *         \p component is invalid
  */
 const char *
 pcmk__time_component_attr(enum pcmk__time_component component)
 {
     switch (component) {
         case pcmk__time_years:
             return PCMK_XA_YEARS;
 
         case pcmk__time_months:
             return PCMK_XA_MONTHS;
 
         case pcmk__time_weeks:
             return PCMK_XA_WEEKS;
 
         case pcmk__time_days:
             return PCMK_XA_DAYS;
 
         case pcmk__time_hours:
             return PCMK_XA_HOURS;
 
         case pcmk__time_minutes:
             return PCMK_XA_MINUTES;
 
         case pcmk__time_seconds:
             return PCMK_XA_SECONDS;
 
         default:
             return NULL;
     }
 }
 
 typedef void (*component_fn_t)(crm_time_t *, int);
 
 /*!
  * \internal
  * \brief Get the addition function corresponding to a time component
  * \param[in] component  Component to check
  *
  * \return Addition function corresponding to \p component, or NULL if
  *         \p component is invalid
  */
 static component_fn_t
 component_fn(enum pcmk__time_component component)
 {
     switch (component) {
         case pcmk__time_years:
             return crm_time_add_years;
 
         case pcmk__time_months:
             return crm_time_add_months;
 
         case pcmk__time_weeks:
             return crm_time_add_weeks;
 
         case pcmk__time_days:
             return crm_time_add_days;
 
         case pcmk__time_hours:
             return crm_time_add_hours;
 
         case pcmk__time_minutes:
             return crm_time_add_minutes;
 
         case pcmk__time_seconds:
             return crm_time_add_seconds;
 
         default:
             return NULL;
     }
 
 }
 
 /*!
  * \internal
  * \brief Add the value of an XML attribute to a time object
  *
  * \param[in,out] t          Time object to add to
  * \param[in]     component  Component of \p t to add to
  * \param[in]     xml        XML with value to add
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__add_time_from_xml(crm_time_t *t, enum pcmk__time_component component,
                         const xmlNode *xml)
 {
     long long value;
     const char *attr = pcmk__time_component_attr(component);
     component_fn_t add = component_fn(component);
 
     if ((t == NULL) || (attr == NULL) || (add == NULL)) {
         return EINVAL;
     }
 
     if (xml == NULL) {
         return pcmk_rc_ok;
     }
 
     if (pcmk__scan_ll(crm_element_value(xml, attr), &value,
                       0LL) != pcmk_rc_ok) {
         return pcmk_rc_unpack_error;
     }
 
     if ((value < INT_MIN) || (value > INT_MAX)) {
         return ERANGE;
     }
 
     if (value != 0LL) {
         add(t, (int) value);
     }
     return pcmk_rc_ok;
 }
 
 crm_time_t *
 crm_time_calculate_duration(const crm_time_t *dt, const crm_time_t *value)
 {
     crm_time_t *utc = NULL;
     crm_time_t *answer = NULL;
 
     if ((dt == NULL) || (value == NULL)) {
         errno = EINVAL;
         return NULL;
     }
 
     utc = crm_get_utc_time(value);
     if (utc == NULL) {
         return NULL;
     }
 
     answer = crm_get_utc_time(dt);
     if (answer == NULL) {
         crm_time_free(utc);
         return NULL;
     }
     answer->duration = TRUE;
 
     crm_time_add_years(answer, -utc->years);
     if(utc->months != 0) {
         crm_time_add_months(answer, -utc->months);
     }
     crm_time_add_days(answer, -utc->days);
     crm_time_add_seconds(answer, -utc->seconds);
 
     crm_time_free(utc);
     return answer;
 }
 
 crm_time_t *
 crm_time_subtract(const crm_time_t *dt, const crm_time_t *value)
 {
     crm_time_t *utc = NULL;
     crm_time_t *answer = NULL;
 
     if ((dt == NULL) || (value == NULL)) {
         errno = EINVAL;
         return NULL;
     }
 
     utc = crm_get_utc_time(value);
     if (utc == NULL) {
         return NULL;
     }
 
     answer = pcmk_copy_time(dt);
     crm_time_add_years(answer, -utc->years);
     if(utc->months != 0) {
         crm_time_add_months(answer, -utc->months);
     }
     crm_time_add_days(answer, -utc->days);
     crm_time_add_seconds(answer, -utc->seconds);
     crm_time_free(utc);
 
     return answer;
 }
 
 /*!
  * \brief Check whether a time object represents a sensible date/time
  *
  * \param[in] dt  Date/time object to check
  *
  * \return \c true if years, days, and seconds are sensible, \c false otherwise
  */
 bool
 crm_time_check(const crm_time_t *dt)
 {
     return (dt != NULL)
            && (dt->days > 0) && (dt->days <= year_days(dt->years))
            && (dt->seconds >= 0) && (dt->seconds < DAY_SECONDS);
 }
 
 #define do_cmp_field(l, r, field)					\
     if(rc == 0) {                                                       \
 		if(l->field > r->field) {				\
 			crm_trace("%s: %d > %d",			\
 				    #field, l->field, r->field);	\
 			rc = 1;                                         \
 		} else if(l->field < r->field) {			\
 			crm_trace("%s: %d < %d",			\
 				    #field, l->field, r->field);	\
 			rc = -1;					\
 		}							\
     }
 
 int
 crm_time_compare(const crm_time_t *a, const crm_time_t *b)
 {
     int rc = 0;
     crm_time_t *t1 = crm_get_utc_time(a);
     crm_time_t *t2 = crm_get_utc_time(b);
 
     if ((t1 == NULL) && (t2 == NULL)) {
         rc = 0;
     } else if (t1 == NULL) {
         rc = -1;
     } else if (t2 == NULL) {
         rc = 1;
     } else {
         do_cmp_field(t1, t2, years);
         do_cmp_field(t1, t2, days);
         do_cmp_field(t1, t2, seconds);
     }
 
     crm_time_free(t1);
     crm_time_free(t2);
     return rc;
 }
 
 /*!
  * \brief Add a given number of seconds to a date/time or duration
  *
  * \param[in,out] a_time  Date/time or duration to add seconds to
  * \param[in]     extra   Number of seconds to add
  */
 void
 crm_time_add_seconds(crm_time_t *a_time, int extra)
 {
     int days = extra / DAY_SECONDS;
 
     pcmk__assert(a_time != NULL);
 
     crm_trace("Adding %d seconds (including %d whole day%s) to %d",
               extra, days, pcmk__plural_s(days), a_time->seconds);
 
     a_time->seconds += extra % DAY_SECONDS;
 
     // Check whether the addition crossed a day boundary
     if (a_time->seconds > DAY_SECONDS) {
         ++days;
         a_time->seconds -= DAY_SECONDS;
 
     } else if (a_time->seconds < 0) {
         --days;
         a_time->seconds += DAY_SECONDS;
     }
 
     crm_time_add_days(a_time, days);
 }
 
 #define ydays(t) (crm_time_leapyear((t)->years)? 366 : 365)
 
 /*!
  * \brief Add days to a date/time
  *
  * \param[in,out] a_time  Time to modify
  * \param[in]     extra   Number of days to add (may be negative to subtract)
  */
 void
 crm_time_add_days(crm_time_t *a_time, int extra)
 {
     pcmk__assert(a_time != NULL);
 
     crm_trace("Adding %d days to %.4d-%.3d", extra, a_time->years, a_time->days);
 
     if (extra > 0) {
         while ((a_time->days + (long long) extra) > ydays(a_time)) {
             if ((a_time->years + 1LL) > INT_MAX) {
                 a_time->days = ydays(a_time); // Clip to latest we can handle
                 return;
             }
             extra -= ydays(a_time);
             a_time->years++;
         }
     } else if (extra < 0) {
         const int min_days = a_time->duration? 0 : 1;
 
         while ((a_time->days + (long long) extra) < min_days) {
             if ((a_time->years - 1) < 1) {
                 a_time->days = 1; // Clip to earliest we can handle (no BCE)
                 return;
             }
             a_time->years--;
             extra += ydays(a_time);
         }
     }
     a_time->days += extra;
 }
 
 void
 crm_time_add_months(crm_time_t * a_time, int extra)
 {
     int lpc;
     uint32_t y, m, d, dmax;
 
     crm_time_get_gregorian(a_time, &y, &m, &d);
     crm_trace("Adding %d months to %.4" PRIu32 "-%.2" PRIu32 "-%.2" PRIu32,
               extra, y, m, d);
 
     if (extra > 0) {
         for (lpc = extra; lpc > 0; lpc--) {
             m++;
             if (m == 13) {
                 m = 1;
                 y++;
             }
         }
     } else {
         for (lpc = -extra; lpc > 0; lpc--) {
             m--;
             if (m == 0) {
                 m = 12;
                 y--;
             }
         }
     }
 
     dmax = crm_time_days_in_month(m, y);
     if (dmax < d) {
         /* Preserve day-of-month unless the month doesn't have enough days */
         d = dmax;
     }
 
     crm_trace("Calculated %.4" PRIu32 "-%.2" PRIu32 "-%.2" PRIu32, y, m, d);
 
     a_time->years = y;
     a_time->days = get_ordinal_days(y, m, d);
 
     crm_time_get_gregorian(a_time, &y, &m, &d);
     crm_trace("Got %.4" PRIu32 "-%.2" PRIu32 "-%.2" PRIu32, y, m, d);
 }
 
 void
 crm_time_add_minutes(crm_time_t * a_time, int extra)
 {
     crm_time_add_seconds(a_time, extra * 60);
 }
 
 void
 crm_time_add_hours(crm_time_t * a_time, int extra)
 {
     crm_time_add_seconds(a_time, extra * HOUR_SECONDS);
 }
 
 void
 crm_time_add_weeks(crm_time_t * a_time, int extra)
 {
     crm_time_add_days(a_time, extra * 7);
 }
 
 void
 crm_time_add_years(crm_time_t * a_time, int extra)
 {
     pcmk__assert(a_time != NULL);
 
     if ((extra > 0) && ((a_time->years + (long long) extra) > INT_MAX)) {
         a_time->years = INT_MAX;
     } else if ((extra < 0) && ((a_time->years + (long long) extra) < 1)) {
         a_time->years = 1; // Clip to earliest we can handle (no BCE)
     } else {
         a_time->years += extra;
     }
 }
 
 static void
 ha_get_tm_time(struct tm *target, const crm_time_t *source)
 {
     *target = (struct tm) {
         .tm_year = source->years - 1900,
         .tm_mday = source->days,
         .tm_sec = source->seconds % 60,
         .tm_min = ( source->seconds / 60 ) % 60,
         .tm_hour = source->seconds / HOUR_SECONDS,
         .tm_isdst = -1, /* don't adjust */
 
 #if defined(HAVE_STRUCT_TM_TM_GMTOFF)
         .tm_gmtoff = source->offset
 #endif
     };
     mktime(target);
 }
 
 /* The high-resolution variant of time object was added to meet an immediate
  * need, and is kept internal API.
  *
  * @TODO The long-term goal is to come up with a clean, unified design for a
  *       time type (or types) that meets all the various needs, to replace
  *       crm_time_t, pcmk__time_hr_t, and struct timespec (in lrmd_cmd_t).
  */
 
 pcmk__time_hr_t *
 pcmk__time_hr_convert(pcmk__time_hr_t *target, const crm_time_t *dt)
 {
     pcmk__time_hr_t *hr_dt = NULL;
 
     if (dt) {
         hr_dt = target;
         if (hr_dt == NULL) {
             hr_dt = pcmk__assert_alloc(1, sizeof(pcmk__time_hr_t));
         }
 
         *hr_dt = (pcmk__time_hr_t) {
             .years = dt->years,
             .months = dt->months,
             .days = dt->days,
             .seconds = dt->seconds,
             .offset = dt->offset,
             .duration = dt->duration
         };
     }
 
     return hr_dt;
 }
 
 void
 pcmk__time_set_hr_dt(crm_time_t *target, const pcmk__time_hr_t *hr_dt)
 {
     pcmk__assert((target != NULL) && (hr_dt != NULL));
     *target = (crm_time_t) {
         .years = hr_dt->years,
         .months = hr_dt->months,
         .days = hr_dt->days,
         .seconds = hr_dt->seconds,
         .offset = hr_dt->offset,
         .duration = hr_dt->duration
     };
 }
 
 /*!
  * \internal
  * \brief Return the current time as a high-resolution time
  *
  * \param[out] epoch  If not NULL, this will be set to seconds since epoch
  *
  * \return Newly allocated high-resolution time set to the current time
  */
 pcmk__time_hr_t *
 pcmk__time_hr_now(time_t *epoch)
 {
     struct timespec tv;
     crm_time_t dt;
     pcmk__time_hr_t *hr;
 
     qb_util_timespec_from_epoch_get(&tv);
     if (epoch != NULL) {
         *epoch = tv.tv_sec;
     }
     crm_time_set_timet(&dt, &(tv.tv_sec));
     hr = pcmk__time_hr_convert(NULL, &dt);
     if (hr != NULL) {
         hr->useconds = tv.tv_nsec / QB_TIME_NS_IN_USEC;
     }
     return hr;
 }
 
 pcmk__time_hr_t *
 pcmk__time_hr_new(const char *date_time)
 {
     pcmk__time_hr_t *hr_dt = NULL;
 
     if (date_time == NULL) {
         hr_dt = pcmk__time_hr_now(NULL);
     } else {
         crm_time_t *dt;
 
         dt = parse_date(date_time);
         hr_dt = pcmk__time_hr_convert(NULL, dt);
         crm_time_free(dt);
     }
     return hr_dt;
 }
 
 void
 pcmk__time_hr_free(pcmk__time_hr_t * hr_dt)
 {
     free(hr_dt);
 }
 
 /*!
  * \internal
  * \brief Expand a date/time format string, including %N for nanoseconds
  *
  * \param[in] format  Date/time format string as per strftime(3) with the
  *                    addition of %N for nanoseconds
  * \param[in] hr_dt   Time value to format
  *
  * \return Newly allocated string with formatted string
  */
 char *
 pcmk__time_format_hr(const char *format, const pcmk__time_hr_t *hr_dt)
 {
     int scanned_pos = 0; // How many characters of format have been parsed
     int printed_pos = 0; // How many characters of format have been processed
     size_t date_len = 0;
 
     char nano_s[10] = { '\0', };
     char date_s[128] = { '\0', };
 
     struct tm tm = { 0, };
     crm_time_t dt = { 0, };
 
     if (format == NULL) {
         return NULL;
     }
     pcmk__time_set_hr_dt(&dt, hr_dt);
     ha_get_tm_time(&tm, &dt);
     sprintf(nano_s, "%06d000", hr_dt->useconds);
 
     while (format[scanned_pos] != '\0') {
         int fmt_pos;            // Index after last character to pass as-is
         int nano_digits = 0;    // Length of %N field width (if any)
         char *tmp_fmt_s = NULL;
         size_t nbytes = 0;
 
         // Look for next format specifier
         const char *mark_s = strchr(&format[scanned_pos], '%');
 
         if (mark_s == NULL) {
             // No more specifiers, so pass remaining string to strftime() as-is
             scanned_pos = strlen(format);
             fmt_pos = scanned_pos;
 
         } else {
             fmt_pos = mark_s - format; // Index of %
 
             // Skip % and any field width
             scanned_pos = fmt_pos + 1;
             while (isdigit(format[scanned_pos])) {
                 scanned_pos++;
             }
 
             switch (format[scanned_pos]) {
                 case '\0': // Literal % and possibly digits at end of string
                     fmt_pos = scanned_pos; // Pass remaining string as-is
                     break;
 
                 case 'N': // %[width]N
                     scanned_pos++;
 
                     // Parse field width
                     nano_digits = atoi(&format[fmt_pos + 1]);
                     nano_digits = QB_MAX(nano_digits, 0);
                     nano_digits = QB_MIN(nano_digits, 6);
                     break;
 
                 default: // Some other specifier
                     if (format[++scanned_pos] != '\0') { // More to parse
                         continue;
                     }
                     fmt_pos = scanned_pos; // Pass remaining string as-is
                     break;
             }
         }
 
         if (date_len >= sizeof(date_s)) {
             return NULL; // No room for remaining string
         }
 
         tmp_fmt_s = strndup(&format[printed_pos], fmt_pos - printed_pos);
 #ifdef HAVE_FORMAT_NONLITERAL
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
 #endif
         nbytes = strftime(&date_s[date_len], sizeof(date_s) - date_len,
                           tmp_fmt_s, &tm);
 #ifdef HAVE_FORMAT_NONLITERAL
 #pragma GCC diagnostic pop
 #endif
         free(tmp_fmt_s);
         if (nbytes == 0) { // Would overflow buffer
             return NULL;
         }
         date_len += nbytes;
         printed_pos = scanned_pos;
         if (nano_digits != 0) {
             int nc = 0;
 
             if (date_len >= sizeof(date_s)) {
                 return NULL; // No room to add nanoseconds
             }
             nc = snprintf(&date_s[date_len], sizeof(date_s) - date_len,
                           "%.*s", nano_digits, nano_s);
 
             if ((nc < 0) || (nc == (sizeof(date_s) - date_len))) {
                 return NULL; // Error or would overflow buffer
             }
             date_len += nc;
         }
     }
 
     return (date_len == 0)? NULL : pcmk__str_copy(date_s);
 }
 
 /*!
  * \internal
  * \brief Return a human-friendly string corresponding to an epoch time value
  *
  * \param[in]  source  Pointer to epoch time value (or \p NULL for current time)
  * \param[in]  flags   Group of \p crm_time_* flags controlling display format
  *                     (0 to use \p ctime() with newline removed)
  *
  * \return String representation of \p source on success (may be empty depending
  *         on \p flags; guaranteed not to be \p NULL)
  *
  * \note The caller is responsible for freeing the return value using \p free().
  */
 char *
 pcmk__epoch2str(const time_t *source, uint32_t flags)
 {
     time_t epoch_time = (source == NULL)? time(NULL) : *source;
 
     if (flags == 0) {
         return pcmk__str_copy(pcmk__trim(ctime(&epoch_time)));
     } else {
         crm_time_t dt;
 
         crm_time_set_timet(&dt, &epoch_time);
         return crm_time_as_string(&dt, flags);
     }
 }
 
 /*!
  * \internal
  * \brief Return a human-friendly string corresponding to seconds-and-
  *        nanoseconds value
  *
  * Time is shown with microsecond resolution if \p crm_time_usecs is in \p
  * flags.
  *
  * \param[in]  ts     Time in seconds and nanoseconds (or \p NULL for current
  *                    time)
  * \param[in]  flags  Group of \p crm_time_* flags controlling display format
  *
  * \return String representation of \p ts on success (may be empty depending on
  *         \p flags; guaranteed not to be \p NULL)
  *
  * \note The caller is responsible for freeing the return value using \p free().
  */
 char *
 pcmk__timespec2str(const struct timespec *ts, uint32_t flags)
 {
     struct timespec tmp_ts;
     crm_time_t dt;
     char result[DATE_MAX] = { 0 };
 
     if (ts == NULL) {
         qb_util_timespec_from_epoch_get(&tmp_ts);
         ts = &tmp_ts;
     }
     crm_time_set_timet(&dt, &ts->tv_sec);
     time_as_string_common(&dt, ts->tv_nsec / QB_TIME_NS_IN_USEC, flags, result);
     return pcmk__str_copy(result);
 }
 
 /*!
  * \internal
  * \brief Given a millisecond interval, return a log-friendly string
  *
  * \param[in] interval_ms  Interval in milliseconds
  *
  * \return Readable version of \p interval_ms
  *
  * \note The return value is a pointer to static memory that will be
  *       overwritten by later calls to this function.
  */
 const char *
 pcmk__readable_interval(guint interval_ms)
 {
 #define MS_IN_S (1000)
 #define MS_IN_M (MS_IN_S * 60)
 #define MS_IN_H (MS_IN_M * 60)
 #define MS_IN_D (MS_IN_H * 24)
 #define MAXSTR sizeof("..d..h..m..s...ms")
     static char str[MAXSTR];
     int offset = 0;
 
     str[0] = '\0';
     if (interval_ms >= MS_IN_D) {
         offset += snprintf(str + offset, MAXSTR - offset, "%ud",
                            interval_ms / MS_IN_D);
         interval_ms -= (interval_ms / MS_IN_D) * MS_IN_D;
     }
     if (interval_ms >= MS_IN_H) {
         offset += snprintf(str + offset, MAXSTR - offset, "%uh",
                            interval_ms / MS_IN_H);
         interval_ms -= (interval_ms / MS_IN_H) * MS_IN_H;
     }
     if (interval_ms >= MS_IN_M) {
         offset += snprintf(str + offset, MAXSTR - offset, "%um",
                            interval_ms / MS_IN_M);
         interval_ms -= (interval_ms / MS_IN_M) * MS_IN_M;
     }
 
     // Ns, N.NNNs, or NNNms
     if (interval_ms >= MS_IN_S) {
         offset += snprintf(str + offset, MAXSTR - offset, "%u",
                            interval_ms / MS_IN_S);
         interval_ms -= (interval_ms / MS_IN_S) * MS_IN_S;
         if (interval_ms > 0) {
             offset += snprintf(str + offset, MAXSTR - offset, ".%03u",
                                interval_ms);
         }
         (void) snprintf(str + offset, MAXSTR - offset, "s");
 
     } else if (interval_ms > 0) {
         (void) snprintf(str + offset, MAXSTR - offset, "%ums", interval_ms);
 
     } else if (str[0] == '\0') {
         strcpy(str, "0s");
     }
     return str;
 }
diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
index 31ea0227d6..e6cff3e26c 100644
--- a/lib/pacemaker/pcmk_graph_producer.c
+++ b/lib/pacemaker/pcmk_graph_producer.c
@@ -1,1107 +1,1108 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 // Convenience macros for logging action properties
 
 #define action_type_str(flags) \
     (pcmk_is_set((flags), pcmk__action_pseudo)? "pseudo-action" : "action")
 
 #define action_optional_str(flags) \
     (pcmk_is_set((flags), pcmk__action_optional)? "optional" : "required")
 
 #define action_runnable_str(flags) \
     (pcmk_is_set((flags), pcmk__action_runnable)? "runnable" : "unrunnable")
 
 #define action_node_str(a) \
     (((a)->node == NULL)? "no node" : (a)->node->priv->name)
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified ID
  *
  * \param[in]     id      Node UUID to add
  * \param[in,out] xml     Parent XML tag to add to
  */
 static xmlNode*
 add_node_to_xml_by_id(const char *id, xmlNode *xml)
 {
     xmlNode *node_xml;
 
     node_xml = pcmk__xe_create(xml, PCMK_XE_NODE);
     crm_xml_add(node_xml, PCMK_XA_ID, id);
 
     return node_xml;
 }
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified node
  *
  * \param[in]     node  Node to add
  * \param[in,out] xml   XML to add node to
  */
 static void
 add_node_to_xml(const pcmk_node_t *node, void *xml)
 {
     add_node_to_xml_by_id(node->priv->id, (xmlNode *) xml);
 }
 
 /*!
  * \internal
  * \brief Count (optionally add to XML) nodes needing maintenance state update
  *
  * \param[in,out] xml        Parent XML tag to add to, if any
  * \param[in]     scheduler  Scheduler data
  *
  * \return Count of nodes added
  * \note Only Pacemaker Remote nodes are considered currently
  */
 static int
 add_maintenance_nodes(xmlNode *xml, const pcmk_scheduler_t *scheduler)
 {
     xmlNode *maintenance = NULL;
     int count = 0;
 
     if (xml != NULL) {
         maintenance = pcmk__xe_create(xml, PCMK__XE_MAINTENANCE);
     }
     for (const GList *iter = scheduler->nodes;
          iter != NULL; iter = iter->next) {
         const pcmk_node_t *node = iter->data;
 
         if (!pcmk__is_pacemaker_remote_node(node)) {
             continue;
         }
         if ((node->details->maintenance
              && !pcmk_is_set(node->priv->flags, pcmk__node_remote_maint))
             || (!node->details->maintenance
                 && pcmk_is_set(node->priv->flags, pcmk__node_remote_maint))) {
 
             if (maintenance != NULL) {
                 crm_xml_add(add_node_to_xml_by_id(node->priv->id,
                                                   maintenance),
                             PCMK__XA_NODE_IN_MAINTENANCE,
                             (node->details->maintenance? "1" : "0"));
             }
             count++;
         }
     }
     crm_trace("%s %d nodes in need of maintenance mode update in state",
               ((maintenance == NULL)? "Counted" : "Added"), count);
     return count;
 }
 
 /*!
  * \internal
  * \brief Add pseudo action with nodes needing maintenance state update
  *
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 add_maintenance_update(pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *action = NULL;
 
     if (add_maintenance_nodes(NULL, scheduler) != 0) {
         action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, scheduler);
         pcmk__set_action_flags(action, pcmk__action_always_in_graph);
     }
 }
 
 /*!
  * \internal
  * \brief Add XML with nodes that an action is expected to bring down
  *
  * If a specified action is expected to bring any nodes down, add an XML block
  * with their UUIDs. When a node is lost, this allows the controller to
  * determine whether it was expected.
  *
  * \param[in,out] xml       Parent XML tag to add to
  * \param[in]     action    Action to check for downed nodes
  */
 static void
 add_downed_nodes(xmlNode *xml, const pcmk_action_t *action)
 {
     CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL),
               return);
 
     if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
 
         /* Shutdown makes the action's node down */
         xmlNode *downed = pcmk__xe_create(xml, PCMK__XE_DOWNED);
         add_node_to_xml_by_id(action->node->priv->id, downed);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
                             pcmk__str_none)) {
 
         /* Fencing makes the action's node and any hosted guest nodes down */
         const char *fence = g_hash_table_lookup(action->meta,
                                                 PCMK__META_STONITH_ACTION);
 
         if (pcmk__is_fencing_action(fence)) {
             xmlNode *downed = pcmk__xe_create(xml, PCMK__XE_DOWNED);
             add_node_to_xml_by_id(action->node->priv->id, downed);
             pe_foreach_guest_node(action->node->priv->scheduler,
                                   action->node, add_node_to_xml, downed);
         }
 
     } else if ((action->rsc != NULL)
                && pcmk_is_set(action->rsc->flags,
                               pcmk__rsc_is_remote_connection)
                && pcmk__str_eq(action->task, PCMK_ACTION_STOP,
                                pcmk__str_none)) {
 
         /* Stopping a remote connection resource makes connected node down,
          * unless it's part of a migration
          */
         GList *iter;
         pcmk_action_t *input;
         bool migrating = false;
 
         for (iter = action->actions_before; iter != NULL; iter = iter->next) {
             input = ((pcmk__related_action_t *) iter->data)->action;
             if ((input->rsc != NULL)
                 && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none)
                 && pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM,
                                 pcmk__str_none)) {
                 migrating = true;
                 break;
             }
         }
         if (!migrating) {
             xmlNode *downed = pcmk__xe_create(xml, PCMK__XE_DOWNED);
             add_node_to_xml_by_id(action->rsc->id, downed);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Create a transition graph operation key for a clone action
  *
  * \param[in] action       Clone action
  * \param[in] interval_ms  Action interval in milliseconds
  *
  * \return Newly allocated string with transition graph operation key
  */
 static char *
 clone_op_key(const pcmk_action_t *action, guint interval_ms)
 {
     if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
         const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
         const char *n_task = g_hash_table_lookup(action->meta,
                                                  "notify_operation");
 
         return pcmk__notify_key(action->rsc->priv->history_id, n_type,
                                 n_task);
     }
     return pcmk__op_key(action->rsc->priv->history_id,
                         pcmk__s(action->cancel_task, action->task),
                         interval_ms);
 }
 
 /*!
  * \internal
  * \brief Add node details to transition graph action XML
  *
  * \param[in]     action  Scheduled action
  * \param[in,out] xml     Transition graph action XML for \p action
  */
 static void
 add_node_details(const pcmk_action_t *action, xmlNode *xml)
 {
     pcmk_node_t *router_node = pcmk__connection_host_for_action(action);
 
     crm_xml_add(xml, PCMK__META_ON_NODE, action->node->priv->name);
     crm_xml_add(xml, PCMK__META_ON_NODE_UUID, action->node->priv->id);
     if (router_node != NULL) {
         crm_xml_add(xml, PCMK__XA_ROUTER_NODE, router_node->priv->name);
     }
 }
 
 /*!
  * \internal
  * \brief Add resource details to transition graph action XML
  *
  * \param[in]     action      Scheduled action
  * \param[in,out] action_xml  Transition graph action XML for \p action
  */
 static void
 add_resource_details(const pcmk_action_t *action, xmlNode *action_xml)
 {
     xmlNode *rsc_xml = NULL;
     const char *attr_list[] = {
         PCMK_XA_CLASS,
         PCMK_XA_PROVIDER,
         PCMK_XA_TYPE,
     };
 
     /* If a resource is locked to a node via PCMK_OPT_SHUTDOWN_LOCK, mark its
      * actions so the controller can preserve the lock when the action
      * completes.
      */
     if (pcmk__action_locks_rsc_to_node(action)) {
         crm_xml_add_ll(action_xml, PCMK_OPT_SHUTDOWN_LOCK,
                        (long long) action->rsc->priv->lock_time);
     }
 
     // List affected resource
 
     rsc_xml = pcmk__xe_create(action_xml,
                               (const char *) action->rsc->priv->xml->name);
     if (pcmk_is_set(action->rsc->flags, pcmk__rsc_removed)
         && (action->rsc->priv->history_id != NULL)) {
         /* Use the numbered instance name here, because if there is more
          * than one instance on a node, we need to make sure the command
          * goes to the right one.
          *
          * This is important even for anonymous clones, because the clone's
          * unique meta-attribute might have just been toggled from on to
          * off.
          */
         crm_debug("Using orphan clone name %s instead of history ID %s",
                   action->rsc->id, action->rsc->priv->history_id);
         crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->priv->history_id);
         crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);
 
     } else if (!pcmk_is_set(action->rsc->flags, pcmk__rsc_unique)) {
         const char *xml_id = pcmk__xe_id(action->rsc->priv->xml);
 
         crm_debug("Using anonymous clone name %s for %s (aka %s)",
                   xml_id, action->rsc->id, action->rsc->priv->history_id);
 
         /* ID is what we'd like client to use
          * LONG_ID is what they might know it as instead
          *
          * LONG_ID is only strictly needed /here/ during the
          * transition period until all nodes in the cluster
          * are running the new software /and/ have rebooted
          * once (meaning that they've only ever spoken to a DC
-         * supporting this feature).
+         * supporting this feature). (@TODO The effect of removing this on
+         * regression tests suggests that it is still needed for unique clones)
          *
          * If anyone toggles the unique flag to 'on', the
          * 'instance free' name will correspond to an orphan
          * and fall into the clause above instead
          */
         crm_xml_add(rsc_xml, PCMK_XA_ID, xml_id);
         if ((action->rsc->priv->history_id != NULL)
             && !pcmk__str_eq(xml_id, action->rsc->priv->history_id,
                              pcmk__str_none)) {
             crm_xml_add(rsc_xml, PCMK__XA_LONG_ID,
                         action->rsc->priv->history_id);
         } else {
             crm_xml_add(rsc_xml, PCMK__XA_LONG_ID, action->rsc->id);
         }
 
     } else {
         pcmk__assert(action->rsc->priv->history_id == NULL);
         crm_xml_add(rsc_xml, PCMK_XA_ID, action->rsc->id);
     }
 
     for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
         crm_xml_add(rsc_xml, attr_list[lpc],
                     g_hash_table_lookup(action->rsc->priv->meta,
                                         attr_list[lpc]));
     }
 }
 
 /*!
  * \internal
  * \brief Add action attributes to transition graph action XML
  *
  * \param[in,out] action      Scheduled action
  * \param[in,out] action_xml  Transition graph action XML for \p action
  */
 static void
 add_action_attributes(pcmk_action_t *action, xmlNode *action_xml)
 {
     xmlNode *args_xml = NULL;
     pcmk_resource_t *rsc = action->rsc;
 
     /* We create free-standing XML to start, so we can sort the attributes
      * before adding it to action_xml, which keeps the scheduler regression
      * test graphs comparable.
      */
     args_xml = pcmk__xe_create(action_xml, PCMK__XE_ATTRIBUTES);
 
     crm_xml_add(args_xml, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
     g_hash_table_foreach(action->extra, hash2field, args_xml);
 
     if ((rsc != NULL) && (action->node != NULL)) {
         // Get the resource instance attributes, evaluated properly for node
         GHashTable *params = pe_rsc_params(rsc, action->node,
                                            rsc->priv->scheduler);
 
         pcmk__substitute_remote_addr(rsc, params);
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
 
     } else if ((rsc != NULL)
                && (rsc->priv->variant <= pcmk__rsc_variant_primitive)) {
         GHashTable *params = pe_rsc_params(rsc, NULL, rsc->priv->scheduler);
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
     }
 
     g_hash_table_foreach(action->meta, hash2metafield, args_xml);
     if (rsc != NULL) {
         pcmk_resource_t *parent = rsc;
 
         while (parent != NULL) {
             parent->priv->cmds->add_graph_meta(parent, args_xml);
             parent = parent->priv->parent;
         }
 
         pcmk__add_guest_meta_to_xml(args_xml, action);
     }
 
     pcmk__xe_sort_attrs(args_xml);
 }
 
 /*!
  * \internal
  * \brief Create the transition graph XML for a scheduled action
  *
  * \param[in,out] parent        Parent XML element to add action to
  * \param[in,out] action        Scheduled action
  * \param[in]     skip_details  If false, add action details as sub-elements
  * \param[in]     scheduler     Scheduler data
  */
 static void
 create_graph_action(xmlNode *parent, pcmk_action_t *action, bool skip_details,
                     const pcmk_scheduler_t *scheduler)
 {
     bool needs_node_info = true;
     bool needs_maintenance_info = false;
     xmlNode *action_xml = NULL;
 
     if ((action == NULL) || (scheduler == NULL)) {
         return;
     }
 
     // Create the top-level element based on task
 
     if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) {
         /* All fences need node info; guest node fences are pseudo-events */
         if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
             action_xml = pcmk__xe_create(parent, PCMK__XE_PSEUDO_EVENT);
         } else {
             action_xml = pcmk__xe_create(parent, PCMK__XE_CRM_EVENT);
         }
 
     } else if (pcmk__str_any_of(action->task,
                                 PCMK_ACTION_DO_SHUTDOWN,
                                 PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) {
         action_xml = pcmk__xe_create(parent, PCMK__XE_CRM_EVENT);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE,
                             pcmk__str_none)) {
         // CIB-only clean-up for shutdown locks
         action_xml = pcmk__xe_create(parent, PCMK__XE_CRM_EVENT);
         crm_xml_add(action_xml, PCMK__XA_MODE, PCMK__VALUE_CIB);
 
     } else if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
         if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES,
                          pcmk__str_none)) {
             needs_maintenance_info = true;
         }
         action_xml = pcmk__xe_create(parent, PCMK__XE_PSEUDO_EVENT);
         needs_node_info = false;
 
     } else {
         action_xml = pcmk__xe_create(parent, PCMK__XE_RSC_OP);
     }
 
     crm_xml_add_int(action_xml, PCMK_XA_ID, action->id);
     crm_xml_add(action_xml, PCMK_XA_OPERATION, action->task);
 
     if ((action->rsc != NULL) && (action->rsc->priv->history_id != NULL)) {
         char *clone_key = NULL;
         guint interval_ms;
 
         if (pcmk__guint_from_hash(action->meta, PCMK_META_INTERVAL, 0,
                                   &interval_ms) != pcmk_rc_ok) {
             interval_ms = 0;
         }
         clone_key = clone_op_key(action, interval_ms);
         crm_xml_add(action_xml, PCMK__XA_OPERATION_KEY, clone_key);
         crm_xml_add(action_xml, "internal_" PCMK__XA_OPERATION_KEY,
                     action->uuid);
         free(clone_key);
     } else {
         crm_xml_add(action_xml, PCMK__XA_OPERATION_KEY, action->uuid);
     }
 
     if (needs_node_info && (action->node != NULL)) {
         add_node_details(action, action_xml);
         pcmk__insert_dup(action->meta, PCMK__META_ON_NODE,
                          action->node->priv->name);
         pcmk__insert_dup(action->meta, PCMK__META_ON_NODE_UUID,
                          action->node->priv->id);
     }
 
     if (skip_details) {
         return;
     }
 
     if ((action->rsc != NULL)
         && !pcmk_is_set(action->flags, pcmk__action_pseudo)) {
 
         // This is a real resource action, so add resource details
         add_resource_details(action, action_xml);
     }
 
     /* List any attributes in effect */
     add_action_attributes(action, action_xml);
 
     /* List any nodes this action is expected to make down */
     if (needs_node_info && (action->node != NULL)) {
         add_downed_nodes(action_xml, action);
     }
 
     if (needs_maintenance_info) {
         add_maintenance_nodes(action_xml, scheduler);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether an action should be added to the transition graph
  *
  * \param[in,out] action  Action to check
  *
  * \return true if action should be added to graph, otherwise false
  */
 static bool
 should_add_action_to_graph(pcmk_action_t *action)
 {
     if (!pcmk_is_set(action->flags, pcmk__action_runnable)) {
         crm_trace("Ignoring action %s (%d): unrunnable",
                   action->uuid, action->id);
         return false;
     }
 
     if (pcmk_is_set(action->flags, pcmk__action_optional)
         && !pcmk_is_set(action->flags, pcmk__action_always_in_graph)) {
         crm_trace("Ignoring action %s (%d): optional",
                   action->uuid, action->id);
         return false;
     }
 
     /* Actions for unmanaged resources should be excluded from the graph,
      * with the exception of monitors and cancellation of recurring monitors.
      */
     if ((action->rsc != NULL)
         && !pcmk_is_set(action->rsc->flags, pcmk__rsc_managed)
         && !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
 
         const char *interval_ms_s;
 
         /* A cancellation of a recurring monitor will get here because the task
          * is cancel rather than monitor, but the interval can still be used to
          * recognize it. The interval has been normalized to milliseconds by
          * this point, so a string comparison is sufficient.
          */
         interval_ms_s = g_hash_table_lookup(action->meta, PCMK_META_INTERVAL);
         if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
             crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
                       action->uuid, action->id, action->rsc->id);
             return false;
         }
     }
 
     /* Always add pseudo-actions, fence actions, and shutdown actions (already
      * determined to be required and runnable by this point)
      */
     if (pcmk_is_set(action->flags, pcmk__action_pseudo)
         || pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH,
                                 PCMK_ACTION_DO_SHUTDOWN, NULL)) {
         return true;
     }
 
     if (action->node == NULL) {
         pcmk__sched_err(action->scheduler,
                         "Skipping action %s (%d) "
                         "because it was not assigned to a node (bug?)",
                         action->uuid, action->id);
         pcmk__log_action("Unassigned", action, false);
         return false;
     }
 
     if (pcmk_is_set(action->flags, pcmk__action_on_dc)) {
         crm_trace("Action %s (%d) should be dumped: "
                   "can run on DC instead of %s",
                   action->uuid, action->id, pcmk__node_name(action->node));
 
     } else if (pcmk__is_guest_or_bundle_node(action->node)
                && !pcmk_is_set(action->node->priv->flags,
                                pcmk__node_remote_reset)) {
         crm_trace("Action %s (%d) should be dumped: "
                   "assuming will be runnable on guest %s",
                   action->uuid, action->id, pcmk__node_name(action->node));
 
     } else if (!action->node->details->online) {
         pcmk__sched_err(action->scheduler,
                         "Skipping action %s (%d) "
                         "because it was scheduled for offline node (bug?)",
                         action->uuid, action->id);
         pcmk__log_action("Offline node", action, false);
         return false;
 
     } else if (action->node->details->unclean) {
         pcmk__sched_err(action->scheduler,
                         "Skipping action %s (%d) "
                         "because it was scheduled for unclean node (bug?)",
                         action->uuid, action->id);
         pcmk__log_action("Unclean node", action, false);
         return false;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether an ordering's flags can change an action
  *
  * \param[in] ordering  Ordering to check
  *
  * \return true if ordering has flags that can change an action, false otherwise
  */
 static bool
 ordering_can_change_actions(const pcmk__related_action_t *ordering)
 {
     return pcmk_any_flags_set(ordering->flags,
                               ~(pcmk__ar_then_implies_first_graphed
                                 |pcmk__ar_first_implies_then_graphed
                                 |pcmk__ar_ordered));
 }
 
 /*!
  * \internal
  * \brief Check whether an action input should be in the transition graph
  *
  * \param[in]     action  Action to check
  * \param[in,out] input   Action input to check
  *
  * \return true if input should be in graph, false otherwise
  * \note This function may not only check an input, but disable it under certian
  *       circumstances (load or anti-colocation orderings that are not needed).
  */
 static bool
 should_add_input_to_graph(const pcmk_action_t *action,
                           pcmk__related_action_t *input)
 {
     if (input->graphed) {
         return true;
     }
 
     if (input->flags == pcmk__ar_none) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "ordering disabled",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pcmk__action_runnable)
                && !ordering_can_change_actions(input)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional and input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pcmk__action_runnable)
                && pcmk_is_set(input->flags, pcmk__ar_min_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "minimum number of instances required but input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->flags, pcmk__ar_unmigratable_then_blocks)
                && !pcmk_is_set(input->action->flags, pcmk__action_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "input blocked if 'then' unmigratable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->flags, pcmk__ar_if_first_unmigratable)
                && pcmk_is_set(input->action->flags, pcmk__action_migratable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): ordering applies "
                   "only if input is unmigratable, but it is migratable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if ((input->flags == pcmk__ar_ordered)
                && pcmk_is_set(input->action->flags, pcmk__action_migratable)
                && pcmk__ends_with(input->action->uuid, "_stop_0")) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional but stop in migration",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (input->flags == pcmk__ar_if_on_same_node_or_target) {
         pcmk_node_t *input_node = input->action->node;
 
         if ((action->rsc != NULL)
             && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO,
                             pcmk__str_none)) {
 
             pcmk_node_t *assigned = action->rsc->priv->assigned_node;
 
             /* For load_stopped -> migrate_to orderings, we care about where
              * the resource has been assigned, not where migrate_to will be
              * executed.
              */
             if (!pcmk__same_node(input_node, assigned)) {
                 crm_trace("Ignoring %s (%d) input %s (%d): "
                           "migration target %s is not same as input node %s",
                           action->uuid, action->id,
                           input->action->uuid, input->action->id,
                           (assigned? assigned->priv->name : "<none>"),
                           (input_node? input_node->priv->name : "<none>"));
                 input->flags = pcmk__ar_none;
                 return false;
             }
 
         } else if (!pcmk__same_node(input_node, action->node)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "not on same node (%s vs %s)",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       (action->node? action->node->priv->name : "<none>"),
                       (input_node? input_node->priv->name : "<none>"));
             input->flags = pcmk__ar_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pcmk__action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "ordering optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->flags = pcmk__ar_none;
             return false;
         }
 
     } else if (input->flags == pcmk__ar_if_required_on_same_node) {
         if (input->action->node && action->node
             && !pcmk__same_node(input->action->node, action->node)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "not on same node (%s vs %s)",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       pcmk__node_name(action->node),
                       pcmk__node_name(input->action->node));
             input->flags = pcmk__ar_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pcmk__action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->flags = pcmk__ar_none;
             return false;
         }
 
     } else if (input->action->rsc
                && input->action->rsc != action->rsc
                && pcmk_is_set(input->action->rsc->flags, pcmk__rsc_failed)
                && !pcmk_is_set(input->action->rsc->flags, pcmk__rsc_managed)
                && pcmk__ends_with(input->action->uuid, "_stop_0")
                && pcmk__is_clone(action->rsc)) {
         crm_warn("Ignoring requirement that %s complete before %s:"
                  " unmanaged failed resources cannot prevent clone shutdown",
                  input->action->uuid, action->uuid);
         return false;
 
     } else if (pcmk_is_set(input->action->flags, pcmk__action_optional)
                && !pcmk_any_flags_set(input->action->flags,
                                       pcmk__action_always_in_graph
                                       |pcmk__action_added_to_graph)
                && !should_add_action_to_graph(input->action)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "input optional",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
     }
 
     crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
               action->uuid, action->id, action_type_str(input->action->flags),
               input->action->uuid, input->action->id,
               action_node_str(input->action),
               action_runnable_str(input->action->flags),
               action_optional_str(input->action->flags), input->flags);
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether an ordering creates an ordering loop
  *
  * \param[in]     init_action  "First" action in ordering
  * \param[in]     action       Callers should always set this the same as
  *                             \p init_action (this function may use a different
  *                             value for recursive calls)
  * \param[in,out] input        Action wrapper for "then" action in ordering
  *
  * \return true if the ordering creates a loop, otherwise false
  */
 bool
 pcmk__graph_has_loop(const pcmk_action_t *init_action,
                      const pcmk_action_t *action, pcmk__related_action_t *input)
 {
     bool has_loop = false;
 
     if (pcmk_is_set(input->action->flags, pcmk__action_detect_loop)) {
         crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->priv->name : "",
                   action->uuid,
                   action->node? action->node->priv->name : "",
                   input->flags);
         return false;
     }
 
     // Don't need to check inputs that won't be used
     if (!should_add_input_to_graph(action, input)) {
         return false;
     }
 
     if (input->action == init_action) {
         crm_debug("Input loop found in %s@%s ->...-> %s@%s",
                   action->uuid,
                   action->node? action->node->priv->name : "",
                   init_action->uuid,
                   init_action->node? init_action->node->priv->name : "");
         return true;
     }
 
     pcmk__set_action_flags(input->action, pcmk__action_detect_loop);
 
     crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
               "for graph loop with %s@%s ",
               action->uuid,
               action->node? action->node->priv->name : "",
               input->action->uuid,
               input->action->node? input->action->node->priv->name : "",
               input->flags,
               init_action->uuid,
               init_action->node? init_action->node->priv->name : "");
 
     // Recursively check input itself for loops
     for (GList *iter = input->action->actions_before;
          iter != NULL; iter = iter->next) {
 
         if (pcmk__graph_has_loop(init_action, input->action,
                                  (pcmk__related_action_t *) iter->data)) {
             // Recursive call already logged a debug message
             has_loop = true;
             break;
         }
     }
 
     pcmk__clear_action_flags(input->action, pcmk__action_detect_loop);
 
     if (!has_loop) {
         crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->priv->name : "",
                   action->uuid,
                   action->node? action->node->priv->name : "",
                   input->flags);
     }
     return has_loop;
 }
 
 /*!
  * \internal
  * \brief Create a synapse XML element for a transition graph
  *
  * \param[in]     action     Action that synapse is for
  * \param[in,out] scheduler  Scheduler data containing graph
  *
  * \return Newly added XML element for new graph synapse
  */
 static xmlNode *
 create_graph_synapse(const pcmk_action_t *action, pcmk_scheduler_t *scheduler)
 {
     int synapse_priority = 0;
     xmlNode *syn = pcmk__xe_create(scheduler->priv->graph, PCMK__XE_SYNAPSE);
 
     crm_xml_add_int(syn, PCMK_XA_ID, scheduler->priv->synapse_count++);
 
     if (action->rsc != NULL) {
         synapse_priority = action->rsc->priv->priority;
     }
     if (action->priority > synapse_priority) {
         synapse_priority = action->priority;
     }
     if (synapse_priority > 0) {
         crm_xml_add_int(syn, PCMK__XA_PRIORITY, synapse_priority);
     }
     return syn;
 }
 
 /*!
  * \internal
  * \brief Add an action to the transition graph XML if appropriate
  *
  * \param[in,out] data       Action to possibly add
  * \param[in,out] user_data  Scheduler data
  *
  * \note This will de-duplicate the action inputs, meaning that the
  *       pcmk__related_action_t:type flags can no longer be relied on to retain
  *       their original settings. That means this MUST be called after
  *       pcmk__apply_orderings() is complete, and nothing after this should rely
  *       on those type flags. (For example, some code looks for type equal to
  *       some flag rather than whether the flag is set, and some code looks for
  *       particular combinations of flags -- such code must be done before
  *       pcmk__create_graph().)
  */
 static void
 add_action_to_graph(gpointer data, gpointer user_data)
 {
     pcmk_action_t *action = (pcmk_action_t *) data;
     pcmk_scheduler_t *scheduler = (pcmk_scheduler_t *) user_data;
 
     xmlNode *syn = NULL;
     xmlNode *set = NULL;
     xmlNode *in = NULL;
 
     /* If we haven't already, de-duplicate inputs (even if we won't be adding
      * the action to the graph, so that crm_simulate's dot graphs don't have
      * duplicates).
      */
     if (!pcmk_is_set(action->flags, pcmk__action_inputs_deduplicated)) {
         pcmk__deduplicate_action_inputs(action);
         pcmk__set_action_flags(action, pcmk__action_inputs_deduplicated);
     }
 
     if (pcmk_is_set(action->flags, pcmk__action_added_to_graph)
         || !should_add_action_to_graph(action)) {
         return; // Already added, or shouldn't be
     }
     pcmk__set_action_flags(action, pcmk__action_added_to_graph);
 
     crm_trace("Adding action %d (%s%s%s) to graph",
               action->id, action->uuid,
               ((action->node == NULL)? "" : " on "),
               ((action->node == NULL)? "" : action->node->priv->name));
 
     syn = create_graph_synapse(action, scheduler);
     set = pcmk__xe_create(syn, PCMK__XE_ACTION_SET);
     in = pcmk__xe_create(syn, PCMK__XE_INPUTS);
 
     create_graph_action(set, action, false, scheduler);
 
     for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
         pcmk__related_action_t *input = lpc->data;
 
         if (should_add_input_to_graph(action, input)) {
             xmlNode *input_xml = pcmk__xe_create(in, PCMK__XE_TRIGGER);
 
             input->graphed = true;
             create_graph_action(input_xml, input->action, true, scheduler);
         }
     }
 }
 
 static int transition_id = 0;
 
 /*!
  * \internal
  * \brief Log a message after calculating a transition
  *
  * \param[in] scheduler  Scheduler data
  * \param[in] filename   Where transition input is stored
  */
 void
 pcmk__log_transition_summary(const pcmk_scheduler_t *scheduler,
                              const char *filename)
 {
     if (pcmk_is_set(scheduler->flags, pcmk__sched_processing_error)
         || pcmk__config_has_error) {
         crm_err("Calculated transition %d (with errors)%s%s",
                 transition_id,
                 (filename == NULL)? "" : ", saving inputs in ",
                 (filename == NULL)? "" : filename);
 
     } else if (pcmk_is_set(scheduler->flags, pcmk__sched_processing_warning)
                || pcmk__config_has_warning) {
         crm_warn("Calculated transition %d (with warnings)%s%s",
                  transition_id,
                  (filename == NULL)? "" : ", saving inputs in ",
                  (filename == NULL)? "" : filename);
 
     } else {
         crm_notice("Calculated transition %d%s%s",
                    transition_id,
                    (filename == NULL)? "" : ", saving inputs in ",
                    (filename == NULL)? "" : filename);
     }
     if (pcmk__config_has_error) {
         crm_notice("Configuration errors found during scheduler processing,"
                    "  please run \"crm_verify -L\" to identify issues");
     }
 }
 
 /*!
  * \internal
  * \brief Add a resource's actions to the transition graph
  *
  * \param[in,out] rsc  Resource whose actions should be added
  */
 void
 pcmk__add_rsc_actions_to_graph(pcmk_resource_t *rsc)
 {
     GList *iter = NULL;
 
     pcmk__assert(rsc != NULL);
 
     pcmk__rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);
 
     // First add the resource's own actions
     g_list_foreach(rsc->priv->actions, add_action_to_graph,
                    rsc->priv->scheduler);
 
     // Then recursively add its children's actions (appropriate to variant)
     for (iter = rsc->priv->children; iter != NULL; iter = iter->next) {
         pcmk_resource_t *child_rsc = (pcmk_resource_t *) iter->data;
 
         child_rsc->priv->cmds->add_actions_to_graph(child_rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Create a transition graph with all cluster actions needed
  *
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__create_graph(pcmk_scheduler_t *scheduler)
 {
     GList *iter = NULL;
     const char *value = NULL;
     long long limit = 0LL;
     GHashTable *config_hash = scheduler->priv->options;
     int rc = pcmk_rc_ok;
 
     transition_id++;
     crm_trace("Creating transition graph %d", transition_id);
 
     scheduler->priv->graph = pcmk__xe_create(NULL, PCMK__XE_TRANSITION_GRAPH);
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_CLUSTER_DELAY);
     crm_xml_add(scheduler->priv->graph, PCMK_OPT_CLUSTER_DELAY, value);
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_STONITH_TIMEOUT);
     crm_xml_add(scheduler->priv->graph, PCMK_OPT_STONITH_TIMEOUT, value);
 
     crm_xml_add(scheduler->priv->graph, PCMK__XA_FAILED_STOP_OFFSET,
                 PCMK_VALUE_INFINITY);
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_start_failure_fatal)) {
         crm_xml_add(scheduler->priv->graph, PCMK__XA_FAILED_START_OFFSET,
                     PCMK_VALUE_INFINITY);
     } else {
         crm_xml_add(scheduler->priv->graph, PCMK__XA_FAILED_START_OFFSET, "1");
     }
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_BATCH_LIMIT);
     crm_xml_add(scheduler->priv->graph, PCMK_OPT_BATCH_LIMIT, value);
 
     crm_xml_add_int(scheduler->priv->graph, "transition_id", transition_id);
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_MIGRATION_LIMIT);
     rc = pcmk__scan_ll(value, &limit, 0LL);
     if (rc != pcmk_rc_ok) {
         crm_warn("Ignoring invalid value '%s' for " PCMK_OPT_MIGRATION_LIMIT
                  ": %s", value, pcmk_rc_str(rc));
     } else if (limit > 0) {
         crm_xml_add(scheduler->priv->graph, PCMK_OPT_MIGRATION_LIMIT, value);
     }
 
     if (scheduler->priv->recheck_by > 0) {
         char *recheck_epoch = NULL;
 
         recheck_epoch = crm_strdup_printf("%llu", (unsigned long long)
                                           scheduler->priv->recheck_by);
         crm_xml_add(scheduler->priv->graph, "recheck-by", recheck_epoch);
         free(recheck_epoch);
     }
 
     /* The following code will de-duplicate action inputs, so nothing past this
      * should rely on the action input type flags retaining their original
      * values.
      */
 
     // Add resource actions to graph
     for (iter = scheduler->priv->resources; iter != NULL; iter = iter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
         pcmk__rsc_trace(rsc, "Processing actions for %s", rsc->id);
         rsc->priv->cmds->add_actions_to_graph(rsc);
     }
 
     // Add pseudo-action for list of nodes with maintenance state update
     add_maintenance_update(scheduler);
 
     // Add non-resource (node) actions
     for (iter = scheduler->priv->actions; iter != NULL; iter = iter->next) {
         pcmk_action_t *action = (pcmk_action_t *) iter->data;
 
         if ((action->rsc != NULL)
             && (action->node != NULL)
             && action->node->details->shutdown
             && !pcmk_is_set(action->rsc->flags, pcmk__rsc_maintenance)
             && !pcmk_any_flags_set(action->flags,
                                    pcmk__action_optional|pcmk__action_runnable)
             && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
             /* Eventually we should just ignore the 'fence' case, but for now
              * it's the best way to detect (in CTS) when CIB resource updates
              * are being lost.
              */
             if (pcmk_is_set(scheduler->flags, pcmk__sched_quorate)
                 || (scheduler->no_quorum_policy == pcmk_no_quorum_ignore)) {
                 const bool managed = pcmk_is_set(action->rsc->flags,
                                                  pcmk__rsc_managed);
                 const bool failed = pcmk_is_set(action->rsc->flags,
                                                 pcmk__rsc_failed);
 
                 crm_crit("Cannot %s %s because of %s:%s%s (%s)",
                          action->node->details->unclean? "fence" : "shut down",
                          pcmk__node_name(action->node), action->rsc->id,
                          (managed? " blocked" : " unmanaged"),
                          (failed? " failed" : ""), action->uuid);
             }
         }
 
         add_action_to_graph((gpointer) action, (gpointer) scheduler);
     }
 
     crm_log_xml_trace(scheduler->priv->graph, "graph");
 }
diff --git a/lib/pacemaker/pcmk_injections.c b/lib/pacemaker/pcmk_injections.c
index d728cfea1e..2f51e5f676 100644
--- a/lib/pacemaker/pcmk_injections.c
+++ b/lib/pacemaker/pcmk_injections.c
@@ -1,796 +1,797 @@
 /*
  * Copyright 2009-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/cib/internal.h>
 #include <crm/common/util.h>
 #include <crm/common/iso8601.h>
 #include <crm/common/xml_internal.h>
 #include <crm/lrmd_events.h>            // lrmd_event_data_t, etc.
 #include <crm/lrmd_internal.h>
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
+// @TODO Replace this with a new scheduler flag
 bool pcmk__simulate_node_config = false;
 
 #define XPATH_NODE_CONFIG   "//" PCMK_XE_NODE "[@" PCMK_XA_UNAME "='%s']"
 #define XPATH_NODE_STATE    "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']"
 #define XPATH_NODE_STATE_BY_ID "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "='%s']"
 #define XPATH_RSC_HISTORY   XPATH_NODE_STATE \
                             "//" PCMK__XE_LRM_RESOURCE "[@" PCMK_XA_ID "='%s']"
 
 
 /*!
  * \internal
  * \brief Inject a fictitious transient node attribute into scheduler input
  *
  * \param[in,out] out       Output object for displaying error messages
  * \param[in,out] cib_node  \c PCMK__XE_NODE_STATE XML to inject attribute into
  * \param[in]     name      Transient node attribute name to inject
  * \param[in]     value     Transient node attribute value to inject
  */
 static void
 inject_transient_attr(pcmk__output_t *out, xmlNode *cib_node,
                       const char *name, const char *value)
 {
     xmlNode *attrs = NULL;
     xmlNode *instance_attrs = NULL;
     const char *node_uuid = pcmk__xe_id(cib_node);
 
     out->message(out, "inject-attr", name, value, cib_node);
 
     attrs = pcmk__xe_first_child(cib_node, PCMK__XE_TRANSIENT_ATTRIBUTES, NULL,
                                  NULL);
     if (attrs == NULL) {
         attrs = pcmk__xe_create(cib_node, PCMK__XE_TRANSIENT_ATTRIBUTES);
         crm_xml_add(attrs, PCMK_XA_ID, node_uuid);
     }
 
     instance_attrs = pcmk__xe_first_child(attrs, PCMK_XE_INSTANCE_ATTRIBUTES,
                                           NULL, NULL);
     if (instance_attrs == NULL) {
         instance_attrs = pcmk__xe_create(attrs, PCMK_XE_INSTANCE_ATTRIBUTES);
         crm_xml_add(instance_attrs, PCMK_XA_ID, node_uuid);
     }
 
     crm_create_nvpair_xml(instance_attrs, NULL, name, value);
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious fail count into a scheduler input
  *
  * \param[in,out] out          Output object for displaying error messages
  * \param[in,out] cib_conn     CIB connection
  * \param[in,out] cib_node     Node state XML to inject into
  * \param[in]     resource     ID of resource for fail count to inject
  * \param[in]     task         Action name for fail count to inject
  * \param[in]     interval_ms  Action interval (in milliseconds) for fail count
  * \param[in]     exit_status  Action result for fail count to inject (if
  *                             \c PCMK_OCF_OK, or \c PCMK_OCF_NOT_RUNNING when
  *                             \p interval_ms is 0, inject nothing)
  * \param[in]     infinity     If true, set fail count to "INFINITY", otherwise
  *                             increase it by 1
  */
 void
 pcmk__inject_failcount(pcmk__output_t *out, cib_t *cib_conn, xmlNode *cib_node,
                        const char *resource, const char *task,
                        guint interval_ms, int exit_status, bool infinity)
 {
     char *name = NULL;
     char *value = NULL;
 
     int failcount = 0;
     xmlNode *output = NULL;
 
     CRM_CHECK((out != NULL) && (cib_conn != NULL) && (cib_node != NULL)
               && (resource != NULL) && (task != NULL), return);
 
     if ((exit_status == PCMK_OCF_OK)
         || ((exit_status == PCMK_OCF_NOT_RUNNING) && (interval_ms == 0))) {
         return;
     }
 
     // Get current failcount and increment it
     name = pcmk__failcount_name(resource, task, interval_ms);
 
     if (cib__get_node_attrs(out, cib_conn, PCMK_XE_STATUS,
                             pcmk__xe_id(cib_node), NULL, NULL, NULL, name,
                             NULL, &output) == pcmk_rc_ok) {
 
         if (crm_element_value_int(output, PCMK_XA_VALUE, &failcount) != 0) {
             failcount = 0;
         }
     }
 
     if (infinity) {
         value = pcmk__str_copy(PCMK_VALUE_INFINITY);
 
     } else {
         value = pcmk__itoa(failcount + 1);
     }
 
     inject_transient_attr(out, cib_node, name, value);
 
     free(name);
     free(value);
     pcmk__xml_free(output);
 
     name = pcmk__lastfailure_name(resource, task, interval_ms);
     value = pcmk__ttoa(time(NULL));
     inject_transient_attr(out, cib_node, name, value);
 
     free(name);
     free(value);
 }
 
 /*!
  * \internal
  * \brief Create a CIB configuration entry for a fictitious node
  *
  * \param[in,out] cib_conn  CIB object to use
  * \param[in]     node      Node name to use
  */
 static void
 create_node_entry(cib_t *cib_conn, const char *node)
 {
     int rc = pcmk_ok;
     char *xpath = crm_strdup_printf(XPATH_NODE_CONFIG, node);
 
     rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath|cib_sync_call);
 
     if (rc == -ENXIO) { // Only add if not already existing
         xmlNode *cib_object = pcmk__xe_create(NULL, PCMK_XE_NODE);
 
         crm_xml_add(cib_object, PCMK_XA_ID, node); // Use node name as ID
         crm_xml_add(cib_object, PCMK_XA_UNAME, node);
         cib_conn->cmds->create(cib_conn, PCMK_XE_NODES, cib_object,
                                cib_sync_call);
         /* Not bothering with subsequent query to see if it exists,
            we'll bomb out later in the call to query_node_uuid()... */
 
         pcmk__xml_free(cib_object);
     }
 
     free(xpath);
 }
 
 /*!
  * \internal
  * \brief Synthesize a fake executor event for an action
  *
  * \param[in] cib_resource  XML for any existing resource action history
  * \param[in] task          Name of action to synthesize
  * \param[in] interval_ms   Interval of action to synthesize
  * \param[in] outcome       Result of action to synthesize
  *
  * \return Newly allocated executor event
  * \note It is the caller's responsibility to free the result with
  *       lrmd_free_event().
  */
 static lrmd_event_data_t *
 create_op(const xmlNode *cib_resource, const char *task, guint interval_ms,
           int outcome)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *xop = NULL;
 
     op = lrmd_new_event(pcmk__xe_id(cib_resource), task, interval_ms);
     lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
     op->params = NULL; // Not needed for simulation purposes
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     // Use a call ID higher than any existing history entries
     op->call_id = 0;
     for (xop = pcmk__xe_first_child(cib_resource, NULL, NULL, NULL);
          xop != NULL; xop = pcmk__xe_next(xop, NULL)) {
 
         int tmp = 0;
 
         crm_element_value_int(xop, PCMK__XA_CALL_ID, &tmp);
         if (tmp > op->call_id) {
             op->call_id = tmp;
         }
     }
     op->call_id++;
 
     return op;
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious resource history entry into a scheduler input
  *
  * \param[in,out] cib_resource  Resource history XML to inject entry into
  * \param[in,out] op            Action result to inject
  * \param[in]     node          Name of node where the action occurred
  * \param[in]     target_rc     Expected result for action to inject
  *
  * \return XML of injected resource history entry
  */
 xmlNode *
 pcmk__inject_action_result(xmlNode *cib_resource, lrmd_event_data_t *op,
                            const char *node, int target_rc)
 {
     return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
                                     target_rc, node, crm_system_name);
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious node into a scheduler input
  *
  * \param[in,out] cib_conn  Scheduler input CIB to inject node into
  * \param[in]     node      Name of node to inject
  * \param[in]     uuid      UUID of node to inject
  *
  * \return XML of \c PCMK__XE_NODE_STATE entry for new node
  * \note If the global pcmk__simulate_node_config has been set to true, a
  *       node entry in the configuration section will be added, as well as a
  *       node state entry in the status section.
  */
 xmlNode *
 pcmk__inject_node(cib_t *cib_conn, const char *node, const char *uuid)
 {
     int rc = pcmk_ok;
     xmlNode *cib_object = NULL;
     char *xpath = crm_strdup_printf(XPATH_NODE_STATE, node);
     bool duplicate = false;
     char *found_uuid = NULL;
 
     if (pcmk__simulate_node_config) {
         create_node_entry(cib_conn, node);
     }
 
     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                cib_xpath|cib_sync_call);
 
     if ((cib_object != NULL) && (pcmk__xe_id(cib_object) == NULL)) {
         crm_err("Detected multiple " PCMK__XE_NODE_STATE " entries for "
                 "xpath=%s, bailing",
                 xpath);
         duplicate = true;
         goto done;
     }
 
     if (rc == -ENXIO) {
         if (uuid == NULL) {
             query_node_uuid(cib_conn, node, &found_uuid, NULL);
         } else {
             found_uuid = strdup(uuid);
         }
 
         if (found_uuid) {
             char *xpath_by_uuid = crm_strdup_printf(XPATH_NODE_STATE_BY_ID,
                                                     found_uuid);
 
             /* It's possible that a PCMK__XE_NODE_STATE entry doesn't have a
              * PCMK_XA_UNAME yet
              */
             rc = cib_conn->cmds->query(cib_conn, xpath_by_uuid, &cib_object,
                                        cib_xpath|cib_sync_call);
 
             if ((cib_object != NULL) && (pcmk__xe_id(cib_object) == NULL)) {
                 crm_err("Can't inject node state for %s because multiple "
                         "state entries found for ID %s", node, found_uuid);
                 duplicate = true;
                 free(xpath_by_uuid);
                 goto done;
 
             } else if (cib_object != NULL) {
                 crm_xml_add(cib_object, PCMK_XA_UNAME, node);
 
                 rc = cib_conn->cmds->modify(cib_conn, PCMK_XE_STATUS,
                                             cib_object, cib_sync_call);
             }
 
             free(xpath_by_uuid);
         }
     }
 
     if (rc == -ENXIO) {
         cib_object = pcmk__xe_create(NULL, PCMK__XE_NODE_STATE);
         crm_xml_add(cib_object, PCMK_XA_ID, found_uuid);
         crm_xml_add(cib_object, PCMK_XA_UNAME, node);
         cib_conn->cmds->create(cib_conn, PCMK_XE_STATUS, cib_object,
                                cib_sync_call);
         pcmk__xml_free(cib_object);
 
         rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                    cib_xpath|cib_sync_call);
         crm_trace("Injecting node state for %s (rc=%d)", node, rc);
     }
 
 done:
     free(found_uuid);
     free(xpath);
 
     if (duplicate) {
         crm_log_xml_warn(cib_object, "Duplicates");
         crm_exit(CRM_EX_SOFTWARE);
         return NULL; // not reached, but makes static analysis happy
     }
 
     pcmk__assert(rc == pcmk_ok);
     return cib_object;
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious node state change into a scheduler input
  *
  * \param[in,out] cib_conn  Scheduler input CIB to inject into
  * \param[in]     node      Name of node to inject change for
  * \param[in]     up        If true, change state to online, otherwise offline
  *
  * \return XML of changed (or added) node state entry
  */
 xmlNode *
 pcmk__inject_node_state_change(cib_t *cib_conn, const char *node, bool up)
 {
     xmlNode *cib_node = pcmk__inject_node(cib_conn, node, NULL);
 
     if (up) {
         pcmk__xe_set_props(cib_node,
                            PCMK__XA_IN_CCM, PCMK_VALUE_TRUE,
                            PCMK_XA_CRMD, PCMK_VALUE_ONLINE,
                            PCMK__XA_JOIN, CRMD_JOINSTATE_MEMBER,
                            PCMK_XA_EXPECTED, CRMD_JOINSTATE_MEMBER,
                            NULL);
     } else {
         pcmk__xe_set_props(cib_node,
                            PCMK__XA_IN_CCM, PCMK_VALUE_FALSE,
                            PCMK_XA_CRMD, PCMK_VALUE_OFFLINE,
                            PCMK__XA_JOIN, CRMD_JOINSTATE_DOWN,
                            PCMK_XA_EXPECTED, CRMD_JOINSTATE_DOWN,
                            NULL);
     }
     crm_xml_add(cib_node, PCMK_XA_CRM_DEBUG_ORIGIN, crm_system_name);
     return cib_node;
 }
 
 /*!
  * \internal
  * \brief Check whether a node has history for a given resource
  *
  * \param[in,out] cib_node  Node state XML to check
  * \param[in]     resource  Resource name to check for
  *
  * \return Resource's \c PCMK__XE_LRM_RESOURCE XML entry beneath \p cib_node if
  *         found, otherwise \c NULL
  */
 static xmlNode *
 find_resource_xml(xmlNode *cib_node, const char *resource)
 {
     const char *node = crm_element_value(cib_node, PCMK_XA_UNAME);
     char *xpath = crm_strdup_printf(XPATH_RSC_HISTORY, node, resource);
     xmlNode *match = get_xpath_object(xpath, cib_node, LOG_TRACE);
 
     free(xpath);
     return match;
 }
 
 /*!
  * \internal
  * \brief Inject a resource history element into a scheduler input
  *
  * \param[in,out] out       Output object for displaying error messages
  * \param[in,out] cib_node  Node state XML to inject resource history entry into
  * \param[in]     resource  ID (in configuration) of resource to inject
  * \param[in]     lrm_name  ID as used in history (could be clone instance)
  * \param[in]     rclass    Resource agent class of resource to inject
  * \param[in]     rtype     Resource agent type of resource to inject
  * \param[in]     rprovider Resource agent provider of resource to inject
  *
  * \return XML of injected resource history element
  * \note If a history element already exists under either \p resource or
  *       \p lrm_name, this will return it rather than injecting a new one.
  */
 xmlNode *
 pcmk__inject_resource_history(pcmk__output_t *out, xmlNode *cib_node,
                               const char *resource, const char *lrm_name,
                               const char *rclass, const char *rtype,
                               const char *rprovider)
 {
     xmlNode *lrm = NULL;
     xmlNode *container = NULL;
     xmlNode *cib_resource = NULL;
 
     cib_resource = find_resource_xml(cib_node, resource);
     if (cib_resource != NULL) {
         /* If an existing LRM history entry uses the resource name,
          * continue using it, even if lrm_name is different.
          */
         return cib_resource;
     }
 
     // Check for history entry under preferred name
     if (strcmp(resource, lrm_name) != 0) {
         cib_resource = find_resource_xml(cib_node, lrm_name);
         if (cib_resource != NULL) {
             return cib_resource;
         }
     }
 
     if ((rclass == NULL) || (rtype == NULL)) {
         // @TODO query configuration for class, provider, type
         out->err(out,
                  "Resource %s not found in the status section of %s "
                  "(supply class and type to continue)",
                  resource, pcmk__xe_id(cib_node));
         return NULL;
 
     } else if (!pcmk__strcase_any_of(rclass,
                                      PCMK_RESOURCE_CLASS_OCF,
                                      PCMK_RESOURCE_CLASS_STONITH,
                                      PCMK_RESOURCE_CLASS_SERVICE,
                                      PCMK_RESOURCE_CLASS_SYSTEMD,
                                      PCMK_RESOURCE_CLASS_LSB, NULL)) {
         out->err(out, "Invalid class for %s: %s", resource, rclass);
         return NULL;
 
     } else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
                && (rprovider == NULL)) {
         // @TODO query configuration for provider
         out->err(out, "Please specify the provider for resource %s", resource);
         return NULL;
     }
 
     crm_info("Injecting new resource %s into node state '%s'",
              lrm_name, pcmk__xe_id(cib_node));
 
     lrm = pcmk__xe_first_child(cib_node, PCMK__XE_LRM, NULL, NULL);
     if (lrm == NULL) {
         const char *node_uuid = pcmk__xe_id(cib_node);
 
         lrm = pcmk__xe_create(cib_node, PCMK__XE_LRM);
         crm_xml_add(lrm, PCMK_XA_ID, node_uuid);
     }
 
     container = pcmk__xe_first_child(lrm, PCMK__XE_LRM_RESOURCES, NULL, NULL);
     if (container == NULL) {
         container = pcmk__xe_create(lrm, PCMK__XE_LRM_RESOURCES);
     }
 
     cib_resource = pcmk__xe_create(container, PCMK__XE_LRM_RESOURCE);
 
     // If we're creating a new entry, use the preferred name
     crm_xml_add(cib_resource, PCMK_XA_ID, lrm_name);
 
     crm_xml_add(cib_resource, PCMK_XA_CLASS, rclass);
     crm_xml_add(cib_resource, PCMK_XA_PROVIDER, rprovider);
     crm_xml_add(cib_resource, PCMK_XA_TYPE, rtype);
 
     return cib_resource;
 }
 
 /*!
  * \internal
  * \brief Inject a ticket attribute into ticket state
  *
  * \param[in,out] out          Output object for displaying error messages
  * \param[in]     ticket_id    Ticket whose state should be changed
  * \param[in]     attr_name    Ticket attribute name to inject
  * \param[in]     attr_value   Boolean value of ticket attribute to inject
  * \param[in,out] cib          CIB object to use
  *
  * \return Standard Pacemaker return code
  */
 static int
 set_ticket_state_attr(pcmk__output_t *out, const char *ticket_id,
                       const char *attr_name, bool attr_value, cib_t *cib)
 {
     int rc = pcmk_rc_ok;
     xmlNode *xml_top = NULL;
     xmlNode *ticket_state_xml = NULL;
 
     // Check for an existing ticket state entry
     rc = pcmk__get_ticket_state(cib, ticket_id, &ticket_state_xml);
 
     if (rc == pcmk_rc_duplicate_id) {
         out->err(out, "Multiple " PCMK__XE_TICKET_STATE "s match ticket_id=%s",
                  ticket_id);
         rc = pcmk_rc_ok;
     }
 
     if (rc == pcmk_rc_ok) { // Ticket state found, use it
         crm_debug("Injecting attribute into existing ticket state %s",
                   ticket_id);
         xml_top = ticket_state_xml;
 
     } else if (rc == ENXIO) { // No ticket state, create it
         xmlNode *xml_obj = NULL;
 
         xml_top = pcmk__xe_create(NULL, PCMK_XE_STATUS);
         xml_obj = pcmk__xe_create(xml_top, PCMK_XE_TICKETS);
         ticket_state_xml = pcmk__xe_create(xml_obj, PCMK__XE_TICKET_STATE);
         crm_xml_add(ticket_state_xml, PCMK_XA_ID, ticket_id);
 
     } else { // Error
         return rc;
     }
 
     // Add the attribute to the ticket state
     pcmk__xe_set_bool_attr(ticket_state_xml, attr_name, attr_value);
     crm_log_xml_debug(xml_top, "Update");
 
     // Commit the change to the CIB
     rc = cib->cmds->modify(cib, PCMK_XE_STATUS, xml_top, cib_sync_call);
     rc = pcmk_legacy2rc(rc);
 
     pcmk__xml_free(xml_top);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Inject a fictitious action into the cluster
  *
  * \param[in,out] out       Output object for displaying error messages
  * \param[in]     spec      Action specification to inject
  * \param[in,out] cib       CIB object for scheduler input
  * \param[in]     scheduler  Scheduler data
  */
 static void
 inject_action(pcmk__output_t *out, const char *spec, cib_t *cib,
               const pcmk_scheduler_t *scheduler)
 {
     int rc;
     int outcome = PCMK_OCF_OK;
     guint interval_ms = 0;
 
     char *key = NULL;
     char *node = NULL;
     char *task = NULL;
     char *resource = NULL;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *rprovider = NULL;
 
     xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     const pcmk_resource_t *rsc = NULL;
     lrmd_event_data_t *op = NULL;
     bool infinity = false;
 
     out->message(out, "inject-spec", spec);
 
     key = pcmk__assert_alloc(1, strlen(spec) + 1);
     node = pcmk__assert_alloc(1, strlen(spec) + 1);
     rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
     if (rc != 3) {
         out->err(out, "Invalid operation spec: %s.  Only found %d fields",
                  spec, rc);
         goto done;
     }
 
     parse_op_key(key, &resource, &task, &interval_ms);
 
     rsc = pe_find_resource(scheduler->priv->resources, resource);
     if (rsc == NULL) {
         out->err(out, "Invalid resource name: %s", resource);
         goto done;
     }
 
     rclass = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
     rtype = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE);
     rprovider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER);
 
     cib_node = pcmk__inject_node(cib, node, NULL);
     pcmk__assert(cib_node != NULL);
 
     if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_none)) {
         infinity = true;
 
     } else if (pcmk__str_eq(task, PCMK_ACTION_START, pcmk__str_none)
                && pcmk_is_set(scheduler->flags,
                               pcmk__sched_start_failure_fatal)) {
         infinity = true;
     }
 
     pcmk__inject_failcount(out, cib, cib_node, resource, task, interval_ms,
                            outcome, infinity);
 
     cib_resource = pcmk__inject_resource_history(out, cib_node,
                                                  resource, resource,
                                                  rclass, rtype, rprovider);
     pcmk__assert(cib_resource != NULL);
 
     op = create_op(cib_resource, task, interval_ms, outcome);
     pcmk__assert(op != NULL);
 
     cib_op = pcmk__inject_action_result(cib_resource, op, node, 0);
     pcmk__assert(cib_op != NULL);
     lrmd_free_event(op);
 
     rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node, cib_sync_call);
     pcmk__assert(rc == pcmk_ok);
 
 done:
     free(task);
     free(node);
     free(key);
 }
 
 /*!
  * \internal
  * \brief Inject fictitious scheduler inputs
  *
  * \param[in,out] scheduler   Scheduler data
  * \param[in,out] cib         CIB object for scheduler input to modify
  * \param[in]     injections  Injections to apply
  */
 void
 pcmk__inject_scheduler_input(pcmk_scheduler_t *scheduler, cib_t *cib,
                              const pcmk_injections_t *injections)
 {
     int rc = pcmk_ok;
     const GList *iter = NULL;
     xmlNode *cib_node = NULL;
     pcmk__output_t *out = scheduler->priv->out;
 
     out->message(out, "inject-modify-config", injections->quorum,
                  injections->watchdog);
     if (injections->quorum != NULL) {
         xmlNode *top = pcmk__xe_create(NULL, PCMK_XE_CIB);
 
         /* crm_xml_add(top, PCMK_XA_DC_UUID, dc_uuid);      */
         crm_xml_add(top, PCMK_XA_HAVE_QUORUM, injections->quorum);
 
         rc = cib->cmds->modify(cib, NULL, top, cib_sync_call);
         pcmk__assert(rc == pcmk_ok);
     }
 
     if (injections->watchdog != NULL) {
         rc = cib__update_node_attr(out, cib, cib_sync_call, PCMK_XE_CRM_CONFIG,
                                    NULL, NULL, NULL, NULL,
                                    PCMK_OPT_HAVE_WATCHDOG, injections->watchdog,
                                    NULL, NULL);
         pcmk__assert(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->node_up; iter != NULL; iter = iter->next) {
         const char *node = (const char *) iter->data;
 
         out->message(out, "inject-modify-node", "Online", node);
 
         cib_node = pcmk__inject_node_state_change(cib, node, true);
         pcmk__assert(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node, cib_sync_call);
         pcmk__assert(rc == pcmk_ok);
         pcmk__xml_free(cib_node);
     }
 
     for (iter = injections->node_down; iter != NULL; iter = iter->next) {
         const char *node = (const char *) iter->data;
         char *xpath = NULL;
 
         out->message(out, "inject-modify-node", "Offline", node);
 
         cib_node = pcmk__inject_node_state_change(cib, node, false);
         pcmk__assert(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node, cib_sync_call);
         pcmk__assert(rc == pcmk_ok);
         pcmk__xml_free(cib_node);
 
         xpath = crm_strdup_printf("//" PCMK__XE_NODE_STATE
                                   "[@" PCMK_XA_UNAME "='%s']"
                                   "/" PCMK__XE_LRM,
                                   node);
         cib->cmds->remove(cib, xpath, NULL, cib_xpath|cib_sync_call);
         free(xpath);
 
         xpath = crm_strdup_printf("//" PCMK__XE_NODE_STATE
                                   "[@" PCMK_XA_UNAME "='%s']"
                                   "/" PCMK__XE_TRANSIENT_ATTRIBUTES,
                                   node);
         cib->cmds->remove(cib, xpath, NULL, cib_xpath|cib_sync_call);
         free(xpath);
     }
 
     for (iter = injections->node_fail; iter != NULL; iter = iter->next) {
         const char *node = (const char *) iter->data;
 
         out->message(out, "inject-modify-node", "Failing", node);
 
         cib_node = pcmk__inject_node_state_change(cib, node, true);
         crm_xml_add(cib_node, PCMK__XA_IN_CCM, PCMK_VALUE_FALSE);
         pcmk__assert(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, PCMK_XE_STATUS, cib_node, cib_sync_call);
         pcmk__assert(rc == pcmk_ok);
         pcmk__xml_free(cib_node);
     }
 
     for (iter = injections->ticket_grant; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Granting", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, PCMK__XA_GRANTED, true, cib);
         pcmk__assert(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->ticket_revoke; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Revoking", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, PCMK__XA_GRANTED, false,
                                    cib);
         pcmk__assert(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->ticket_standby; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Standby", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, PCMK_XA_STANDBY, true, cib);
         pcmk__assert(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->ticket_activate; iter != NULL; iter = iter->next) {
         const char *ticket_id = (const char *) iter->data;
 
         out->message(out, "inject-modify-ticket", "Activating", ticket_id);
 
         rc = set_ticket_state_attr(out, ticket_id, PCMK_XA_STANDBY, false, cib);
         pcmk__assert(rc == pcmk_rc_ok);
     }
 
     for (iter = injections->op_inject; iter != NULL; iter = iter->next) {
         inject_action(out, (const char *) iter->data, cib, scheduler);
     }
 
     if (!out->is_quiet(out)) {
         out->end_list(out);
     }
 }
 
 void
 pcmk_free_injections(pcmk_injections_t *injections)
 {
     if (injections == NULL) {
         return;
     }
 
     g_list_free_full(injections->node_up, g_free);
     g_list_free_full(injections->node_down, g_free);
     g_list_free_full(injections->node_fail, g_free);
     g_list_free_full(injections->op_fail, g_free);
     g_list_free_full(injections->op_inject, g_free);
     g_list_free_full(injections->ticket_grant, g_free);
     g_list_free_full(injections->ticket_revoke, g_free);
     g_list_free_full(injections->ticket_standby, g_free);
     g_list_free_full(injections->ticket_activate, g_free);
     free(injections->quorum);
     free(injections->watchdog);
 
     free(injections);
 }
diff --git a/lib/pacemaker/pcmk_sched_actions.c b/lib/pacemaker/pcmk_sched_actions.c
index e6ed67c0d9..60f6ec236a 100644
--- a/lib/pacemaker/pcmk_sched_actions.c
+++ b/lib/pacemaker/pcmk_sched_actions.c
@@ -1,1942 +1,1956 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <sys/param.h>
 #include <glib.h>
 
 #include <crm/lrmd_internal.h>
 #include <crm/common/scheduler_internal.h>
 #include <pacemaker-internal.h>
 #include "libpacemaker_private.h"
 
 /*!
  * \internal
  * \brief Get the action flags relevant to ordering constraints
  *
  * \param[in,out] action  Action to check
  * \param[in]     node    Node that *other* action in the ordering is on
  *                        (used only for clone resource actions)
  *
  * \return Action flags that should be used for orderings
  */
 static uint32_t
 action_flags_for_ordering(pcmk_action_t *action, const pcmk_node_t *node)
 {
     bool runnable = false;
     uint32_t flags;
 
     // For non-resource actions, return the action flags
     if (action->rsc == NULL) {
         return action->flags;
     }
 
     /* For non-clone resources, or a clone action not assigned to a node,
      * return the flags as determined by the resource method without a node
      * specified.
      */
     flags = action->rsc->priv->cmds->action_flags(action, NULL);
     if ((node == NULL) || !pcmk__is_clone(action->rsc)) {
         return flags;
     }
 
     /* Otherwise (i.e., for clone resource actions on a specific node), first
      * remember whether the non-node-specific action is runnable.
      */
     runnable = pcmk_is_set(flags, pcmk__action_runnable);
 
     // Then recheck the resource method with the node
     flags = action->rsc->priv->cmds->action_flags(action, node);
 
     /* For clones in ordering constraints, the node-specific "runnable" doesn't
      * matter, just the non-node-specific setting (i.e., is the action runnable
      * anywhere).
      *
      * This applies only to runnable, and only for ordering constraints. This
      * function shouldn't be used for other types of constraints without
      * changes. Not very satisfying, but it's logical and appears to work well.
      */
     if (runnable && !pcmk_is_set(flags, pcmk__action_runnable)) {
         pcmk__set_raw_action_flags(flags, action->rsc->id,
                                    pcmk__action_runnable);
     }
     return flags;
 }
 
 /*!
  * \internal
  * \brief Get action UUID that should be used with a resource ordering
  *
  * When an action is ordered relative to an action for a collective resource
  * (clone, group, or bundle), it actually needs to be ordered after all
  * instances of the collective have completed the relevant action (for example,
  * given "start CLONE then start RSC", RSC must wait until all instances of
  * CLONE have started). Given the UUID and resource of the first action in an
  * ordering, this returns the UUID of the action that should actually be used
  * for ordering (for example, "CLONE_started_0" instead of "CLONE_start_0").
  *
  * \param[in] first_uuid    UUID of first action in ordering
  * \param[in] first_rsc     Resource of first action in ordering
  *
  * \return Newly allocated copy of UUID to use with ordering
  * \note It is the caller's responsibility to free the return value.
  */
 static char *
 action_uuid_for_ordering(const char *first_uuid,
                          const pcmk_resource_t *first_rsc)
 {
     guint interval_ms = 0;
     char *uuid = NULL;
     char *rid = NULL;
     char *first_task_str = NULL;
     enum pcmk__action_type first_task = pcmk__action_unspecified;
     enum pcmk__action_type remapped_task = pcmk__action_unspecified;
 
     // Only non-notify actions for collective resources need remapping
     if ((strstr(first_uuid, PCMK_ACTION_NOTIFY) != NULL)
         || (first_rsc->priv->variant < pcmk__rsc_variant_group)) {
         goto done;
     }
 
     // Only non-recurring actions need remapping
     pcmk__assert(parse_op_key(first_uuid, &rid, &first_task_str, &interval_ms));
     if (interval_ms > 0) {
         goto done;
     }
 
     first_task = pcmk__parse_action(first_task_str);
     switch (first_task) {
         case pcmk__action_stop:
         case pcmk__action_start:
         case pcmk__action_notify:
         case pcmk__action_promote:
         case pcmk__action_demote:
             remapped_task = first_task + 1;
             break;
         case pcmk__action_stopped:
         case pcmk__action_started:
         case pcmk__action_notified:
         case pcmk__action_promoted:
         case pcmk__action_demoted:
             remapped_task = first_task;
             break;
         case pcmk__action_monitor:
         case pcmk__action_shutdown:
         case pcmk__action_fence:
             break;
         default:
             crm_err("Unknown action '%s' in ordering", first_task_str);
             break;
     }
 
     if (remapped_task != pcmk__action_unspecified) {
         /* If a clone or bundle has notifications enabled, the ordering will be
          * relative to when notifications have been sent for the remapped task.
          */
         if (pcmk_is_set(first_rsc->flags, pcmk__rsc_notify)
             && (pcmk__is_clone(first_rsc) || pcmk__is_bundled(first_rsc))) {
             uuid = pcmk__notify_key(rid, "confirmed-post",
                                     pcmk__action_text(remapped_task));
         } else {
             uuid = pcmk__op_key(rid, pcmk__action_text(remapped_task), 0);
         }
         pcmk__rsc_trace(first_rsc,
                         "Remapped action UUID %s to %s for ordering purposes",
                         first_uuid, uuid);
     }
 
 done:
     free(first_task_str);
     free(rid);
     return (uuid != NULL)? uuid : pcmk__str_copy(first_uuid);
 }
 
 /*!
  * \internal
  * \brief Get actual action that should be used with an ordering
  *
  * When an action is ordered relative to an action for a collective resource
  * (clone, group, or bundle), it actually needs to be ordered after all
  * instances of the collective have completed the relevant action (for example,
  * given "start CLONE then start RSC", RSC must wait until all instances of
  * CLONE have started). Given the first action in an ordering, this returns the
  * the action that should actually be used for ordering (for example, the
  * started action instead of the start action).
  *
  * \param[in] action  First action in an ordering
  *
  * \return Actual action that should be used for the ordering
  */
 static pcmk_action_t *
 action_for_ordering(pcmk_action_t *action)
 {
     pcmk_action_t *result = action;
     pcmk_resource_t *rsc = action->rsc;
 
     if (rsc == NULL) {
         return result;
     }
 
     if ((rsc->priv->variant >= pcmk__rsc_variant_group)
         && (action->uuid != NULL)) {
         char *uuid = action_uuid_for_ordering(action->uuid, rsc);
 
         result = find_first_action(rsc->priv->actions, uuid, NULL, NULL);
         if (result == NULL) {
             crm_warn("Not remapping %s to %s because %s does not have "
                      "remapped action", action->uuid, uuid, rsc->id);
             result = action;
         }
         free(uuid);
     }
     return result;
 }
 
 /*!
  * \internal
  * \brief Wrapper for update_ordered_actions() method for readability
  *
  * \param[in,out] rsc        Resource to call method for
  * \param[in,out] first      'First' action in an ordering
  * \param[in,out] then       'Then' action in an ordering
  * \param[in]     node       If not NULL, limit scope of ordering to this
  *                           node (only used when interleaving instances)
  * \param[in]     flags      Action flags for \p first for ordering purposes
  * \param[in]     filter     Action flags to limit scope of certain updates
  *                           (may include pcmk__action_optional to affect only
  *                           mandatory actions, and pe_action_runnable to
  *                           affect only runnable actions)
  * \param[in]     type       Group of enum pcmk__action_relation_flags to apply
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Group of enum pcmk__updated flags indicating what was updated
  */
 static inline uint32_t
 update(pcmk_resource_t *rsc, pcmk_action_t *first, pcmk_action_t *then,
        const pcmk_node_t *node, uint32_t flags, uint32_t filter, uint32_t type,
        pcmk_scheduler_t *scheduler)
 {
     return rsc->priv->cmds->update_ordered_actions(first, then, node, flags,
                                                    filter, type, scheduler);
 }
 
 /*!
  * \internal
  * \brief Update flags for ordering's actions appropriately for ordering's flags
  *
  * \param[in,out] first        First action in an ordering
  * \param[in,out] then         Then action in an ordering
  * \param[in]     first_flags  Action flags for \p first for ordering purposes
  * \param[in]     then_flags   Action flags for \p then for ordering purposes
  * \param[in,out] order        Action wrapper for \p first in ordering
  * \param[in,out] scheduler    Scheduler data
  *
  * \return Group of enum pcmk__updated flags
  */
 static uint32_t
 update_action_for_ordering_flags(pcmk_action_t *first, pcmk_action_t *then,
                                  uint32_t first_flags, uint32_t then_flags,
                                  pcmk__related_action_t *order,
                                  pcmk_scheduler_t *scheduler)
 {
     uint32_t changed = pcmk__updated_none;
 
     /* The node will only be used for clones. If interleaved, node will be NULL,
      * otherwise the ordering scope will be limited to the node. Normally, the
      * whole 'then' clone should restart if 'first' is restarted, so then->node
      * is needed.
      */
     pcmk_node_t *node = then->node;
 
     if (pcmk_is_set(order->flags, pcmk__ar_first_implies_same_node_then)) {
         /* For unfencing, only instances of 'then' on the same node as 'first'
          * (the unfencing operation) should restart, so reset node to
          * first->node, at which point this case is handled like a normal
          * pcmk__ar_first_implies_then.
          */
         pcmk__clear_relation_flags(order->flags,
                                    pcmk__ar_first_implies_same_node_then);
         pcmk__set_relation_flags(order->flags, pcmk__ar_first_implies_then);
         node = first->node;
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: mapped "
                         "pcmk__ar_first_implies_same_node_then to "
                         "pcmk__ar_first_implies_then on %s",
                         first->uuid, then->uuid, pcmk__node_name(node));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_first_implies_then)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node,
                               first_flags & pcmk__action_optional,
                               pcmk__action_optional,
                               pcmk__ar_first_implies_then, scheduler);
         } else if (!pcmk_is_set(first_flags, pcmk__action_optional)
                    && pcmk_is_set(then->flags, pcmk__action_optional)) {
             pcmk__clear_action_flags(then, pcmk__action_optional);
             pcmk__set_updated_flags(changed, first, pcmk__updated_then);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after pcmk__ar_first_implies_then",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_intermediate_stop)
         && (then->rsc != NULL)) {
         enum pcmk__action_flags restart = pcmk__action_optional
                                           |pcmk__action_runnable;
 
         changed |= update(then->rsc, first, then, node, first_flags, restart,
                           pcmk__ar_intermediate_stop, scheduler);
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after pcmk__ar_intermediate_stop",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_then_implies_first)) {
         if (first->rsc != NULL) {
             changed |= update(first->rsc, first, then, node, first_flags,
                               pcmk__action_optional,
                               pcmk__ar_then_implies_first, scheduler);
         } else if (!pcmk_is_set(first_flags, pcmk__action_optional)
                    && pcmk_is_set(first->flags, pcmk__action_runnable)) {
             pcmk__clear_action_flags(first, pcmk__action_runnable);
             pcmk__set_updated_flags(changed, first, pcmk__updated_first);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after pcmk__ar_then_implies_first",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_promoted_then_implies_first)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node,
                               first_flags & pcmk__action_optional,
                               pcmk__action_optional,
                               pcmk__ar_promoted_then_implies_first, scheduler);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after "
                         "pcmk__ar_promoted_then_implies_first",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_min_runnable)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node, first_flags,
                               pcmk__action_runnable, pcmk__ar_min_runnable,
                               scheduler);
 
         } else if (pcmk_is_set(first_flags, pcmk__action_runnable)) {
             // We have another runnable instance of "first"
             then->runnable_before++;
 
             /* Mark "then" as runnable if it requires a certain number of
              * "before" instances to be runnable, and they now are.
              */
             if ((then->runnable_before >= then->required_runnable_before)
                 && !pcmk_is_set(then->flags, pcmk__action_runnable)) {
 
                 pcmk__set_action_flags(then, pcmk__action_runnable);
                 pcmk__set_updated_flags(changed, first, pcmk__updated_then);
             }
         }
         pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_min_runnable",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_nested_remote_probe)
         && (then->rsc != NULL)) {
 
         if (!pcmk_is_set(first_flags, pcmk__action_runnable)
             && (first->rsc != NULL)
             && (first->rsc->priv->active_nodes != NULL)) {
 
             pcmk__rsc_trace(then->rsc,
                             "%s then %s: ignoring because first is stopping",
                             first->uuid, then->uuid);
             order->flags = pcmk__ar_none;
         } else {
             changed |= update(then->rsc, first, then, node, first_flags,
                               pcmk__action_runnable,
                               pcmk__ar_unrunnable_first_blocks, scheduler);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after pcmk__ar_nested_remote_probe",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_unrunnable_first_blocks)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node, first_flags,
                               pcmk__action_runnable,
                               pcmk__ar_unrunnable_first_blocks, scheduler);
 
         } else if (!pcmk_is_set(first_flags, pcmk__action_runnable)
                    && pcmk_is_set(then->flags, pcmk__action_runnable)) {
 
             pcmk__clear_action_flags(then, pcmk__action_runnable);
             pcmk__set_updated_flags(changed, first, pcmk__updated_then);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after pcmk__ar_unrunnable_first_blocks",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_unmigratable_then_blocks)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node, first_flags,
                               pcmk__action_optional,
                               pcmk__ar_unmigratable_then_blocks, scheduler);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after "
                         "pcmk__ar_unmigratable_then_blocks",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_first_else_then)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node, first_flags,
                               pcmk__action_optional, pcmk__ar_first_else_then,
                               scheduler);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after pcmk__ar_first_else_then",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_ordered)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node, first_flags,
                               pcmk__action_runnable, pcmk__ar_ordered,
                               scheduler);
         }
         pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_ordered",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_asymmetric)) {
         if (then->rsc != NULL) {
             changed |= update(then->rsc, first, then, node, first_flags,
                               pcmk__action_runnable, pcmk__ar_asymmetric,
                               scheduler);
         }
         pcmk__rsc_trace(then->rsc, "%s then %s: %s after pcmk__ar_asymmetric",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     if (pcmk_is_set(first->flags, pcmk__action_runnable)
         && pcmk_is_set(order->flags, pcmk__ar_first_implies_then_graphed)
         && !pcmk_is_set(first_flags, pcmk__action_optional)) {
 
         pcmk__rsc_trace(then->rsc, "%s will be in graph because %s is required",
                         then->uuid, first->uuid);
         pcmk__set_action_flags(then, pcmk__action_always_in_graph);
         // Don't bother marking 'then' as changed just for this
     }
 
     if (pcmk_is_set(order->flags, pcmk__ar_then_implies_first_graphed)
         && !pcmk_is_set(then_flags, pcmk__action_optional)) {
 
         pcmk__rsc_trace(then->rsc, "%s will be in graph because %s is required",
                         first->uuid, then->uuid);
         pcmk__set_action_flags(first, pcmk__action_always_in_graph);
         // Don't bother marking 'first' as changed just for this
     }
 
     if (pcmk_any_flags_set(order->flags, pcmk__ar_first_implies_then
                                          |pcmk__ar_then_implies_first
                                          |pcmk__ar_intermediate_stop)
         && (first->rsc != NULL)
         && !pcmk_is_set(first->rsc->flags, pcmk__rsc_managed)
         && pcmk_is_set(first->rsc->flags, pcmk__rsc_blocked)
         && !pcmk_is_set(first->flags, pcmk__action_runnable)
         && pcmk__str_eq(first->task, PCMK_ACTION_STOP, pcmk__str_none)) {
 
+        /* @TODO This seems odd; why wouldn't an unrunnable "first" already
+         * block "then" before this? Note that the unmanaged-stop-{1,2}
+         * scheduler regression tests and the test CIB for T209 have tests for
+         * "stop then stop" relations that would be good for checking any
+         * changes.
+         */
         if (pcmk_is_set(then->flags, pcmk__action_runnable)) {
             pcmk__clear_action_flags(then, pcmk__action_runnable);
             pcmk__set_updated_flags(changed, first, pcmk__updated_then);
         }
         pcmk__rsc_trace(then->rsc,
                         "%s then %s: %s after checking whether first "
                         "is blocked, unmanaged, unrunnable stop",
                         first->uuid, then->uuid,
                         (changed? "changed" : "unchanged"));
     }
 
     return changed;
 }
 
 // Convenience macros for logging action properties
 
 #define action_type_str(flags) \
     (pcmk_is_set((flags), pcmk__action_pseudo)? "pseudo-action" : "action")
 
 #define action_optional_str(flags) \
     (pcmk_is_set((flags), pcmk__action_optional)? "optional" : "required")
 
 #define action_runnable_str(flags) \
     (pcmk_is_set((flags), pcmk__action_runnable)? "runnable" : "unrunnable")
 
 #define action_node_str(a) \
     (((a)->node == NULL)? "no node" : (a)->node->priv->name)
 
 /*!
  * \internal
  * \brief Update an action's flags for all orderings where it is "then"
  *
  * \param[in,out] then       Action to update
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__update_action_for_orderings(pcmk_action_t *then,
                                   pcmk_scheduler_t *scheduler)
 {
     GList *lpc = NULL;
     uint32_t changed = pcmk__updated_none;
     int last_flags = then->flags;
 
     pcmk__rsc_trace(then->rsc, "Updating %s %s (%s %s) on %s",
                     action_type_str(then->flags), then->uuid,
                     action_optional_str(then->flags),
                     action_runnable_str(then->flags), action_node_str(then));
 
     if (then->required_runnable_before > 0) {
         /* Initialize current known "runnable before" actions. As
          * update_action_for_ordering_flags() is called for each of then's
          * before actions, this number will increment as runnable 'first'
          * actions are encountered.
          */
         then->runnable_before = 0;
 
         /* The pcmk__ar_min_runnable clause of
          * update_action_for_ordering_flags() (called below)
          * will reset runnable if appropriate.
          */
         pcmk__clear_action_flags(then, pcmk__action_runnable);
     }
 
     for (lpc = then->actions_before; lpc != NULL; lpc = lpc->next) {
         pcmk__related_action_t *other = lpc->data;
         pcmk_action_t *first = other->action;
 
         pcmk_node_t *then_node = then->node;
         pcmk_node_t *first_node = first->node;
 
         const uint32_t target = pcmk__rsc_node_assigned;
 
         if ((first->rsc != NULL)
             && pcmk__is_group(first->rsc)
             && pcmk__str_eq(first->task, PCMK_ACTION_START, pcmk__str_none)) {
 
             first_node = first->rsc->priv->fns->location(first->rsc, NULL,
                                                          target);
             if (first_node != NULL) {
                 pcmk__rsc_trace(first->rsc, "Found %s for 'first' %s",
                                 pcmk__node_name(first_node), first->uuid);
             }
         }
 
         if (pcmk__is_group(then->rsc)
             && pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)) {
 
             then_node = then->rsc->priv->fns->location(then->rsc, NULL, target);
             if (then_node != NULL) {
                 pcmk__rsc_trace(then->rsc, "Found %s for 'then' %s",
                                 pcmk__node_name(then_node), then->uuid);
             }
         }
 
         // Disable constraint if it only applies when on same node, but isn't
         if (pcmk_is_set(other->flags, pcmk__ar_if_on_same_node)
             && (first_node != NULL) && (then_node != NULL)
             && !pcmk__same_node(first_node, then_node)) {
 
             pcmk__rsc_trace(then->rsc,
                             "Disabled ordering %s on %s then %s on %s: "
                             "not same node",
                             other->action->uuid, pcmk__node_name(first_node),
                             then->uuid, pcmk__node_name(then_node));
             other->flags = pcmk__ar_none;
             continue;
         }
 
         pcmk__clear_updated_flags(changed, then, pcmk__updated_first);
 
         if ((first->rsc != NULL)
             && pcmk_is_set(other->flags, pcmk__ar_then_cancels_first)
             && !pcmk_is_set(then->flags, pcmk__action_optional)) {
 
             /* 'then' is required, so we must abandon 'first'
              * (e.g. a required stop cancels any agent reload).
              */
             pcmk__set_action_flags(other->action, pcmk__action_optional);
             if (!strcmp(first->task, PCMK_ACTION_RELOAD_AGENT)) {
                 pcmk__clear_rsc_flags(first->rsc, pcmk__rsc_reload);
             }
         }
 
         if ((first->rsc != NULL) && (then->rsc != NULL)
             && (first->rsc != then->rsc) && !is_parent(then->rsc, first->rsc)) {
             first = action_for_ordering(first);
         }
         if (first != other->action) {
             pcmk__rsc_trace(then->rsc, "Ordering %s after %s instead of %s",
                             then->uuid, first->uuid, other->action->uuid);
         }
 
         pcmk__rsc_trace(then->rsc,
                         "%s (%#.6x) then %s (%#.6x): type=%#.6x node=%s",
                         first->uuid, first->flags, then->uuid, then->flags,
                         other->flags, action_node_str(first));
 
         if (first == other->action) {
             /* 'first' was not remapped (e.g. from 'start' to 'running'), which
              * could mean it is a non-resource action, a primitive resource
              * action, or already expanded.
              */
             uint32_t first_flags, then_flags;
 
             first_flags = action_flags_for_ordering(first, then_node);
             then_flags = action_flags_for_ordering(then, first_node);
 
             changed |= update_action_for_ordering_flags(first, then,
                                                         first_flags, then_flags,
                                                         other, scheduler);
 
             /* 'first' was for a complex resource (clone, group, etc),
              * create a new dependency if necessary
              */
         } else if (order_actions(first, then, other->flags)) {
             /* This was the first time 'first' and 'then' were associated,
              * start again to get the new actions_before list
              */
             pcmk__set_updated_flags(changed, then, pcmk__updated_then);
             pcmk__rsc_trace(then->rsc,
                             "Disabled ordering %s then %s in favor of %s "
                             "then %s",
                             other->action->uuid, then->uuid, first->uuid,
                             then->uuid);
             other->flags = pcmk__ar_none;
         }
 
 
         if (pcmk_is_set(changed, pcmk__updated_first)) {
             crm_trace("Re-processing %s and its 'after' actions "
                       "because it changed", first->uuid);
             for (GList *lpc2 = first->actions_after; lpc2 != NULL;
                  lpc2 = lpc2->next) {
                 pcmk__related_action_t *other = lpc2->data;
 
                 pcmk__update_action_for_orderings(other->action, scheduler);
             }
             pcmk__update_action_for_orderings(first, scheduler);
         }
     }
 
     if (then->required_runnable_before > 0) {
         if (last_flags == then->flags) {
             pcmk__clear_updated_flags(changed, then, pcmk__updated_then);
         } else {
             pcmk__set_updated_flags(changed, then, pcmk__updated_then);
         }
     }
 
     if (pcmk_is_set(changed, pcmk__updated_then)) {
         crm_trace("Re-processing %s and its 'after' actions because it changed",
                   then->uuid);
         if (pcmk_is_set(last_flags, pcmk__action_runnable)
             && !pcmk_is_set(then->flags, pcmk__action_runnable)) {
             pcmk__block_colocation_dependents(then);
         }
         pcmk__update_action_for_orderings(then, scheduler);
         for (lpc = then->actions_after; lpc != NULL; lpc = lpc->next) {
             pcmk__related_action_t *other = lpc->data;
 
             pcmk__update_action_for_orderings(other->action, scheduler);
         }
     }
 }
 
 static inline bool
 is_primitive_action(const pcmk_action_t *action)
 {
     return (action != NULL) && pcmk__is_primitive(action->rsc);
 }
 
 /*!
  * \internal
  * \brief Clear a single action flag and set reason text
  *
  * \param[in,out] action  Action whose flag should be cleared
  * \param[in]     flag    Action flag that should be cleared
  * \param[in]     reason  Action that is the reason why flag is being cleared
  */
 #define clear_action_flag_because(action, flag, reason) do {                \
         if (pcmk_is_set((action)->flags, (flag))) {                         \
             pcmk__clear_action_flags(action, flag);                         \
             if ((action)->rsc != (reason)->rsc) {                           \
                 char *reason_text = pe__action2reason((reason), (flag));    \
                 pe_action_set_reason((action), reason_text, false);         \
                 free(reason_text);                                          \
             }                                                               \
         }                                                                   \
     } while (0)
 
 /*!
  * \internal
  * \brief Update actions in an asymmetric ordering
  *
  * If the "first" action in an asymmetric ordering is unrunnable, make the
  * "second" action unrunnable as well, if appropriate.
  *
  * \param[in]     first  'First' action in an asymmetric ordering
  * \param[in,out] then   'Then' action in an asymmetric ordering
  */
 static void
 handle_asymmetric_ordering(const pcmk_action_t *first, pcmk_action_t *then)
 {
     /* Only resource actions after an unrunnable 'first' action need updates for
      * asymmetric ordering.
      */
     if ((then->rsc == NULL)
         || pcmk_is_set(first->flags, pcmk__action_runnable)) {
         return;
     }
 
     // Certain optional 'then' actions are unaffected by unrunnable 'first'
     if (pcmk_is_set(then->flags, pcmk__action_optional)) {
         enum rsc_role_e then_rsc_role;
 
         then_rsc_role = then->rsc->priv->fns->state(then->rsc, TRUE);
 
         if ((then_rsc_role == pcmk_role_stopped)
             && pcmk__str_eq(then->task, PCMK_ACTION_STOP, pcmk__str_none)) {
             /* If 'then' should stop after 'first' but is already stopped, the
              * ordering is irrelevant.
              */
             return;
         } else if ((then_rsc_role >= pcmk_role_started)
             && pcmk__str_eq(then->task, PCMK_ACTION_START, pcmk__str_none)
             && pe__rsc_running_on_only(then->rsc, then->node)) {
             /* Similarly if 'then' should start after 'first' but is already
              * started on a single node.
              */
             return;
         }
     }
 
     // 'First' can't run, so 'then' can't either
     clear_action_flag_because(then, pcmk__action_optional, first);
     clear_action_flag_because(then, pcmk__action_runnable, first);
 }
 
 /*!
  * \internal
  * \brief Set action bits appropriately when pcmk__ar_intermediate_stop is used
  *
  * \param[in,out] first   'First' action in ordering
  * \param[in,out] then    'Then' action in ordering
  * \param[in]     filter  What action flags to care about
  *
  * \note pcmk__ar_intermediate_stop is set for "stop resource before starting
  *       it" and "stop later group member before stopping earlier group member"
  */
 static void
 handle_restart_ordering(pcmk_action_t *first, pcmk_action_t *then,
                         uint32_t filter)
 {
     const char *reason = NULL;
 
     pcmk__assert(is_primitive_action(first) && is_primitive_action(then));
 
     // We need to update the action in two cases:
 
     // ... if 'then' is required
     if (pcmk_is_set(filter, pcmk__action_optional)
         && !pcmk_is_set(then->flags, pcmk__action_optional)) {
         reason = "restart";
     }
 
     /* ... if 'then' is unrunnable action on same resource (if a resource
      * should restart but can't start, we still want to stop)
      */
     if (pcmk_is_set(filter, pcmk__action_runnable)
         && !pcmk_is_set(then->flags, pcmk__action_runnable)
         && pcmk_is_set(then->rsc->flags, pcmk__rsc_managed)
         && (first->rsc == then->rsc)) {
         reason = "stop";
     }
 
     if (reason == NULL) {
         return;
     }
 
     pcmk__rsc_trace(first->rsc, "Handling %s -> %s for %s",
                     first->uuid, then->uuid, reason);
 
     // Make 'first' required if it is runnable
     if (pcmk_is_set(first->flags, pcmk__action_runnable)) {
         clear_action_flag_because(first, pcmk__action_optional, then);
     }
 
     // Make 'first' required if 'then' is required
     if (!pcmk_is_set(then->flags, pcmk__action_optional)) {
         clear_action_flag_because(first, pcmk__action_optional, then);
     }
 
     // Make 'first' unmigratable if 'then' is unmigratable
     if (!pcmk_is_set(then->flags, pcmk__action_migratable)) {
         clear_action_flag_because(first, pcmk__action_migratable, then);
     }
 
     // Make 'then' unrunnable if 'first' is required but unrunnable
     if (!pcmk_is_set(first->flags, pcmk__action_optional)
         && !pcmk_is_set(first->flags, pcmk__action_runnable)) {
         clear_action_flag_because(then, pcmk__action_runnable, first);
     }
 }
 
 /*!
  * \internal
  * \brief Update two actions according to an ordering between them
  *
  * Given information about an ordering of two actions, update the actions' flags
  * (and runnable_before members if appropriate) as appropriate for the ordering.
  * Effects may cascade to other orderings involving the actions as well.
  *
  * \param[in,out] first      'First' action in an ordering
  * \param[in,out] then       'Then' action in an ordering
  * \param[in]     node       If not NULL, limit scope of ordering to this node
  *                           (ignored)
  * \param[in]     flags      Action flags for \p first for ordering purposes
  * \param[in]     filter     Action flags to limit scope of certain updates (may
  *                           include pcmk__action_optional to affect only
  *                           mandatory actions, and pcmk__action_runnable to
  *                           affect only runnable actions)
  * \param[in]     type       Group of enum pcmk__action_relation_flags to apply
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Group of enum pcmk__updated flags indicating what was updated
  */
 uint32_t
 pcmk__update_ordered_actions(pcmk_action_t *first, pcmk_action_t *then,
                              const pcmk_node_t *node, uint32_t flags,
                              uint32_t filter, uint32_t type,
                              pcmk_scheduler_t *scheduler)
 {
     uint32_t changed = pcmk__updated_none;
     uint32_t then_flags = 0U;
     uint32_t first_flags = 0U;
 
     pcmk__assert((first != NULL) && (then != NULL) && (scheduler != NULL));
 
     then_flags = then->flags;
     first_flags = first->flags;
     if (pcmk_is_set(type, pcmk__ar_asymmetric)) {
         handle_asymmetric_ordering(first, then);
     }
 
     if (pcmk_is_set(type, pcmk__ar_then_implies_first)
         && !pcmk_is_set(then_flags, pcmk__action_optional)) {
         // Then is required, and implies first should be, too
 
         if (pcmk_is_set(filter, pcmk__action_optional)
             && !pcmk_is_set(flags, pcmk__action_optional)
             && pcmk_is_set(first_flags, pcmk__action_optional)) {
             clear_action_flag_because(first, pcmk__action_optional, then);
         }
 
         if (pcmk_is_set(flags, pcmk__action_migratable)
             && !pcmk_is_set(then->flags, pcmk__action_migratable)) {
             clear_action_flag_because(first, pcmk__action_migratable, then);
         }
     }
 
     if (pcmk_is_set(type, pcmk__ar_promoted_then_implies_first)
         && (then->rsc != NULL)
         && (then->rsc->priv->orig_role == pcmk_role_promoted)
         && pcmk_is_set(filter, pcmk__action_optional)
         && !pcmk_is_set(then->flags, pcmk__action_optional)) {
 
         clear_action_flag_because(first, pcmk__action_optional, then);
 
         if (pcmk_is_set(first->flags, pcmk__action_migratable)
             && !pcmk_is_set(then->flags, pcmk__action_migratable)) {
             clear_action_flag_because(first, pcmk__action_migratable, then);
         }
     }
 
     if (pcmk_is_set(type, pcmk__ar_unmigratable_then_blocks)
         && pcmk_is_set(filter, pcmk__action_optional)) {
 
         if (!pcmk_all_flags_set(then->flags, pcmk__action_migratable
                                              |pcmk__action_runnable)) {
             clear_action_flag_because(first, pcmk__action_runnable, then);
         }
 
         if (!pcmk_is_set(then->flags, pcmk__action_optional)) {
             clear_action_flag_because(first, pcmk__action_optional, then);
         }
     }
 
     if (pcmk_is_set(type, pcmk__ar_first_else_then)
         && pcmk_is_set(filter, pcmk__action_optional)
         && !pcmk_is_set(first->flags, pcmk__action_runnable)) {
 
         clear_action_flag_because(then, pcmk__action_migratable, first);
         pcmk__clear_action_flags(then, pcmk__action_pseudo);
     }
 
     if (pcmk_is_set(type, pcmk__ar_unrunnable_first_blocks)
         && pcmk_is_set(filter, pcmk__action_runnable)
         && pcmk_is_set(then->flags, pcmk__action_runnable)
         && !pcmk_is_set(flags, pcmk__action_runnable)) {
 
         clear_action_flag_because(then, pcmk__action_runnable, first);
         clear_action_flag_because(then, pcmk__action_migratable, first);
     }
 
     if (pcmk_is_set(type, pcmk__ar_first_implies_then)
         && pcmk_is_set(filter, pcmk__action_optional)
         && pcmk_is_set(then->flags, pcmk__action_optional)
         && !pcmk_is_set(flags, pcmk__action_optional)
         && !pcmk_is_set(first->flags, pcmk__action_migratable)) {
 
         clear_action_flag_because(then, pcmk__action_optional, first);
     }
 
     if (pcmk_is_set(type, pcmk__ar_intermediate_stop)) {
         handle_restart_ordering(first, then, filter);
     }
 
     if (then_flags != then->flags) {
         pcmk__set_updated_flags(changed, first, pcmk__updated_then);
         pcmk__rsc_trace(then->rsc,
                         "%s on %s: flags are now %#.6x (was %#.6x) "
                         "because of 'first' %s (%#.6x)",
                         then->uuid, pcmk__node_name(then->node),
                         then->flags, then_flags, first->uuid, first->flags);
 
         if ((then->rsc != NULL) && (then->rsc->priv->parent != NULL)) {
             // Required to handle "X_stop then X_start" for cloned groups
             pcmk__update_action_for_orderings(then, scheduler);
         }
     }
 
     if (first_flags != first->flags) {
         pcmk__set_updated_flags(changed, first, pcmk__updated_first);
         pcmk__rsc_trace(first->rsc,
                         "%s on %s: flags are now %#.6x (was %#.6x) "
                         "because of 'then' %s (%#.6x)",
                         first->uuid, pcmk__node_name(first->node),
                         first->flags, first_flags, then->uuid, then->flags);
     }
 
     return changed;
 }
 
 /*!
  * \internal
  * \brief Trace-log an action (optionally with its dependent actions)
  *
  * \param[in] pre_text  If not NULL, prefix the log with this plus ": "
  * \param[in] action    Action to log
  * \param[in] details   If true, recursively log dependent actions
  */
 void
 pcmk__log_action(const char *pre_text, const pcmk_action_t *action,
                  bool details)
 {
     const char *node_uname = NULL;
     const char *node_uuid = NULL;
     const char *desc = NULL;
 
     CRM_CHECK(action != NULL, return);
 
     if (!pcmk_is_set(action->flags, pcmk__action_pseudo)) {
         if (action->node != NULL) {
             node_uname = action->node->priv->name;
             node_uuid = action->node->priv->id;
         } else {
             node_uname = "<none>";
         }
     }
 
     switch (pcmk__parse_action(action->task)) {
         case pcmk__action_fence:
         case pcmk__action_shutdown:
             if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
                 desc = "Pseudo ";
             } else if (pcmk_is_set(action->flags, pcmk__action_optional)) {
                 desc = "Optional ";
             } else if (!pcmk_is_set(action->flags, pcmk__action_runnable)) {
                 desc = "!!Non-Startable!! ";
             } else {
                desc = "(Provisional) ";
             }
             crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
                       ((pre_text == NULL)? "" : pre_text),
                       ((pre_text == NULL)? "" : ": "),
                       desc, action->id, action->uuid,
                       (node_uname? "\ton " : ""), (node_uname? node_uname : ""),
                       (node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
                       (node_uuid? ")" : ""));
             break;
         default:
             if (pcmk_is_set(action->flags, pcmk__action_optional)) {
                 desc = "Optional ";
             } else if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
                 desc = "Pseudo ";
             } else if (!pcmk_is_set(action->flags, pcmk__action_runnable)) {
                 desc = "!!Non-Startable!! ";
             } else {
                desc = "(Provisional) ";
             }
             crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
                       ((pre_text == NULL)? "" : pre_text),
                       ((pre_text == NULL)? "" : ": "),
                       desc, action->id, action->uuid,
                       (action->rsc? action->rsc->id : "<none>"),
                       (node_uname? "\ton " : ""), (node_uname? node_uname : ""),
                       (node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
                       (node_uuid? ")" : ""));
             break;
     }
 
     if (details) {
         const GList *iter = NULL;
         const pcmk__related_action_t *other = NULL;
 
         crm_trace("\t\t====== Preceding Actions");
         for (iter = action->actions_before; iter != NULL; iter = iter->next) {
             other = (const pcmk__related_action_t *) iter->data;
             pcmk__log_action("\t\t", other->action, false);
         }
         crm_trace("\t\t====== Subsequent Actions");
         for (iter = action->actions_after; iter != NULL; iter = iter->next) {
             other = (const pcmk__related_action_t *) iter->data;
             pcmk__log_action("\t\t", other->action, false);
         }
         crm_trace("\t\t====== End");
 
     } else {
         crm_trace("\t\t(before=%d, after=%d)",
                   g_list_length(action->actions_before),
                   g_list_length(action->actions_after));
     }
 }
 
 /*!
  * \internal
  * \brief Create a new shutdown action for a node
  *
  * \param[in,out] node  Node being shut down
  *
  * \return Newly created shutdown action for \p node
  */
 pcmk_action_t *
 pcmk__new_shutdown_action(pcmk_node_t *node)
 {
     char *shutdown_id = NULL;
     pcmk_action_t *shutdown_op = NULL;
 
     pcmk__assert(node != NULL);
 
     shutdown_id = crm_strdup_printf("%s-%s", PCMK_ACTION_DO_SHUTDOWN,
                                     node->priv->name);
 
     shutdown_op = custom_action(NULL, shutdown_id, PCMK_ACTION_DO_SHUTDOWN,
                                 node, FALSE, node->priv->scheduler);
 
     pcmk__order_stops_before_shutdown(node, shutdown_op);
     pcmk__insert_meta(shutdown_op, PCMK__META_OP_NO_WAIT, PCMK_VALUE_TRUE);
     return shutdown_op;
 }
 
 /*!
  * \internal
  * \brief Calculate and add an operation digest to XML
  *
  * Calculate an operation digest, which enables us to later determine when a
  * restart is needed due to the resource's parameters being changed, and add it
  * to given XML.
  *
  * \param[in]     op      Operation result from executor
  * \param[in,out] update  XML to add digest to
  */
 static void
 add_op_digest_to_xml(const lrmd_event_data_t *op, xmlNode *update)
 {
     char *digest = NULL;
     xmlNode *args_xml = NULL;
 
     if (op->params == NULL) {
         return;
     }
     args_xml = pcmk__xe_create(NULL, PCMK_XE_PARAMETERS);
     g_hash_table_foreach(op->params, hash2field, args_xml);
     pcmk__filter_op_for_digest(args_xml);
     digest = pcmk__digest_operation(args_xml);
     crm_xml_add(update, PCMK__XA_OP_DIGEST, digest);
     pcmk__xml_free(args_xml);
     free(digest);
 }
 
 #define FAKE_TE_ID     "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 
 /*!
  * \internal
  * \brief Create XML for resource operation history update
  *
  * \param[in,out] parent          Parent XML node to add to
  * \param[in,out] op              Operation event data
  * \param[in]     caller_version  DC feature set
  * \param[in]     target_rc       Expected result of operation
  * \param[in]     node            Name of node on which operation was performed
  * \param[in]     origin          Arbitrary description of update source
  *
  * \return Newly created XML node for history update
  */
 xmlNode *
 pcmk__create_history_xml(xmlNode *parent, lrmd_event_data_t *op,
                          const char *caller_version, int target_rc,
                          const char *node, const char *origin)
 {
     char *key = NULL;
     char *magic = NULL;
     char *op_id = NULL;
     char *op_id_additional = NULL;
     char *local_user_data = NULL;
     const char *exit_reason = NULL;
 
     xmlNode *xml_op = NULL;
     const char *task = NULL;
 
     CRM_CHECK(op != NULL, return NULL);
     crm_trace("Creating history XML for %s-interval %s action for %s on %s "
               "(DC version: %s, origin: %s)",
               pcmk__readable_interval(op->interval_ms), op->op_type, op->rsc_id,
               ((node == NULL)? "no node" : node), caller_version, origin);
 
     task = op->op_type;
 
     /* Record a successful agent reload as a start, and a failed one as a
      * monitor, to make life easier for the scheduler when determining the
      * current state.
      *
      * @COMPAT We should check "reload" here only if the operation was for a
      * pre-OCF-1.1 resource agent, but we don't know that here, and we should
      * only ever get results for actions scheduled by us, so we can reasonably
      * assume any "reload" is actually a pre-1.1 agent reload.
+     *
+     * @TODO This remapping can make log messages with task confusing for users
+     * (for example, an "Initiating reload ..." followed by "... start ...
+     * confirmed"). Either do this remapping in the scheduler if possible, or
+     * store the original task in a new XML attribute for later logging.
      */
     if (pcmk__str_any_of(task, PCMK_ACTION_RELOAD, PCMK_ACTION_RELOAD_AGENT,
                          NULL)) {
         if (op->op_status == PCMK_EXEC_DONE) {
             task = PCMK_ACTION_START;
         } else {
             task = PCMK_ACTION_MONITOR;
         }
     }
 
     key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
     if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
         const char *n_type = crm_meta_value(op->params, "notify_type");
         const char *n_task = crm_meta_value(op->params, "notify_operation");
 
         CRM_LOG_ASSERT(n_type != NULL);
         CRM_LOG_ASSERT(n_task != NULL);
         op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);
 
         if (op->op_status != PCMK_EXEC_PENDING) {
             /* Ignore notify errors.
              *
              * @TODO It might be better to keep the correct result here, and
              * ignore it in process_graph_event().
              */
             lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
         }
 
     /* Migration history is preserved separately, which usually matters for
      * multiple nodes and is important for future cluster transitions.
      */
     } else if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
                                 PCMK_ACTION_MIGRATE_FROM, NULL)) {
         op_id = strdup(key);
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
         if (op->interval_ms == 0) {
             /* Ensure 'last' gets updated, in case PCMK_META_RECORD_PENDING is
              * true
              */
             op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
         }
         exit_reason = op->exit_reason;
 
     } else if (op->interval_ms > 0) {
         op_id = strdup(key);
 
     } else {
         op_id = pcmk__op_key(op->rsc_id, "last", 0);
     }
 
   again:
     xml_op = pcmk__xe_first_child(parent, PCMK__XE_LRM_RSC_OP, PCMK_XA_ID,
                                   op_id);
     if (xml_op == NULL) {
         xml_op = pcmk__xe_create(parent, PCMK__XE_LRM_RSC_OP);
     }
 
     if (op->user_data == NULL) {
         crm_debug("Generating fake transition key for: " PCMK__OP_FMT
                   " %d from %s", op->rsc_id, op->op_type, op->interval_ms,
                   op->call_id, origin);
         local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
                                                FAKE_TE_ID);
         op->user_data = local_user_data;
     }
 
     if (magic == NULL) {
         magic = crm_strdup_printf("%d:%d;%s", op->op_status, op->rc,
                                   (const char *) op->user_data);
     }
 
     crm_xml_add(xml_op, PCMK_XA_ID, op_id);
     crm_xml_add(xml_op, PCMK__XA_OPERATION_KEY, key);
     crm_xml_add(xml_op, PCMK_XA_OPERATION, task);
     crm_xml_add(xml_op, PCMK_XA_CRM_DEBUG_ORIGIN, origin);
     crm_xml_add(xml_op, PCMK_XA_CRM_FEATURE_SET, caller_version);
     crm_xml_add(xml_op, PCMK__XA_TRANSITION_KEY, op->user_data);
     crm_xml_add(xml_op, PCMK__XA_TRANSITION_MAGIC, magic);
     crm_xml_add(xml_op, PCMK_XA_EXIT_REASON, pcmk__s(exit_reason, ""));
     crm_xml_add(xml_op, PCMK__META_ON_NODE, node); // For context during triage
 
     crm_xml_add_int(xml_op, PCMK__XA_CALL_ID, op->call_id);
     crm_xml_add_int(xml_op, PCMK__XA_RC_CODE, op->rc);
     crm_xml_add_int(xml_op, PCMK__XA_OP_STATUS, op->op_status);
     crm_xml_add_ms(xml_op, PCMK_META_INTERVAL, op->interval_ms);
 
     if ((op->t_run > 0) || (op->t_rcchange > 0) || (op->exec_time > 0)
         || (op->queue_time > 0)) {
 
         crm_trace("Timing data (" PCMK__OP_FMT "): "
                   "last=%lld change=%lld exec=%u queue=%u",
                   op->rsc_id, op->op_type, op->interval_ms,
                   (long long) op->t_run, (long long) op->t_rcchange,
                   op->exec_time, op->queue_time);
 
         if ((op->interval_ms > 0) && (op->t_rcchange > 0)) {
             // Recurring ops may have changed rc after initial run
             crm_xml_add_ll(xml_op, PCMK_XA_LAST_RC_CHANGE,
                            (long long) op->t_rcchange);
         } else {
             crm_xml_add_ll(xml_op, PCMK_XA_LAST_RC_CHANGE,
                            (long long) op->t_run);
         }
 
         crm_xml_add_int(xml_op, PCMK_XA_EXEC_TIME, op->exec_time);
         crm_xml_add_int(xml_op, PCMK_XA_QUEUE_TIME, op->queue_time);
     }
 
     if (pcmk__str_any_of(op->op_type, PCMK_ACTION_MIGRATE_TO,
                          PCMK_ACTION_MIGRATE_FROM, NULL)) {
         /* Record PCMK__META_MIGRATE_SOURCE and PCMK__META_MIGRATE_TARGET always
          * for migrate ops.
          */
         const char *name = PCMK__META_MIGRATE_SOURCE;
 
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
 
         name = PCMK__META_MIGRATE_TARGET;
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
     }
 
     add_op_digest_to_xml(op, xml_op);
 
     if (op_id_additional) {
         free(op_id);
         op_id = op_id_additional;
         op_id_additional = NULL;
         goto again;
     }
 
     if (local_user_data) {
         free(local_user_data);
         op->user_data = NULL;
     }
     free(magic);
     free(op_id);
     free(key);
     return xml_op;
 }
 
 /*!
  * \internal
  * \brief Check whether an action shutdown-locks a resource to a node
  *
  * If the PCMK_OPT_SHUTDOWN_LOCK cluster property is set, resources will not be
  * recovered on a different node if cleanly stopped, and may start only on that
  * same node. This function checks whether that applies to a given action, so
  * that the transition graph can be marked appropriately.
  *
  * \param[in] action  Action to check
  *
  * \return true if \p action locks its resource to the action's node,
  *         otherwise false
  */
 bool
 pcmk__action_locks_rsc_to_node(const pcmk_action_t *action)
 {
     // Only resource actions taking place on resource's lock node are locked
     if ((action == NULL) || (action->rsc == NULL)
         || !pcmk__same_node(action->node, action->rsc->priv->lock_node)) {
         return false;
     }
 
     /* During shutdown, only stops are locked (otherwise, another action such as
      * a demote would cause the controller to clear the lock)
      */
     if (action->node->details->shutdown && (action->task != NULL)
         && (strcmp(action->task, PCMK_ACTION_STOP) != 0)) {
         return false;
     }
 
     return true;
 }
 
 /* lowest to highest */
 static gint
 sort_action_id(gconstpointer a, gconstpointer b)
 {
     const pcmk__related_action_t *action_wrapper2 = a;
     const pcmk__related_action_t *action_wrapper1 = b;
 
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
     if (action_wrapper1->action->id < action_wrapper2->action->id) {
         return 1;
     }
     if (action_wrapper1->action->id > action_wrapper2->action->id) {
         return -1;
     }
     return 0;
 }
 
 /*!
  * \internal
  * \brief Remove any duplicate action inputs, merging action flags
  *
  * \param[in,out] action  Action whose inputs should be checked
  */
 void
 pcmk__deduplicate_action_inputs(pcmk_action_t *action)
 {
     GList *item = NULL;
     GList *next = NULL;
     pcmk__related_action_t *last_input = NULL;
 
     action->actions_before = g_list_sort(action->actions_before,
                                          sort_action_id);
     for (item = action->actions_before; item != NULL; item = next) {
         pcmk__related_action_t *input = item->data;
 
         next = item->next;
         if ((last_input != NULL)
             && (input->action->id == last_input->action->id)) {
             crm_trace("Input %s (%d) duplicate skipped for action %s (%d)",
                       input->action->uuid, input->action->id,
                       action->uuid, action->id);
 
             /* For the purposes of scheduling, the ordering flags no longer
              * matter, but crm_simulate looks at certain ones when creating a
              * dot graph. Combining the flags is sufficient for that purpose.
              */
             pcmk__set_relation_flags(last_input->flags, input->flags);
             if (input->graphed) {
                 last_input->graphed = true;
             }
 
             free(item->data);
             action->actions_before = g_list_delete_link(action->actions_before,
                                                         item);
         } else {
             last_input = input;
             input->graphed = false;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Output all scheduled actions
  *
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__output_actions(pcmk_scheduler_t *scheduler)
 {
     pcmk__output_t *out = scheduler->priv->out;
 
     // Output node (non-resource) actions
     for (GList *iter = scheduler->priv->actions;
          iter != NULL; iter = iter->next) {
 
         char *node_name = NULL;
         char *task = NULL;
         pcmk_action_t *action = (pcmk_action_t *) iter->data;
 
         if (action->rsc != NULL) {
             continue; // Resource actions will be output later
 
         } else if (pcmk_is_set(action->flags, pcmk__action_optional)) {
             continue; // This action was not scheduled
         }
 
         if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN,
                          pcmk__str_none)) {
             task = strdup("Shutdown");
 
         } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
                                 pcmk__str_none)) {
             const char *op = g_hash_table_lookup(action->meta,
                                                  PCMK__META_STONITH_ACTION);
 
             task = crm_strdup_printf("Fence (%s)", op);
 
         } else {
             continue; // Don't display other node action types
         }
 
         if (pcmk__is_guest_or_bundle_node(action->node)) {
             const pcmk_resource_t *remote = action->node->priv->remote;
 
             node_name = crm_strdup_printf("%s (resource: %s)",
                                           pcmk__node_name(action->node),
                                           remote->priv->launcher->id);
         } else if (action->node != NULL) {
             node_name = crm_strdup_printf("%s", pcmk__node_name(action->node));
         }
 
         out->message(out, "node-action", task, node_name, action->reason);
 
         free(node_name);
         free(task);
     }
 
     // Output resource actions
     for (GList *iter = scheduler->priv->resources;
          iter != NULL; iter = iter->next) {
 
         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
         rsc->priv->cmds->output_actions(rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Get action name needed to compare digest for configuration changes
  *
  * \param[in] task         Action name from history
  * \param[in] interval_ms  Action interval (in milliseconds)
  *
  * \return Action name whose digest should be compared
  */
 static const char *
 task_for_digest(const char *task, guint interval_ms)
 {
     /* Certain actions need to be compared against the parameters used to start
      * the resource.
      */
     if ((interval_ms == 0)
         && pcmk__str_any_of(task, PCMK_ACTION_MONITOR, PCMK_ACTION_MIGRATE_FROM,
                             PCMK_ACTION_PROMOTE, NULL)) {
         task = PCMK_ACTION_START;
     }
     return task;
 }
 
 /*!
  * \internal
  * \brief Check whether only sanitized parameters to an action changed
  *
  * When collecting CIB files for troubleshooting, crm_report will mask
  * sensitive resource parameters. If simulations were run using that, affected
  * resources would appear to need a restart, which would complicate
  * troubleshooting. To avoid that, we save a "secure digest" of non-sensitive
  * parameters. This function used that digest to check whether only masked
  * parameters are different.
  *
  * \param[in] xml_op       Resource history entry with secure digest
  * \param[in] digest_data  Operation digest information being compared
  * \param[in] scheduler    Scheduler data
  *
  * \return true if only sanitized parameters changed, otherwise false
  */
 static bool
 only_sanitized_changed(const xmlNode *xml_op,
                        const pcmk__op_digest_t *digest_data,
                        const pcmk_scheduler_t *scheduler)
 {
     const char *digest_secure = NULL;
 
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_sanitized)) {
         // The scheduler is not being run as a simulation
         return false;
     }
 
     digest_secure = crm_element_value(xml_op, PCMK__XA_OP_SECURE_DIGEST);
 
     return (digest_data->rc != pcmk__digest_match) && (digest_secure != NULL)
            && (digest_data->digest_secure_calc != NULL)
            && (strcmp(digest_data->digest_secure_calc, digest_secure) == 0);
 }
 
 /*!
  * \internal
  * \brief Force a restart due to a configuration change
  *
  * \param[in,out] rsc          Resource that action is for
  * \param[in]     task         Name of action whose configuration changed
  * \param[in]     interval_ms  Action interval (in milliseconds)
  * \param[in,out] node         Node where resource should be restarted
  */
 static void
 force_restart(pcmk_resource_t *rsc, const char *task, guint interval_ms,
               pcmk_node_t *node)
 {
     char *key = pcmk__op_key(rsc->id, task, interval_ms);
     pcmk_action_t *required = custom_action(rsc, key, task, NULL, FALSE,
                                             rsc->priv->scheduler);
 
     pe_action_set_reason(required, "resource definition change", true);
     trigger_unfencing(rsc, node, "Device parameters changed", NULL,
                       rsc->priv->scheduler);
 }
 
 /*!
  * \internal
  * \brief Schedule a reload of a resource on a node
  *
  * \param[in,out] data       Resource to reload
  * \param[in]     user_data  Where resource should be reloaded
  */
 static void
 schedule_reload(gpointer data, gpointer user_data)
 {
     pcmk_resource_t *rsc = data;
     const pcmk_node_t *node = user_data;
 
     pcmk_action_t *reload = NULL;
 
     // For collective resources, just call recursively for children
     if (rsc->priv->variant > pcmk__rsc_variant_primitive) {
         g_list_foreach(rsc->priv->children, schedule_reload, user_data);
         return;
     }
 
     // Skip the reload in certain situations
     if ((node == NULL)
         || !pcmk_is_set(rsc->flags, pcmk__rsc_managed)
         || pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
         pcmk__rsc_trace(rsc, "Skip reload of %s:%s%s %s",
                         rsc->id,
                         pcmk_is_set(rsc->flags, pcmk__rsc_managed)? "" : " unmanaged",
                         pcmk_is_set(rsc->flags, pcmk__rsc_failed)? " failed" : "",
                         (node == NULL)? "inactive" : node->priv->name);
         return;
     }
 
     /* If a resource's configuration changed while a start was pending,
      * force a full restart instead of a reload.
      */
     if (pcmk_is_set(rsc->flags, pcmk__rsc_start_pending)) {
         pcmk__rsc_trace(rsc,
                         "%s: preventing agent reload because start pending",
                         rsc->id);
         custom_action(rsc, stop_key(rsc), PCMK_ACTION_STOP, node, FALSE,
                       rsc->priv->scheduler);
         return;
     }
 
     // Schedule the reload
     pcmk__set_rsc_flags(rsc, pcmk__rsc_reload);
     reload = custom_action(rsc, reload_key(rsc), PCMK_ACTION_RELOAD_AGENT, node,
                            FALSE, rsc->priv->scheduler);
     pe_action_set_reason(reload, "resource definition change", FALSE);
 
     // Set orderings so that a required stop or demote cancels the reload
     pcmk__new_ordering(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
                        pcmk__ar_ordered|pcmk__ar_then_cancels_first,
                        rsc->priv->scheduler);
     pcmk__new_ordering(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
                        pcmk__ar_ordered|pcmk__ar_then_cancels_first,
                        rsc->priv->scheduler);
 }
 
 /*!
  * \internal
  * \brief Handle any configuration change for an action
  *
  * Given an action from resource history, if the resource's configuration
  * changed since the action was done, schedule any actions needed (restart,
  * reload, unfencing, rescheduling recurring actions, etc.).
  *
  * \param[in,out] rsc     Resource that action is for
  * \param[in,out] node    Node that action was on
  * \param[in]     xml_op  Action XML from resource history
  *
  * \return true if action configuration changed, otherwise false
  */
 bool
 pcmk__check_action_config(pcmk_resource_t *rsc, pcmk_node_t *node,
                           const xmlNode *xml_op)
 {
     guint interval_ms = 0;
     const char *task = NULL;
     const pcmk__op_digest_t *digest_data = NULL;
 
     CRM_CHECK((rsc != NULL) && (node != NULL) && (xml_op != NULL),
               return false);
 
     task = crm_element_value(xml_op, PCMK_XA_OPERATION);
     CRM_CHECK(task != NULL, return false);
 
     crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);
 
     // If this is a recurring action, check whether it has been orphaned
     if (interval_ms > 0) {
         if (pcmk__find_action_config(rsc, task, interval_ms, false) != NULL) {
             pcmk__rsc_trace(rsc,
                             "%s-interval %s for %s on %s is in configuration",
                             pcmk__readable_interval(interval_ms), task, rsc->id,
                             pcmk__node_name(node));
         } else if (pcmk_is_set(rsc->priv->scheduler->flags,
                                pcmk__sched_cancel_removed_actions)) {
             pcmk__schedule_cancel(rsc,
                                   crm_element_value(xml_op, PCMK__XA_CALL_ID),
                                   task, interval_ms, node, "orphan");
             return true;
         } else {
             pcmk__rsc_debug(rsc, "%s-interval %s for %s on %s is orphaned",
                             pcmk__readable_interval(interval_ms), task, rsc->id,
                             pcmk__node_name(node));
             return true;
         }
     }
 
     crm_trace("Checking %s-interval %s for %s on %s for configuration changes",
               pcmk__readable_interval(interval_ms), task, rsc->id,
               pcmk__node_name(node));
     task = task_for_digest(task, interval_ms);
     digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
                                         rsc->priv->scheduler);
 
     if (only_sanitized_changed(xml_op, digest_data, rsc->priv->scheduler)) {
         if (!pcmk__is_daemon && (rsc->priv->scheduler->priv->out != NULL)) {
             pcmk__output_t *out = rsc->priv->scheduler->priv->out;
 
             out->info(out,
                       "Only 'private' parameters to %s-interval %s for %s "
                       "on %s changed: %s",
                       pcmk__readable_interval(interval_ms), task, rsc->id,
                       pcmk__node_name(node),
                       crm_element_value(xml_op, PCMK__XA_TRANSITION_MAGIC));
         }
         return false;
     }
 
     switch (digest_data->rc) {
         case pcmk__digest_restart:
             crm_log_xml_debug(digest_data->params_restart, "params:restart");
             force_restart(rsc, task, interval_ms, node);
             return true;
 
         case pcmk__digest_unknown:
         case pcmk__digest_mismatch:
             // Changes that can potentially be handled by an agent reload
 
             if (interval_ms > 0) {
                 /* Recurring actions aren't reloaded per se, they are just
                  * re-scheduled so the next run uses the new parameters.
                  * The old instance will be cancelled automatically.
                  */
                 crm_log_xml_debug(digest_data->params_all, "params:reschedule");
                 pcmk__reschedule_recurring(rsc, task, interval_ms, node);
 
             } else if (crm_element_value(xml_op,
                                          PCMK__XA_OP_RESTART_DIGEST) != NULL) {
                 // Agent supports reload, so use it
                 trigger_unfencing(rsc, node,
                                   "Device parameters changed (reload)", NULL,
                                   rsc->priv->scheduler);
                 crm_log_xml_debug(digest_data->params_all, "params:reload");
                 schedule_reload((gpointer) rsc, (gpointer) node);
 
             } else {
                 pcmk__rsc_trace(rsc,
                                 "Restarting %s "
                                 "because agent doesn't support reload",
                                 rsc->id);
                 crm_log_xml_debug(digest_data->params_restart,
                                   "params:restart");
                 force_restart(rsc, task, interval_ms, node);
             }
             return true;
 
         default:
             break;
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Create a list of resource's action history entries, sorted by call ID
  *
  * \param[in]  rsc_entry    Resource's \c PCMK__XE_LRM_RSC_OP status XML
  * \param[out] start_index  Where to store index of start-like action, if any
  * \param[out] stop_index   Where to store index of stop action, if any
  */
 static GList *
 rsc_history_as_list(const xmlNode *rsc_entry, int *start_index, int *stop_index)
 {
     GList *ops = NULL;
 
     for (xmlNode *rsc_op = pcmk__xe_first_child(rsc_entry, PCMK__XE_LRM_RSC_OP,
                                                 NULL, NULL);
          rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op, PCMK__XE_LRM_RSC_OP)) {
 
         ops = g_list_prepend(ops, rsc_op);
     }
     ops = g_list_sort(ops, sort_op_by_callid);
     calculate_active_ops(ops, start_index, stop_index);
     return ops;
 }
 
 /*!
  * \internal
  * \brief Process a resource's action history from the CIB status
  *
  * Given a resource's action history, if the resource's configuration
  * changed since the actions were done, schedule any actions needed (restart,
  * reload, unfencing, rescheduling recurring actions, clean-up, etc.).
  * (This also cancels recurring actions for maintenance mode, which is not
  * entirely related but convenient to do here.)
  *
  * \param[in]     rsc_entry  Resource's \c PCMK__XE_LRM_RSC_OP status XML
  * \param[in,out] rsc        Resource whose history is being processed
  * \param[in,out] node       Node whose history is being processed
  */
 static void
 process_rsc_history(const xmlNode *rsc_entry, pcmk_resource_t *rsc,
                     pcmk_node_t *node)
 {
     int offset = -1;
     int stop_index = 0;
     int start_index = 0;
     GList *sorted_op_list = NULL;
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
         if (pcmk__is_anonymous_clone(pe__const_top_resource(rsc, false))) {
+            /* @TODO Should this be done for bundled primitives as well? Added
+             * by 2ac43ae31
+             */
             pcmk__rsc_trace(rsc,
                             "Skipping configuration check "
                             "for orphaned clone instance %s",
                             rsc->id);
         } else {
             pcmk__rsc_trace(rsc,
                             "Skipping configuration check and scheduling "
                             "clean-up for orphaned resource %s", rsc->id);
             pcmk__schedule_cleanup(rsc, node, false);
         }
         return;
     }
 
     if (pe_find_node_id(rsc->priv->active_nodes,
                         node->priv->id) == NULL) {
         if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, false)) {
             pcmk__schedule_cleanup(rsc, node, false);
         }
         pcmk__rsc_trace(rsc,
                         "Skipping configuration check for %s "
                         "because no longer active on %s",
                         rsc->id, pcmk__node_name(node));
         return;
     }
 
     pcmk__rsc_trace(rsc, "Checking for configuration changes for %s on %s",
                     rsc->id, pcmk__node_name(node));
 
     if (pcmk__rsc_agent_changed(rsc, node, rsc_entry, true)) {
         pcmk__schedule_cleanup(rsc, node, false);
     }
 
     sorted_op_list = rsc_history_as_list(rsc_entry, &start_index, &stop_index);
     if (start_index < stop_index) {
         return; // Resource is stopped
     }
 
     for (GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
         xmlNode *rsc_op = (xmlNode *) iter->data;
         const char *task = NULL;
         guint interval_ms = 0;
 
         if (++offset < start_index) {
             // Skip actions that happened before a start
             continue;
         }
 
         task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
         crm_element_value_ms(rsc_op, PCMK_META_INTERVAL, &interval_ms);
 
         if ((interval_ms > 0)
             && (pcmk_is_set(rsc->flags, pcmk__rsc_maintenance)
                 || node->details->maintenance)) {
             // Maintenance mode cancels recurring operations
             pcmk__schedule_cancel(rsc,
                                   crm_element_value(rsc_op, PCMK__XA_CALL_ID),
                                   task, interval_ms, node, "maintenance mode");
 
         } else if ((interval_ms > 0)
                    || pcmk__strcase_any_of(task, PCMK_ACTION_MONITOR,
                                            PCMK_ACTION_START,
                                            PCMK_ACTION_PROMOTE,
                                            PCMK_ACTION_MIGRATE_FROM, NULL)) {
             /* If a resource operation failed, and the operation's definition
              * has changed, clear any fail count so they can be retried fresh.
              */
 
             if (pe__bundle_needs_remote_name(rsc)) {
                 /* We haven't assigned resources to nodes yet, so if the
                  * REMOTE_CONTAINER_HACK is used, we may calculate the digest
                  * based on the literal "#uname" value rather than the properly
                  * substituted value. That would mistakenly make the action
                  * definition appear to have been changed. Defer the check until
                  * later in this case.
                  */
                 pe__add_param_check(rsc_op, rsc, node, pcmk__check_active,
                                     rsc->priv->scheduler);
 
             } else if (pcmk__check_action_config(rsc, node, rsc_op)
                        && (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
                                             NULL) != 0)) {
                 pe__clear_failcount(rsc, node, "action definition changed",
                                     rsc->priv->scheduler);
             }
         }
     }
     g_list_free(sorted_op_list);
 }
 
 /*!
  * \internal
  * \brief Process a node's action history from the CIB status
  *
  * Given a node's resource history, if the resource's configuration changed
  * since the actions were done, schedule any actions needed (restart,
  * reload, unfencing, rescheduling recurring actions, clean-up, etc.).
  * (This also cancels recurring actions for maintenance mode, which is not
  * entirely related but convenient to do here.)
  *
  * \param[in,out] node      Node whose history is being processed
  * \param[in]     lrm_rscs  Node's \c PCMK__XE_LRM_RESOURCES from CIB status XML
  */
 static void
 process_node_history(pcmk_node_t *node, const xmlNode *lrm_rscs)
 {
     crm_trace("Processing node history for %s", pcmk__node_name(node));
     for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rscs,
                                                          PCMK__XE_LRM_RESOURCE,
                                                          NULL, NULL);
          rsc_entry != NULL;
          rsc_entry = pcmk__xe_next(rsc_entry, PCMK__XE_LRM_RESOURCE)) {
 
         if (rsc_entry->children != NULL) {
             GList *result = pcmk__rscs_matching_id(pcmk__xe_id(rsc_entry),
                                                    node->priv->scheduler);
 
             for (GList *iter = result; iter != NULL; iter = iter->next) {
                 pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
                 if (pcmk__is_primitive(rsc)) {
                     process_rsc_history(rsc_entry, rsc, node);
                 }
             }
             g_list_free(result);
         }
     }
 }
 
 // XPath to find a node's resource history
 #define XPATH_NODE_HISTORY "/" PCMK_XE_CIB "/" PCMK_XE_STATUS   \
                            "/" PCMK__XE_NODE_STATE              \
                            "[@" PCMK_XA_UNAME "='%s']"          \
                            "/" PCMK__XE_LRM "/" PCMK__XE_LRM_RESOURCES
 
 /*!
  * \internal
  * \brief Process any resource configuration changes in the CIB status
  *
  * Go through all nodes' resource history, and if a resource's configuration
  * changed since its actions were done, schedule any actions needed (restart,
  * reload, unfencing, rescheduling recurring actions, clean-up, etc.).
  * (This also cancels recurring actions for maintenance mode, which is not
  * entirely related but convenient to do here.)
  *
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__handle_rsc_config_changes(pcmk_scheduler_t *scheduler)
 {
     crm_trace("Check resource and action configuration for changes");
 
     /* Rather than iterate through the status section, iterate through the nodes
      * and search for the appropriate status subsection for each. This skips
      * orphaned nodes and lets us eliminate some cases before searching the XML.
      */
     for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
         pcmk_node_t *node = (pcmk_node_t *) iter->data;
 
         /* Don't bother checking actions for a node that can't run actions ...
          * unless it's in maintenance mode, in which case we still need to
          * cancel any existing recurring monitors.
          */
         if (node->details->maintenance
             || pcmk__node_available(node, false, false)) {
 
             char *xpath = NULL;
             xmlNode *history = NULL;
 
             xpath = crm_strdup_printf(XPATH_NODE_HISTORY, node->priv->name);
             history = get_xpath_object(xpath, scheduler->input, LOG_NEVER);
             free(xpath);
 
             process_node_history(node, history);
         }
     }
 }
diff --git a/lib/pacemaker/pcmk_sched_location.c b/lib/pacemaker/pcmk_sched_location.c
index a3915b5435..22b76940d2 100644
--- a/lib/pacemaker/pcmk_sched_location.c
+++ b/lib/pacemaker/pcmk_sched_location.c
@@ -1,781 +1,783 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdbool.h>
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/common/rules_internal.h>
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 /*!
  * \internal
  * \brief Parse a role configuration for a location constraint
  *
  * \param[in]  role_spec  Role specification
  * \param[out] role       Where to store parsed role
  *
  * \return true if role specification is valid, otherwise false
  */
 static bool
 parse_location_role(const char *role_spec, enum rsc_role_e *role)
 {
     if (role_spec == NULL) {
         *role = pcmk_role_unknown;
         return true;
     }
 
     *role = pcmk_parse_role(role_spec);
     switch (*role) {
         case pcmk_role_unknown:
             return false;
 
         case pcmk_role_started:
         case pcmk_role_unpromoted:
             /* Any promotable clone instance cannot be promoted without being in
              * the unpromoted role first. Therefore, any constraint for the
              * started or unpromoted role applies to every role.
              */
             *role = pcmk_role_unknown;
             break;
 
         default:
             break;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Get the score attribute name (if any) used for a rule
  *
  * \param[in]  rule_xml    Rule XML
  * \param[out] allocated   If the score attribute name needs to be allocated,
  *                         this will be set to the non-const equivalent of the
  *                         return value (should be set to NULL when passed)
  * \param[in]  rule_input  Values used to evaluate rule criteria
  *
  * \return Score attribute name used for rule, or NULL if none
  * \note The caller is responsible for freeing \p *allocated if it is non-NULL.
  */
 static const char *
 score_attribute_name(const xmlNode *rule_xml, char **allocated,
                      const pcmk_rule_input_t *rule_input)
 {
     const char *name = NULL;
 
     name = crm_element_value(rule_xml, PCMK_XA_SCORE_ATTRIBUTE);
     if (name == NULL) {
         return NULL;
     }
 
     /* A score attribute name may use submatches extracted from a
      * resource ID regular expression. For example, if score-attribute is
      * "loc-\1", rsc-pattern is "ip-(.*)", and the resource ID is "ip-db", then
      * the score attribute name is "loc-db".
      */
     if ((rule_input->rsc_id != NULL) && (rule_input->rsc_id_nmatches > 0)) {
         *allocated = pcmk__replace_submatches(name, rule_input->rsc_id,
                                               rule_input->rsc_id_submatches,
                                               rule_input->rsc_id_nmatches);
         if (*allocated != NULL) {
             name = *allocated;
         }
     }
     return name;
 }
 
 /*!
  * \internal
  * \brief Parse a score from a rule without a score attribute
  *
  * \param[in]  rule_xml    Rule XML
  * \param[out] score       Where to store parsed score
  *
  * \return Standard Pacemaker return code
  */
 static int
 score_from_rule(const xmlNode *rule_xml, int *score)
 {
     int rc = pcmk_rc_ok;
     const char *score_s = crm_element_value(rule_xml, PCMK_XA_SCORE);
 
     if (score_s == NULL) { // Not possible with schema validation enabled
         pcmk__config_err("Ignoring location constraint rule %s because "
                          "neither " PCMK_XA_SCORE " nor "
                          PCMK_XA_SCORE_ATTRIBUTE " was specified",
                          pcmk__xe_id(rule_xml));
         return pcmk_rc_unpack_error;
     }
 
     rc = pcmk_parse_score(score_s, score, 0);
     if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled
         pcmk__config_err("Ignoring location constraint rule %s because "
                          "'%s' is not a valid " PCMK_XA_SCORE ": %s",
                          pcmk__xe_id(rule_xml), score_s, pcmk_rc_str(rc));
         return pcmk_rc_unpack_error;
     }
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Get a rule score from a node attribute
  *
  * \param[in]  constraint_id  Location constraint ID (for logging only)
  * \param[in]  attr_name      Name of node attribute with score
  * \param[in]  node           Node to get attribute for
  * \param[in]  rsc            Resource being located
  * \param[out] score          Where to store parsed score
  *
  * \return Standard Pacemaker return code (pcmk_rc_ok if a valid score was
  *         parsed, ENXIO if the node attribute was unset, and some other value
  *         if the node attribute value was invalid)
  */
 static int
 score_from_attr(const char *constraint_id, const char *attr_name,
                 const pcmk_node_t *node, const pcmk_resource_t *rsc, int *score)
 {
     int rc = pcmk_rc_ok;
     const char *target = NULL;
     const char *score_s = NULL;
 
     target = g_hash_table_lookup(rsc->priv->meta,
                                  PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
     score_s = pcmk__node_attr(node, attr_name, target, pcmk__rsc_node_current);
     if (pcmk__str_empty(score_s)) {
         crm_info("Ignoring location %s for %s on %s "
                  "because it has no node attribute %s",
                  constraint_id, rsc->id, pcmk__node_name(node), attr_name);
         return ENXIO;
     }
 
     rc = pcmk_parse_score(score_s, score, 0);
     if (rc != pcmk_rc_ok) {
         crm_warn("Ignoring location %s for node %s because node "
                  "attribute %s value '%s' is not a valid score: %s",
                  constraint_id, pcmk__node_name(node), attr_name,
                  score_s, pcmk_rc_str(rc));
         return rc;
     }
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Generate a location constraint from a rule
  *
  * \param[in,out] rsc            Resource that constraint is for
  * \param[in]     rule_xml       Rule XML (sub-element of location constraint)
  * \param[in]     discovery      Value of \c PCMK_XA_RESOURCE_DISCOVERY for
  *                               constraint
  * \param[out]    next_change    Where to set when rule evaluation will change
  * \param[in,out] rule_input     Values used to evaluate rule criteria
  *                               (node-specific values will be overwritten by
  *                               this function)
  * \param[in]     constraint_id  ID of location constraint (for logging only)
  *
  * \return true if rule is valid, otherwise false
  */
 static bool
 generate_location_rule(pcmk_resource_t *rsc, xmlNode *rule_xml,
                        const char *discovery, crm_time_t *next_change,
                        pcmk_rule_input_t *rule_input, const char *constraint_id)
 {
     const char *rule_id = NULL;
     const char *score_attr = NULL;
     const char *boolean = NULL;
     const char *role_spec = NULL;
 
     GList *iter = NULL;
     int score = 0;
     char *local_score_attr = NULL;
     pcmk__location_t *location_rule = NULL;
     enum rsc_role_e role = pcmk_role_unknown;
     enum pcmk__combine combine = pcmk__combine_unknown;
 
     rule_xml = pcmk__xe_resolve_idref(rule_xml, rsc->priv->scheduler->input);
     if (rule_xml == NULL) {
         return false; // Error already logged
     }
 
     rule_id = crm_element_value(rule_xml, PCMK_XA_ID);
     if (rule_id == NULL) {
         pcmk__config_err("Ignoring location constraint '%s' because its rule "
                          "has no " PCMK_XA_ID,
                          constraint_id);
         return false;
     }
 
     boolean = crm_element_value(rule_xml, PCMK_XA_BOOLEAN_OP);
     role_spec = crm_element_value(rule_xml, PCMK_XA_ROLE);
 
     if (parse_location_role(role_spec, &role)) {
         crm_trace("Setting rule %s role filter to %s", rule_id, role_spec);
     } else {
         pcmk__config_err("Ignoring location constraint '%s' because rule '%s' "
                          "has invalid " PCMK_XA_ROLE " '%s'",
                          constraint_id, rule_id, role_spec);
         return false;
     }
 
     combine = pcmk__parse_combine(boolean);
     switch (combine) {
         case pcmk__combine_and:
         case pcmk__combine_or:
             break;
 
         default: // Not possible with schema validation enabled
             pcmk__config_err("Ignoring location constraint '%s' because rule "
                              "'%s' has invalid " PCMK_XA_BOOLEAN_OP " '%s'",
                              constraint_id, rule_id, boolean);
             return false;
     }
 
     /* Users may configure the rule with either a score or the name of a
      * node attribute whose value should be used as the constraint score for
      * that node.
      */
     score_attr = score_attribute_name(rule_xml, &local_score_attr, rule_input);
     if ((score_attr == NULL)
         && (score_from_rule(rule_xml, &score) != pcmk_rc_ok)) {
         return false;
     }
 
     location_rule = pcmk__new_location(rule_id, rsc, 0, discovery, NULL);
     CRM_CHECK(location_rule != NULL, return NULL);
 
     location_rule->role_filter = role;
 
     for (iter = rsc->priv->scheduler->nodes;
          iter != NULL; iter = iter->next) {
 
         pcmk_node_t *node = iter->data;
         pcmk_node_t *local = NULL;
 
         rule_input->node_attrs = node->priv->attrs;
         rule_input->rsc_params = pe_rsc_params(rsc, node,
                                                rsc->priv->scheduler);
 
         if (pcmk_evaluate_rule(rule_xml, rule_input,
                                next_change) != pcmk_rc_ok) {
             continue;
         }
 
         if ((score_attr != NULL)
             && (score_from_attr(constraint_id, score_attr, node, rsc,
                                 &score) != pcmk_rc_ok)) {
             continue; // Message already logged
         }
 
         local = pe__copy_node(node);
         location_rule->nodes = g_list_prepend(location_rule->nodes, local);
         local->assign->score = score;
         pcmk__rsc_trace(rsc,
                         "Location %s score for %s on %s is %s via rule %s",
                         constraint_id, rsc->id, pcmk__node_name(node),
                         pcmk_readable_score(score), rule_id);
     }
 
     free(local_score_attr);
 
     if (location_rule->nodes == NULL) {
         crm_trace("No matching nodes for location constraint rule %s", rule_id);
     } else {
         crm_trace("Location constraint rule %s matched %d nodes",
                   rule_id, g_list_length(location_rule->nodes));
     }
     return true;
 }
 
 static void
 unpack_rsc_location(xmlNode *xml_obj, pcmk_resource_t *rsc,
                     const char *role_spec, const char *score,
                     char *rsc_id_match, int rsc_id_nmatches,
                     regmatch_t *rsc_id_submatches)
 {
     const char *rsc_id = crm_element_value(xml_obj, PCMK_XA_RSC);
     const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
     const char *node = crm_element_value(xml_obj, PCMK_XA_NODE);
     const char *discovery = crm_element_value(xml_obj,
                                               PCMK_XA_RESOURCE_DISCOVERY);
 
     if (rsc == NULL) {
         pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                           "does not exist", id, rsc_id);
         return;
     }
 
     if (score == NULL) {
         score = crm_element_value(xml_obj, PCMK_XA_SCORE);
     }
 
     if ((node != NULL) && (score != NULL)) {
         int score_i = 0;
         int rc = pcmk_rc_ok;
         pcmk_node_t *match = pcmk_find_node(rsc->priv->scheduler, node);
         enum rsc_role_e role = pcmk_role_unknown;
         pcmk__location_t *location = NULL;
 
         if (match == NULL) {
             crm_info("Ignoring location constraint %s "
                      "because '%s' is not a known node",
                      pcmk__s(id, "without ID"), node);
             return;
         }
 
         rc = pcmk_parse_score(score, &score_i, 0);
         if (rc != pcmk_rc_ok) { // Not possible with schema validation enabled
             pcmk__config_err("Ignoring location constraint %s "
                              "because '%s' is not a valid score", id, score);
             return;
         }
 
         if (role_spec == NULL) {
             role_spec = crm_element_value(xml_obj, PCMK_XA_ROLE);
         }
         if (parse_location_role(role_spec, &role)) {
             crm_trace("Setting location constraint %s role filter: %s",
                       id, role_spec);
         } else { // Not possible with schema validation enabled
             pcmk__config_err("Ignoring location constraint %s "
                              "because '%s' is not a valid " PCMK_XA_ROLE,
                              id, role_spec);
             return;
         }
 
         location = pcmk__new_location(id, rsc, score_i, discovery, match);
         if (location == NULL) {
             return; // Error already logged
         }
         location->role_filter = role;
 
     } else {
         crm_time_t *next_change = crm_time_new_undefined();
         xmlNode *rule_xml = pcmk__xe_first_child(xml_obj, PCMK_XE_RULE, NULL,
                                                  NULL);
         pcmk_rule_input_t rule_input = {
             .now = rsc->priv->scheduler->priv->now,
             .rsc_meta = rsc->priv->meta,
             .rsc_id = rsc_id_match,
             .rsc_id_submatches = rsc_id_submatches,
             .rsc_id_nmatches = rsc_id_nmatches,
         };
 
         generate_location_rule(rsc, rule_xml, discovery, next_change,
                                &rule_input, id);
 
         /* If there is a point in the future when the evaluation of a rule will
          * change, make sure the scheduler is re-run by that time.
          */
         if (crm_time_is_defined(next_change)) {
             time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
             pe__update_recheck_time(t, rsc->priv->scheduler,
                                     "location rule evaluation");
         }
         crm_time_free(next_change);
     }
 }
 
 static void
 unpack_simple_location(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
 {
     const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
     const char *value = crm_element_value(xml_obj, PCMK_XA_RSC);
 
     if (value) {
         pcmk_resource_t *rsc;
 
         rsc = pcmk__find_constraint_resource(scheduler->priv->resources, value);
         unpack_rsc_location(xml_obj, rsc, NULL, NULL, NULL, 0, NULL);
     }
 
     value = crm_element_value(xml_obj, PCMK_XA_RSC_PATTERN);
     if (value) {
         regex_t regex;
         bool invert = false;
 
         if (value[0] == '!') {
             value++;
             invert = true;
         }
 
         if (regcomp(&regex, value, REG_EXTENDED) != 0) {
             pcmk__config_err("Ignoring constraint '%s' because "
                              PCMK_XA_RSC_PATTERN
                              " has invalid value '%s'", id, value);
             return;
         }
 
         for (GList *iter = scheduler->priv->resources;
              iter != NULL; iter = iter->next) {
 
             pcmk_resource_t *r = iter->data;
             int nregs = 0;
             regmatch_t *pmatch = NULL;
             int status;
 
             if (regex.re_nsub > 0) {
                 nregs = regex.re_nsub + 1;
             } else {
                 nregs = 1;
             }
             pmatch = pcmk__assert_alloc(nregs, sizeof(regmatch_t));
 
             status = regexec(&regex, r->id, nregs, pmatch, 0);
 
             if (!invert && (status == 0)) {
                 crm_debug("'%s' matched '%s' for %s", r->id, value, id);
                 unpack_rsc_location(xml_obj, r, NULL, NULL, r->id, nregs,
                                     pmatch);
 
             } else if (invert && (status != 0)) {
                 crm_debug("'%s' is an inverted match of '%s' for %s",
                           r->id, value, id);
                 unpack_rsc_location(xml_obj, r, NULL, NULL, NULL, 0, NULL);
 
             } else {
                 crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
             }
 
             free(pmatch);
         }
 
+        // @TODO Maybe log a notice if we did not match any resources
+
         regfree(&regex);
     }
 }
 
 // \return Standard Pacemaker return code
 static int
 unpack_location_tags(xmlNode *xml_obj, xmlNode **expanded_xml,
                      pcmk_scheduler_t *scheduler)
 {
     const char *id = NULL;
     const char *rsc_id = NULL;
     const char *state = NULL;
     pcmk_resource_t *rsc = NULL;
     pcmk__idref_t *tag = NULL;
     xmlNode *rsc_set = NULL;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return EINVAL);
 
     id = pcmk__xe_id(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
                          xml_obj->name);
         return pcmk_rc_unpack_error;
     }
 
     // Check whether there are any resource sets with template or tag references
     *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
     if (*expanded_xml != NULL) {
         crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION);
         return pcmk_rc_ok;
     }
 
     rsc_id = crm_element_value(xml_obj, PCMK_XA_RSC);
     if (rsc_id == NULL) {
         return pcmk_rc_ok;
     }
 
     if (!pcmk__valid_resource_or_tag(scheduler, rsc_id, &rsc, &tag)) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, rsc_id);
         return pcmk_rc_unpack_error;
 
     } else if (rsc != NULL) {
         // No template is referenced
         return pcmk_rc_ok;
     }
 
     state = crm_element_value(xml_obj, PCMK_XA_ROLE);
 
     *expanded_xml = pcmk__xml_copy(NULL, xml_obj);
 
     /* Convert any template or tag reference into constraint
      * PCMK_XE_RESOURCE_SET
      */
     if (!pcmk__tag_to_set(*expanded_xml, &rsc_set, PCMK_XA_RSC,
                           false, scheduler)) {
         pcmk__xml_free(*expanded_xml);
         *expanded_xml = NULL;
         return pcmk_rc_unpack_error;
     }
 
     if (rsc_set != NULL) {
         if (state != NULL) {
             /* Move PCMK_XA_RSC_ROLE into converted PCMK_XE_RESOURCE_SET as
              * PCMK_XA_ROLE attribute
              */
             crm_xml_add(rsc_set, PCMK_XA_ROLE, state);
             pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_ROLE);
         }
         crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_LOCATION);
 
     } else {
         // No sets
         pcmk__xml_free(*expanded_xml);
         *expanded_xml = NULL;
     }
 
     return pcmk_rc_ok;
 }
 
 // \return Standard Pacemaker return code
 static int
 unpack_location_set(xmlNode *location, xmlNode *set,
                     pcmk_scheduler_t *scheduler)
 {
     xmlNode *xml_rsc = NULL;
     pcmk_resource_t *resource = NULL;
     const char *set_id;
     const char *role;
     const char *local_score;
 
     CRM_CHECK(set != NULL, return EINVAL);
 
     set_id = pcmk__xe_id(set);
     if (set_id == NULL) {
         pcmk__config_err("Ignoring " PCMK_XE_RESOURCE_SET " without "
                          PCMK_XA_ID " in constraint '%s'",
                          pcmk__s(pcmk__xe_id(location), "(missing ID)"));
         return pcmk_rc_unpack_error;
     }
 
     role = crm_element_value(set, PCMK_XA_ROLE);
     local_score = crm_element_value(set, PCMK_XA_SCORE);
 
     for (xml_rsc = pcmk__xe_first_child(set, PCMK_XE_RESOURCE_REF, NULL, NULL);
          xml_rsc != NULL;
          xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
         resource = pcmk__find_constraint_resource(scheduler->priv->resources,
                                                   pcmk__xe_id(xml_rsc));
         if (resource == NULL) {
             pcmk__config_err("%s: No resource found for %s",
                              set_id, pcmk__xe_id(xml_rsc));
             return pcmk_rc_unpack_error;
         }
 
         unpack_rsc_location(location, resource, role, local_score, NULL, 0,
                             NULL);
     }
 
     return pcmk_rc_ok;
 }
 
 void
 pcmk__unpack_location(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
 {
     xmlNode *set = NULL;
     bool any_sets = false;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     if (unpack_location_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
         return;
     }
 
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
     }
 
     for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL);
          set != NULL; set = pcmk__xe_next(set, PCMK_XE_RESOURCE_SET)) {
 
         any_sets = true;
         set = pcmk__xe_resolve_idref(set, scheduler->input);
         if ((set == NULL) // Configuration error, message already logged
             || (unpack_location_set(xml_obj, set, scheduler) != pcmk_rc_ok)) {
 
             if (expanded_xml) {
                 pcmk__xml_free(expanded_xml);
             }
             return;
         }
     }
 
     if (expanded_xml) {
         pcmk__xml_free(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (!any_sets) {
         unpack_simple_location(xml_obj, scheduler);
     }
 }
 
 /*!
  * \internal
  * \brief Add a new location constraint to scheduler data
  *
  * \param[in]     id             XML ID of location constraint
  * \param[in,out] rsc            Resource in location constraint
  * \param[in]     node_score     Constraint score
  * \param[in]     probe_mode     When resource should be probed on node
  * \param[in]     node           Node in constraint (or NULL if rule-based)
  *
  * \return Newly allocated location constraint on success, otherwise NULL
  * \note The result will be added to the cluster (via \p rsc) and should not be
  *       freed separately.
  */
 pcmk__location_t *
 pcmk__new_location(const char *id, pcmk_resource_t *rsc,
                    int node_score, const char *probe_mode, pcmk_node_t *node)
 {
     pcmk__location_t *new_con = NULL;
 
     CRM_CHECK((node != NULL) || (node_score == 0), return NULL);
 
     if (id == NULL) {
         pcmk__config_err("Invalid constraint: no ID specified");
         return NULL;
     }
 
     if (rsc == NULL) {
         pcmk__config_err("Invalid constraint %s: no resource specified", id);
         return NULL;
     }
 
     new_con = pcmk__assert_alloc(1, sizeof(pcmk__location_t));
     new_con->id = pcmk__str_copy(id);
     new_con->rsc = rsc;
     new_con->nodes = NULL;
     new_con->role_filter = pcmk_role_unknown;
 
     if (pcmk__str_eq(probe_mode, PCMK_VALUE_ALWAYS,
                      pcmk__str_null_matches|pcmk__str_casei)) {
         new_con->probe_mode = pcmk__probe_always;
 
     } else if (pcmk__str_eq(probe_mode, PCMK_VALUE_NEVER, pcmk__str_casei)) {
         new_con->probe_mode = pcmk__probe_never;
 
     } else if (pcmk__str_eq(probe_mode, PCMK_VALUE_EXCLUSIVE,
                             pcmk__str_casei)) {
         new_con->probe_mode = pcmk__probe_exclusive;
         pcmk__set_rsc_flags(rsc, pcmk__rsc_exclusive_probes);
 
     } else {
         pcmk__config_err("Invalid " PCMK_XA_RESOURCE_DISCOVERY " value %s "
                          "in location constraint", probe_mode);
     }
 
     if (node != NULL) {
         pcmk_node_t *copy = pe__copy_node(node);
 
         copy->assign->score = node_score;
         new_con->nodes = g_list_prepend(NULL, copy);
     }
 
     rsc->priv->scheduler->priv->location_constraints =
         g_list_prepend(rsc->priv->scheduler->priv->location_constraints,
                        new_con);
     rsc->priv->location_constraints =
         g_list_prepend(rsc->priv->location_constraints, new_con);
 
     return new_con;
 }
 
 /*!
  * \internal
  * \brief Apply all location constraints
  *
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__apply_locations(pcmk_scheduler_t *scheduler)
 {
     for (GList *iter = scheduler->priv->location_constraints;
          iter != NULL; iter = iter->next) {
         pcmk__location_t *location = iter->data;
 
         location->rsc->priv->cmds->apply_location(location->rsc, location);
     }
 }
 
 /*!
  * \internal
  * \brief Apply a location constraint to a resource's allowed node scores
  *
  * \param[in,out] rsc         Resource to apply constraint to
  * \param[in,out] location    Location constraint to apply
  *
  * \note This does not consider the resource's children, so the resource's
  *       apply_location() method should be used instead in most cases.
  */
 void
 pcmk__apply_location(pcmk_resource_t *rsc, pcmk__location_t *location)
 {
     bool need_role = false;
 
     pcmk__assert((rsc != NULL) && (location != NULL));
 
     // If a role was specified, ensure constraint is applicable
     need_role = (location->role_filter > pcmk_role_unknown);
     if (need_role && (location->role_filter != rsc->priv->next_role)) {
         pcmk__rsc_trace(rsc,
                         "Not applying %s to %s because role will be %s not %s",
                         location->id, rsc->id,
                         pcmk_role_text(rsc->priv->next_role),
                         pcmk_role_text(location->role_filter));
         return;
     }
 
     if (location->nodes == NULL) {
         pcmk__rsc_trace(rsc, "Not applying %s to %s because no nodes match",
                         location->id, rsc->id);
         return;
     }
 
     for (GList *iter = location->nodes; iter != NULL; iter = iter->next) {
         pcmk_node_t *node = iter->data;
         pcmk_node_t *allowed_node = NULL;
 
         allowed_node = g_hash_table_lookup(rsc->priv->allowed_nodes,
                                            node->priv->id);
 
         pcmk__rsc_trace(rsc, "Applying %s%s%s to %s score on %s: %c %s",
                         location->id,
                         (need_role? " for role " : ""),
                         (need_role? pcmk_role_text(location->role_filter) : ""),
                         rsc->id, pcmk__node_name(node),
                         ((allowed_node == NULL)? '=' : '+'),
                         pcmk_readable_score(node->assign->score));
 
         if (allowed_node == NULL) {
             allowed_node = pe__copy_node(node);
             g_hash_table_insert(rsc->priv->allowed_nodes,
                                 (gpointer) allowed_node->priv->id,
                                 allowed_node);
         } else {
             allowed_node->assign->score =
                 pcmk__add_scores(allowed_node->assign->score,
                                  node->assign->score);
         }
 
         if (allowed_node->assign->probe_mode < location->probe_mode) {
             if (location->probe_mode == pcmk__probe_exclusive) {
                 pcmk__set_rsc_flags(rsc, pcmk__rsc_exclusive_probes);
             }
             /* exclusive > never > always... always is default */
             allowed_node->assign->probe_mode = location->probe_mode;
         }
     }
 }
diff --git a/lib/pacemaker/pcmk_sched_ordering.c b/lib/pacemaker/pcmk_sched_ordering.c
index 5bc2ffc919..2308be795e 100644
--- a/lib/pacemaker/pcmk_sched_ordering.c
+++ b/lib/pacemaker/pcmk_sched_ordering.c
@@ -1,1496 +1,1499 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <inttypes.h>               // PRIx32
 #include <stdbool.h>
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <pacemaker-internal.h>
 #include "libpacemaker_private.h"
 
 enum pe_order_kind {
     pe_order_kind_optional,
     pe_order_kind_mandatory,
     pe_order_kind_serialize,
 };
 
 enum ordering_symmetry {
     ordering_asymmetric,        // the only relation in an asymmetric ordering
     ordering_symmetric,         // the normal relation in a symmetric ordering
     ordering_symmetric_inverse, // the inverse relation in a symmetric ordering
 };
 
+// @TODO de-functionize this for readability and possibly better log messages
 #define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do {                  \
         __rsc = pcmk__find_constraint_resource(scheduler->priv->resources,  \
                                                __name);                     \
         if (__rsc == NULL) {                                                \
             pcmk__config_err("%s: No resource found for %s", __set, __name);\
             return pcmk_rc_unpack_error;                                    \
         }                                                                   \
     } while (0)
 
 static const char *
 invert_action(const char *action)
 {
     if (pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
         return PCMK_ACTION_STOP;
 
     } else if (pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
         return PCMK_ACTION_START;
 
     } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
         return PCMK_ACTION_DEMOTE;
 
     } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
         return PCMK_ACTION_PROMOTE;
 
     } else if (pcmk__str_eq(action, PCMK_ACTION_PROMOTED, pcmk__str_none)) {
         return PCMK_ACTION_DEMOTED;
 
     } else if (pcmk__str_eq(action, PCMK_ACTION_DEMOTED, pcmk__str_none)) {
         return PCMK_ACTION_PROMOTED;
 
     } else if (pcmk__str_eq(action, PCMK_ACTION_RUNNING, pcmk__str_none)) {
         return PCMK_ACTION_STOPPED;
 
     } else if (pcmk__str_eq(action, PCMK_ACTION_STOPPED, pcmk__str_none)) {
         return PCMK_ACTION_RUNNING;
     }
     pcmk__config_warn("Unknown action '%s' specified in order constraint",
                       action);
     return NULL;
 }
 
 static enum pe_order_kind
 get_ordering_type(const xmlNode *xml_obj)
 {
     enum pe_order_kind kind_e = pe_order_kind_mandatory;
     const char *kind = crm_element_value(xml_obj, PCMK_XA_KIND);
 
     if (kind == NULL) {
         const char *score = crm_element_value(xml_obj, PCMK_XA_SCORE);
 
         kind_e = pe_order_kind_mandatory;
 
         if (score) {
             // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
             int score_i = 0;
 
             (void) pcmk_parse_score(score, &score_i, 0);
             if (score_i == 0) {
                 kind_e = pe_order_kind_optional;
             }
             pcmk__warn_once(pcmk__wo_order_score,
                             "Support for '" PCMK_XA_SCORE "' in "
                             PCMK_XE_RSC_ORDER " is deprecated and will be "
                             "removed in a future release "
                             "(use '" PCMK_XA_KIND "' instead)");
         }
 
     } else if (pcmk__str_eq(kind, PCMK_VALUE_MANDATORY, pcmk__str_none)) {
         kind_e = pe_order_kind_mandatory;
 
     } else if (pcmk__str_eq(kind, PCMK_VALUE_OPTIONAL, pcmk__str_none)) {
         kind_e = pe_order_kind_optional;
 
     } else if (pcmk__str_eq(kind, PCMK_VALUE_SERIALIZE, pcmk__str_none)) {
         kind_e = pe_order_kind_serialize;
 
     } else {
         pcmk__config_err("Resetting '" PCMK_XA_KIND "' for constraint %s to "
                          "'" PCMK_VALUE_MANDATORY "' because '%s' is not valid",
                          pcmk__s(pcmk__xe_id(xml_obj), "missing ID"), kind);
     }
     return kind_e;
 }
 
 /*!
  * \internal
  * \brief Get ordering symmetry from XML
  *
  * \param[in] xml_obj               Ordering XML
  * \param[in] parent_kind           Default ordering kind
  * \param[in] parent_symmetrical_s  Parent element's \c PCMK_XA_SYMMETRICAL
  *                                  setting, if any
  *
  * \retval ordering_symmetric   Ordering is symmetric
  * \retval ordering_asymmetric  Ordering is asymmetric
  */
 static enum ordering_symmetry
 get_ordering_symmetry(const xmlNode *xml_obj, enum pe_order_kind parent_kind,
                       const char *parent_symmetrical_s)
 {
     int rc = pcmk_rc_ok;
     bool symmetric = false;
     enum pe_order_kind kind = parent_kind; // Default to parent's kind
 
     // Check ordering XML for explicit kind
     if ((crm_element_value(xml_obj, PCMK_XA_KIND) != NULL)
         || (crm_element_value(xml_obj, PCMK_XA_SCORE) != NULL)) {
         kind = get_ordering_type(xml_obj);
     }
 
     // Check ordering XML (and parent) for explicit PCMK_XA_SYMMETRICAL setting
     rc = pcmk__xe_get_bool_attr(xml_obj, PCMK_XA_SYMMETRICAL, &symmetric);
 
     if (rc != pcmk_rc_ok && parent_symmetrical_s != NULL) {
         symmetric = crm_is_true(parent_symmetrical_s);
         rc = pcmk_rc_ok;
     }
 
     if (rc == pcmk_rc_ok) {
         if (symmetric) {
             if (kind == pe_order_kind_serialize) {
                 pcmk__config_warn("Ignoring " PCMK_XA_SYMMETRICAL
                                   " for '%s' because not valid with "
                                   PCMK_XA_KIND " of '" PCMK_VALUE_SERIALIZE "'",
                                   pcmk__xe_id(xml_obj));
             } else {
                 return ordering_symmetric;
             }
         }
         return ordering_asymmetric;
     }
 
     // Use default symmetry
     if (kind == pe_order_kind_serialize) {
         return ordering_asymmetric;
     }
     return ordering_symmetric;
 }
 
 /*!
  * \internal
  * \brief Get ordering flags appropriate to ordering kind
  *
  * \param[in] kind      Ordering kind
  * \param[in] first     Action name for 'first' action
  * \param[in] symmetry  This ordering's symmetry role
  *
  * \return Minimal ordering flags appropriate to \p kind
  */
 static uint32_t
 ordering_flags_for_kind(enum pe_order_kind kind, const char *first,
                         enum ordering_symmetry symmetry)
 {
     uint32_t flags = pcmk__ar_none; // so we trace-log all flags set
 
     switch (kind) {
         case pe_order_kind_optional:
             pcmk__set_relation_flags(flags, pcmk__ar_ordered);
             break;
 
         case pe_order_kind_serialize:
             /* This flag is not used anywhere directly but means the relation
              * will not match an equality comparison against pcmk__ar_none or
              * pcmk__ar_ordered.
              */
             pcmk__set_relation_flags(flags, pcmk__ar_serialize);
             break;
 
         case pe_order_kind_mandatory:
             pcmk__set_relation_flags(flags, pcmk__ar_ordered);
             switch (symmetry) {
                 case ordering_asymmetric:
                     pcmk__set_relation_flags(flags, pcmk__ar_asymmetric);
                     break;
 
                 case ordering_symmetric:
                     pcmk__set_relation_flags(flags,
                                              pcmk__ar_first_implies_then);
                     if (pcmk__strcase_any_of(first, PCMK_ACTION_START,
                                              PCMK_ACTION_PROMOTE, NULL)) {
                         pcmk__set_relation_flags(flags,
                                                  pcmk__ar_unrunnable_first_blocks);
                     }
                     break;
 
                 case ordering_symmetric_inverse:
                     pcmk__set_relation_flags(flags,
                                              pcmk__ar_then_implies_first);
                     break;
             }
             break;
     }
     return flags;
 }
 
 /*!
  * \internal
  * \brief Find resource corresponding to ID specified in ordering
  *
  * \param[in] xml            Ordering XML
  * \param[in] resource_attr  XML attribute name for resource ID
  * \param[in] scheduler      Scheduler data
  *
  * \return Resource corresponding to \p id, or NULL if none
  */
 static pcmk_resource_t *
 get_ordering_resource(const xmlNode *xml, const char *resource_attr,
                       const pcmk_scheduler_t *scheduler)
 {
     pcmk_resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(xml, resource_attr);
 
     if (rsc_id == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without %s",
                          pcmk__xe_id(xml), resource_attr);
         return NULL;
     }
 
     rsc = pcmk__find_constraint_resource(scheduler->priv->resources, rsc_id);
     if (rsc == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", pcmk__xe_id(xml), rsc_id);
         return NULL;
     }
 
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Determine minimum number of 'first' instances required in ordering
  *
  * \param[in] rsc  'First' resource in ordering
  * \param[in] xml  Ordering XML
  *
  * \return Minimum 'first' instances required (or 0 if not applicable)
  */
 static int
 get_minimum_first_instances(const pcmk_resource_t *rsc, const xmlNode *xml)
 {
     const char *clone_min = NULL;
     bool require_all = false;
 
     if (!pcmk__is_clone(rsc)) {
         return 0;
     }
 
     clone_min = g_hash_table_lookup(rsc->priv->meta, PCMK_META_CLONE_MIN);
     if (clone_min != NULL) {
         int clone_min_int = 0;
 
         pcmk__scan_min_int(clone_min, &clone_min_int, 0);
         return clone_min_int;
     }
 
     /* @COMPAT 1.1.13:
      * PCMK_XA_REQUIRE_ALL=PCMK_VALUE_FALSE is deprecated equivalent of
      * PCMK_META_CLONE_MIN=1
      */
     if (pcmk__xe_get_bool_attr(xml, PCMK_XA_REQUIRE_ALL,
                                &require_all) != ENODATA) {
         pcmk__warn_once(pcmk__wo_require_all,
                         "Support for " PCMK_XA_REQUIRE_ALL " in ordering "
                         "constraints is deprecated and will be removed in a "
                         "future release (use " PCMK_META_CLONE_MIN " clone "
                         "meta-attribute instead)");
         if (!require_all) {
             return 1;
         }
     }
 
     return 0;
 }
 
 /*!
  * \internal
  * \brief Create orderings for a constraint with \c PCMK_META_CLONE_MIN > 0
  *
  * \param[in]     id            Ordering ID
  * \param[in,out] rsc_first     'First' resource in ordering (a clone)
  * \param[in]     action_first  'First' action in ordering
  * \param[in]     rsc_then      'Then' resource in ordering
  * \param[in]     action_then   'Then' action in ordering
  * \param[in]     flags         Ordering flags
  * \param[in]     clone_min     Minimum required instances of 'first'
  */
 static void
 clone_min_ordering(const char *id,
                    pcmk_resource_t *rsc_first, const char *action_first,
                    pcmk_resource_t *rsc_then, const char *action_then,
                    uint32_t flags, int clone_min)
 {
     // Create a pseudo-action for when the minimum instances are active
     char *task = crm_strdup_printf(PCMK_ACTION_CLONE_ONE_OR_MORE ":%s", id);
     pcmk_action_t *clone_min_met = get_pseudo_op(task,
                                                  rsc_first->priv->scheduler);
 
     free(task);
 
     /* Require the pseudo-action to have the required number of actions to be
      * considered runnable before allowing the pseudo-action to be runnable.
      */
     clone_min_met->required_runnable_before = clone_min;
 
     // Order the actions for each clone instance before the pseudo-action
     for (GList *iter = rsc_first->priv->children;
          iter != NULL; iter = iter->next) {
 
         pcmk_resource_t *child = iter->data;
 
         pcmk__new_ordering(child, pcmk__op_key(child->id, action_first, 0),
                            NULL, NULL, NULL, clone_min_met,
                            pcmk__ar_min_runnable
                            |pcmk__ar_first_implies_then_graphed,
                            rsc_first->priv->scheduler);
     }
 
     // Order "then" action after the pseudo-action (if runnable)
     pcmk__new_ordering(NULL, NULL, clone_min_met, rsc_then,
                        pcmk__op_key(rsc_then->id, action_then, 0),
                        NULL, flags|pcmk__ar_unrunnable_first_blocks,
                        rsc_first->priv->scheduler);
 }
 
 /*!
  * \internal
  * \brief Create new ordering for inverse of symmetric constraint
  *
  * \param[in]     id            Ordering ID (for logging only)
  * \param[in]     kind          Ordering kind
  * \param[in]     rsc_first     'First' resource in ordering (a clone)
  * \param[in]     action_first  'First' action in ordering
  * \param[in,out] rsc_then      'Then' resource in ordering
  * \param[in]     action_then   'Then' action in ordering
  */
 static void
 inverse_ordering(const char *id, enum pe_order_kind kind,
                  pcmk_resource_t *rsc_first, const char *action_first,
                  pcmk_resource_t *rsc_then, const char *action_then)
 {
     action_then = invert_action(action_then);
     action_first = invert_action(action_first);
     if ((action_then == NULL) || (action_first == NULL)) {
         pcmk__config_warn("Cannot invert constraint '%s' "
                           "(please specify inverse manually)", id);
     } else {
         uint32_t flags = ordering_flags_for_kind(kind, action_first,
                                                  ordering_symmetric_inverse);
 
         pcmk__order_resource_actions(rsc_then, action_then, rsc_first,
                                      action_first, flags);
     }
 }
 
 static void
 unpack_simple_rsc_order(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
 {
     pcmk_resource_t *rsc_then = NULL;
     pcmk_resource_t *rsc_first = NULL;
     int min_required_before = 0;
     enum pe_order_kind kind = pe_order_kind_mandatory;
     uint32_t flags = pcmk__ar_none;
     enum ordering_symmetry symmetry;
 
     const char *action_then = NULL;
     const char *action_first = NULL;
     const char *id = NULL;
 
     CRM_CHECK(xml_obj != NULL, return);
 
     id = crm_element_value(xml_obj, PCMK_XA_ID);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " PCMK_XA_ID,
                          xml_obj->name);
         return;
     }
 
     rsc_first = get_ordering_resource(xml_obj, PCMK_XA_FIRST, scheduler);
     if (rsc_first == NULL) {
         return;
     }
 
     rsc_then = get_ordering_resource(xml_obj, PCMK_XA_THEN, scheduler);
     if (rsc_then == NULL) {
         return;
     }
 
     action_first = crm_element_value(xml_obj, PCMK_XA_FIRST_ACTION);
     if (action_first == NULL) {
         action_first = PCMK_ACTION_START;
     }
 
     action_then = crm_element_value(xml_obj, PCMK_XA_THEN_ACTION);
     if (action_then == NULL) {
         action_then = action_first;
     }
 
     kind = get_ordering_type(xml_obj);
 
     symmetry = get_ordering_symmetry(xml_obj, kind, NULL);
     flags = ordering_flags_for_kind(kind, action_first, symmetry);
 
     /* If there is a minimum number of instances that must be runnable before
      * the 'then' action is runnable, we use a pseudo-action for convenience:
      * minimum number of clone instances have runnable actions ->
      * pseudo-action is runnable -> dependency is runnable.
      */
     min_required_before = get_minimum_first_instances(rsc_first, xml_obj);
     if (min_required_before > 0) {
         clone_min_ordering(id, rsc_first, action_first, rsc_then, action_then,
                            flags, min_required_before);
     } else {
         pcmk__order_resource_actions(rsc_first, action_first, rsc_then,
                                      action_then, flags);
     }
 
     if (symmetry == ordering_symmetric) {
         inverse_ordering(id, kind, rsc_first, action_first,
                          rsc_then, action_then);
     }
 }
 
 /*!
  * \internal
  * \brief Create a new ordering between two actions
  *
  * \param[in,out] first_rsc          Resource for 'first' action (if NULL and
  *                                   \p first_action is a resource action, that
  *                                   resource will be used)
  * \param[in,out] first_action_task  Action key for 'first' action (if NULL and
  *                                   \p first_action is not NULL, its UUID will
  *                                   be used)
  * \param[in,out] first_action       'first' action (if NULL, \p first_rsc and
  *                                   \p first_action_task must be set)
  *
  * \param[in]     then_rsc           Resource for 'then' action (if NULL and
  *                                   \p then_action is a resource action, that
  *                                   resource will be used)
  * \param[in,out] then_action_task   Action key for 'then' action (if NULL and
  *                                   \p then_action is not NULL, its UUID will
  *                                   be used)
  * \param[in]     then_action        'then' action (if NULL, \p then_rsc and
  *                                   \p then_action_task must be set)
  *
  * \param[in]     flags              Group of enum pcmk__action_relation_flags
  * \param[in,out] sched              Scheduler data to add ordering to
  *
  * \note This function takes ownership of first_action_task and
  *       then_action_task, which do not need to be freed by the caller.
  */
 void
 pcmk__new_ordering(pcmk_resource_t *first_rsc, char *first_action_task,
                    pcmk_action_t *first_action, pcmk_resource_t *then_rsc,
                    char *then_action_task, pcmk_action_t *then_action,
                    uint32_t flags, pcmk_scheduler_t *sched)
 {
     pcmk__action_relation_t *order = NULL;
 
     // One of action or resource must be specified for each side
     CRM_CHECK(((first_action != NULL) || (first_rsc != NULL))
               && ((then_action != NULL) || (then_rsc != NULL)),
               free(first_action_task); free(then_action_task); return);
 
     if ((first_rsc == NULL) && (first_action != NULL)) {
         first_rsc = first_action->rsc;
     }
     if ((then_rsc == NULL) && (then_action != NULL)) {
         then_rsc = then_action->rsc;
     }
 
     order = pcmk__assert_alloc(1, sizeof(pcmk__action_relation_t));
 
     order->id = sched->priv->next_ordering_id++;
     order->flags = flags;
     order->rsc1 = first_rsc;
     order->rsc2 = then_rsc;
     order->action1 = first_action;
     order->action2 = then_action;
     order->task1 = first_action_task;
     order->task2 = then_action_task;
 
     if ((order->task1 == NULL) && (first_action != NULL)) {
         order->task1 = strdup(first_action->uuid);
     }
 
     if ((order->task2 == NULL) && (then_action != NULL)) {
         order->task2 = strdup(then_action->uuid);
     }
 
     if ((order->rsc1 == NULL) && (first_action != NULL)) {
         order->rsc1 = first_action->rsc;
     }
 
     if ((order->rsc2 == NULL) && (then_action != NULL)) {
         order->rsc2 = then_action->rsc;
     }
 
     pcmk__rsc_trace(first_rsc, "Created ordering %d for %s then %s",
                     (sched->priv->next_ordering_id - 1),
                     pcmk__s(order->task1, "an underspecified action"),
                     pcmk__s(order->task2, "an underspecified action"));
 
     sched->priv->ordering_constraints =
         g_list_prepend(sched->priv->ordering_constraints, order);
     pcmk__order_migration_equivalents(order);
 }
 
 /*!
  * \brief Unpack a set in an ordering constraint
  *
  * \param[in]     set                   Set XML to unpack
  * \param[in]     parent_kind           \c PCMK_XE_RSC_ORDER XML \c PCMK_XA_KIND
  *                                      attribute
  * \param[in]     parent_symmetrical_s  \c PCMK_XE_RSC_ORDER XML
  *                                      \c PCMK_XA_SYMMETRICAL attribute
  * \param[in,out] scheduler             Scheduler data
  *
  * \return Standard Pacemaker return code
  */
 static int
 unpack_order_set(const xmlNode *set, enum pe_order_kind parent_kind,
                  const char *parent_symmetrical_s, pcmk_scheduler_t *scheduler)
 {
     GList *set_iter = NULL;
     GList *resources = NULL;
 
     pcmk_resource_t *last = NULL;
     pcmk_resource_t *resource = NULL;
 
     int local_kind = parent_kind;
     bool sequential = false;
     uint32_t flags = pcmk__ar_ordered;
     enum ordering_symmetry symmetry;
 
     char *key = NULL;
     const char *id = pcmk__xe_id(set);
     const char *action = crm_element_value(set, PCMK_XA_ACTION);
     const char *sequential_s = crm_element_value(set, PCMK_XA_SEQUENTIAL);
     const char *kind_s = crm_element_value(set, PCMK_XA_KIND);
 
     if (action == NULL) {
         action = PCMK_ACTION_START;
     }
 
     if (kind_s) {
         local_kind = get_ordering_type(set);
     }
     if (sequential_s == NULL) {
         sequential_s = "1";
     }
 
     sequential = crm_is_true(sequential_s);
 
     symmetry = get_ordering_symmetry(set, parent_kind, parent_symmetrical_s);
     flags = ordering_flags_for_kind(local_kind, action, symmetry);
 
     for (const xmlNode *xml_rsc = pcmk__xe_first_child(set,
                                                        PCMK_XE_RESOURCE_REF,
                                                        NULL, NULL);
          xml_rsc != NULL;
          xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
         EXPAND_CONSTRAINT_IDREF(id, resource, pcmk__xe_id(xml_rsc));
         resources = g_list_append(resources, resource);
     }
 
     if (pcmk__list_of_1(resources)) {
         crm_trace("Single set: %s", id);
         goto done;
     }
 
     set_iter = resources;
     while (set_iter != NULL) {
         resource = (pcmk_resource_t *) set_iter->data;
         set_iter = set_iter->next;
 
         key = pcmk__op_key(resource->id, action, 0);
 
         if (local_kind == pe_order_kind_serialize) {
             /* Serialize before everything that comes after */
 
             for (GList *iter = set_iter; iter != NULL; iter = iter->next) {
                 pcmk_resource_t *then_rsc = iter->data;
                 char *then_key = pcmk__op_key(then_rsc->id, action, 0);
 
                 pcmk__new_ordering(resource, strdup(key), NULL, then_rsc,
                                    then_key, NULL, flags, scheduler);
             }
 
         } else if (sequential) {
             if (last != NULL) {
                 pcmk__order_resource_actions(last, action, resource, action,
                                              flags);
             }
             last = resource;
         }
         free(key);
     }
 
     if (symmetry == ordering_asymmetric) {
         goto done;
     }
 
     last = NULL;
     action = invert_action(action);
 
     flags = ordering_flags_for_kind(local_kind, action,
                                     ordering_symmetric_inverse);
 
     set_iter = resources;
     while (set_iter != NULL) {
         resource = (pcmk_resource_t *) set_iter->data;
         set_iter = set_iter->next;
 
         if (sequential) {
             if (last != NULL) {
                 pcmk__order_resource_actions(resource, action, last, action,
                                              flags);
             }
             last = resource;
         }
     }
 
   done:
     g_list_free(resources);
     return pcmk_rc_ok;
 }
 
 /*!
  * \brief Order two resource sets relative to each other
  *
  * \param[in]     id         Ordering ID (for logging)
  * \param[in]     set1       First listed set
  * \param[in]     set2       Second listed set
  * \param[in]     kind       Ordering kind
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     symmetry   Which ordering symmetry applies to this relation
  *
  * \return Standard Pacemaker return code
  */
 static int
 order_rsc_sets(const char *id, const xmlNode *set1, const xmlNode *set2,
                enum pe_order_kind kind, pcmk_scheduler_t *scheduler,
                enum ordering_symmetry symmetry)
 {
 
     const xmlNode *xml_rsc = NULL;
     const xmlNode *xml_rsc_2 = NULL;
 
     pcmk_resource_t *rsc_1 = NULL;
     pcmk_resource_t *rsc_2 = NULL;
 
     const char *action_1 = crm_element_value(set1, PCMK_XA_ACTION);
     const char *action_2 = crm_element_value(set2, PCMK_XA_ACTION);
 
     uint32_t flags = pcmk__ar_none;
 
     bool require_all = true;
 
     (void) pcmk__xe_get_bool_attr(set1, PCMK_XA_REQUIRE_ALL, &require_all);
 
     if (action_1 == NULL) {
         action_1 = PCMK_ACTION_START;
     }
 
     if (action_2 == NULL) {
         action_2 = PCMK_ACTION_START;
     }
 
     if (symmetry == ordering_symmetric_inverse) {
         action_1 = invert_action(action_1);
         action_2 = invert_action(action_2);
     }
 
     if (pcmk__str_eq(PCMK_ACTION_STOP, action_1, pcmk__str_none)
         || pcmk__str_eq(PCMK_ACTION_DEMOTE, action_1, pcmk__str_none)) {
         /* Assuming: A -> ( B || C) -> D
          * The one-or-more logic only applies during the start/promote phase.
          * During shutdown neither B nor can shutdown until D is down, so simply
          * turn require_all back on.
          */
         require_all = true;
     }
 
     flags = ordering_flags_for_kind(kind, action_1, symmetry);
 
     /* If we have an unordered set1, whether it is sequential or not is
      * irrelevant in regards to set2.
      */
     if (!require_all) {
         char *task = crm_strdup_printf(PCMK_ACTION_ONE_OR_MORE ":%s",
                                        pcmk__xe_id(set1));
         pcmk_action_t *unordered_action = get_pseudo_op(task, scheduler);
 
         free(task);
         unordered_action->required_runnable_before = 1;
 
         for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
                                             NULL);
              xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
 
             /* Add an ordering constraint between every element in set1 and the
              * pseudo action. If any action in set1 is runnable the pseudo
              * action will be runnable.
              */
             pcmk__new_ordering(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
                                NULL, NULL, NULL, unordered_action,
                                pcmk__ar_min_runnable
                                |pcmk__ar_first_implies_then_graphed,
                                scheduler);
         }
         for (xml_rsc_2 = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
                                               NULL);
              xml_rsc_2 != NULL;
              xml_rsc_2 = pcmk__xe_next(xml_rsc_2, PCMK_XE_RESOURCE_REF)) {
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2));
 
             /* Add an ordering constraint between the pseudo-action and every
              * element in set2. If the pseudo-action is runnable, every action
              * in set2 will be runnable.
              */
             pcmk__new_ordering(NULL, NULL, unordered_action,
                                rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
                                NULL, flags|pcmk__ar_unrunnable_first_blocks,
                                scheduler);
         }
 
         return pcmk_rc_ok;
     }
 
     if (pcmk__xe_attr_is_true(set1, PCMK_XA_SEQUENTIAL)) {
         if (symmetry == ordering_symmetric_inverse) {
             // Get the first one
             xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
                                            NULL);
             if (xml_rsc != NULL) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
             }
 
         } else {
             // Get the last one
             const char *rid = NULL;
 
             for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF,
                                                 NULL, NULL);
                  xml_rsc != NULL;
                  xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
                 rid = pcmk__xe_id(xml_rsc);
             }
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
         }
     }
 
     if (pcmk__xe_attr_is_true(set2, PCMK_XA_SEQUENTIAL)) {
         if (symmetry == ordering_symmetric_inverse) {
             // Get the last one
             const char *rid = NULL;
 
             for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF,
                                                 NULL, NULL);
                  xml_rsc != NULL;
                  xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
                 rid = pcmk__xe_id(xml_rsc);
             }
             EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
 
         } else {
             // Get the first one
             xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
                                            NULL);
             if (xml_rsc != NULL) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc));
             }
         }
     }
 
     if ((rsc_1 != NULL) && (rsc_2 != NULL)) {
         pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2, flags);
 
     } else if (rsc_1 != NULL) {
         for (xml_rsc = pcmk__xe_first_child(set2, PCMK_XE_RESOURCE_REF, NULL,
                                             NULL);
              xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc));
             pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
                                          flags);
         }
 
     } else if (rsc_2 != NULL) {
         for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
                                             NULL);
              xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
             pcmk__order_resource_actions(rsc_1, action_1, rsc_2, action_2,
                                          flags);
         }
 
     } else {
         for (xml_rsc = pcmk__xe_first_child(set1, PCMK_XE_RESOURCE_REF, NULL,
                                             NULL);
              xml_rsc != NULL;
              xml_rsc = pcmk__xe_next(xml_rsc, PCMK_XE_RESOURCE_REF)) {
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, pcmk__xe_id(xml_rsc));
 
             for (xmlNode *xml_rsc_2 = pcmk__xe_first_child(set2,
                                                            PCMK_XE_RESOURCE_REF,
                                                            NULL, NULL);
                  xml_rsc_2 != NULL;
                  xml_rsc_2 = pcmk__xe_next(xml_rsc_2, PCMK_XE_RESOURCE_REF)) {
 
                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, pcmk__xe_id(xml_rsc_2));
                 pcmk__order_resource_actions(rsc_1, action_1, rsc_2,
                                              action_2, flags);
             }
         }
     }
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief If an ordering constraint uses resource tags, expand them
  *
  * \param[in,out] xml_obj       Ordering constraint XML
  * \param[out]    expanded_xml  Equivalent XML with tags expanded
  * \param[in]     scheduler     Scheduler data
  *
  * \return Standard Pacemaker return code (specifically, pcmk_rc_ok on success,
  *         and pcmk_rc_unpack_error on invalid configuration)
  */
 static int
 unpack_order_tags(xmlNode *xml_obj, xmlNode **expanded_xml,
                   const pcmk_scheduler_t *scheduler)
 {
     const char *id_first = NULL;
     const char *id_then = NULL;
     const char *action_first = NULL;
     const char *action_then = NULL;
 
     pcmk_resource_t *rsc_first = NULL;
     pcmk_resource_t *rsc_then = NULL;
     pcmk__idref_t *tag_first = NULL;
     pcmk__idref_t *tag_then = NULL;
 
     xmlNode *rsc_set_first = NULL;
     xmlNode *rsc_set_then = NULL;
     bool any_sets = false;
 
     // Check whether there are any resource sets with template or tag references
     *expanded_xml = pcmk__expand_tags_in_sets(xml_obj, scheduler);
     if (*expanded_xml != NULL) {
         crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER);
         return pcmk_rc_ok;
     }
 
     id_first = crm_element_value(xml_obj, PCMK_XA_FIRST);
     id_then = crm_element_value(xml_obj, PCMK_XA_THEN);
     if ((id_first == NULL) || (id_then == NULL)) {
         return pcmk_rc_ok;
     }
 
     if (!pcmk__valid_resource_or_tag(scheduler, id_first, &rsc_first,
                                      &tag_first)) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag",
                          pcmk__xe_id(xml_obj), id_first);
         return pcmk_rc_unpack_error;
     }
 
     if (!pcmk__valid_resource_or_tag(scheduler, id_then, &rsc_then,
                                      &tag_then)) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag",
                          pcmk__xe_id(xml_obj), id_then);
         return pcmk_rc_unpack_error;
     }
 
     if ((rsc_first != NULL) && (rsc_then != NULL)) {
         // Neither side references a template or tag
         return pcmk_rc_ok;
     }
 
     action_first = crm_element_value(xml_obj, PCMK_XA_FIRST_ACTION);
     action_then = crm_element_value(xml_obj, PCMK_XA_THEN_ACTION);
 
     *expanded_xml = pcmk__xml_copy(NULL, xml_obj);
 
     /* Convert template/tag reference in PCMK_XA_FIRST into constraint
      * PCMK_XE_RESOURCE_SET
      */
     if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_first, PCMK_XA_FIRST, true,
                           scheduler)) {
         pcmk__xml_free(*expanded_xml);
         *expanded_xml = NULL;
         return pcmk_rc_unpack_error;
     }
 
     if (rsc_set_first != NULL) {
         if (action_first != NULL) {
             /* Move PCMK_XA_FIRST_ACTION into converted PCMK_XE_RESOURCE_SET as
              * PCMK_XA_ACTION
              */
             crm_xml_add(rsc_set_first, PCMK_XA_ACTION, action_first);
             pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_FIRST_ACTION);
         }
         any_sets = true;
     }
 
     /* Convert template/tag reference in PCMK_XA_THEN into constraint
      * PCMK_XE_RESOURCE_SET
      */
     if (!pcmk__tag_to_set(*expanded_xml, &rsc_set_then, PCMK_XA_THEN, true,
                           scheduler)) {
         pcmk__xml_free(*expanded_xml);
         *expanded_xml = NULL;
         return pcmk_rc_unpack_error;
     }
 
     if (rsc_set_then != NULL) {
         if (action_then != NULL) {
             /* Move PCMK_XA_THEN_ACTION into converted PCMK_XE_RESOURCE_SET as
              * PCMK_XA_ACTION
              */
             crm_xml_add(rsc_set_then, PCMK_XA_ACTION, action_then);
             pcmk__xe_remove_attr(*expanded_xml, PCMK_XA_THEN_ACTION);
         }
         any_sets = true;
     }
 
     if (any_sets) {
         crm_log_xml_trace(*expanded_xml, "Expanded " PCMK_XE_RSC_ORDER);
     } else {
         pcmk__xml_free(*expanded_xml);
         *expanded_xml = NULL;
     }
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Unpack ordering constraint XML
  *
  * \param[in,out] xml_obj    Ordering constraint XML to unpack
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__unpack_ordering(xmlNode *xml_obj, pcmk_scheduler_t *scheduler)
 {
     xmlNode *set = NULL;
     xmlNode *last = NULL;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     const char *id = crm_element_value(xml_obj, PCMK_XA_ID);
     const char *invert = crm_element_value(xml_obj, PCMK_XA_SYMMETRICAL);
     enum pe_order_kind kind = get_ordering_type(xml_obj);
 
     enum ordering_symmetry symmetry = get_ordering_symmetry(xml_obj, kind,
                                                             NULL);
 
     // Expand any resource tags in the constraint XML
     if (unpack_order_tags(xml_obj, &expanded_xml, scheduler) != pcmk_rc_ok) {
         return;
     }
     if (expanded_xml != NULL) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
     }
 
     // If the constraint has resource sets, unpack them
     for (set = pcmk__xe_first_child(xml_obj, PCMK_XE_RESOURCE_SET, NULL, NULL);
          set != NULL; set = pcmk__xe_next(set, PCMK_XE_RESOURCE_SET)) {
 
         set = pcmk__xe_resolve_idref(set, scheduler->input);
         if ((set == NULL) // Configuration error, message already logged
             || (unpack_order_set(set, kind, invert, scheduler) != pcmk_rc_ok)) {
 
             if (expanded_xml != NULL) {
                 pcmk__xml_free(expanded_xml);
             }
             return;
         }
 
         if (last != NULL) {
 
             if (order_rsc_sets(id, last, set, kind, scheduler,
                                symmetry) != pcmk_rc_ok) {
                 if (expanded_xml != NULL) {
                     pcmk__xml_free(expanded_xml);
                 }
                 return;
             }
 
             if ((symmetry == ordering_symmetric)
                 && (order_rsc_sets(id, set, last, kind, scheduler,
                                    ordering_symmetric_inverse) != pcmk_rc_ok)) {
                 if (expanded_xml != NULL) {
                     pcmk__xml_free(expanded_xml);
                 }
                 return;
             }
 
         }
         last = set;
     }
 
     if (expanded_xml) {
         pcmk__xml_free(expanded_xml);
         xml_obj = orig_xml;
     }
 
     // If the constraint has no resource sets, unpack it as a simple ordering
     if (last == NULL) {
         return unpack_simple_rsc_order(xml_obj, scheduler);
     }
 }
 
 static bool
 ordering_is_invalid(pcmk_action_t *action, pcmk__related_action_t *input)
 {
     /* Prevent user-defined ordering constraints between resources
      * running in a guest node and the resource that defines that node.
      */
     if (!pcmk_is_set(input->flags, pcmk__ar_guest_allowed)
         && (input->action->rsc != NULL)
         && pcmk__rsc_corresponds_to_guest(action->rsc, input->action->node)) {
 
         pcmk__config_warn("Invalid ordering constraint between %s and %s",
                           input->action->rsc->id, action->rsc->id);
         return true;
     }
 
     /* If there's an order like
      * "rscB_stop node2"-> "load_stopped_node2" -> "rscA_migrate_to node1"
      *
      * then rscA is being migrated from node1 to node2, while rscB is being
      * migrated from node2 to node1. If there would be a graph loop,
      * break the order "load_stopped_node2" -> "rscA_migrate_to node1".
      */
     if ((input->flags == pcmk__ar_if_on_same_node_or_target)
         && (action->rsc != NULL)
         && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO, pcmk__str_none)
         && pcmk__graph_has_loop(action, action, input)) {
         return true;
     }
 
     return false;
 }
 
 void
 pcmk__disable_invalid_orderings(pcmk_scheduler_t *scheduler)
 {
     for (GList *iter = scheduler->priv->actions;
          iter != NULL; iter = iter->next) {
 
         pcmk_action_t *action = (pcmk_action_t *) iter->data;
         pcmk__related_action_t *input = NULL;
 
         for (GList *input_iter = action->actions_before;
              input_iter != NULL; input_iter = input_iter->next) {
 
             input = input_iter->data;
             if (ordering_is_invalid(action, input)) {
                 input->flags = pcmk__ar_none;
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief Order stops on a node before the node's shutdown
  *
  * \param[in,out] node         Node being shut down
  * \param[in]     shutdown_op  Shutdown action for node
  */
 void
 pcmk__order_stops_before_shutdown(pcmk_node_t *node, pcmk_action_t *shutdown_op)
 {
     for (GList *iter = node->priv->scheduler->priv->actions;
          iter != NULL; iter = iter->next) {
 
         pcmk_action_t *action = (pcmk_action_t *) iter->data;
 
         // Only stops on the node shutting down are relevant
         if (!pcmk__same_node(action->node, node)
             || !pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
             continue;
         }
 
         // Resources and nodes in maintenance mode won't be touched
 
         if (pcmk_is_set(action->rsc->flags, pcmk__rsc_maintenance)) {
             pcmk__rsc_trace(action->rsc,
                             "Not ordering %s before shutdown of %s because "
                             "resource in maintenance mode",
                             action->uuid, pcmk__node_name(node));
             continue;
 
         } else if (node->details->maintenance) {
             pcmk__rsc_trace(action->rsc,
                             "Not ordering %s before shutdown of %s because "
                             "node in maintenance mode",
                             action->uuid, pcmk__node_name(node));
             continue;
         }
 
         /* Don't touch a resource that is unmanaged or blocked, to avoid
          * blocking the shutdown (though if another action depends on this one,
          * we may still end up blocking)
+         *
+         * @TODO This "if" looks wrong, create a regression test for these cases
          */
         if (!pcmk_any_flags_set(action->rsc->flags,
                                 pcmk__rsc_managed|pcmk__rsc_blocked)) {
             pcmk__rsc_trace(action->rsc,
                             "Not ordering %s before shutdown of %s because "
                             "resource is unmanaged or blocked",
                             action->uuid, pcmk__node_name(node));
             continue;
         }
 
         pcmk__rsc_trace(action->rsc, "Ordering %s before shutdown of %s",
                         action->uuid, pcmk__node_name(node));
         pcmk__clear_action_flags(action, pcmk__action_optional);
         pcmk__new_ordering(action->rsc, NULL, action, NULL,
                            strdup(PCMK_ACTION_DO_SHUTDOWN), shutdown_op,
                            pcmk__ar_ordered|pcmk__ar_unrunnable_first_blocks,
                            node->priv->scheduler);
     }
 }
 
 /*!
  * \brief Find resource actions matching directly or as child
  *
  * \param[in] rsc           Resource to check
  * \param[in] original_key  Action key to search for (possibly referencing
  *                          parent of \rsc)
  *
  * \return Newly allocated list of matching actions
  * \note It is the caller's responsibility to free the result with g_list_free()
  */
 static GList *
 find_actions_by_task(const pcmk_resource_t *rsc, const char *original_key)
 {
     // Search under given task key directly
     GList *list = find_actions(rsc->priv->actions, original_key, NULL);
 
     if (list == NULL) {
         // Search again using this resource's ID
         char *key = NULL;
         char *task = NULL;
         guint interval_ms = 0;
 
         CRM_CHECK(parse_op_key(original_key, NULL, &task, &interval_ms),
                   return NULL);
         key = pcmk__op_key(rsc->id, task, interval_ms);
         list = find_actions(rsc->priv->actions, key, NULL);
         free(key);
         free(task);
     }
     return list;
 }
 
 /*!
  * \internal
  * \brief Order relevant resource actions after a given action
  *
  * \param[in,out] first_action  Action to order after (or NULL if none runnable)
  * \param[in]     rsc           Resource whose actions should be ordered
  * \param[in,out] order         Ordering constraint being applied
  */
 static void
 order_resource_actions_after(pcmk_action_t *first_action,
                              const pcmk_resource_t *rsc,
                              pcmk__action_relation_t *order)
 {
     GList *then_actions = NULL;
     uint32_t flags = pcmk__ar_none;
 
     CRM_CHECK((rsc != NULL) && (order != NULL), return);
 
     flags = order->flags;
     pcmk__rsc_trace(rsc, "Applying ordering %d for 'then' resource %s",
                     order->id, rsc->id);
 
     if (order->action2 != NULL) {
         then_actions = g_list_prepend(NULL, order->action2);
 
     } else {
         then_actions = find_actions_by_task(rsc, order->task2);
     }
 
     if (then_actions == NULL) {
         pcmk__rsc_trace(rsc, "Ignoring ordering %d: no %s actions found for %s",
                         order->id, order->task2, rsc->id);
         return;
     }
 
     if ((first_action != NULL) && (first_action->rsc == rsc)
         && pcmk_is_set(first_action->flags, pcmk__action_migration_abort)) {
 
         pcmk__rsc_trace(rsc,
                         "Detected dangling migration ordering (%s then %s %s)",
                         first_action->uuid, order->task2, rsc->id);
         pcmk__clear_relation_flags(flags, pcmk__ar_first_implies_then);
     }
 
     if ((first_action == NULL)
         && !pcmk_is_set(flags, pcmk__ar_first_implies_then)) {
 
         pcmk__rsc_debug(rsc,
                         "Ignoring ordering %d for %s: No first action found",
                         order->id, rsc->id);
         g_list_free(then_actions);
         return;
     }
 
     for (GList *iter = then_actions; iter != NULL; iter = iter->next) {
         pcmk_action_t *then_action_iter = (pcmk_action_t *) iter->data;
 
         if (first_action != NULL) {
             order_actions(first_action, then_action_iter, flags);
         } else {
             pcmk__clear_action_flags(then_action_iter, pcmk__action_runnable);
             crm_warn("%s of %s is unrunnable because there is no %s of %s "
                      "to order it after", then_action_iter->task, rsc->id,
                      order->task1, order->rsc1->id);
         }
     }
 
     g_list_free(then_actions);
 }
 
 static void
 rsc_order_first(pcmk_resource_t *first_rsc, pcmk__action_relation_t *order)
 {
     GList *first_actions = NULL;
     pcmk_action_t *first_action = order->action1;
     pcmk_resource_t *then_rsc = order->rsc2;
 
     pcmk__assert(first_rsc != NULL);
     pcmk__rsc_trace(first_rsc, "Applying ordering constraint %d (first: %s)",
                     order->id, first_rsc->id);
 
     if (first_action != NULL) {
         first_actions = g_list_prepend(NULL, first_action);
 
     } else {
         first_actions = find_actions_by_task(first_rsc, order->task1);
     }
 
     if ((first_actions == NULL) && (first_rsc == then_rsc)) {
         pcmk__rsc_trace(first_rsc,
                         "Ignoring constraint %d: first (%s for %s) not found",
                         order->id, order->task1, first_rsc->id);
 
     } else if (first_actions == NULL) {
         char *key = NULL;
         char *op_type = NULL;
         guint interval_ms = 0;
         enum rsc_role_e first_role;
 
         parse_op_key(order->task1, NULL, &op_type, &interval_ms);
         key = pcmk__op_key(first_rsc->id, op_type, interval_ms);
 
         first_role = first_rsc->priv->fns->state(first_rsc, TRUE);
         if ((first_role == pcmk_role_stopped)
             && pcmk__str_eq(op_type, PCMK_ACTION_STOP, pcmk__str_none)) {
             free(key);
             pcmk__rsc_trace(first_rsc,
                             "Ignoring constraint %d: first (%s for %s) "
                             "not found",
                             order->id, order->task1, first_rsc->id);
 
         } else if ((first_role == pcmk_role_unpromoted)
                    && pcmk__str_eq(op_type, PCMK_ACTION_DEMOTE,
                                    pcmk__str_none)) {
             free(key);
             pcmk__rsc_trace(first_rsc,
                             "Ignoring constraint %d: first (%s for %s) "
                             "not found",
                             order->id, order->task1, first_rsc->id);
 
         } else {
             pcmk__rsc_trace(first_rsc,
                             "Creating first (%s for %s) for constraint %d ",
                             order->task1, first_rsc->id, order->id);
             first_action = custom_action(first_rsc, key, op_type, NULL, TRUE,
                                          first_rsc->priv->scheduler);
             first_actions = g_list_prepend(NULL, first_action);
         }
 
         free(op_type);
     }
 
     if (then_rsc == NULL) {
         if (order->action2 == NULL) {
             pcmk__rsc_trace(first_rsc, "Ignoring constraint %d: then not found",
                             order->id);
             return;
         }
         then_rsc = order->action2->rsc;
     }
     for (GList *iter = first_actions; iter != NULL; iter = iter->next) {
         first_action = iter->data;
 
         if (then_rsc == NULL) {
             order_actions(first_action, order->action2, order->flags);
 
         } else {
             order_resource_actions_after(first_action, then_rsc, order);
         }
     }
 
     g_list_free(first_actions);
 }
 
 // GFunc to call pcmk__block_colocation_dependents()
 static void
 block_colocation_dependents(gpointer data, gpointer user_data)
 {
     pcmk__block_colocation_dependents(data);
 }
 
 // GFunc to call pcmk__update_action_for_orderings()
 static void
 update_action_for_orderings(gpointer data, gpointer user_data)
 {
     pcmk__update_action_for_orderings((pcmk_action_t *) data,
                                       (pcmk_scheduler_t *) user_data);
 }
 
 /*!
  * \internal
  * \brief Apply all ordering constraints
  *
  * \param[in,out] sched  Scheduler data
  */
 void
 pcmk__apply_orderings(pcmk_scheduler_t *sched)
 {
     crm_trace("Applying ordering constraints");
 
     /* Ordering constraints need to be processed in the order they were created.
      * rsc_order_first() and order_resource_actions_after() require the relevant
      * actions to already exist in some cases, but rsc_order_first() will create
      * the 'first' action in certain cases. Thus calling rsc_order_first() can
      * change the behavior of later-created orderings.
      *
      * Also, g_list_append() should be avoided for performance reasons, so we
      * prepend orderings when creating them and reverse the list here.
      *
      * @TODO This is brittle and should be carefully redesigned so that the
      * order of creation doesn't matter, and the reverse becomes unneeded.
      */
     sched->priv->ordering_constraints =
         g_list_reverse(sched->priv->ordering_constraints);
 
     for (GList *iter = sched->priv->ordering_constraints;
          iter != NULL; iter = iter->next) {
 
         pcmk__action_relation_t *order = iter->data;
         pcmk_resource_t *rsc = order->rsc1;
 
         if (rsc != NULL) {
             rsc_order_first(rsc, order);
             continue;
         }
 
         rsc = order->rsc2;
         if (rsc != NULL) {
             order_resource_actions_after(order->action1, rsc, order);
 
         } else {
             crm_trace("Applying ordering constraint %d (non-resource actions)",
                       order->id);
             order_actions(order->action1, order->action2, order->flags);
         }
     }
 
     g_list_foreach(sched->priv->actions, block_colocation_dependents, NULL);
 
     crm_trace("Ordering probes");
     pcmk__order_probes(sched);
 
     crm_trace("Updating %d actions", g_list_length(sched->priv->actions));
     g_list_foreach(sched->priv->actions, update_action_for_orderings, sched);
 
     pcmk__disable_invalid_orderings(sched);
 }
 
 /*!
  * \internal
  * \brief Order a given action after each action in a given list
  *
  * \param[in,out] after  "After" action
  * \param[in,out] list   List of "before" actions
  */
 void
 pcmk__order_after_each(pcmk_action_t *after, GList *list)
 {
     const char *after_desc = (after->task == NULL)? after->uuid : after->task;
 
     for (GList *iter = list; iter != NULL; iter = iter->next) {
         pcmk_action_t *before = (pcmk_action_t *) iter->data;
         const char *before_desc = before->task? before->task : before->uuid;
 
         crm_debug("Ordering %s on %s before %s on %s",
                   before_desc, pcmk__node_name(before->node),
                   after_desc, pcmk__node_name(after->node));
         order_actions(before, after, pcmk__ar_ordered);
     }
 }
 
 /*!
  * \internal
  * \brief Order promotions and demotions for restarts of a clone or bundle
  *
  * \param[in,out] rsc  Clone or bundle to order
  */
 void
 pcmk__promotable_restart_ordering(pcmk_resource_t *rsc)
 {
     // Order start and promote after all instances are stopped
     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
                                  rsc, PCMK_ACTION_START,
                                  pcmk__ar_ordered);
     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
                                  rsc, PCMK_ACTION_PROMOTE,
                                  pcmk__ar_ordered);
 
     // Order stop, start, and promote after all instances are demoted
     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
                                  rsc, PCMK_ACTION_STOP,
                                  pcmk__ar_ordered);
     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
                                  rsc, PCMK_ACTION_START,
                                  pcmk__ar_ordered);
     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTED,
                                  rsc, PCMK_ACTION_PROMOTE,
                                  pcmk__ar_ordered);
 
     // Order promote after all instances are started
     pcmk__order_resource_actions(rsc, PCMK_ACTION_RUNNING,
                                  rsc, PCMK_ACTION_PROMOTE,
                                  pcmk__ar_ordered);
 
     // Order demote after all instances are demoted
     pcmk__order_resource_actions(rsc, PCMK_ACTION_DEMOTE,
                                  rsc, PCMK_ACTION_DEMOTED,
                                  pcmk__ar_ordered);
 }
diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c
index ad68c6b49f..d95eef25a4 100644
--- a/lib/pacemaker/pcmk_sched_primitive.c
+++ b/lib/pacemaker/pcmk_sched_primitive.c
@@ -1,1716 +1,1717 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdbool.h>
 #include <stdint.h>                 // uint8_t, uint32_t
 
 #include <crm/common/xml.h>
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 static void stop_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
                           bool optional);
 static void start_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
                            bool optional);
 static void demote_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
                             bool optional);
 static void promote_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
                              bool optional);
 static void assert_role_error(pcmk_resource_t *rsc, pcmk_node_t *node,
                               bool optional);
 
 #define RSC_ROLE_MAX    (pcmk_role_promoted + 1)
 
 static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
     /* This array lists the immediate next role when transitioning from one role
      * to a target role. For example, when going from Stopped to Promoted, the
      * next role is Unpromoted, because the resource must be started before it
      * can be promoted. The current state then becomes Started, which is fed
      * into this array again, giving a next role of Promoted.
      *
      * Current role       Immediate next role   Final target role
      * ------------       -------------------   -----------------
      */
     /* Unknown */       { pcmk_role_unknown,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_stopped,    /* Started */
                           pcmk_role_stopped,    /* Unpromoted */
                           pcmk_role_stopped,    /* Promoted */
                         },
     /* Stopped */       { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_started,    /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_unpromoted, /* Promoted */
                         },
     /* Started */       { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_started,    /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_promoted,   /* Promoted */
                         },
     /* Unpromoted */    { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_stopped,    /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_promoted,   /* Promoted */
                         },
     /* Promoted  */     { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_unpromoted, /* Stopped */
                           pcmk_role_unpromoted, /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_promoted,   /* Promoted */
                         },
 };
 
 /*!
  * \internal
  * \brief Function to schedule actions needed for a role change
  *
  * \param[in,out] rsc       Resource whose role is changing
  * \param[in,out] node      Node where resource will be in its next role
  * \param[in]     optional  Whether scheduled actions should be optional
  */
 typedef void (*rsc_transition_fn)(pcmk_resource_t *rsc, pcmk_node_t *node,
                                   bool optional);
 
 static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
     /* This array lists the function needed to transition directly from one role
      * to another. NULL indicates that nothing is needed.
      *
      * Current role         Transition function             Next role
      * ------------         -------------------             ----------
      */
     /* Unknown */       {   assert_role_error,              /* Unknown */
                             stop_resource,                  /* Stopped */
                             assert_role_error,              /* Started */
                             assert_role_error,              /* Unpromoted */
                             assert_role_error,              /* Promoted */
                         },
     /* Stopped */       {   assert_role_error,              /* Unknown */
                             NULL,                           /* Stopped */
                             start_resource,                 /* Started */
                             start_resource,                 /* Unpromoted */
                             assert_role_error,              /* Promoted */
                         },
     /* Started */       {   assert_role_error,              /* Unknown */
                             stop_resource,                  /* Stopped */
                             NULL,                           /* Started */
                             NULL,                           /* Unpromoted */
                             promote_resource,               /* Promoted */
                         },
     /* Unpromoted */    {   assert_role_error,              /* Unknown */
                             stop_resource,                  /* Stopped */
                             stop_resource,                  /* Started */
                             NULL,                           /* Unpromoted */
                             promote_resource,               /* Promoted */
                         },
     /* Promoted  */     {   assert_role_error,              /* Unknown */
                             demote_resource,                /* Stopped */
                             demote_resource,                /* Started */
                             demote_resource,                /* Unpromoted */
                             NULL,                           /* Promoted */
                         },
 };
 
 /*!
  * \internal
  * \brief Get a list of a resource's allowed nodes sorted by node score
  *
  * \param[in] rsc  Resource to check
  *
  * \return List of allowed nodes sorted by node score
  */
 static GList *
 sorted_allowed_nodes(const pcmk_resource_t *rsc)
 {
     if (rsc->priv->allowed_nodes != NULL) {
         GList *nodes = g_hash_table_get_values(rsc->priv->allowed_nodes);
 
         if (nodes != NULL) {
             return pcmk__sort_nodes(nodes, pcmk__current_node(rsc));
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Assign a resource to its best allowed node, if possible
  *
  * \param[in,out] rsc           Resource to choose a node for
  * \param[in]     prefer        If not \c NULL, prefer this node when all else
  *                              equal
  * \param[in]     stop_if_fail  If \c true and \p rsc can't be assigned to a
  *                              node, set next role to stopped and update
  *                              existing actions
  *
  * \return true if \p rsc could be assigned to a node, otherwise false
  *
  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
  *       completely undo the assignment. A successful assignment can be either
  *       undone or left alone as final. A failed assignment has the same effect
  *       as calling pcmk__unassign_resource(); there are no side effects on
  *       roles or actions.
  */
 static bool
 assign_best_node(pcmk_resource_t *rsc, const pcmk_node_t *prefer,
                  bool stop_if_fail)
 {
     GList *nodes = NULL;
     pcmk_node_t *chosen = NULL;
     pcmk_node_t *best = NULL;
     const pcmk_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);
 
     if (prefer == NULL) {
         prefer = most_free_node;
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
         // We've already finished assignment of resources to nodes
         return rsc->priv->assigned_node != NULL;
     }
 
     // Sort allowed nodes by score
     nodes = sorted_allowed_nodes(rsc);
     if (nodes != NULL) {
         best = (pcmk_node_t *) nodes->data; // First node has best score
     }
 
     if ((prefer != NULL) && (nodes != NULL)) {
         // Get the allowed node version of prefer
         chosen = g_hash_table_lookup(rsc->priv->allowed_nodes,
                                      prefer->priv->id);
 
         if (chosen == NULL) {
             pcmk__rsc_trace(rsc, "Preferred node %s for %s was unknown",
                             pcmk__node_name(prefer), rsc->id);
 
         /* Favor the preferred node as long as its score is at least as good as
          * the best allowed node's.
          *
          * An alternative would be to favor the preferred node even if the best
          * node is better, when the best node's score is less than INFINITY.
          */
         } else if (chosen->assign->score < best->assign->score) {
             pcmk__rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
                             pcmk__node_name(chosen), rsc->id);
             chosen = NULL;
 
         } else if (!pcmk__node_available(chosen, true, false)) {
             pcmk__rsc_trace(rsc, "Preferred node %s for %s was unavailable",
                             pcmk__node_name(chosen), rsc->id);
             chosen = NULL;
 
         } else {
             pcmk__rsc_trace(rsc,
                             "Chose preferred node %s for %s "
                             "(ignoring %d candidates)",
                             pcmk__node_name(chosen), rsc->id,
                             g_list_length(nodes));
         }
     }
 
     if ((chosen == NULL) && (best != NULL)) {
         /* Either there is no preferred node, or the preferred node is not
          * suitable, but another node is allowed to run the resource.
          */
 
         chosen = best;
 
         if (!pcmk__is_unique_clone(rsc->priv->parent)
             && (chosen->assign->score > 0) // Zero not acceptable
             && pcmk__node_available(chosen, false, false)) {
             /* If the resource is already running on a node, prefer that node if
              * it is just as good as the chosen node.
              *
              * We don't do this for unique clone instances, because
              * pcmk__assign_instances() has already assigned instances to their
              * running nodes when appropriate, and if we get here, we don't want
              * remaining unassigned instances to prefer a node that's already
              * running another instance.
              */
             pcmk_node_t *running = pcmk__current_node(rsc);
 
             if (running == NULL) {
                 // Nothing to do
 
             } else if (!pcmk__node_available(running, true, false)) {
                 pcmk__rsc_trace(rsc,
                                 "Current node for %s (%s) can't run resources",
                                 rsc->id, pcmk__node_name(running));
 
             } else {
                 int nodes_with_best_score = 1;
 
                 for (GList *iter = nodes->next; iter; iter = iter->next) {
                     pcmk_node_t *allowed = (pcmk_node_t *) iter->data;
 
                     if (allowed->assign->score != chosen->assign->score) {
                         // The nodes are sorted by score, so no more are equal
                         break;
                     }
                     if (pcmk__same_node(allowed, running)) {
                         // Scores are equal, so prefer the current node
                         chosen = allowed;
                     }
                     nodes_with_best_score++;
                 }
 
                 if (nodes_with_best_score > 1) {
                     uint8_t log_level = LOG_INFO;
 
                     if (chosen->assign->score >= PCMK_SCORE_INFINITY) {
                         log_level = LOG_WARNING;
                     }
                     do_crm_log(log_level,
                                "Chose %s for %s from %d nodes with score %s",
                                pcmk__node_name(chosen), rsc->id,
                                nodes_with_best_score,
                                pcmk_readable_score(chosen->assign->score));
                 }
             }
         }
 
         pcmk__rsc_trace(rsc, "Chose %s for %s from %d candidates",
                         pcmk__node_name(chosen), rsc->id, g_list_length(nodes));
     }
 
     pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
     g_list_free(nodes);
     return rsc->priv->assigned_node != NULL;
 }
 
 /*!
  * \internal
  * \brief Apply a "this with" colocation to a node's allowed node scores
  *
  * \param[in,out] colocation  Colocation to apply
  * \param[in,out] rsc         Resource being assigned
  */
 static void
 apply_this_with(pcmk__colocation_t *colocation, pcmk_resource_t *rsc)
 {
     GHashTable *archive = NULL;
     pcmk_resource_t *other = colocation->primary;
 
     // In certain cases, we will need to revert the node scores
     if ((colocation->dependent_role >= pcmk_role_promoted)
         || ((colocation->score < 0)
             && (colocation->score > -PCMK_SCORE_INFINITY))) {
         archive = pcmk__copy_node_table(rsc->priv->allowed_nodes);
     }
 
     if (pcmk_is_set(other->flags, pcmk__rsc_unassigned)) {
         pcmk__rsc_trace(rsc,
                         "%s: Assigning colocation %s primary %s first"
                         "(score=%d role=%s)",
                         rsc->id, colocation->id, other->id,
                         colocation->score,
                         pcmk_role_text(colocation->dependent_role));
         other->priv->cmds->assign(other, NULL, true);
     }
 
     // Apply the colocation score to this resource's allowed node scores
     rsc->priv->cmds->apply_coloc_score(rsc, other, colocation, true);
     if ((archive != NULL)
         && !pcmk__any_node_available(rsc->priv->allowed_nodes)) {
         pcmk__rsc_info(rsc,
                        "%s: Reverting scores from colocation with %s "
                        "because no nodes allowed",
                        rsc->id, other->id);
         g_hash_table_destroy(rsc->priv->allowed_nodes);
         rsc->priv->allowed_nodes = archive;
         archive = NULL;
     }
     if (archive != NULL) {
         g_hash_table_destroy(archive);
     }
 }
 
 /*!
  * \internal
  * \brief Update a Pacemaker Remote node once its connection has been assigned
  *
  * \param[in] connection  Connection resource that has been assigned
  */
 static void
 remote_connection_assigned(const pcmk_resource_t *connection)
 {
     pcmk_node_t *remote_node = pcmk_find_node(connection->priv->scheduler,
                                               connection->id);
 
     CRM_CHECK(remote_node != NULL, return);
 
     if ((connection->priv->assigned_node != NULL)
         && (connection->priv->next_role != pcmk_role_stopped)) {
 
         crm_trace("Pacemaker Remote node %s will be online",
                   remote_node->priv->id);
         remote_node->details->online = TRUE;
         if (!pcmk_is_set(remote_node->priv->flags, pcmk__node_seen)) {
             // Avoid unnecessary fence, since we will attempt connection
             remote_node->details->unclean = FALSE;
         }
 
     } else {
         crm_trace("Pacemaker Remote node %s will be shut down "
                   "(%sassigned connection's next role is %s)",
                   remote_node->priv->id,
                   ((connection->priv->assigned_node == NULL)? "un" : ""),
                   pcmk_role_text(connection->priv->next_role));
         remote_node->details->shutdown = TRUE;
     }
 }
 
 /*!
  * \internal
  * \brief Assign a primitive resource to a node
  *
  * \param[in,out] rsc           Resource to assign to a node
  * \param[in]     prefer        Node to prefer, if all else is equal
  * \param[in]     stop_if_fail  If \c true and \p rsc can't be assigned to a
  *                              node, set next role to stopped and update
  *                              existing actions
  *
  * \return Node that \p rsc is assigned to, if assigned entirely to one node
  *
  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
  *       completely undo the assignment. A successful assignment can be either
  *       undone or left alone as final. A failed assignment has the same effect
  *       as calling pcmk__unassign_resource(); there are no side effects on
  *       roles or actions.
  */
 pcmk_node_t *
 pcmk__primitive_assign(pcmk_resource_t *rsc, const pcmk_node_t *prefer,
                        bool stop_if_fail)
 {
     GList *this_with_colocations = NULL;
     GList *with_this_colocations = NULL;
     GList *iter = NULL;
     pcmk_resource_t *parent = NULL;
     pcmk__colocation_t *colocation = NULL;
     pcmk_scheduler_t *scheduler = NULL;
 
     pcmk__assert(pcmk__is_primitive(rsc));
     scheduler = rsc->priv->scheduler;
     parent = rsc->priv->parent;
 
     // Never assign a child without parent being assigned first
     if ((parent != NULL) && !pcmk_is_set(parent->flags, pcmk__rsc_assigning)) {
         pcmk__rsc_debug(rsc, "%s: Assigning parent %s first",
                         rsc->id, parent->id);
         parent->priv->cmds->assign(parent, prefer, stop_if_fail);
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
         // Assignment has already been done
         const char *node_name = "no node";
 
         if (rsc->priv->assigned_node != NULL) {
             node_name = pcmk__node_name(rsc->priv->assigned_node);
         }
         pcmk__rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
         return rsc->priv->assigned_node;
     }
 
     // Ensure we detect assignment loops
     if (pcmk_is_set(rsc->flags, pcmk__rsc_assigning)) {
         pcmk__rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
         return NULL;
     }
     pcmk__set_rsc_flags(rsc, pcmk__rsc_assigning);
 
     pe__show_node_scores(true, rsc, "Pre-assignment",
                          rsc->priv->allowed_nodes, scheduler);
 
     this_with_colocations = pcmk__this_with_colocations(rsc);
     with_this_colocations = pcmk__with_this_colocations(rsc);
 
     // Apply mandatory colocations first, to satisfy as many as possible
     for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score <= -PCMK_SCORE_INFINITY)
             || (colocation->score >= PCMK_SCORE_INFINITY)) {
             apply_this_with(colocation, rsc);
         }
     }
     for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score <= -PCMK_SCORE_INFINITY)
             || (colocation->score >= PCMK_SCORE_INFINITY)) {
             pcmk__add_dependent_scores(colocation, rsc);
         }
     }
 
     pe__show_node_scores(true, rsc, "Mandatory-colocations",
                          rsc->priv->allowed_nodes, scheduler);
 
     // Then apply optional colocations
     for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score > -PCMK_SCORE_INFINITY)
             && (colocation->score < PCMK_SCORE_INFINITY)) {
             apply_this_with(colocation, rsc);
         }
     }
     for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score > -PCMK_SCORE_INFINITY)
             && (colocation->score < PCMK_SCORE_INFINITY)) {
             pcmk__add_dependent_scores(colocation, rsc);
         }
     }
 
     g_list_free(this_with_colocations);
     g_list_free(with_this_colocations);
 
     if (rsc->priv->next_role == pcmk_role_stopped) {
         pcmk__rsc_trace(rsc,
                         "Banning %s from all nodes because it will be stopped",
                         rsc->id);
         resource_location(rsc, NULL, -PCMK_SCORE_INFINITY,
                           PCMK_META_TARGET_ROLE, scheduler);
 
     } else if ((rsc->priv->next_role > rsc->priv->orig_role)
                && !pcmk_is_set(scheduler->flags, pcmk__sched_quorate)
                && (scheduler->no_quorum_policy == pcmk_no_quorum_freeze)) {
         crm_notice("Resource %s cannot be elevated from %s to %s due to "
                    PCMK_OPT_NO_QUORUM_POLICY "=" PCMK_VALUE_FREEZE,
                    rsc->id, pcmk_role_text(rsc->priv->orig_role),
                    pcmk_role_text(rsc->priv->next_role));
         pe__set_next_role(rsc, rsc->priv->orig_role,
                           PCMK_OPT_NO_QUORUM_POLICY "=" PCMK_VALUE_FREEZE);
     }
 
     pe__show_node_scores(!pcmk_is_set(scheduler->flags,
                                       pcmk__sched_output_scores),
                          rsc, __func__, rsc->priv->allowed_nodes, scheduler);
 
     // Unmanage resource if fencing is enabled but no device is configured
     if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)
         && !pcmk_is_set(scheduler->flags, pcmk__sched_have_fencing)) {
         pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
         // Unmanaged resources stay on their current node
         const char *reason = NULL;
         pcmk_node_t *assign_to = NULL;
 
         pe__set_next_role(rsc, rsc->priv->orig_role, "unmanaged");
         assign_to = pcmk__current_node(rsc);
         if (assign_to == NULL) {
             reason = "inactive";
         } else if (rsc->priv->orig_role == pcmk_role_promoted) {
             reason = "promoted";
         } else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
             reason = "failed";
         } else {
             reason = "active";
         }
         pcmk__rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
                        (assign_to? assign_to->priv->name : "no node"),
                        reason);
         pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);
 
     } else if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_all)) {
         // Must stop at some point, but be consistent with stop_if_fail
         if (stop_if_fail) {
             pcmk__rsc_debug(rsc,
                             "Forcing %s to stop: " PCMK_OPT_STOP_ALL_RESOURCES,
                             rsc->id);
         }
         pcmk__assign_resource(rsc, NULL, true, stop_if_fail);
 
     } else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
         // Assignment failed
         if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
             pcmk__rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
         } else if ((rsc->priv->active_nodes != NULL) && stop_if_fail) {
             pcmk__rsc_info(rsc, "Stopping removed resource %s", rsc->id);
         }
     }
 
     pcmk__clear_rsc_flags(rsc, pcmk__rsc_assigning);
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
         remote_connection_assigned(rsc);
     }
 
     return rsc->priv->assigned_node;
 }
 
 /*!
  * \internal
  * \brief Schedule actions to bring resource down and back to current role
  *
  * \param[in,out] rsc           Resource to restart
  * \param[in,out] current       Node that resource should be brought down on
  * \param[in]     need_stop     Whether the resource must be stopped
  * \param[in]     need_promote  Whether the resource must be promoted
  *
  * \return Role that resource would have after scheduled actions are taken
  */
 static void
 schedule_restart_actions(pcmk_resource_t *rsc, pcmk_node_t *current,
                          bool need_stop, bool need_promote)
 {
     enum rsc_role_e role = rsc->priv->orig_role;
     enum rsc_role_e next_role;
     rsc_transition_fn fn = NULL;
 
     pcmk__set_rsc_flags(rsc, pcmk__rsc_restarting);
 
     // Bring resource down to a stop on its current node
     while (role != pcmk_role_stopped) {
         next_role = rsc_state_matrix[role][pcmk_role_stopped];
         pcmk__rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
                         (need_stop? "required" : "optional"), rsc->id,
                         pcmk_role_text(role), pcmk_role_text(next_role));
         fn = rsc_action_matrix[role][next_role];
         if (fn == NULL) {
             break;
         }
         fn(rsc, current, !need_stop);
         role = next_role;
     }
 
     // Bring resource up to its next role on its next node
     while ((rsc->priv->orig_role <= rsc->priv->next_role)
            && (role != rsc->priv->orig_role)
            && !pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
         bool required = need_stop;
 
         next_role = rsc_state_matrix[role][rsc->priv->orig_role];
         if ((next_role == pcmk_role_promoted) && need_promote) {
             required = true;
         }
         pcmk__rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
                         (required? "required" : "optional"), rsc->id,
                         pcmk_role_text(role), pcmk_role_text(next_role));
         fn = rsc_action_matrix[role][next_role];
         if (fn == NULL) {
             break;
         }
         fn(rsc, rsc->priv->assigned_node, !required);
         role = next_role;
     }
 
     pcmk__clear_rsc_flags(rsc, pcmk__rsc_restarting);
 }
 
 /*!
  * \internal
  * \brief If a resource's next role is not explicitly specified, set a default
  *
  * \param[in,out] rsc  Resource to set next role for
  *
  * \return "explicit" if next role was explicitly set, otherwise "implicit"
  */
 static const char *
 set_default_next_role(pcmk_resource_t *rsc)
 {
     if (rsc->priv->next_role != pcmk_role_unknown) {
         return "explicit";
     }
 
     if (rsc->priv->assigned_node == NULL) {
         pe__set_next_role(rsc, pcmk_role_stopped, "assignment");
     } else {
         pe__set_next_role(rsc, pcmk_role_started, "assignment");
     }
     return "implicit";
 }
 
 /*!
  * \internal
  * \brief Create an action to represent an already pending start
  *
  * \param[in,out] rsc  Resource to create start action for
  */
 static void
 create_pending_start(pcmk_resource_t *rsc)
 {
     pcmk_action_t *start = NULL;
 
     pcmk__rsc_trace(rsc,
                     "Creating action for %s to represent already pending start",
                     rsc->id);
     start = start_action(rsc, rsc->priv->assigned_node, TRUE);
     pcmk__set_action_flags(start, pcmk__action_always_in_graph);
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to take a resource to its next role
  *
  * \param[in,out] rsc  Resource to schedule actions for
  */
 static void
 schedule_role_transition_actions(pcmk_resource_t *rsc)
 {
     enum rsc_role_e role = rsc->priv->orig_role;
 
     while (role != rsc->priv->next_role) {
         enum rsc_role_e next_role =
             rsc_state_matrix[role][rsc->priv->next_role];
         rsc_transition_fn fn = NULL;
 
         pcmk__rsc_trace(rsc,
                         "Creating action to take %s from %s to %s "
                         "(ending at %s)",
                         rsc->id, pcmk_role_text(role),
                         pcmk_role_text(next_role),
                         pcmk_role_text(rsc->priv->next_role));
         fn = rsc_action_matrix[role][next_role];
         if (fn == NULL) {
             break;
         }
         fn(rsc, rsc->priv->assigned_node, false);
         role = next_role;
     }
 }
 
 /*!
  * \internal
  * \brief Create all actions needed for a given primitive resource
  *
  * \param[in,out] rsc  Primitive resource to create actions for
  */
 void
 pcmk__primitive_create_actions(pcmk_resource_t *rsc)
 {
     bool need_stop = false;
     bool need_promote = false;
     bool is_moving = false;
     bool allow_migrate = false;
     bool multiply_active = false;
 
     pcmk_node_t *current = NULL;
     pcmk_node_t *migration_target = NULL;
     unsigned int num_all_active = 0;
     unsigned int num_clean_active = 0;
     const char *next_role_source = NULL;
 
     pcmk__assert(pcmk__is_primitive(rsc));
 
     next_role_source = set_default_next_role(rsc);
     pcmk__rsc_trace(rsc,
                     "Creating all actions for %s transition from %s to %s "
                     "(%s) on %s",
                     rsc->id, pcmk_role_text(rsc->priv->orig_role),
                     pcmk_role_text(rsc->priv->next_role), next_role_source,
                     pcmk__node_name(rsc->priv->assigned_node));
 
     current = rsc->priv->fns->active_node(rsc, &num_all_active,
                                           &num_clean_active);
 
     g_list_foreach(rsc->priv->dangling_migration_sources,
                    pcmk__abort_dangling_migration, rsc);
 
     if ((current != NULL) && (rsc->priv->assigned_node != NULL)
         && !pcmk__same_node(current, rsc->priv->assigned_node)
         && (rsc->priv->next_role >= pcmk_role_started)) {
 
         pcmk__rsc_trace(rsc, "Moving %s from %s to %s",
                         rsc->id, pcmk__node_name(current),
                         pcmk__node_name(rsc->priv->assigned_node));
         is_moving = true;
         allow_migrate = pcmk__rsc_can_migrate(rsc, current);
 
         // This is needed even if migrating (though I'm not sure why ...)
         need_stop = true;
     }
 
     // Check whether resource is partially migrated and/or multiply active
     migration_target = rsc->priv->partial_migration_target;
     if ((rsc->priv->partial_migration_source != NULL)
         && (migration_target != NULL) && allow_migrate && (num_all_active == 2)
         && pcmk__same_node(current, rsc->priv->partial_migration_source)
         && pcmk__same_node(rsc->priv->assigned_node, migration_target)) {
         /* A partial migration is in progress, and the migration target remains
          * the same as when the migration began.
          */
         pcmk__rsc_trace(rsc,
                         "Partial migration of %s from %s to %s will continue",
                         rsc->id,
                         pcmk__node_name(rsc->priv->partial_migration_source),
                         pcmk__node_name(migration_target));
 
     } else if ((rsc->priv->partial_migration_source != NULL)
                || (migration_target != NULL)) {
         // A partial migration is in progress but can't be continued
 
         if (num_all_active > 2) {
             // The resource is migrating *and* multiply active!
             crm_notice("Forcing recovery of %s because it is migrating "
                        "from %s to %s and possibly active elsewhere",
                        rsc->id,
                        pcmk__node_name(rsc->priv->partial_migration_source),
                        pcmk__node_name(migration_target));
         } else {
             // The migration source or target isn't available
             crm_notice("Forcing recovery of %s because it can no longer "
                        "migrate from %s to %s",
                        rsc->id,
                        pcmk__node_name(rsc->priv->partial_migration_source),
                        pcmk__node_name(migration_target));
         }
         need_stop = true;
         rsc->priv->partial_migration_source = NULL;
         rsc->priv->partial_migration_target = NULL;
         allow_migrate = false;
 
     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
         multiply_active = (num_all_active > 1);
     } else {
         /* If a resource has PCMK_META_REQUIRES set to PCMK_VALUE_NOTHING or
          * PCMK_VALUE_QUORUM, don't consider it active on unclean nodes (similar
          * to how all resources behave when PCMK_OPT_STONITH_ENABLED is false).
          * We can start such resources elsewhere before fencing completes, and
          * if we considered the resource active on the failed node, we would
          * attempt recovery for being active on multiple nodes.
          */
         multiply_active = (num_clean_active > 1);
     }
 
     if (multiply_active) {
         const char *class = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
 
         // Resource was (possibly) incorrectly multiply active
         pcmk__sched_err(rsc->priv->scheduler,
                         "%s resource %s might be active on %u nodes (%s)",
                         pcmk__s(class, "Untyped"), rsc->id, num_all_active,
                         pcmk__multiply_active_text(rsc));
         crm_notice("For more information, see \"What are multiply active "
                    "resources?\" at "
                    "https://projects.clusterlabs.org/w/clusterlabs/faq/");
 
         switch (rsc->priv->multiply_active_policy) {
             case pcmk__multiply_active_restart:
                 need_stop = true;
                 break;
             case pcmk__multiply_active_unexpected:
                 need_stop = true; // stop_resource() will skip expected node
                 pcmk__set_rsc_flags(rsc, pcmk__rsc_stop_unexpected);
                 break;
             default:
                 break;
         }
 
     } else {
         pcmk__clear_rsc_flags(rsc, pcmk__rsc_stop_unexpected);
     }
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_start_pending)) {
         create_pending_start(rsc);
     }
 
     if (is_moving) {
         // Remaining tests are only for resources staying where they are
 
     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
         if (pcmk_is_set(rsc->flags, pcmk__rsc_stop_if_failed)) {
             need_stop = true;
             pcmk__rsc_trace(rsc, "Recovering %s", rsc->id);
         } else {
             pcmk__rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
             if (rsc->priv->next_role == pcmk_role_promoted) {
                 need_promote = true;
             }
         }
 
     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
         pcmk__rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
         need_stop = true;
 
     } else if ((rsc->priv->orig_role > pcmk_role_started)
                && (current != NULL)
                && (rsc->priv->assigned_node != NULL)) {
         pcmk_action_t *start = NULL;
 
         pcmk__rsc_trace(rsc, "Creating start action for promoted resource %s",
                         rsc->id);
         start = start_action(rsc, rsc->priv->assigned_node, TRUE);
         if (!pcmk_is_set(start->flags, pcmk__action_optional)) {
             // Recovery of a promoted resource
             pcmk__rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
             need_stop = true;
         }
     }
 
     // Create any actions needed to bring resource down and back up to same role
     schedule_restart_actions(rsc, current, need_stop, need_promote);
 
     // Create any actions needed to take resource from this role to the next
     schedule_role_transition_actions(rsc);
 
     pcmk__create_recurring_actions(rsc);
 
     if (allow_migrate) {
         pcmk__create_migration_actions(rsc, current);
     }
 }
 
 /*!
  * \internal
  * \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
  *
  * \param[in] rsc  Resource to check
  */
 static void
 rsc_avoids_remote_nodes(const pcmk_resource_t *rsc)
 {
     GHashTableIter iter;
     pcmk_node_t *node = NULL;
 
     g_hash_table_iter_init(&iter, rsc->priv->allowed_nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
         if (node->priv->remote != NULL) {
             node->assign->score = -PCMK_SCORE_INFINITY;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Return allowed nodes as (possibly sorted) list
  *
  * Convert a resource's hash table of allowed nodes to a list. If printing to
  * stdout, sort the list, to keep action ID numbers consistent for regression
  * test output (while avoiding the performance hit on a live cluster).
  *
  * \param[in] rsc       Resource to check for allowed nodes
  *
  * \return List of resource's allowed nodes
  * \note Callers should take care not to rely on the list being sorted.
  */
 static GList *
 allowed_nodes_as_list(const pcmk_resource_t *rsc)
 {
     GList *allowed_nodes = NULL;
 
     if (rsc->priv->allowed_nodes != NULL) {
         allowed_nodes = g_hash_table_get_values(rsc->priv->allowed_nodes);
     }
 
     if (!pcmk__is_daemon) {
         allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
     }
 
     return allowed_nodes;
 }
 
 /*!
  * \internal
  * \brief Create implicit constraints needed for a primitive resource
  *
  * \param[in,out] rsc  Primitive resource to create implicit constraints for
  */
 void
 pcmk__primitive_internal_constraints(pcmk_resource_t *rsc)
 {
     GList *allowed_nodes = NULL;
     bool check_unfencing = false;
     bool check_utilization = false;
     pcmk_scheduler_t *scheduler = NULL;
 
     pcmk__assert(pcmk__is_primitive(rsc));
     scheduler = rsc->priv->scheduler;
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
         pcmk__rsc_trace(rsc,
                         "Skipping implicit constraints for unmanaged resource "
                         "%s", rsc->id);
         return;
     }
 
     // Whether resource requires unfencing
     check_unfencing = !pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)
                       && pcmk_is_set(scheduler->flags,
                                      pcmk__sched_enable_unfencing)
                       && pcmk_is_set(rsc->flags, pcmk__rsc_needs_unfencing);
 
     // Whether a non-default placement strategy is used
     check_utilization = (g_hash_table_size(rsc->priv->utilization) > 0)
                          && !pcmk__str_eq(scheduler->priv->placement_strategy,
                                           PCMK_VALUE_DEFAULT, pcmk__str_casei);
 
     // Order stops before starts (i.e. restart)
     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0), NULL,
                        rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0), NULL,
                        pcmk__ar_ordered
                        |pcmk__ar_first_implies_then
                        |pcmk__ar_intermediate_stop, scheduler);
 
     // Promotable ordering: demote before stop, start before promote
     if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
                     pcmk__rsc_promotable)
         || (rsc->priv->orig_role > pcmk_role_unpromoted)) {
 
         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0),
                            NULL,
                            rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
                            NULL,
                            pcmk__ar_promoted_then_implies_first, scheduler);
 
         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
                            NULL,
                            rsc, pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0),
                            NULL,
                            pcmk__ar_unrunnable_first_blocks, scheduler);
     }
 
     // Don't clear resource history if probing on same node
     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
                        NULL, rsc,
                        pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
                        NULL,
                        pcmk__ar_if_on_same_node|pcmk__ar_then_cancels_first,
                        scheduler);
 
     // Certain checks need allowed nodes
     if (check_unfencing || check_utilization
         || (rsc->priv->launcher != NULL)) {
 
         allowed_nodes = allowed_nodes_as_list(rsc);
     }
 
     if (check_unfencing) {
         g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
     }
 
     if (check_utilization) {
         pcmk__create_utilization_constraints(rsc, allowed_nodes);
     }
 
     if (rsc->priv->launcher != NULL) {
         pcmk_resource_t *remote_rsc = NULL;
 
         if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
             // rsc is the implicit remote connection for a guest or bundle node
 
             /* Guest resources are not allowed to run on Pacemaker Remote nodes,
              * to avoid nesting remotes. However, bundles are allowed.
              */
             if (!pcmk_is_set(rsc->flags, pcmk__rsc_remote_nesting_allowed)) {
                 rsc_avoids_remote_nodes(rsc->priv->launcher);
             }
 
             /* If someone cleans up a guest or bundle node's launcher, we will
              * likely schedule a (re-)probe of the launcher and recovery of the
              * connection. Order the connection stop after the launcher probe,
              * so that if we detect the launcher running, we will trigger a new
              * transition and avoid the unnecessary recovery.
              */
             pcmk__order_resource_actions(rsc->priv->launcher,
                                          PCMK_ACTION_MONITOR,
                                          rsc, PCMK_ACTION_STOP,
                                          pcmk__ar_ordered);
 
         /* A user can specify that a resource must start on a Pacemaker Remote
          * node by explicitly configuring it with the PCMK__META_CONTAINER
          * meta-attribute. This is of questionable merit, since location
          * constraints can accomplish the same thing. But we support it, so here
          * we check whether a resource (that is not itself a remote connection)
          * has PCMK__META_CONTAINER set to a remote node or guest node resource.
          */
         } else if (pcmk_is_set(rsc->priv->launcher->flags,
                                pcmk__rsc_is_remote_connection)) {
             remote_rsc = rsc->priv->launcher;
         } else  {
             remote_rsc =
                 pe__resource_contains_guest_node(scheduler,
                                                  rsc->priv->launcher);
         }
 
         if (remote_rsc != NULL) {
             /* Force the resource on the Pacemaker Remote node instead of
              * colocating the resource with the launcher.
              */
             for (GList *item = allowed_nodes; item; item = item->next) {
                 pcmk_node_t *node = item->data;
 
                 if (node->priv->remote != remote_rsc) {
                     node->assign->score = -PCMK_SCORE_INFINITY;
                 }
             }
 
         } else {
             /* This resource is either launched by a resource that does NOT
              * represent a Pacemaker Remote node, or a Pacemaker Remote
              * connection resource for a guest node or bundle.
              */
             int score;
 
             crm_trace("Order and colocate %s relative to its launcher %s",
                       rsc->id, rsc->priv->launcher->id);
 
             pcmk__new_ordering(rsc->priv->launcher,
                                pcmk__op_key(rsc->priv->launcher->id,
                                             PCMK_ACTION_START, 0),
                                NULL, rsc,
                                pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
                                NULL,
                                pcmk__ar_first_implies_then
                                |pcmk__ar_unrunnable_first_blocks, scheduler);
 
             pcmk__new_ordering(rsc,
                                pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
                                NULL,
                                rsc->priv->launcher,
                                pcmk__op_key(rsc->priv->launcher->id,
                                             PCMK_ACTION_STOP, 0),
                                NULL, pcmk__ar_then_implies_first, scheduler);
 
-            if (pcmk_is_set(rsc->flags, pcmk__rsc_remote_nesting_allowed)) {
+            if (pcmk_is_set(rsc->flags, pcmk__rsc_remote_nesting_allowed)
+                /* @TODO: && non-bundle Pacemaker Remote nodes exist */) {
                 score = 10000;    /* Highly preferred but not essential */
             } else {
                 score = PCMK_SCORE_INFINITY; // Force to run on same host
             }
             pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
                                  rsc->priv->launcher, NULL, NULL,
                                  pcmk__coloc_influence);
         }
     }
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)
         || pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
         /* Remote connections and fencing devices are not allowed to run on
          * Pacemaker Remote nodes
          */
         rsc_avoids_remote_nodes(rsc);
     }
     g_list_free(allowed_nodes);
 }
 
 /*!
  * \internal
  * \brief Apply a colocation's score to node scores or resource priority
  *
  * Given a colocation constraint, apply its score to the dependent's
  * allowed node scores (if we are still placing resources) or priority (if
  * we are choosing promotable clone instance roles).
  *
  * \param[in,out] dependent      Dependent resource in colocation
  * \param[in]     primary        Primary resource in colocation
  * \param[in]     colocation     Colocation constraint to apply
  * \param[in]     for_dependent  true if called on behalf of dependent
  *
  * \return The score added to the dependent's priority
  */
 int
 pcmk__primitive_apply_coloc_score(pcmk_resource_t *dependent,
                                   const pcmk_resource_t *primary,
                                   const pcmk__colocation_t *colocation,
                                   bool for_dependent)
 {
     enum pcmk__coloc_affects filter_results;
 
     pcmk__assert((dependent != NULL) && (primary != NULL)
                  && (colocation != NULL));
 
     if (for_dependent) {
         // Always process on behalf of primary resource
         return primary->priv->cmds->apply_coloc_score(dependent, primary,
                                                       colocation, false);
     }
 
     filter_results = pcmk__colocation_affects(dependent, primary, colocation,
                                               false);
     pcmk__rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
                     ((colocation->score > 0)? "Colocating" : "Anti-colocating"),
                     dependent->id, primary->id, colocation->id,
                     colocation->score,
                     filter_results);
 
     switch (filter_results) {
         case pcmk__coloc_affects_role:
             return pcmk__apply_coloc_to_priority(dependent, primary,
                                                  colocation);
 
         case pcmk__coloc_affects_location:
             pcmk__apply_coloc_to_scores(dependent, primary, colocation);
             return 0;
 
         default: // pcmk__coloc_affects_nothing
             return 0;
     }
 }
 
 /* Primitive implementation of
  * pcmk__assignment_methods_t:with_this_colocations()
  */
 void
 pcmk__with_primitive_colocations(const pcmk_resource_t *rsc,
                                  const pcmk_resource_t *orig_rsc, GList **list)
 {
     const pcmk_resource_t *parent = NULL;
 
     pcmk__assert(pcmk__is_primitive(rsc) && (list != NULL));
     parent = rsc->priv->parent;
 
     if (rsc == orig_rsc) {
         /* For the resource itself, add all of its own colocations and relevant
          * colocations from its parent (if any).
          */
         pcmk__add_with_this_list(list, rsc->priv->with_this_colocations,
                                  orig_rsc);
         if (parent != NULL) {
             parent->priv->cmds->with_this_colocations(parent, orig_rsc, list);
         }
     } else {
         // For an ancestor, add only explicitly configured constraints
         for (GList *iter = rsc->priv->with_this_colocations;
              iter != NULL; iter = iter->next) {
             pcmk__colocation_t *colocation = iter->data;
 
             if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
                 pcmk__add_with_this(list, colocation, orig_rsc);
             }
         }
     }
 }
 
 /* Primitive implementation of
  * pcmk__assignment_methods_t:this_with_colocations()
  */
 void
 pcmk__primitive_with_colocations(const pcmk_resource_t *rsc,
                                  const pcmk_resource_t *orig_rsc, GList **list)
 {
     const pcmk_resource_t *parent = NULL;
 
     pcmk__assert(pcmk__is_primitive(rsc) && (list != NULL));
     parent = rsc->priv->parent;
 
     if (rsc == orig_rsc) {
         /* For the resource itself, add all of its own colocations and relevant
          * colocations from its parent (if any).
          */
         pcmk__add_this_with_list(list, rsc->priv->this_with_colocations,
                                  orig_rsc);
         if (parent != NULL) {
             parent->priv->cmds->this_with_colocations(parent, orig_rsc, list);
         }
     } else {
         // For an ancestor, add only explicitly configured constraints
         for (GList *iter = rsc->priv->this_with_colocations;
              iter != NULL; iter = iter->next) {
             pcmk__colocation_t *colocation = iter->data;
 
             if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
                 pcmk__add_this_with(list, colocation, orig_rsc);
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief Return action flags for a given primitive resource action
  *
  * \param[in,out] action  Action to get flags for
  * \param[in]     node    If not NULL, limit effects to this node (ignored)
  *
  * \return Flags appropriate to \p action on \p node
  */
 uint32_t
 pcmk__primitive_action_flags(pcmk_action_t *action, const pcmk_node_t *node)
 {
     pcmk__assert(action != NULL);
     return (uint32_t) action->flags;
 }
 
 /*!
  * \internal
  * \brief Check whether a node is a multiply active resource's expected node
  *
  * \param[in] rsc  Resource to check
  * \param[in] node  Node to check
  *
  * \return \c true if \p rsc is multiply active with
  *         \c PCMK_META_MULTIPLE_ACTIVE set to \c PCMK_VALUE_STOP_UNEXPECTED,
  *         and \p node is the node where it will remain active
  * \note This assumes that the resource's next role cannot be changed to stopped
  *       after this is called, which should be reasonable if status has already
  *       been unpacked and resources have been assigned to nodes.
  */
 static bool
 is_expected_node(const pcmk_resource_t *rsc, const pcmk_node_t *node)
 {
     return pcmk_all_flags_set(rsc->flags,
                               pcmk__rsc_stop_unexpected|pcmk__rsc_restarting)
            && (rsc->priv->next_role > pcmk_role_stopped)
            && pcmk__same_node(rsc->priv->assigned_node, node);
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to stop a resource wherever it is active
  *
  * \param[in,out] rsc       Resource being stopped
  * \param[in]     node      Node where resource is being stopped (ignored)
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 stop_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
 {
     for (GList *iter = rsc->priv->active_nodes;
          iter != NULL; iter = iter->next) {
 
         pcmk_node_t *current = (pcmk_node_t *) iter->data;
         pcmk_action_t *stop = NULL;
 
         if (is_expected_node(rsc, current)) {
             /* We are scheduling restart actions for a multiply active resource
              * with PCMK_META_MULTIPLE_ACTIVE=PCMK_VALUE_STOP_UNEXPECTED, and
              * this is where it should not be stopped.
              */
             pcmk__rsc_trace(rsc,
                             "Skipping stop of multiply active resource %s "
                             "on expected node %s",
                             rsc->id, pcmk__node_name(current));
             continue;
         }
 
         if (rsc->priv->partial_migration_target != NULL) {
             // Continue migration if node originally was and remains target
             if (pcmk__same_node(current, rsc->priv->partial_migration_target)
                 && pcmk__same_node(current, rsc->priv->assigned_node)) {
                 pcmk__rsc_trace(rsc,
                                 "Skipping stop of %s on %s "
                                 "because partial migration there will continue",
                                 rsc->id, pcmk__node_name(current));
                 continue;
             } else {
                 pcmk__rsc_trace(rsc,
                                 "Forcing stop of %s on %s "
                                 "because migration target changed",
                                 rsc->id, pcmk__node_name(current));
                 optional = false;
             }
         }
 
         pcmk__rsc_trace(rsc, "Scheduling stop of %s on %s",
                         rsc->id, pcmk__node_name(current));
         stop = stop_action(rsc, current, optional);
 
         if (rsc->priv->assigned_node == NULL) {
             pe_action_set_reason(stop, "node availability", true);
         } else if (pcmk_all_flags_set(rsc->flags, pcmk__rsc_restarting
                                                   |pcmk__rsc_stop_unexpected)) {
             /* We are stopping a multiply active resource on a node that is
              * not its expected node, and we are still scheduling restart
              * actions, so the stop is for being multiply active.
              */
             pe_action_set_reason(stop, "being multiply active", true);
         }
 
         if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
             pcmk__clear_action_flags(stop, pcmk__action_runnable);
         }
 
         if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_unfencing)) {
             pcmk_action_t *unfence = pe_fence_op(current, PCMK_ACTION_ON, true,
                                                  NULL, false,
                                                  rsc->priv->scheduler);
 
             order_actions(stop, unfence, pcmk__ar_then_implies_first);
             if (!pcmk__node_unfenced(current)) {
                 pcmk__sched_err(rsc->priv->scheduler,
                                 "Stopping %s until %s can be unfenced",
                                 rsc->id, pcmk__node_name(current));
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to start a resource on a node
  *
  * \param[in,out] rsc       Resource being started
  * \param[in,out] node      Node where resource should be started
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 start_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
 {
     pcmk_action_t *start = NULL;
 
     pcmk__assert(node != NULL);
 
     pcmk__rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
                     (optional? "optional" : "required"), rsc->id,
                     pcmk__node_name(node), node->assign->score);
     start = start_action(rsc, node, TRUE);
 
     pcmk__order_vs_unfence(rsc, node, start, pcmk__ar_first_implies_then);
 
     if (pcmk_is_set(start->flags, pcmk__action_runnable) && !optional) {
         pcmk__clear_action_flags(start, pcmk__action_optional);
     }
 
     if (is_expected_node(rsc, node)) {
         /* This could be a problem if the start becomes necessary for other
          * reasons later.
          */
         pcmk__rsc_trace(rsc,
                         "Start of multiply active resouce %s "
                         "on expected node %s will be a pseudo-action",
                         rsc->id, pcmk__node_name(node));
         pcmk__set_action_flags(start, pcmk__action_pseudo);
     }
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to promote a resource on a node
  *
  * \param[in,out] rsc       Resource being promoted
  * \param[in]     node      Node where resource should be promoted
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 promote_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
 {
     GList *iter = NULL;
     GList *action_list = NULL;
     bool runnable = true;
 
     pcmk__assert(node != NULL);
 
     // Any start must be runnable for promotion to be runnable
     action_list = pe__resource_actions(rsc, node, PCMK_ACTION_START, true);
     for (iter = action_list; iter != NULL; iter = iter->next) {
         pcmk_action_t *start = (pcmk_action_t *) iter->data;
 
         if (!pcmk_is_set(start->flags, pcmk__action_runnable)) {
             runnable = false;
         }
     }
     g_list_free(action_list);
 
     if (runnable) {
         pcmk_action_t *promote = promote_action(rsc, node, optional);
 
         pcmk__rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
                         (optional? "optional" : "required"), rsc->id,
                         pcmk__node_name(node));
 
         if (is_expected_node(rsc, node)) {
             /* This could be a problem if the promote becomes necessary for
              * other reasons later.
              */
             pcmk__rsc_trace(rsc,
                             "Promotion of multiply active resouce %s "
                             "on expected node %s will be a pseudo-action",
                             rsc->id, pcmk__node_name(node));
             pcmk__set_action_flags(promote, pcmk__action_pseudo);
         }
     } else {
         pcmk__rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
                         rsc->id, pcmk__node_name(node));
         action_list = pe__resource_actions(rsc, node, PCMK_ACTION_PROMOTE,
                                            true);
         for (iter = action_list; iter != NULL; iter = iter->next) {
             pcmk_action_t *promote = (pcmk_action_t *) iter->data;
 
             pcmk__clear_action_flags(promote, pcmk__action_runnable);
         }
         g_list_free(action_list);
     }
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to demote a resource wherever it is active
  *
  * \param[in,out] rsc       Resource being demoted
  * \param[in]     node      Node where resource should be demoted (ignored)
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 demote_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
 {
     /* Since this will only be called for a primitive (possibly as an instance
      * of a collective resource), the resource is multiply active if it is
      * running on more than one node, so we want to demote on all of them as
      * part of recovery, regardless of which one is the desired node.
      */
     for (GList *iter = rsc->priv->active_nodes;
          iter != NULL; iter = iter->next) {
 
         pcmk_node_t *current = (pcmk_node_t *) iter->data;
 
         if (is_expected_node(rsc, current)) {
             pcmk__rsc_trace(rsc,
                             "Skipping demote of multiply active resource %s "
                             "on expected node %s",
                             rsc->id, pcmk__node_name(current));
         } else {
             pcmk__rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
                             (optional? "optional" : "required"), rsc->id,
                             pcmk__node_name(current));
             demote_action(rsc, current, optional);
         }
     }
 }
 
 static void
 assert_role_error(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
 {
     pcmk__assert(false);
 }
 
 /*!
  * \internal
  * \brief Schedule cleanup of a resource
  *
  * \param[in,out] rsc       Resource to clean up
  * \param[in]     node      Node to clean up on
  * \param[in]     optional  Whether clean-up should be optional
  */
 void
 pcmk__schedule_cleanup(pcmk_resource_t *rsc, const pcmk_node_t *node,
                        bool optional)
 {
     /* If the cleanup is required, its orderings are optional, because they're
      * relevant only if both actions are required. Conversely, if the cleanup is
      * optional, the orderings make the then action required if the first action
      * becomes required.
      */
     uint32_t flag = optional? pcmk__ar_first_implies_then : pcmk__ar_ordered;
 
     CRM_CHECK((rsc != NULL) && (node != NULL), return);
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
         pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
                         rsc->id, pcmk__node_name(node));
         return;
     }
 
     if (node->details->unclean || !node->details->online) {
         pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
                         rsc->id, pcmk__node_name(node));
         return;
     }
 
     crm_notice("Scheduling clean-up of %s on %s",
                rsc->id, pcmk__node_name(node));
     delete_action(rsc, node, optional);
 
     // stop -> clean-up -> start
     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
                                  rsc, PCMK_ACTION_DELETE, flag);
     pcmk__order_resource_actions(rsc, PCMK_ACTION_DELETE,
                                  rsc, PCMK_ACTION_START, flag);
 }
 
 /*!
  * \internal
  * \brief Add primitive meta-attributes relevant to graph actions to XML
  *
  * \param[in]     rsc  Primitive resource whose meta-attributes should be added
  * \param[in,out] xml  Transition graph action attributes XML to add to
  */
 void
 pcmk__primitive_add_graph_meta(const pcmk_resource_t *rsc, xmlNode *xml)
 {
     char *name = NULL;
     char *value = NULL;
     const pcmk_resource_t *parent = NULL;
 
     pcmk__assert(pcmk__is_primitive(rsc) && (xml != NULL));
 
     /* Clone instance numbers get set internally as meta-attributes, and are
      * needed in the transition graph (for example, to tell unique clone
      * instances apart).
      */
     value = g_hash_table_lookup(rsc->priv->meta, PCMK__META_CLONE);
     if (value != NULL) {
         name = crm_meta_name(PCMK__META_CLONE);
         crm_xml_add(xml, name, value);
         free(name);
     }
 
     // Not sure if this one is really needed ...
     value = g_hash_table_lookup(rsc->priv->meta, PCMK_META_REMOTE_NODE);
     if (value != NULL) {
         name = crm_meta_name(PCMK_META_REMOTE_NODE);
         crm_xml_add(xml, name, value);
         free(name);
     }
 
     /* The PCMK__META_CONTAINER meta-attribute can be set on the primitive
      * itself or one of its ancestors, so check them all and keep the highest.
      */
     for (parent = rsc; parent != NULL; parent = parent->priv->parent) {
         if (parent->priv->launcher != NULL) {
             crm_xml_add(xml, CRM_META "_" PCMK__META_CONTAINER,
                         parent->priv->launcher->id);
         }
     }
 
     /* Bundle replica children will get their external-ip set internally as a
      * meta-attribute. The graph action needs it, but under a different naming
      * convention than other meta-attributes.
      */
     value = g_hash_table_lookup(rsc->priv->meta, "external-ip");
     if (value != NULL) {
         crm_xml_add(xml, "pcmk_external_ip", value);
     }
 }
 
 // Primitive implementation of pcmk__assignment_methods_t:add_utilization()
 void
 pcmk__primitive_add_utilization(const pcmk_resource_t *rsc,
                                 const pcmk_resource_t *orig_rsc,
                                 GList *all_rscs, GHashTable *utilization)
 {
     pcmk__assert(pcmk__is_primitive(rsc) && (orig_rsc != NULL)
                  && (utilization != NULL));
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
         return;
     }
 
     pcmk__rsc_trace(orig_rsc,
                     "%s: Adding primitive %s as colocated utilization",
                     orig_rsc->id, rsc->id);
     pcmk__release_node_capacity(utilization, rsc);
 }
 
 /*!
  * \internal
  * \brief Get epoch time of node's shutdown attribute (or now if none)
  *
  * \param[in,out] node  Node to check
  *
  * \return Epoch time corresponding to shutdown attribute if set or now if not
  */
 static time_t
 shutdown_time(pcmk_node_t *node)
 {
     const char *shutdown = pcmk__node_attr(node, PCMK__NODE_ATTR_SHUTDOWN, NULL,
                                            pcmk__rsc_node_current);
     time_t result = 0;
 
     if (shutdown != NULL) {
         long long result_ll;
         int rc = pcmk__scan_ll(shutdown, &result_ll, 0LL);
 
         if (rc == pcmk_rc_ok) {
             result = (time_t) result_ll;
         } else {
             crm_warn("Ignoring invalid value '%s' for %s "
                      PCMK__NODE_ATTR_SHUTDOWN " attribute: %s",
                      shutdown, pcmk__node_name(node), pcmk_rc_str(rc));
         }
     }
     return (result == 0)? get_effective_time(node->priv->scheduler) : result;
 }
 
 /*!
  * \internal
  * \brief Ban a resource from a node if it's not locked to the node
  *
  * \param[in]     data       Node to check
  * \param[in,out] user_data  Resource to check
  */
 static void
 ban_if_not_locked(gpointer data, gpointer user_data)
 {
     const pcmk_node_t *node = (const pcmk_node_t *) data;
     pcmk_resource_t *rsc = (pcmk_resource_t *) user_data;
 
     if (!pcmk__same_node(node, rsc->priv->lock_node)) {
         resource_location(rsc, node, -PCMK_SCORE_INFINITY,
                           PCMK_OPT_SHUTDOWN_LOCK, rsc->priv->scheduler);
     }
 }
 
 // Primitive implementation of pcmk__assignment_methods_t:shutdown_lock()
 void
 pcmk__primitive_shutdown_lock(pcmk_resource_t *rsc)
 {
     pcmk_scheduler_t *scheduler = NULL;
 
     pcmk__assert(pcmk__is_primitive(rsc));
     scheduler = rsc->priv->scheduler;
 
     // Fence devices and remote connections can't be locked
     if (pcmk_any_flags_set(rsc->flags, pcmk__rsc_fence_device
                                        |pcmk__rsc_is_remote_connection)) {
         return;
     }
 
     if (rsc->priv->lock_node != NULL) {
         // The lock was obtained from resource history
 
         if (rsc->priv->active_nodes != NULL) {
             /* The resource was started elsewhere even though it is now
              * considered locked. This shouldn't be possible, but as a
              * failsafe, we don't want to disturb the resource now.
              */
             pcmk__rsc_info(rsc,
                            "Cancelling shutdown lock "
                            "because %s is already active", rsc->id);
             pe__clear_resource_history(rsc, rsc->priv->lock_node);
             rsc->priv->lock_node = NULL;
             rsc->priv->lock_time = 0;
         }
 
     // Only a resource active on exactly one node can be locked
     } else if (pcmk__list_of_1(rsc->priv->active_nodes)) {
         pcmk_node_t *node = rsc->priv->active_nodes->data;
 
         if (node->details->shutdown) {
             if (node->details->unclean) {
                 pcmk__rsc_debug(rsc,
                                 "Not locking %s to unclean %s for shutdown",
                                 rsc->id, pcmk__node_name(node));
             } else {
                 rsc->priv->lock_node = node;
                 rsc->priv->lock_time = shutdown_time(node);
             }
         }
     }
 
     if (rsc->priv->lock_node == NULL) {
         // No lock needed
         return;
     }
 
     if (scheduler->priv->shutdown_lock_ms > 0U) {
         time_t lock_expiration = rsc->priv->lock_time
                                  + pcmk__timeout_ms2s(scheduler->priv->shutdown_lock_ms);
 
         pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
                        rsc->id, pcmk__node_name(rsc->priv->lock_node),
                        (long long) lock_expiration);
         pe__update_recheck_time(++lock_expiration, scheduler,
                                 "shutdown lock expiration");
     } else {
         pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown",
                        rsc->id, pcmk__node_name(rsc->priv->lock_node));
     }
 
     // If resource is locked to one node, ban it from all other nodes
     g_list_foreach(scheduler->nodes, ban_if_not_locked, rsc);
 }
diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c
index 8960c298f5..de30f4d8f1 100644
--- a/lib/pacemaker/pcmk_scheduler.c
+++ b/lib/pacemaker/pcmk_scheduler.c
@@ -1,880 +1,883 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/cib/internal.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 #include <crm/common/scheduler_internal.h>
 
 #include <glib.h>
 
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 #include "libpacemaker_private.h"
 
 CRM_TRACE_INIT_DATA(pacemaker);
 
 /*!
  * \internal
  * \brief Do deferred action checks after assignment
  *
  * When unpacking the resource history, the scheduler checks for resource
  * configurations that have changed since an action was run. However, at that
  * time, bundles using the REMOTE_CONTAINER_HACK don't have their final
  * parameter information, so instead they add a deferred check to a list. This
  * function processes one entry in that list.
  *
  * \param[in,out] rsc     Resource that action history is for
  * \param[in,out] node    Node that action history is for
  * \param[in]     rsc_op  Action history entry
  * \param[in]     check   Type of deferred check to do
  */
 static void
 check_params(pcmk_resource_t *rsc, pcmk_node_t *node, const xmlNode *rsc_op,
              enum pcmk__check_parameters check)
 {
     const char *reason = NULL;
     pcmk__op_digest_t *digest_data = NULL;
 
     switch (check) {
         case pcmk__check_active:
             if (pcmk__check_action_config(rsc, node, rsc_op)
                 && pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
                                     NULL)) {
                 reason = "action definition changed";
             }
             break;
 
         case pcmk__check_last_failure:
             digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
                                                 rsc->priv->scheduler);
             switch (digest_data->rc) {
                 case pcmk__digest_unknown:
                     crm_trace("Resource %s history entry %s on %s has "
                               "no digest to compare",
                               rsc->id, pcmk__xe_id(rsc_op), node->priv->id);
                     break;
                 case pcmk__digest_match:
                     break;
                 default:
                     reason = "resource parameters have changed";
                     break;
             }
             break;
     }
     if (reason != NULL) {
         pe__clear_failcount(rsc, node, reason, rsc->priv->scheduler);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a resource has failcount clearing scheduled on a node
  *
  * \param[in] node  Node to check
  * \param[in] rsc   Resource to check
  *
  * \return true if \p rsc has failcount clearing scheduled on \p node,
  *         otherwise false
  */
 static bool
 failcount_clear_action_exists(const pcmk_node_t *node,
                               const pcmk_resource_t *rsc)
 {
     GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
                                        TRUE);
 
     if (list != NULL) {
         g_list_free(list);
         return true;
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Ban a resource from a node if it reached its failure threshold there
  *
  * \param[in,out] data       Resource to check failure threshold for
  * \param[in]     user_data  Node to check resource on
  */
 static void
 check_failure_threshold(gpointer data, gpointer user_data)
 {
     pcmk_resource_t *rsc = data;
     const pcmk_node_t *node = user_data;
 
     // If this is a collective resource, apply recursively to children instead
     if (rsc->priv->children != NULL) {
         g_list_foreach(rsc->priv->children, check_failure_threshold,
                        user_data);
         return;
     }
 
     if (!failcount_clear_action_exists(node, rsc)) {
         /* Don't force the resource away from this node due to a failcount
          * that's going to be cleared.
          *
          * @TODO Failcount clearing can be scheduled in
          * pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
          * schedule_resource_actions() via check_params(). This runs well before
          * then, so it cannot detect those, meaning we might check the migration
          * threshold when we shouldn't. Worst case, we stop or move the
          * resource, then move it back in the next transition.
          */
         pcmk_resource_t *failed = NULL;
 
         if (pcmk__threshold_reached(rsc, node, &failed)) {
             resource_location(failed, node, -PCMK_SCORE_INFINITY,
                               "__fail_limit__", rsc->priv->scheduler);
         }
     }
 }
 
 /*!
  * \internal
  * \brief If resource has exclusive discovery, ban node if not allowed
  *
  * Location constraints have a PCMK_XA_RESOURCE_DISCOVERY option that allows
  * users to specify where probes are done for the affected resource. If this is
  * set to \c exclusive, probes will only be done on nodes listed in exclusive
  * constraints. This function bans the resource from the node if the node is not
  * listed.
  *
  * \param[in,out] data       Resource to check
  * \param[in]     user_data  Node to check resource on
  */
 static void
 apply_exclusive_discovery(gpointer data, gpointer user_data)
 {
     pcmk_resource_t *rsc = data;
     const pcmk_node_t *node = user_data;
 
+    /* @TODO This checks rsc and the top rsc, but should probably check all
+     * ancestors (a cloned group could have it set on the group)
+     */
     if (pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)
         || pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
                        pcmk__rsc_exclusive_probes)) {
         pcmk_node_t *match = NULL;
 
         // If this is a collective resource, apply recursively to children
         g_list_foreach(rsc->priv->children, apply_exclusive_discovery,
                        user_data);
 
         match = g_hash_table_lookup(rsc->priv->allowed_nodes,
                                     node->priv->id);
         if ((match != NULL)
             && (match->assign->probe_mode != pcmk__probe_exclusive)) {
             match->assign->score = -PCMK_SCORE_INFINITY;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Apply stickiness to a resource if appropriate
  *
  * \param[in,out] data       Resource to check for stickiness
  * \param[in]     user_data  Ignored
  */
 static void
 apply_stickiness(gpointer data, gpointer user_data)
 {
     pcmk_resource_t *rsc = data;
     pcmk_node_t *node = NULL;
 
     // If this is a collective resource, apply recursively to children instead
     if (rsc->priv->children != NULL) {
         g_list_foreach(rsc->priv->children, apply_stickiness, NULL);
         return;
     }
 
     /* A resource is sticky if it is managed, has stickiness configured, and is
      * active on a single node.
      */
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)
         || (rsc->priv->stickiness < 1)
         || !pcmk__list_of_1(rsc->priv->active_nodes)) {
         return;
     }
 
     node = rsc->priv->active_nodes->data;
 
     /* In a symmetric cluster, stickiness can always be used. In an
      * asymmetric cluster, we have to check whether the resource is still
      * allowed on the node, so we don't keep the resource somewhere it is no
      * longer explicitly enabled.
      */
     if (!pcmk_is_set(rsc->priv->scheduler->flags,
                      pcmk__sched_symmetric_cluster)
         && (g_hash_table_lookup(rsc->priv->allowed_nodes,
                                 node->priv->id) == NULL)) {
         pcmk__rsc_debug(rsc,
                         "Ignoring %s stickiness because the cluster is "
                         "asymmetric and %s is not explicitly allowed",
                         rsc->id, pcmk__node_name(node));
         return;
     }
 
     pcmk__rsc_debug(rsc, "Resource %s has %d stickiness on %s",
                     rsc->id, rsc->priv->stickiness, pcmk__node_name(node));
     resource_location(rsc, node, rsc->priv->stickiness, "stickiness",
                       rsc->priv->scheduler);
 }
 
 /*!
  * \internal
  * \brief Apply shutdown locks for all resources as appropriate
  *
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 apply_shutdown_locks(pcmk_scheduler_t *scheduler)
 {
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_shutdown_lock)) {
         return;
     }
     for (GList *iter = scheduler->priv->resources;
          iter != NULL; iter = iter->next) {
 
         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
         rsc->priv->cmds->shutdown_lock(rsc);
     }
 }
 
 /*
  * \internal
  * \brief Apply node-specific scheduling criteria
  *
  * After the CIB has been unpacked, process node-specific scheduling criteria
  * including shutdown locks, location constraints, resource stickiness,
  * migration thresholds, and exclusive resource discovery.
  */
 static void
 apply_node_criteria(pcmk_scheduler_t *scheduler)
 {
     crm_trace("Applying node-specific scheduling criteria");
     apply_shutdown_locks(scheduler);
     pcmk__apply_locations(scheduler);
     g_list_foreach(scheduler->priv->resources, apply_stickiness, NULL);
 
     for (GList *node_iter = scheduler->nodes; node_iter != NULL;
          node_iter = node_iter->next) {
 
         for (GList *rsc_iter = scheduler->priv->resources;
              rsc_iter != NULL; rsc_iter = rsc_iter->next) {
 
             check_failure_threshold(rsc_iter->data, node_iter->data);
             apply_exclusive_discovery(rsc_iter->data, node_iter->data);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Assign resources to nodes
  *
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 assign_resources(pcmk_scheduler_t *scheduler)
 {
     GList *iter = NULL;
 
     crm_trace("Assigning resources to nodes");
 
     if (!pcmk__str_eq(scheduler->priv->placement_strategy, PCMK_VALUE_DEFAULT,
                       pcmk__str_casei)) {
         pcmk__sort_resources(scheduler);
     }
     pcmk__show_node_capacities("Original", scheduler);
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_have_remote_nodes)) {
         /* Assign remote connection resources first (which will also assign any
          * colocation dependencies). If the connection is migrating, always
          * prefer the partial migration target.
          */
         for (iter = scheduler->priv->resources;
              iter != NULL; iter = iter->next) {
 
             pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
             const pcmk_node_t *target = rsc->priv->partial_migration_target;
 
             if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
                 pcmk__rsc_trace(rsc, "Assigning remote connection resource '%s'",
                                 rsc->id);
                 rsc->priv->cmds->assign(rsc, target, true);
             }
         }
     }
 
     /* now do the rest of the resources */
     for (iter = scheduler->priv->resources; iter != NULL; iter = iter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
         if (!pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
             pcmk__rsc_trace(rsc, "Assigning %s resource '%s'",
                             rsc->priv->xml->name, rsc->id);
             rsc->priv->cmds->assign(rsc, NULL, true);
         }
     }
 
     pcmk__show_node_capacities("Remaining", scheduler);
 }
 
 /*!
  * \internal
  * \brief Schedule fail count clearing on online nodes if resource is orphaned
  *
  * \param[in,out] data       Resource to check
  * \param[in]     user_data  Ignored
  */
 static void
 clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
 {
     pcmk_resource_t *rsc = data;
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
         return;
     }
     crm_trace("Clear fail counts for orphaned resource %s", rsc->id);
 
     /* There's no need to recurse into rsc->private->children because those
      * should just be unassigned clone instances.
      */
 
     for (GList *iter = rsc->priv->scheduler->nodes;
          iter != NULL; iter = iter->next) {
 
         pcmk_node_t *node = (pcmk_node_t *) iter->data;
         pcmk_action_t *clear_op = NULL;
 
         if (!node->details->online) {
             continue;
         }
         if (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective, NULL) == 0) {
             continue;
         }
 
         clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
                                        rsc->priv->scheduler);
 
         /* We can't use order_action_then_stop() here because its
          * pcmk__ar_guest_allowed breaks things
          */
         pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
                            NULL, pcmk__ar_ordered, rsc->priv->scheduler);
     }
 }
 
 /*!
  * \internal
  * \brief Schedule any resource actions needed
  *
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 schedule_resource_actions(pcmk_scheduler_t *scheduler)
 {
     // Process deferred action checks
     pe__foreach_param_check(scheduler, check_params);
     pe__free_param_checks(scheduler);
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_probe_resources)) {
         crm_trace("Scheduling probes");
         pcmk__schedule_probes(scheduler);
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_removed_resources)) {
         g_list_foreach(scheduler->priv->resources, clear_failcounts_if_orphaned,
                        NULL);
     }
 
     crm_trace("Scheduling resource actions");
     for (GList *iter = scheduler->priv->resources;
          iter != NULL; iter = iter->next) {
 
         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 
         rsc->priv->cmds->create_actions(rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a resource or any of its descendants are managed
  *
  * \param[in] rsc  Resource to check
  *
  * \return true if resource or any descendant is managed, otherwise false
  */
 static bool
 is_managed(const pcmk_resource_t *rsc)
 {
     if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
         return true;
     }
     for (GList *iter = rsc->priv->children;
          iter != NULL; iter = iter->next) {
 
         if (is_managed((pcmk_resource_t *) iter->data)) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Check whether any resources in the cluster are managed
  *
  * \param[in] scheduler  Scheduler data
  *
  * \return true if any resource is managed, otherwise false
  */
 static bool
 any_managed_resources(const pcmk_scheduler_t *scheduler)
 {
     for (const GList *iter = scheduler->priv->resources;
          iter != NULL; iter = iter->next) {
         if (is_managed((const pcmk_resource_t *) iter->data)) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Check whether a node requires fencing
  *
  * \param[in] node          Node to check
  * \param[in] have_managed  Whether any resource in cluster is managed
  *
  * \return true if \p node should be fenced, otherwise false
  */
 static bool
 needs_fencing(const pcmk_node_t *node, bool have_managed)
 {
     return have_managed && node->details->unclean
            && pe_can_fence(node->priv->scheduler, node);
 }
 
 /*!
  * \internal
  * \brief Check whether a node requires shutdown
  *
  * \param[in] node          Node to check
  *
  * \return true if \p node should be shut down, otherwise false
  */
 static bool
 needs_shutdown(const pcmk_node_t *node)
 {
     if (pcmk__is_pacemaker_remote_node(node)) {
        /* Do not send shutdown actions for Pacemaker Remote nodes.
         * @TODO We might come up with a good use for this in the future.
         */
         return false;
     }
     return node->details->online && node->details->shutdown;
 }
 
 /*!
  * \internal
  * \brief Track and order non-DC fencing
  *
  * \param[in,out] list       List of existing non-DC fencing actions
  * \param[in,out] action     Fencing action to prepend to \p list
  * \param[in]     scheduler  Scheduler data
  *
  * \return (Possibly new) head of \p list
  */
 static GList *
 add_nondc_fencing(GList *list, pcmk_action_t *action,
                   const pcmk_scheduler_t *scheduler)
 {
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_concurrent_fencing)
         && (list != NULL)) {
         /* Concurrent fencing is disabled, so order each non-DC
          * fencing in a chain. If there is any DC fencing or
          * shutdown, it will be ordered after the last action in the
          * chain later.
          */
         order_actions((pcmk_action_t *) list->data, action, pcmk__ar_ordered);
     }
     return g_list_prepend(list, action);
 }
 
 /*!
  * \internal
  * \brief Schedule a node for fencing
  *
  * \param[in,out] node      Node that requires fencing
  */
 static pcmk_action_t *
 schedule_fencing(pcmk_node_t *node)
 {
     pcmk_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
                                          FALSE, node->priv->scheduler);
 
     pcmk__sched_warn(node->priv->scheduler, "Scheduling node %s for fencing",
                      pcmk__node_name(node));
     pcmk__order_vs_fence(fencing, node->priv->scheduler);
     return fencing;
 }
 
 /*!
  * \internal
  * \brief Create and order node fencing and shutdown actions
  *
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 schedule_fencing_and_shutdowns(pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *dc_down = NULL;
     bool integrity_lost = false;
     bool have_managed = any_managed_resources(scheduler);
     GList *fencing_ops = NULL;
     GList *shutdown_ops = NULL;
 
     crm_trace("Scheduling fencing and shutdowns as needed");
     if (!have_managed) {
         crm_notice("No fencing will be done until there are resources "
                    "to manage");
     }
 
     // Check each node for whether it needs fencing or shutdown
     for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
         pcmk_node_t *node = (pcmk_node_t *) iter->data;
         pcmk_action_t *fencing = NULL;
         const bool is_dc = pcmk__same_node(node, scheduler->dc_node);
 
         /* Guest nodes are "fenced" by recovering their container resource,
          * so handle them separately.
          */
         if (pcmk__is_guest_or_bundle_node(node)) {
             if (pcmk_is_set(node->priv->flags, pcmk__node_remote_reset)
                 && have_managed && pe_can_fence(scheduler, node)) {
                 pcmk__fence_guest(node);
             }
             continue;
         }
 
         if (needs_fencing(node, have_managed)) {
             fencing = schedule_fencing(node);
 
             // Track DC and non-DC fence actions separately
             if (is_dc) {
                 dc_down = fencing;
             } else {
                 fencing_ops = add_nondc_fencing(fencing_ops, fencing,
                                                 scheduler);
             }
 
         } else if (needs_shutdown(node)) {
             pcmk_action_t *down_op = pcmk__new_shutdown_action(node);
 
             // Track DC and non-DC shutdown actions separately
             if (is_dc) {
                 dc_down = down_op;
             } else {
                 shutdown_ops = g_list_prepend(shutdown_ops, down_op);
             }
         }
 
         if ((fencing == NULL) && node->details->unclean) {
             integrity_lost = true;
             pcmk__config_warn("Node %s is unclean but cannot be fenced",
                               pcmk__node_name(node));
         }
     }
 
     if (integrity_lost) {
         if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
             pcmk__config_warn("Resource functionality and data integrity "
                               "cannot be guaranteed (configure, enable, "
                               "and test fencing to correct this)");
 
         } else if (!pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
             crm_notice("Unclean nodes will not be fenced until quorum is "
                        "attained or " PCMK_OPT_NO_QUORUM_POLICY " is set to "
                        PCMK_VALUE_IGNORE);
         }
     }
 
     if (dc_down != NULL) {
         /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
          * DC elections. However, we don't want to order non-DC shutdowns before
          * a DC *fencing*, because even though we don't want a node that's
          * shutting down to become DC, the DC fencing could be ordered before a
          * clone stop that's also ordered before the shutdowns, thus leading to
          * a graph loop.
          */
         if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
                          pcmk__str_none)) {
             pcmk__order_after_each(dc_down, shutdown_ops);
         }
 
         // Order any non-DC fencing before any DC fencing or shutdown
 
         if (pcmk_is_set(scheduler->flags, pcmk__sched_concurrent_fencing)) {
             /* With concurrent fencing, order each non-DC fencing action
              * separately before any DC fencing or shutdown.
              */
             pcmk__order_after_each(dc_down, fencing_ops);
         } else if (fencing_ops != NULL) {
             /* Without concurrent fencing, the non-DC fencing actions are
              * already ordered relative to each other, so we just need to order
              * the DC fencing after the last action in the chain (which is the
              * first item in the list).
              */
             order_actions((pcmk_action_t *) fencing_ops->data, dc_down,
                           pcmk__ar_ordered);
         }
     }
     g_list_free(fencing_ops);
     g_list_free(shutdown_ops);
 }
 
 static void
 log_resource_details(pcmk_scheduler_t *scheduler)
 {
     pcmk__output_t *out = scheduler->priv->out;
     GList *all = NULL;
 
     /* Due to the `crm_mon --node=` feature, out->message() for all the
      * resource-related messages expects a list of nodes that we are allowed to
      * output information for. Here, we create a wildcard to match all nodes.
      */
     all = g_list_prepend(all, (gpointer) "*");
 
     for (GList *item = scheduler->priv->resources;
          item != NULL; item = item->next) {
 
         pcmk_resource_t *rsc = (pcmk_resource_t *) item->data;
 
         // Log all resources except inactive orphans
         if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)
             || (rsc->priv->orig_role != pcmk_role_stopped)) {
             out->message(out, (const char *) rsc->priv->xml->name, 0UL,
                          rsc, all, all);
         }
     }
 
     g_list_free(all);
 }
 
 static void
 log_all_actions(pcmk_scheduler_t *scheduler)
 {
     /* This only ever outputs to the log, so ignore whatever output object was
      * previously set and just log instead.
      */
     pcmk__output_t *prev_out = scheduler->priv->out;
     pcmk__output_t *out = NULL;
 
     if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
         return;
     }
 
     pe__register_messages(out);
     pcmk__register_lib_messages(out);
     pcmk__output_set_log_level(out, LOG_NOTICE);
     scheduler->priv->out = out;
 
     out->begin_list(out, NULL, NULL, "Actions");
     pcmk__output_actions(scheduler);
     out->end_list(out);
     out->finish(out, CRM_EX_OK, true, NULL);
     pcmk__output_free(out);
 
     scheduler->priv->out = prev_out;
 }
 
 /*!
  * \internal
  * \brief Log all required but unrunnable actions at trace level
  *
  * \param[in] scheduler  Scheduler data
  */
 static void
 log_unrunnable_actions(const pcmk_scheduler_t *scheduler)
 {
     const uint64_t flags = pcmk__action_optional
                            |pcmk__action_runnable
                            |pcmk__action_pseudo;
 
     crm_trace("Required but unrunnable actions:");
     for (const GList *iter = scheduler->priv->actions;
          iter != NULL; iter = iter->next) {
 
         const pcmk_action_t *action = (const pcmk_action_t *) iter->data;
 
         if (!pcmk_any_flags_set(action->flags, flags)) {
             pcmk__log_action("\t", action, true);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Unpack the CIB for scheduling
  *
  * \param[in,out] cib        CIB XML to unpack (may be NULL if already unpacked)
  * \param[in]     flags      Scheduler flags to set in addition to defaults
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 unpack_cib(xmlNode *cib, unsigned long long flags, pcmk_scheduler_t *scheduler)
 {
     if (pcmk_is_set(scheduler->flags, pcmk__sched_have_status)) {
         crm_trace("Reusing previously calculated cluster status");
         pcmk__set_scheduler_flags(scheduler, flags);
         return;
     }
 
     pcmk__assert(cib != NULL);
     crm_trace("Calculating cluster status");
 
     /* This will zero the entire struct without freeing anything first, so
      * callers should never call pcmk__schedule_actions() with a populated data
      * set unless pcmk__sched_have_status is set (i.e. cluster_status() was
      * previously called, whether directly or via pcmk__schedule_actions()).
      */
     set_working_set_defaults(scheduler);
 
     pcmk__set_scheduler_flags(scheduler, flags);
     scheduler->input = cib;
     cluster_status(scheduler); // Sets pcmk__sched_have_status
 }
 
 /*!
  * \internal
  * \brief Run the scheduler for a given CIB
  *
  * \param[in,out] cib        CIB XML to use as scheduler input
  * \param[in]     flags      Scheduler flags to set in addition to defaults
  * \param[in,out] scheduler  Scheduler data
  */
 void
 pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
                        pcmk_scheduler_t *scheduler)
 {
     unpack_cib(cib, flags, scheduler);
     pcmk__set_assignment_methods(scheduler);
     pcmk__apply_node_health(scheduler);
     pcmk__unpack_constraints(scheduler);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_validate_only)) {
         return;
     }
 
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_location_only)
         && pcmk__is_daemon) {
         log_resource_details(scheduler);
     }
 
     apply_node_criteria(scheduler);
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_location_only)) {
         return;
     }
 
     pcmk__create_internal_constraints(scheduler);
     pcmk__handle_rsc_config_changes(scheduler);
     assign_resources(scheduler);
     schedule_resource_actions(scheduler);
 
     /* Remote ordering constraints need to happen prior to calculating fencing
      * because it is one more place we can mark nodes as needing fencing.
      */
     pcmk__order_remote_connection_actions(scheduler);
 
     schedule_fencing_and_shutdowns(scheduler);
     pcmk__apply_orderings(scheduler);
     log_all_actions(scheduler);
     pcmk__create_graph(scheduler);
 
     if (get_crm_log_level() == LOG_TRACE) {
         log_unrunnable_actions(scheduler);
     }
 }
 
 /*!
  * \internal
  * \brief Initialize scheduler data
  *
  * Make our own copies of the CIB XML and date/time object, if they're not
  * \c NULL. This way we don't have to take ownership of the objects passed via
  * the API.
  *
  * This function is most useful for public API functions that want the caller
  * to retain ownership of the CIB object
  *
  * \param[in,out] out        Output object
  * \param[in]     input      The CIB XML to check (if \c NULL, use current CIB)
  * \param[in]     date       Date and time to use in the scheduler (if \c NULL,
  *                           use current date and time).  This can be used for
  *                           checking whether a rule is in effect at a certa
  *                           date and time.
  * \param[out]    scheduler  Where to store initialized scheduler data
  *
  * \return Standard Pacemaker return code
  */
 int
 pcmk__init_scheduler(pcmk__output_t *out, xmlNodePtr input, const crm_time_t *date,
                      pcmk_scheduler_t **scheduler)
 {
     // Allows for cleaner syntax than dereferencing the scheduler argument
     pcmk_scheduler_t *new_scheduler = NULL;
 
     new_scheduler = pe_new_working_set();
     if (new_scheduler == NULL) {
         return ENOMEM;
     }
 
     pcmk__set_scheduler_flags(new_scheduler, pcmk__sched_no_counts);
 
     // Populate the scheduler data
 
     // Make our own copy of the given input or fetch the CIB and use that
     if (input != NULL) {
         new_scheduler->input = pcmk__xml_copy(NULL, input);
         if (new_scheduler->input == NULL) {
             out->err(out, "Failed to copy input XML");
             pe_free_working_set(new_scheduler);
             return ENOMEM;
         }
 
     } else {
         int rc = cib__signon_query(out, NULL, &(new_scheduler->input));
 
         if (rc != pcmk_rc_ok) {
             pe_free_working_set(new_scheduler);
             return rc;
         }
     }
 
     // Make our own copy of the given crm_time_t object; otherwise
     // cluster_status() populates with the current time
     if (date != NULL) {
         // pcmk_copy_time() guarantees non-NULL
         new_scheduler->priv->now = pcmk_copy_time(date);
     }
 
     // Unpack everything
     cluster_status(new_scheduler);
     *scheduler = new_scheduler;
 
     return pcmk_rc_ok;
 }
diff --git a/lib/pengine/bundle.c b/lib/pengine/bundle.c
index f19fbd11c0..fc2d1a3d80 100644
--- a/lib/pengine/bundle.c
+++ b/lib/pengine/bundle.c
@@ -1,2091 +1,2097 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <ctype.h>
 #include <stdint.h>
 
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <crm/common/xml.h>
 #include <crm/common/output.h>
 #include <crm/common/xml_internal.h>
 #include <pe_status_private.h>
 
 enum pe__bundle_mount_flags {
     pe__bundle_mount_none       = 0x00,
 
     // mount instance-specific subdirectory rather than source directly
     pe__bundle_mount_subdir     = 0x01
 };
 
 typedef struct {
     char *source;
     char *target;
     char *options;
     uint32_t flags; // bitmask of pe__bundle_mount_flags
 } pe__bundle_mount_t;
 
 typedef struct {
     char *source;
     char *target;
 } pe__bundle_port_t;
 
 enum pe__container_agent {
     PE__CONTAINER_AGENT_UNKNOWN,
     PE__CONTAINER_AGENT_DOCKER,
     PE__CONTAINER_AGENT_PODMAN,
 };
 
 #define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
 #define PE__CONTAINER_AGENT_DOCKER_S  "docker"
 #define PE__CONTAINER_AGENT_PODMAN_S  "podman"
 
 typedef struct pe__bundle_variant_data_s {
         int promoted_max;
         int nreplicas;
         int nreplicas_per_host;
         char *prefix;
         char *image;
         const char *ip_last;
         char *host_network;
         char *host_netmask;
         char *control_port;
         char *container_network;
         char *ip_range_start;
         gboolean add_host;
         gchar *container_host_options;
         char *container_command;
         char *launcher_options;
         const char *attribute_target;
 
         pcmk_resource_t *child;
 
         GList *replicas;    // pcmk__bundle_replica_t *
         GList *ports;       // pe__bundle_port_t *
         GList *mounts;      // pe__bundle_mount_t *
 
+        /* @TODO Maybe use a more object-oriented design instead, with a set of
+         * methods that are different per type rather than switching on this
+         */
         enum pe__container_agent agent_type;
 } pe__bundle_variant_data_t;
 
 #define get_bundle_variant_data(data, rsc) do { \
         pcmk__assert(pcmk__is_bundle(rsc));     \
         data = rsc->priv->variant_opaque;       \
     } while (0)
 
 /*!
  * \internal
  * \brief Get maximum number of bundle replicas allowed to run
  *
  * \param[in] rsc  Bundle or bundled resource to check
  *
  * \return Maximum replicas for bundle corresponding to \p rsc
  */
 int
 pe__bundle_max(const pcmk_resource_t *rsc)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
     return bundle_data->nreplicas;
 }
 
 /*!
  * \internal
  * \brief Get the resource inside a bundle
  *
  * \param[in] bundle  Bundle to check
  *
  * \return Resource inside \p bundle if any, otherwise NULL
  */
 pcmk_resource_t *
 pe__bundled_resource(const pcmk_resource_t *rsc)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
     return bundle_data->child;
 }
 
 /*!
  * \internal
  * \brief Get containerized resource corresponding to a given bundle container
  *
  * \param[in] instance  Collective instance that might be a bundle container
  *
  * \return Bundled resource instance inside \p instance if it is a bundle
  *         container instance, otherwise NULL
  */
 const pcmk_resource_t *
 pe__get_rsc_in_container(const pcmk_resource_t *instance)
 {
     const pe__bundle_variant_data_t *data = NULL;
     const pcmk_resource_t *top = pe__const_top_resource(instance, true);
 
     if (!pcmk__is_bundle(top)) {
         return NULL;
     }
     get_bundle_variant_data(data, top);
 
     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
         const pcmk__bundle_replica_t *replica = iter->data;
 
         if (instance == replica->container) {
             return replica->child;
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Check whether a given node is created by a bundle
  *
  * \param[in] bundle  Bundle resource to check
  * \param[in] node    Node to check
  *
  * \return true if \p node is an instance of \p bundle, otherwise false
  */
 bool
 pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
                             const pcmk_node_t *node)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
 
         if (pcmk__same_node(node, replica->node)) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Get the container of a bundle's first replica
  *
  * \param[in] bundle  Bundle resource to get container for
  *
  * \return Container resource from first replica of \p bundle if any,
  *         otherwise NULL
  */
 pcmk_resource_t *
 pe__first_container(const pcmk_resource_t *bundle)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
     const pcmk__bundle_replica_t *replica = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     if (bundle_data->replicas == NULL) {
         return NULL;
     }
     replica = bundle_data->replicas->data;
     return replica->container;
 }
 
 /*!
  * \internal
  * \brief Iterate over bundle replicas
  *
  * \param[in,out] bundle     Bundle to iterate over
  * \param[in]     fn         Function to call for each replica (its return value
  *                           indicates whether to continue iterating)
  * \param[in,out] user_data  Pointer to pass to \p fn
  */
 void
 pe__foreach_bundle_replica(pcmk_resource_t *bundle,
                            bool (*fn)(pcmk__bundle_replica_t *, void *),
                            void *user_data)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
         if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
             break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Iterate over const bundle replicas
  *
  * \param[in]     bundle     Bundle to iterate over
  * \param[in]     fn         Function to call for each replica (its return value
  *                           indicates whether to continue iterating)
  * \param[in,out] user_data  Pointer to pass to \p fn
  */
 void
 pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
                                  bool (*fn)(const pcmk__bundle_replica_t *,
                                             void *),
                                  void *user_data)
 {
     const pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, bundle);
     for (const GList *iter = bundle_data->replicas; iter != NULL;
          iter = iter->next) {
 
         if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
             break;
         }
     }
 }
 
 static char *
 next_ip(const char *last_ip)
 {
     unsigned int oct1 = 0;
     unsigned int oct2 = 0;
     unsigned int oct3 = 0;
     unsigned int oct4 = 0;
     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 
     if (rc != 4) {
         /*@ TODO check for IPv6 */
         return NULL;
 
     } else if (oct3 > 253) {
         return NULL;
 
     } else if (oct4 > 253) {
         ++oct3;
         oct4 = 1;
 
     } else {
         ++oct4;
     }
 
     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 }
 
 static void
 allocate_ip(pe__bundle_variant_data_t *data, pcmk__bundle_replica_t *replica,
             GString *buffer)
 {
     if(data->ip_range_start == NULL) {
         return;
 
     } else if(data->ip_last) {
         replica->ipaddr = next_ip(data->ip_last);
 
     } else {
         replica->ipaddr = strdup(data->ip_range_start);
     }
 
     data->ip_last = replica->ipaddr;
     switch (data->agent_type) {
         case PE__CONTAINER_AGENT_DOCKER:
         case PE__CONTAINER_AGENT_PODMAN:
             if (data->add_host) {
                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
                                        data->prefix, replica->offset,
                                        replica->ipaddr);
             } else {
                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
                                        replica->ipaddr, data->prefix,
                                        replica->offset);
             }
             break;
 
         default: // PE__CONTAINER_AGENT_UNKNOWN
             break;
     }
 }
 
 static xmlNode *
 create_resource(const char *name, const char *provider, const char *kind)
 {
     xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);
 
     crm_xml_add(rsc, PCMK_XA_ID, name);
     crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
     crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
     crm_xml_add(rsc, PCMK_XA_TYPE, kind);
 
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Check whether cluster can manage resource inside container
  *
  * \param[in,out] data  Container variant data
  *
  * \return TRUE if networking configuration is acceptable, FALSE otherwise
  *
  * \note The resource is manageable if an IP range or control port has been
  *       specified. If a control port is used without an IP range, replicas per
  *       host must be 1.
  */
 static bool
 valid_network(pe__bundle_variant_data_t *data)
 {
     if(data->ip_range_start) {
         return TRUE;
     }
     if(data->control_port) {
         if(data->nreplicas_per_host > 1) {
             pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
                              "requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
                              data->prefix);
             data->nreplicas_per_host = 1;
             // @TODO to be sure:
             // pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
         }
         return TRUE;
     }
     return FALSE;
 }
 
 static int
 create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
                    pcmk__bundle_replica_t *replica)
 {
     if(data->ip_range_start) {
         char *id = NULL;
         xmlNode *xml_ip = NULL;
         xmlNode *xml_obj = NULL;
 
         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
         pcmk__xml_sanitize_id(id);
         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
         free(id);
 
         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
         pcmk__xe_set_id(xml_obj, "%s-attributes-%d",
                         data->prefix, replica->offset);
 
         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
         if(data->host_network) {
             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
         }
 
         if(data->host_netmask) {
             crm_create_nvpair_xml(xml_obj, NULL,
                                   "cidr_netmask", data->host_netmask);
 
         } else {
             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
         }
 
         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
         crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
                           "60s", NULL);
 
         // TODO: Other ops? Timeouts and intervals from underlying resource?
 
         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
                                 parent->priv->scheduler) != pcmk_rc_ok) {
             return pcmk_rc_unpack_error;
         }
 
         parent->priv->children = g_list_append(parent->priv->children,
                                                replica->ip);
     }
     return pcmk_rc_ok;
 }
 
 static const char*
 container_agent_str(enum pe__container_agent t)
 {
     switch (t) {
         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
         default: // PE__CONTAINER_AGENT_UNKNOWN
             break;
     }
     return PE__CONTAINER_AGENT_UNKNOWN_S;
 }
 
 static int
 create_container_resource(pcmk_resource_t *parent,
                           const pe__bundle_variant_data_t *data,
                           pcmk__bundle_replica_t *replica)
 {
     char *id = NULL;
     xmlNode *xml_container = NULL;
     xmlNode *xml_obj = NULL;
 
     // Agent-specific
     const char *hostname_opt = NULL;
     const char *env_opt = NULL;
     const char *agent_str = NULL;
 
     GString *buffer = NULL;
     GString *dbuffer = NULL;
 
     // Where syntax differences are drop-in replacements, set them now
     switch (data->agent_type) {
         case PE__CONTAINER_AGENT_DOCKER:
         case PE__CONTAINER_AGENT_PODMAN:
             hostname_opt = "-h ";
             env_opt = "-e ";
             break;
         default:    // PE__CONTAINER_AGENT_UNKNOWN
             return pcmk_rc_unpack_error;
     }
     agent_str = container_agent_str(data->agent_type);
 
     buffer = g_string_sized_new(4096);
 
     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
                            replica->offset);
     pcmk__xml_sanitize_id(id);
     xml_container = create_resource(id, "heartbeat", agent_str);
     free(id);
 
     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
     pcmk__xe_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 
     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
     crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);
 
     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
         g_string_append(buffer, " --restart=no");
     }
 
     /* Set a container hostname only if we have an IP to map it to. The user can
      * set -h or --uts=host themselves if they want a nicer name for logs, but
      * this makes applications happy who need their  hostname to match the IP
      * they bind to.
      */
     if (data->ip_range_start != NULL) {
         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
                                replica->offset);
     }
     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 
     if (data->container_network != NULL) {
         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
     }
 
     if (data->control_port != NULL) {
         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
                        data->control_port, NULL);
     } else {
         g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
                                env_opt, DEFAULT_REMOTE_PORT);
     }
 
     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
         char *source = NULL;
 
         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
                                        replica->offset);
             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
         }
 
         switch (data->agent_type) {
             case PE__CONTAINER_AGENT_DOCKER:
             case PE__CONTAINER_AGENT_PODMAN:
                 pcmk__g_strcat(buffer,
                                " -v ", pcmk__s(source, mount->source),
                                ":", mount->target, NULL);
 
                 if (mount->options != NULL) {
                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
                 }
                 break;
             default:
                 break;
         }
         free(source);
     }
 
     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 
         switch (data->agent_type) {
             case PE__CONTAINER_AGENT_DOCKER:
             case PE__CONTAINER_AGENT_PODMAN:
                 if (replica->ipaddr != NULL) {
                     pcmk__g_strcat(buffer,
                                    " -p ", replica->ipaddr, ":", port->source,
                                    ":", port->target, NULL);
 
                 } else if (!pcmk__str_eq(data->container_network,
                                          PCMK_VALUE_HOST, pcmk__str_none)) {
                     // No need to do port mapping if net == host
                     pcmk__g_strcat(buffer,
                                    " -p ", port->source, ":", port->target,
                                    NULL);
                 }
                 break;
             default:
                 break;
         }
     }
 
     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
      * it would cause restarts during rolling upgrades.
      *
      * In a previous version of the container resource creation logic, if
      * data->launcher_options is not NULL, we append
      * (" %s", data->launcher_options) even if data->launcher_options is an
      * empty string. Likewise for data->container_host_options. Using
      *
      *     pcmk__add_word(buffer, 0, data->launcher_options)
      *
      * removes that extra trailing space, causing a resource definition change.
      */
     if (data->launcher_options != NULL) {
         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
     }
 
     if (data->container_host_options != NULL) {
         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
     }
 
     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
                           (const char *) buffer->str);
     g_string_free(buffer, TRUE);
 
     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
     if (dbuffer != NULL) {
         g_string_free(dbuffer, TRUE);
     }
 
     if (replica->child != NULL) {
         if (data->container_command != NULL) {
             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                   data->container_command);
         } else {
             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                   SBIN_DIR "/" PCMK__SERVER_REMOTED);
         }
 
         /* TODO: Allow users to specify their own?
          *
          * We just want to know if the container is alive; we'll monitor the
          * child independently.
          */
         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 #if 0
         /* @TODO Consider supporting the use case where we can start and stop
          * resources, but not proxy local commands (such as setting node
          * attributes), by running the local executor in stand-alone mode.
          * However, this would probably be better done via ACLs as with other
          * Pacemaker Remote nodes.
          */
     } else if ((child != NULL) && data->untrusted) {
         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                               CRM_DAEMON_DIR "/" PCMK__SERVER_EXECD);
         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 #endif
     } else {
         if (data->container_command != NULL) {
             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
                                   data->container_command);
         }
 
         /* TODO: Allow users to specify their own?
          *
          * We don't know what's in the container, so we just want to know if it
          * is alive.
          */
         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
     }
 
     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
     crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
                       "60s", NULL);
 
     // TODO: Other ops? Timeouts and intervals from underlying resource?
     if (pe__unpack_resource(xml_container, &replica->container, parent,
                             parent->priv->scheduler) != pcmk_rc_ok) {
         return pcmk_rc_unpack_error;
     }
     pcmk__set_rsc_flags(replica->container, pcmk__rsc_replica_container);
     parent->priv->children = g_list_append(parent->priv->children,
                                            replica->container);
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \brief Ban a node from a resource's (and its children's) allowed nodes list
  *
  * \param[in,out] rsc    Resource to modify
  * \param[in]     uname  Name of node to ban
  */
 static void
 disallow_node(pcmk_resource_t *rsc, const char *uname)
 {
     gpointer match = g_hash_table_lookup(rsc->priv->allowed_nodes, uname);
 
     if (match) {
         ((pcmk_node_t *) match)->assign->score = -PCMK_SCORE_INFINITY;
         ((pcmk_node_t *) match)->assign->probe_mode = pcmk__probe_never;
     }
     g_list_foreach(rsc->priv->children, (GFunc) disallow_node,
                    (gpointer) uname);
 }
 
 static int
 create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
                        pcmk__bundle_replica_t *replica)
 {
     if (replica->child && valid_network(data)) {
         GHashTableIter gIter;
         pcmk_node_t *node = NULL;
         xmlNode *xml_remote = NULL;
         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
         char *port_s = NULL;
         const char *uname = NULL;
         const char *connect_name = NULL;
         pcmk_scheduler_t *scheduler = parent->priv->scheduler;
 
         if (pe_find_resource(scheduler->priv->resources, id) != NULL) {
             free(id);
             // The biggest hammer we have
             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
                                    replica->child->id, replica->offset);
             //@TODO return error instead of asserting?
             pcmk__assert(pe_find_resource(scheduler->priv->resources,
                                           id) == NULL);
         }
 
         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
          * connection does not have its own IP is a magic string that we use to
          * support nested remotes (i.e. a bundle running on a remote node).
          */
         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 
         if (data->control_port == NULL) {
             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
         }
 
         /* This sets replica->container as replica->remote's container, which is
          * similar to what happens with guest nodes. This is how the scheduler
          * knows that the bundle node is fenced by recovering the container, and
          * that remote should be ordered relative to the container.
          */
         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
                                           NULL, NULL, NULL,
                                           connect_name, (data->control_port?
                                           data->control_port : port_s));
         free(port_s);
 
         /* Abandon our created ID, and pull the copy from the XML, because we
          * need something that will get freed during scheduler data cleanup to
          * use as the node ID and uname.
          */
         free(id);
         id = NULL;
         uname = pcmk__xe_id(xml_remote);
 
         /* Ensure a node has been created for the guest (it may have already
          * been, if it has a permanent node attribute), and ensure its weight is
          * -INFINITY so no other resources can run on it.
          */
         node = pcmk_find_node(scheduler, uname);
         if (node == NULL) {
             node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
                                   -PCMK_SCORE_INFINITY, scheduler);
         } else {
             node->assign->score = -PCMK_SCORE_INFINITY;
         }
         node->assign->probe_mode = pcmk__probe_never;
 
         /* unpack_remote_nodes() ensures that each remote node and guest node
          * has a pcmk_node_t entry. Ideally, it would do the same for bundle
          * nodes. Unfortunately, a bundle has to be mostly unpacked before it's
          * obvious what nodes will be needed, so we do it just above.
          *
          * Worse, that means that the node may have been utilized while
          * unpacking other resources, without our weight correction. The most
          * likely place for this to happen is when pe__unpack_resource() calls
          * resource_location() to set a default score in symmetric clusters.
          * This adds a node *copy* to each resource's allowed nodes, and these
          * copies will have the wrong weight.
          *
          * As a hacky workaround, fix those copies here.
          *
          * @TODO Possible alternative: ensure bundles are unpacked before other
          * resources, so the weight is correct before any copies are made.
          */
         g_list_foreach(scheduler->priv->resources,
                        (GFunc) disallow_node, (gpointer) uname);
 
         replica->node = pe__copy_node(node);
         replica->node->assign->score = 500;
         replica->node->assign->probe_mode = pcmk__probe_exclusive;
 
         /* Ensure the node shows up as allowed and with the correct discovery set */
         if (replica->child->priv->allowed_nodes != NULL) {
             g_hash_table_destroy(replica->child->priv->allowed_nodes);
         }
         replica->child->priv->allowed_nodes =
             pcmk__strkey_table(NULL, pcmk__free_node_copy);
         g_hash_table_insert(replica->child->priv->allowed_nodes,
                             (gpointer) replica->node->priv->id,
                             pe__copy_node(replica->node));
 
         {
             const pcmk_resource_t *parent = replica->child->priv->parent;
             pcmk_node_t *copy = pe__copy_node(replica->node);
 
             copy->assign->score = -PCMK_SCORE_INFINITY;
             g_hash_table_insert(parent->priv->allowed_nodes,
                                 (gpointer) replica->node->priv->id, copy);
         }
         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
                                 scheduler) != pcmk_rc_ok) {
             return pcmk_rc_unpack_error;
         }
 
         g_hash_table_iter_init(&gIter, replica->remote->priv->allowed_nodes);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
             if (pcmk__is_pacemaker_remote_node(node)) {
                 /* Remote resources can only run on 'normal' cluster node */
                 node->assign->score = -PCMK_SCORE_INFINITY;
             }
         }
 
         replica->node->priv->remote = replica->remote;
 
         // Ensure pcmk__is_guest_or_bundle_node() functions correctly
         replica->remote->priv->launcher = replica->container;
 
         /* A bundle's #kind is closer to "container" (guest node) than the
          * "remote" set by pe_create_node().
          */
         pcmk__insert_dup(replica->node->priv->attrs,
                          CRM_ATTR_KIND, "container");
 
         /* One effect of this is that unpack_launcher() will add
          * replica->remote to replica->container's launched resources, which
          * will make pe__resource_contains_guest_node() true for
          * replica->container.
          *
          * replica->child does NOT get added to replica->container's launched
          * resources. The only noticeable effect if it did would be for its
          * fail count to be taken into account when checking
          * replica->container's migration threshold.
          */
         parent->priv->children = g_list_append(parent->priv->children,
                                                replica->remote);
     }
     return pcmk_rc_ok;
 }
 
 static int
 create_replica_resources(pcmk_resource_t *parent,
                          pe__bundle_variant_data_t *data,
                          pcmk__bundle_replica_t *replica)
 {
     int rc = pcmk_rc_ok;
 
     rc = create_container_resource(parent, data, replica);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     rc = create_ip_resource(parent, data, replica);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     rc = create_remote_resource(parent, data, replica);
     if (rc != pcmk_rc_ok) {
         return rc;
     }
 
     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
         pcmk__insert_meta(replica->child->priv, "external-ip", replica->ipaddr);
     }
 
     if (replica->remote != NULL) {
         /*
          * Allow the remote connection resource to be allocated to a
          * different node than the one on which the container is active.
          *
          * This makes it possible to have Pacemaker Remote nodes running
          * containers with the remote executor inside in order to start
          * services inside those containers.
          */
         pcmk__set_rsc_flags(replica->remote, pcmk__rsc_remote_nesting_allowed);
     }
     return rc;
 }
 
 static void
 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
           const char *target, const char *options, uint32_t flags)
 {
     pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
                                                    sizeof(pe__bundle_mount_t));
 
     mount->source = pcmk__str_copy(source);
     mount->target = pcmk__str_copy(target);
     mount->options = pcmk__str_copy(options);
     mount->flags = flags;
     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 }
 
 static void
 mount_free(pe__bundle_mount_t *mount)
 {
     free(mount->source);
     free(mount->target);
     free(mount->options);
     free(mount);
 }
 
 static void
 port_free(pe__bundle_port_t *port)
 {
     free(port->source);
     free(port->target);
     free(port);
 }
 
 static pcmk__bundle_replica_t *
 replica_for_remote(pcmk_resource_t *remote)
 {
     pcmk_resource_t *top = remote;
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     if (top == NULL) {
         return NULL;
     }
     while (top->priv->parent != NULL) {
         top = top->priv->parent;
     }
 
     get_bundle_variant_data(bundle_data, top);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         if (replica->remote == remote) {
             return replica;
         }
     }
     CRM_LOG_ASSERT(FALSE);
     return NULL;
 }
 
 bool
 pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
 {
     const char *value;
     GHashTable *params = NULL;
 
     if (rsc == NULL) {
         return false;
     }
 
     // Use NULL node since pcmk__bundle_expand() uses that to set value
     params = pe_rsc_params(rsc, NULL, rsc->priv->scheduler);
     value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);
 
     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
            && xml_contains_remote_node(rsc->priv->xml);
 }
 
 const char *
 pe__add_bundle_remote_name(pcmk_resource_t *rsc, xmlNode *xml,
                            const char *field)
 {
     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 
     pcmk_node_t *node = NULL;
     pcmk__bundle_replica_t *replica = NULL;
 
     if (!pe__bundle_needs_remote_name(rsc)) {
         return NULL;
     }
 
     replica = replica_for_remote(rsc);
     if (replica == NULL) {
         return NULL;
     }
 
     node = replica->container->priv->assigned_node;
     if (node == NULL) {
         /* If it won't be running anywhere after the
          * transition, go with where it's running now.
          */
         node = pcmk__current_node(replica->container);
     }
 
     if(node == NULL) {
         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
         return NULL;
     }
 
     crm_trace("Setting address for bundle connection %s to bundle host %s",
               rsc->id, pcmk__node_name(node));
     if(xml != NULL && field != NULL) {
         crm_xml_add(xml, field, node->priv->name);
     }
 
     return node->priv->name;
 }
 
 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
                                    "Bundle mount", pcmk__xe_id(mount_xml),  \
                                    flags, (flags_to_set), #flags_to_set);   \
     } while (0)
 
 gboolean
 pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     const char *value = NULL;
     xmlNode *xml_obj = NULL;
     const xmlNode *xml_child = NULL;
     xmlNode *xml_resource = NULL;
     pe__bundle_variant_data_t *bundle_data = NULL;
     bool need_log_mount = TRUE;
 
     pcmk__assert(rsc != NULL);
     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 
     bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
     rsc->priv->variant_opaque = bundle_data;
     bundle_data->prefix = strdup(rsc->id);
 
     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_DOCKER, NULL,
                                    NULL);
     if (xml_obj != NULL) {
         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
     }
 
     if (xml_obj == NULL) {
         xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_PODMAN, NULL,
                                        NULL);
         if (xml_obj != NULL) {
             bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
         }
     }
 
     if (xml_obj == NULL) {
         return FALSE;
     }
 
     // Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
     value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
 
     /* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
      * otherwise
      */
     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
         bundle_data->nreplicas = bundle_data->promoted_max;
     } else {
         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
     }
 
     /*
      * Communication between containers on the same host via the
      * floating IPs only works if the container is started with:
      *   --userland-proxy=false --ip-masq=false
      */
     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
     if (bundle_data->nreplicas_per_host == 1) {
         pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
     }
 
     bundle_data->container_command =
         crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
     bundle_data->launcher_options = crm_element_value_copy(xml_obj,
                                                            PCMK_XA_OPTIONS);
     bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
     bundle_data->container_network = crm_element_value_copy(xml_obj,
                                                             PCMK_XA_NETWORK);
 
     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_NETWORK, NULL,
                                    NULL);
     if(xml_obj) {
         bundle_data->ip_range_start =
             crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
         bundle_data->host_netmask =
             crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
         bundle_data->host_network =
             crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
         bundle_data->control_port =
             crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
         value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
             bundle_data->add_host = TRUE;
         }
 
         for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
                                               NULL, NULL);
              xml_child != NULL;
              xml_child = pcmk__xe_next(xml_child, PCMK_XE_PORT_MAPPING)) {
 
             pe__bundle_port_t *port =
                 pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
 
             port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);
 
             if(port->source == NULL) {
                 port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
             } else {
                 port->target = crm_element_value_copy(xml_child,
                                                       PCMK_XA_INTERNAL_PORT);
             }
 
             if(port->source != NULL && strlen(port->source) > 0) {
                 if(port->target == NULL) {
                     port->target = strdup(port->source);
                 }
                 bundle_data->ports = g_list_append(bundle_data->ports, port);
 
             } else {
                 pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
                                  pcmk__xe_id(xml_child));
                 port_free(port);
             }
         }
     }
 
     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_STORAGE, NULL,
                                    NULL);
     for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
                                           NULL, NULL);
          xml_child != NULL;
          xml_child = pcmk__xe_next(xml_child, PCMK_XE_STORAGE_MAPPING)) {
 
         const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
         const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
         const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
         int flags = pe__bundle_mount_none;
 
         if (source == NULL) {
             source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
             pe__set_bundle_mount_flags(xml_child, flags,
                                        pe__bundle_mount_subdir);
         }
 
         if (source && target) {
             mount_add(bundle_data, source, target, options, flags);
             if (strcmp(target, "/var/log") == 0) {
                 need_log_mount = FALSE;
             }
         } else {
             pcmk__config_err("Invalid mount directive %s",
                              pcmk__xe_id(xml_child));
         }
     }
 
     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_PRIMITIVE, NULL,
                                    NULL);
     if (xml_obj && valid_network(bundle_data)) {
         const char *suffix = NULL;
         char *value = NULL;
         xmlNode *xml_set = NULL;
 
         xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);
 
         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
          * part of the resource name, so that bundles don't restart in a rolling
          * upgrade. (It also avoids needing to change regression tests.)
          */
         suffix = (const char *) xml_resource->name;
         if (bundle_data->promoted_max > 0) {
             suffix = "master";
         }
 
         pcmk__xe_set_id(xml_resource, "%s-%s", bundle_data->prefix, suffix);
 
         xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
         pcmk__xe_set_id(xml_set, "%s-%s-meta",
                         bundle_data->prefix, xml_resource->name);
 
         crm_create_nvpair_xml(xml_set, NULL,
                               PCMK_META_ORDERED, PCMK_VALUE_TRUE);
 
         value = pcmk__itoa(bundle_data->nreplicas);
         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
         free(value);
 
         value = pcmk__itoa(bundle_data->nreplicas_per_host);
         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
         free(value);
 
         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
 
         if (bundle_data->promoted_max) {
             crm_create_nvpair_xml(xml_set, NULL,
                                   PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);
 
             value = pcmk__itoa(bundle_data->promoted_max);
             crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
             free(value);
         }
 
         //crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
         pcmk__xml_copy(xml_resource, xml_obj);
 
     } else if(xml_obj) {
         pcmk__config_err("Cannot control %s inside %s without either "
                          PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
                          rsc->id, pcmk__xe_id(xml_obj));
         return FALSE;
     }
 
     if(xml_resource) {
         int lpc = 0;
         GList *childIter = NULL;
         pe__bundle_port_t *port = NULL;
         GString *buffer = NULL;
 
         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
                                 scheduler) != pcmk_rc_ok) {
             return FALSE;
         }
 
         /* Currently, we always map the default authentication key location
          * into the same location inside the container.
          *
          * Ideally, we would respect the host's PCMK_authkey_location, but:
          * - it may be different on different nodes;
          * - the actual connection will do extra checking to make sure the key
          *   file exists and is readable, that we can't do here on the DC
          * - tools such as crm_resource and crm_simulate may not have the same
          *   environment variables as the cluster, causing operation digests to
          *   differ
          *
          * Always using the default location inside the container is fine,
          * because we control the pacemaker_remote environment, and it avoids
          * having to pass another environment variable to the container.
          *
          * @TODO A better solution may be to have only pacemaker_remote use the
          * environment variable, and have the cluster nodes use a new
          * cluster option for key location. This would introduce the limitation
          * of the location being the same on all cluster nodes, but that's
          * reasonable.
          */
         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
 
         if (need_log_mount) {
             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
                       pe__bundle_mount_subdir);
         }
 
         port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
         if(bundle_data->control_port) {
             port->source = strdup(bundle_data->control_port);
         } else {
             /* If we wanted to respect PCMK_remote_port, we could use
              * crm_default_remote_port() here and elsewhere in this file instead
              * of DEFAULT_REMOTE_PORT.
              *
              * However, it gains nothing, since we control both the container
              * environment and the connection resource parameters, and the user
              * can use a different port if desired by setting
              * PCMK_XA_CONTROL_PORT.
              */
             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
         }
         port->target = strdup(port->source);
         bundle_data->ports = g_list_append(bundle_data->ports, port);
 
         buffer = g_string_sized_new(1024);
         for (childIter = bundle_data->child->priv->children;
              childIter != NULL; childIter = childIter->next) {
 
             pcmk__bundle_replica_t *replica = NULL;
 
             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
             replica->child = childIter->data;
             pcmk__set_rsc_flags(replica->child, pcmk__rsc_exclusive_probes);
             replica->offset = lpc++;
 
             // Ensure the child's notify gets set based on the underlying primitive's value
             if (pcmk_is_set(replica->child->flags, pcmk__rsc_notify)) {
                 pcmk__set_rsc_flags(bundle_data->child, pcmk__rsc_notify);
             }
 
             allocate_ip(bundle_data, replica, buffer);
             bundle_data->replicas = g_list_append(bundle_data->replicas,
                                                   replica);
             bundle_data->attribute_target =
                 g_hash_table_lookup(replica->child->priv->meta,
                                     PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
         }
         bundle_data->container_host_options = g_string_free(buffer, FALSE);
 
         if (bundle_data->attribute_target) {
             pcmk__insert_dup(rsc->priv->meta,
                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
                              bundle_data->attribute_target);
             pcmk__insert_dup(bundle_data->child->priv->meta,
                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
                              bundle_data->attribute_target);
         }
 
     } else {
         // Just a naked container, no pacemaker-remote
         GString *buffer = g_string_sized_new(1024);
 
         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
             pcmk__bundle_replica_t *replica = NULL;
 
             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
             replica->offset = lpc;
             allocate_ip(bundle_data, replica, buffer);
             bundle_data->replicas = g_list_append(bundle_data->replicas,
                                                   replica);
         }
         bundle_data->container_host_options = g_string_free(buffer, FALSE);
     }
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
             pcmk__config_err("Failed unpacking resource %s", rsc->id);
             rsc->priv->fns->free(rsc);
             return FALSE;
         }
 
         /* Utilization needs special handling for bundles. It makes no sense for
          * the inner primitive to have utilization, because it is tied
          * one-to-one to the guest node created by the container resource -- and
          * there's no way to set capacities for that guest node anyway.
          *
          * What the user really wants is to configure utilization for the
          * container. However, the schema only allows utilization for
          * primitives, and the container resource is implicit anyway, so the
          * user can *only* configure utilization for the inner primitive. If
          * they do, move the primitive's utilization values to the container.
          *
          * @TODO This means that bundles without an inner primitive can't have
          * utilization. An alternative might be to allow utilization values in
          * the top-level bundle XML in the schema, and copy those to each
          * container.
          */
         if (replica->child != NULL) {
             GHashTable *empty = replica->container->priv->utilization;
 
             replica->container->priv->utilization =
                 replica->child->priv->utilization;
 
             replica->child->priv->utilization = empty;
         }
     }
 
     if (bundle_data->child) {
         rsc->priv->children = g_list_append(rsc->priv->children,
                                             bundle_data->child);
     }
     return TRUE;
 }
 
 static int
 replica_resource_active(pcmk_resource_t *rsc, gboolean all)
 {
     if (rsc) {
         gboolean child_active = rsc->priv->fns->active(rsc, all);
 
         if (child_active && !all) {
             return TRUE;
         } else if (!child_active && all) {
             return FALSE;
         }
     }
     return -1;
 }
 
 gboolean
 pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     GList *iter = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
         int rsc_active;
 
         rsc_active = replica_resource_active(replica->ip, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->child, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->container, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
 
         rsc_active = replica_resource_active(replica->remote, all);
         if (rsc_active >= 0) {
             return (gboolean) rsc_active;
         }
     }
 
     /* If "all" is TRUE, we've already checked that no resources were inactive,
      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
      * so return FALSE.
      */
     return all;
 }
 
 /*!
  * \internal
  * \brief Find the bundle replica corresponding to a given node
  *
  * \param[in] bundle  Top-level bundle resource
  * \param[in] node    Node to search for
  *
  * \return Bundle replica if found, NULL otherwise
  */
 pcmk_resource_t *
 pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     pcmk__assert((bundle != NULL) && (node != NULL));
 
     get_bundle_variant_data(bundle_data, bundle);
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
 
         pcmk__assert((replica != NULL) && (replica->node != NULL));
         if (pcmk__same_node(replica->node, node)) {
             return replica->child;
         }
     }
     return NULL;
 }
 
 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__bundle_xml(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     pe__bundle_variant_data_t *bundle_data = NULL;
     int rc = pcmk_rc_no_output;
     gboolean printed_header = FALSE;
     gboolean print_everything = TRUE;
 
     const char *desc = NULL;
 
     pcmk__assert(rsc != NULL);
     get_bundle_variant_data(bundle_data, rsc);
 
     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
         pcmk_resource_t *ip = replica->ip;
         pcmk_resource_t *child = replica->child;
         pcmk_resource_t *container = replica->container;
         pcmk_resource_t *remote = replica->remote;
         char *id = NULL;
         gboolean print_ip, print_child, print_ctnr, print_remote;
 
         pcmk__assert(replica != NULL);
 
         if (pcmk__rsc_filtered_by_node(container, only_node)) {
             continue;
         }
 
         print_ip = (ip != NULL)
                    && !ip->priv->fns->is_filtered(ip, only_rsc,
                                                   print_everything);
         print_child = (child != NULL)
                       && !child->priv->fns->is_filtered(child, only_rsc,
                                                         print_everything);
         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
                                                         print_everything);
         print_remote = (remote != NULL)
                        && !remote->priv->fns->is_filtered(remote, only_rsc,
                                                           print_everything);
 
         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
             continue;
         }
 
         if (!printed_header) {
             const char *type = container_agent_str(bundle_data->agent_type);
             const char *unique = pcmk__flag_text(rsc->flags, pcmk__rsc_unique);
             const char *maintenance = pcmk__flag_text(rsc->flags,
                                                       pcmk__rsc_maintenance);
             const char *managed = pcmk__flag_text(rsc->flags,
                                                   pcmk__rsc_managed);
             const char *failed = pcmk__flag_text(rsc->flags, pcmk__rsc_failed);
 
             printed_header = TRUE;
 
             desc = pe__resource_description(rsc, show_opts);
 
             rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE,
                                           PCMK_XA_ID, rsc->id,
                                           PCMK_XA_TYPE, type,
                                           PCMK_XA_IMAGE, bundle_data->image,
                                           PCMK_XA_UNIQUE, unique,
                                           PCMK_XA_MAINTENANCE, maintenance,
                                           PCMK_XA_MANAGED, managed,
                                           PCMK_XA_FAILED, failed,
                                           PCMK_XA_DESCRIPTION, desc,
                                           NULL);
             pcmk__assert(rc == pcmk_rc_ok);
         }
 
         id = pcmk__itoa(replica->offset);
         rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA,
                                       PCMK_XA_ID, id,
                                       NULL);
         free(id);
         pcmk__assert(rc == pcmk_rc_ok);
 
         if (print_ip) {
             out->message(out, (const char *) ip->priv->xml->name, show_opts,
                          ip, only_node, only_rsc);
         }
 
         if (print_child) {
             out->message(out, (const char *) child->priv->xml->name,
                          show_opts, child, only_node, only_rsc);
         }
 
         if (print_ctnr) {
             out->message(out, (const char *) container->priv->xml->name,
                          show_opts, container, only_node, only_rsc);
         }
 
         if (print_remote) {
             out->message(out, (const char *) remote->priv->xml->name,
                          show_opts, remote, only_node, only_rsc);
         }
 
         pcmk__output_xml_pop_parent(out); // replica
     }
 
     if (printed_header) {
         pcmk__output_xml_pop_parent(out); // bundle
     }
 
     return rc;
 }
 
 static void
 pe__bundle_replica_output_html(pcmk__output_t *out,
                                pcmk__bundle_replica_t *replica,
                                pcmk_node_t *node, uint32_t show_opts)
 {
     pcmk_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     pe__common_output_html(out, rsc, buffer, node, show_opts);
 }
 
 /*!
  * \internal
  * \brief Get a string describing a resource's unmanaged state or lack thereof
  *
  * \param[in] rsc  Resource to describe
  *
  * \return A string indicating that a resource is in maintenance mode or
  *         otherwise unmanaged, or an empty string otherwise
  */
 static const char *
 get_unmanaged_str(const pcmk_resource_t *rsc)
 {
     if (pcmk_is_set(rsc->flags, pcmk__rsc_maintenance)) {
         return " (maintenance)";
     }
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
         return " (unmanaged)";
     }
     return "";
 }
 
 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__bundle_html(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     const char *desc = NULL;
     pe__bundle_variant_data_t *bundle_data = NULL;
     int rc = pcmk_rc_no_output;
     gboolean print_everything = TRUE;
 
     pcmk__assert(rsc != NULL);
     get_bundle_variant_data(bundle_data, rsc);
 
     desc = pe__resource_description(rsc, show_opts);
 
     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
         pcmk_resource_t *ip = replica->ip;
         pcmk_resource_t *child = replica->child;
         pcmk_resource_t *container = replica->container;
         pcmk_resource_t *remote = replica->remote;
         gboolean print_ip, print_child, print_ctnr, print_remote;
 
         pcmk__assert(replica != NULL);
 
         if (pcmk__rsc_filtered_by_node(container, only_node)) {
             continue;
         }
 
         print_ip = (ip != NULL)
                    && !ip->priv->fns->is_filtered(ip, only_rsc,
                                                   print_everything);
         print_child = (child != NULL)
                       && !child->priv->fns->is_filtered(child, only_rsc,
                                                         print_everything);
         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
                                                         print_everything);
         print_remote = (remote != NULL)
                        && !remote->priv->fns->is_filtered(remote, only_rsc,
                                                           print_everything);
 
         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
             /* The text output messages used below require pe_print_implicit to
              * be set to do anything.
              */
             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
 
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
             }
 
             if (print_ip) {
                 out->message(out, (const char *) ip->priv->xml->name,
                              new_show_opts, ip, only_node, only_rsc);
             }
 
             if (print_child) {
                 out->message(out, (const char *) child->priv->xml->name,
                              new_show_opts, child, only_node, only_rsc);
             }
 
             if (print_ctnr) {
                 out->message(out, (const char *) container->priv->xml->name,
                              new_show_opts, container, only_node, only_rsc);
             }
 
             if (print_remote) {
                 out->message(out, (const char *) remote->priv->xml->name,
                              new_show_opts, remote, only_node, only_rsc);
             }
 
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 out->end_list(out);
             }
         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
             continue;
         } else {
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             pe__bundle_replica_output_html(out, replica,
                                            pcmk__current_node(container),
                                            show_opts);
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 static void
 pe__bundle_replica_output_text(pcmk__output_t *out,
                                pcmk__bundle_replica_t *replica,
                                pcmk_node_t *node, uint32_t show_opts)
 {
     const pcmk_resource_t *rsc = replica->child;
 
     int offset = 0;
     char buffer[LINE_MAX];
 
     if(rsc == NULL) {
         rsc = replica->container;
     }
 
     if (replica->remote) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->remote));
     } else {
         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
                            rsc_printable_id(replica->container));
     }
     if (replica->ipaddr) {
         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
                            replica->ipaddr);
     }
 
     pe__common_output_text(out, rsc, buffer, node, show_opts);
 }
 
 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__bundle_text(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     const char *desc = NULL;
     pe__bundle_variant_data_t *bundle_data = NULL;
     int rc = pcmk_rc_no_output;
     gboolean print_everything = TRUE;
 
     desc = pe__resource_description(rsc, show_opts);
 
     pcmk__assert(rsc != NULL);
     get_bundle_variant_data(bundle_data, rsc);
 
     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
 
     for (GList *gIter = bundle_data->replicas; gIter != NULL;
          gIter = gIter->next) {
         pcmk__bundle_replica_t *replica = gIter->data;
         pcmk_resource_t *ip = replica->ip;
         pcmk_resource_t *child = replica->child;
         pcmk_resource_t *container = replica->container;
         pcmk_resource_t *remote = replica->remote;
         gboolean print_ip, print_child, print_ctnr, print_remote;
 
         pcmk__assert(replica != NULL);
 
         if (pcmk__rsc_filtered_by_node(container, only_node)) {
             continue;
         }
 
         print_ip = (ip != NULL)
                    && !ip->priv->fns->is_filtered(ip, only_rsc,
                                                   print_everything);
         print_child = (child != NULL)
                       && !child->priv->fns->is_filtered(child, only_rsc,
                                                         print_everything);
         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
                                                         print_everything);
         print_remote = (remote != NULL)
                        && !remote->priv->fns->is_filtered(remote, only_rsc,
                                                           print_everything);
 
         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
             /* The text output messages used below require pe_print_implicit to
              * be set to do anything.
              */
             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
 
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             if (pcmk__list_of_multiple(bundle_data->replicas)) {
                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
             }
 
             out->begin_list(out, NULL, NULL, NULL);
 
             if (print_ip) {
                 out->message(out, (const char *) ip->priv->xml->name,
                              new_show_opts, ip, only_node, only_rsc);
             }
 
             if (print_child) {
                 out->message(out, (const char *) child->priv->xml->name,
                              new_show_opts, child, only_node, only_rsc);
             }
 
             if (print_ctnr) {
                 out->message(out, (const char *) container->priv->xml->name,
                              new_show_opts, container, only_node, only_rsc);
             }
 
             if (print_remote) {
                 out->message(out, (const char *) remote->priv->xml->name,
                              new_show_opts, remote, only_node, only_rsc);
             }
 
             out->end_list(out);
         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
             continue;
         } else {
             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
                                      (bundle_data->nreplicas > 1)? " set" : "",
                                      rsc->id, bundle_data->image,
                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
                                      get_unmanaged_str(rsc));
 
             pe__bundle_replica_output_text(out, replica,
                                            pcmk__current_node(container),
                                            show_opts);
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 static void
 free_bundle_replica(pcmk__bundle_replica_t *replica)
 {
     if (replica == NULL) {
         return;
     }
 
     pcmk__free_node_copy(replica->node);
     replica->node = NULL;
 
     if (replica->ip) {
         pcmk__xml_free(replica->ip->priv->xml);
         replica->ip->priv->xml = NULL;
         replica->ip->priv->fns->free(replica->ip);
     }
     if (replica->container) {
         pcmk__xml_free(replica->container->priv->xml);
         replica->container->priv->xml = NULL;
         replica->container->priv->fns->free(replica->container);
     }
     if (replica->remote) {
         pcmk__xml_free(replica->remote->priv->xml);
         replica->remote->priv->xml = NULL;
         replica->remote->priv->fns->free(replica->remote);
     }
     free(replica->ipaddr);
     free(replica);
 }
 
 void
 pe__free_bundle(pcmk_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
     CRM_CHECK(rsc != NULL, return);
 
     get_bundle_variant_data(bundle_data, rsc);
     pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
 
     free(bundle_data->prefix);
     free(bundle_data->image);
     free(bundle_data->control_port);
     free(bundle_data->host_network);
     free(bundle_data->host_netmask);
     free(bundle_data->ip_range_start);
     free(bundle_data->container_network);
     free(bundle_data->launcher_options);
     free(bundle_data->container_command);
     g_free(bundle_data->container_host_options);
 
     g_list_free_full(bundle_data->replicas,
                      (GDestroyNotify) free_bundle_replica);
     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
     g_list_free(rsc->priv->children);
 
     if(bundle_data->child) {
         pcmk__xml_free(bundle_data->child->priv->xml);
         bundle_data->child->priv->xml = NULL;
         bundle_data->child->priv->fns->free(bundle_data->child);
     }
     common_free(rsc);
 }
 
 enum rsc_role_e
 pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
 {
     enum rsc_role_e container_role = pcmk_role_unknown;
     return container_role;
 }
 
 /*!
  * \brief Get the number of configured replicas in a bundle
  *
  * \param[in] rsc  Bundle resource
  *
  * \return Number of configured replicas, or 0 on error
  */
 int
 pe_bundle_replicas(const pcmk_resource_t *rsc)
 {
     if (pcmk__is_bundle(rsc)) {
         pe__bundle_variant_data_t *bundle_data = NULL;
 
         get_bundle_variant_data(bundle_data, rsc);
         return bundle_data->nreplicas;
     }
     return 0;
 }
 
 void
 pe__count_bundle(pcmk_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
         pcmk__bundle_replica_t *replica = item->data;
 
         if (replica->ip) {
             replica->ip->priv->fns->count(replica->ip);
         }
         if (replica->child) {
             replica->child->priv->fns->count(replica->child);
         }
         if (replica->container) {
             replica->container->priv->fns->count(replica->container);
         }
         if (replica->remote) {
             replica->remote->priv->fns->count(replica->remote);
         }
     }
 }
 
 gboolean
 pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                        gboolean check_parent)
 {
     gboolean passes = FALSE;
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
         passes = TRUE;
     } else {
         get_bundle_variant_data(bundle_data, rsc);
 
         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
             pcmk__bundle_replica_t *replica = gIter->data;
             pcmk_resource_t *ip = replica->ip;
             pcmk_resource_t *child = replica->child;
             pcmk_resource_t *container = replica->container;
             pcmk_resource_t *remote = replica->remote;
 
             if ((ip != NULL)
                 && !ip->priv->fns->is_filtered(ip, only_rsc, FALSE)) {
                 passes = TRUE;
                 break;
             }
             if ((child != NULL)
                 && !child->priv->fns->is_filtered(child, only_rsc, FALSE)) {
                 passes = TRUE;
                 break;
             }
             if (!container->priv->fns->is_filtered(container, only_rsc,
                                                    FALSE)) {
                 passes = TRUE;
                 break;
             }
             if ((remote != NULL)
                 && !remote->priv->fns->is_filtered(remote, only_rsc, FALSE)) {
                 passes = TRUE;
                 break;
             }
         }
     }
 
     return !passes;
 }
 
 /*!
  * \internal
  * \brief Get a list of a bundle's containers
  *
  * \param[in] bundle  Bundle resource
  *
  * \return Newly created list of \p bundle's containers
  * \note It is the caller's responsibility to free the result with
  *       g_list_free().
  */
 GList *
 pe__bundle_containers(const pcmk_resource_t *bundle)
 {
+    /* @TODO It would be more efficient to do this once when unpacking the
+     * bundle, creating a new GList* in the variant data
+     */
     GList *containers = NULL;
     const pe__bundle_variant_data_t *data = NULL;
 
     get_bundle_variant_data(data, bundle);
     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
 
         containers = g_list_append(containers, replica->container);
     }
     return containers;
 }
 
 // Bundle implementation of pcmk__rsc_methods_t:active_node()
 pcmk_node_t *
 pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
                        unsigned int *count_clean)
 {
     pcmk_node_t *active = NULL;
     pcmk_node_t *node = NULL;
     pcmk_resource_t *container = NULL;
     GList *containers = NULL;
     GList *iter = NULL;
     GHashTable *nodes = NULL;
     const pe__bundle_variant_data_t *data = NULL;
 
     if (count_all != NULL) {
         *count_all = 0;
     }
     if (count_clean != NULL) {
         *count_clean = 0;
     }
     if (rsc == NULL) {
         return NULL;
     }
 
     /* For the purposes of this method, we only care about where the bundle's
      * containers are active, so build a list of active containers.
      */
     get_bundle_variant_data(data, rsc);
     for (iter = data->replicas; iter != NULL; iter = iter->next) {
         pcmk__bundle_replica_t *replica = iter->data;
 
         if (replica->container->priv->active_nodes != NULL) {
             containers = g_list_append(containers, replica->container);
         }
     }
     if (containers == NULL) {
         return NULL;
     }
 
     /* If the bundle has only a single active container, just use that
      * container's method. If live migration is ever supported for bundle
      * containers, this will allow us to prefer the migration source when there
      * is only one container and it is migrating. For now, this just lets us
      * avoid creating the nodes table.
      */
     if (pcmk__list_of_1(containers)) {
         container = containers->data;
         node = container->priv->fns->active_node(container, count_all,
                                                  count_clean);
         g_list_free(containers);
         return node;
     }
 
     // Add all containers' active nodes to a hash table (for uniqueness)
     nodes = g_hash_table_new(NULL, NULL);
     for (iter = containers; iter != NULL; iter = iter->next) {
         container = iter->data;
         for (GList *node_iter = container->priv->active_nodes;
              node_iter != NULL; node_iter = node_iter->next) {
 
             node = node_iter->data;
 
             // If insert returns true, we haven't counted this node yet
             if (g_hash_table_insert(nodes, (gpointer) node->details,
                                     (gpointer) node)
                 && !pe__count_active_node(rsc, node, &active, count_all,
                                           count_clean)) {
                 goto done;
             }
         }
     }
 
 done:
     g_list_free(containers);
     g_hash_table_destroy(nodes);
     return active;
 }
 
 /*!
  * \internal
  * \brief Get maximum bundle resource instances per node
  *
  * \param[in] rsc  Bundle resource to check
  *
  * \return Maximum number of \p rsc instances that can be active on one node
  */
 unsigned int
 pe__bundle_max_per_node(const pcmk_resource_t *rsc)
 {
     pe__bundle_variant_data_t *bundle_data = NULL;
 
     get_bundle_variant_data(bundle_data, rsc);
     pcmk__assert(bundle_data->nreplicas_per_host >= 0);
     return (unsigned int) bundle_data->nreplicas_per_host;
 }
diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c
index 24953cd131..f8db1db221 100644
--- a/lib/pengine/clone.c
+++ b/lib/pengine/clone.c
@@ -1,1256 +1,1258 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdint.h>
 
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <pe_status_private.h>
 #include <crm/common/xml.h>
 #include <crm/common/output.h>
 #include <crm/common/xml_internal.h>
 #include <crm/common/scheduler_internal.h>
 
 typedef struct clone_variant_data_s {
     int clone_max;
     int clone_node_max;
 
     int promoted_max;
     int promoted_node_max;
 
     int total_clones;
 
     uint32_t flags; // Group of enum pcmk__clone_flags
 
     notify_data_t *stop_notify;
     notify_data_t *start_notify;
     notify_data_t *demote_notify;
     notify_data_t *promote_notify;
 
     xmlNode *xml_obj_child;
 } clone_variant_data_t;
 
 #define get_clone_variant_data(data, rsc) do {  \
         pcmk__assert(pcmk__is_clone(rsc));      \
         data = rsc->priv->variant_opaque;       \
     } while (0)
 
 /*!
  * \internal
  * \brief Return the maximum number of clone instances allowed to be run
  *
  * \param[in] clone  Clone or clone instance to check
  *
  * \return Maximum instances for \p clone
  */
 int
 pe__clone_max(const pcmk_resource_t *clone)
 {
     const clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
     return clone_data->clone_max;
 }
 
 /*!
  * \internal
  * \brief Return the maximum number of clone instances allowed per node
  *
  * \param[in] clone  Promotable clone or clone instance to check
  *
  * \return Maximum allowed instances per node for \p clone
  */
 int
 pe__clone_node_max(const pcmk_resource_t *clone)
 {
     const clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
     return clone_data->clone_node_max;
 }
 
 /*!
  * \internal
  * \brief Return the maximum number of clone instances allowed to be promoted
  *
  * \param[in] clone  Promotable clone or clone instance to check
  *
  * \return Maximum promoted instances for \p clone
  */
 int
 pe__clone_promoted_max(const pcmk_resource_t *clone)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
     return clone_data->promoted_max;
 }
 
 /*!
  * \internal
  * \brief Return the maximum number of clone instances allowed to be promoted
  *
  * \param[in] clone  Promotable clone or clone instance to check
  *
  * \return Maximum promoted instances for \p clone
  */
 int
 pe__clone_promoted_node_max(const pcmk_resource_t *clone)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, pe__const_top_resource(clone, false));
     return clone_data->promoted_node_max;
 }
 
 static GList *
 sorted_hash_table_values(GHashTable *table)
 {
     GList *retval = NULL;
     GHashTableIter iter;
     gpointer key, value;
 
     g_hash_table_iter_init(&iter, table);
     while (g_hash_table_iter_next(&iter, &key, &value)) {
         if (!g_list_find_custom(retval, value, (GCompareFunc) strcmp)) {
             retval = g_list_prepend(retval, (char *) value);
         }
     }
 
     retval = g_list_sort(retval, (GCompareFunc) strcmp);
     return retval;
 }
 
 static GList *
 nodes_with_status(GHashTable *table, const char *status)
 {
     GList *retval = NULL;
     GHashTableIter iter;
     gpointer key, value;
 
     g_hash_table_iter_init(&iter, table);
     while (g_hash_table_iter_next(&iter, &key, &value)) {
         if (!strcmp((char *) value, status)) {
             retval = g_list_prepend(retval, key);
         }
     }
 
     retval = g_list_sort(retval, (GCompareFunc) pcmk__numeric_strcasecmp);
     return retval;
 }
 
 static GString *
 node_list_to_str(const GList *list)
 {
     GString *retval = NULL;
 
     for (const GList *iter = list; iter != NULL; iter = iter->next) {
         pcmk__add_word(&retval, 1024, (const char *) iter->data);
     }
 
     return retval;
 }
 
 static void
 clone_header(pcmk__output_t *out, int *rc, const pcmk_resource_t *rsc,
              clone_variant_data_t *clone_data, const char *desc)
 {
     GString *attrs = NULL;
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
         pcmk__add_separated_word(&attrs, 64, "promotable", ", ");
     }
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
         pcmk__add_separated_word(&attrs, 64, "unique", ", ");
     }
 
     if (pe__resource_is_disabled(rsc)) {
         pcmk__add_separated_word(&attrs, 64, "disabled", ", ");
     }
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_maintenance)) {
         pcmk__add_separated_word(&attrs, 64, "maintenance", ", ");
 
     } else if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
         pcmk__add_separated_word(&attrs, 64, "unmanaged", ", ");
     }
 
     if (attrs != NULL) {
         PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s] (%s)%s%s%s",
                                  rsc->id,
                                  pcmk__xe_id(clone_data->xml_obj_child),
                                  (const char *) attrs->str, desc ? " (" : "",
                                  desc ? desc : "", desc ? ")" : "");
         g_string_free(attrs, TRUE);
     } else {
         PCMK__OUTPUT_LIST_HEADER(out, FALSE, *rc, "Clone Set: %s [%s]%s%s%s",
                                  rsc->id,
                                  pcmk__xe_id(clone_data->xml_obj_child),
                                  desc ? " (" : "", desc ? desc : "",
                                  desc ? ")" : "");
     }
 }
 
 void
 pe__force_anon(const char *standard, pcmk_resource_t *rsc, const char *rid,
                pcmk_scheduler_t *scheduler)
 {
     if (pcmk__is_clone(rsc)) {
         clone_variant_data_t *clone_data = rsc->priv->variant_opaque;
 
         pcmk__config_warn("Ignoring " PCMK_META_GLOBALLY_UNIQUE " for %s "
                           "because %s resources such as %s can be used only as "
                           "anonymous clones", rsc->id, standard, rid);
 
         clone_data->clone_node_max = 1;
         clone_data->clone_max = QB_MIN(clone_data->clone_max,
                                        g_list_length(scheduler->nodes));
     }
 }
 
 pcmk_resource_t *
 pe__create_clone_child(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     gboolean as_orphan = FALSE;
     char *inc_num = NULL;
     char *inc_max = NULL;
     pcmk_resource_t *child_rsc = NULL;
     xmlNode *child_copy = NULL;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     CRM_CHECK(clone_data->xml_obj_child != NULL, return FALSE);
 
     if (clone_data->total_clones >= clone_data->clone_max) {
         // If we've already used all available instances, this is an orphan
         as_orphan = TRUE;
     }
 
     // Allocate instance numbers in numerical order (starting at 0)
     inc_num = pcmk__itoa(clone_data->total_clones);
     inc_max = pcmk__itoa(clone_data->clone_max);
 
     child_copy = pcmk__xml_copy(NULL, clone_data->xml_obj_child);
 
     crm_xml_add(child_copy, PCMK__META_CLONE, inc_num);
 
     if (pe__unpack_resource(child_copy, &child_rsc, rsc,
                             scheduler) != pcmk_rc_ok) {
         goto bail;
     }
 /*  child_rsc->globally_unique = rsc->globally_unique; */
 
     pcmk__assert(child_rsc != NULL);
     clone_data->total_clones += 1;
     pcmk__rsc_trace(child_rsc, "Setting clone attributes for: %s",
                     child_rsc->id);
     rsc->priv->children = g_list_append(rsc->priv->children, child_rsc);
     if (as_orphan) {
         pe__set_resource_flags_recursive(child_rsc, pcmk__rsc_removed);
     }
 
     pcmk__insert_meta(child_rsc->priv, PCMK_META_CLONE_MAX, inc_max);
     pcmk__rsc_trace(rsc, "Added %s instance %s", rsc->id, child_rsc->id);
 
   bail:
     free(inc_num);
     free(inc_max);
 
     return child_rsc;
 }
 
 /*!
  * \internal
  * \brief Unpack a nonnegative integer value from a resource meta-attribute
  *
  * \param[in]  rsc              Resource with meta-attribute
  * \param[in]  meta_name        Name of meta-attribute to unpack
  * \param[in]  deprecated_name  If not NULL, try unpacking this
  *                              if \p meta_name is unset
  * \param[in]  default_value    Value to use if unset
  *
  * \return Integer parsed from resource's specified meta-attribute if a valid
  *         nonnegative integer, \p default_value if unset, or 0 if invalid
  */
 static int
 unpack_meta_int(const pcmk_resource_t *rsc, const char *meta_name,
                 const char *deprecated_name, int default_value)
 {
     int integer = default_value;
     const char *value = g_hash_table_lookup(rsc->priv->meta, meta_name);
 
     if ((value == NULL) && (deprecated_name != NULL)) {
         value = g_hash_table_lookup(rsc->priv->meta, deprecated_name);
 
         if (value != NULL) {
             if (pcmk__str_eq(deprecated_name, PCMK__META_PROMOTED_MAX_LEGACY,
                              pcmk__str_none)) {
                 pcmk__warn_once(pcmk__wo_clone_master_max,
                                 "Support for the " PCMK__META_PROMOTED_MAX_LEGACY
                                 " meta-attribute (such as in %s) is deprecated "
                                 "and will be removed in a future release. Use the "
                                 PCMK_META_PROMOTED_MAX " meta-attribute instead.",
                                 rsc->id);
             } else if (pcmk__str_eq(deprecated_name, PCMK__META_PROMOTED_NODE_MAX_LEGACY,
                                     pcmk__str_none)) {
                 pcmk__warn_once(pcmk__wo_clone_master_node_max,
                                 "Support for the " PCMK__META_PROMOTED_NODE_MAX_LEGACY
                                 " meta-attribute (such as in %s) is deprecated "
                                 "and will be removed in a future release. Use the "
                                 PCMK_META_PROMOTED_NODE_MAX " meta-attribute instead.",
                                 rsc->id);
             }
         }
     }
     if (value != NULL) {
         pcmk__scan_min_int(value, &integer, 0);
     }
     return integer;
 }
 
 gboolean
 clone_unpack(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     int lpc = 0;
     xmlNode *a_child = NULL;
     xmlNode *xml_obj = rsc->priv->xml;
     clone_variant_data_t *clone_data = NULL;
 
     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 
     clone_data = pcmk__assert_alloc(1, sizeof(clone_variant_data_t));
     rsc->priv->variant_opaque = clone_data;
 
     if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
         // Use 1 as default but 0 for minimum and invalid
         // @COMPAT PCMK__META_PROMOTED_MAX_LEGACY deprecated since 2.0.0
         clone_data->promoted_max =
             unpack_meta_int(rsc, PCMK_META_PROMOTED_MAX,
                             PCMK__META_PROMOTED_MAX_LEGACY, 1);
 
         // Use 1 as default but 0 for minimum and invalid
         // @COMPAT PCMK__META_PROMOTED_NODE_MAX_LEGACY deprecated since 2.0.0
         clone_data->promoted_node_max =
             unpack_meta_int(rsc, PCMK_META_PROMOTED_NODE_MAX,
                             PCMK__META_PROMOTED_NODE_MAX_LEGACY, 1);
     }
 
     // Use 1 as default but 0 for minimum and invalid
     clone_data->clone_node_max = unpack_meta_int(rsc, PCMK_META_CLONE_NODE_MAX,
                                                  NULL, 1);
 
     /* Use number of nodes (but always at least 1, which is handy for crm_verify
      * for a CIB without nodes) as default, but 0 for minimum and invalid
+     *
+     * @TODO Exclude bundle nodes when counting
      */
     clone_data->clone_max = unpack_meta_int(rsc, PCMK_META_CLONE_MAX, NULL,
                                             QB_MAX(1, g_list_length(scheduler->nodes)));
 
     if (crm_is_true(g_hash_table_lookup(rsc->priv->meta,
                                         PCMK_META_ORDERED))) {
         clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
                                                "Clone", rsc->id,
                                                clone_data->flags,
                                                pcmk__clone_ordered,
                                                "pcmk__clone_ordered");
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unique)
         && (clone_data->clone_node_max > 1)) {
 
         pcmk__config_err("Ignoring " PCMK_META_CLONE_NODE_MAX " of %d for %s "
                          "because anonymous clones support only one instance "
                          "per node", clone_data->clone_node_max, rsc->id);
         clone_data->clone_node_max = 1;
     }
 
     pcmk__rsc_trace(rsc, "Options for %s", rsc->id);
     pcmk__rsc_trace(rsc, "\tClone max: %d", clone_data->clone_max);
     pcmk__rsc_trace(rsc, "\tClone node max: %d", clone_data->clone_node_max);
     pcmk__rsc_trace(rsc, "\tClone is unique: %s",
                     pcmk__flag_text(rsc->flags, pcmk__rsc_unique));
     pcmk__rsc_trace(rsc, "\tClone is promotable: %s",
                     pcmk__flag_text(rsc->flags, pcmk__rsc_promotable));
 
     // Clones may contain a single group or primitive
     for (a_child = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
          a_child != NULL; a_child = pcmk__xe_next(a_child, NULL)) {
 
         if (pcmk__str_any_of((const char *) a_child->name,
                              PCMK_XE_PRIMITIVE, PCMK_XE_GROUP, NULL)) {
             clone_data->xml_obj_child = a_child;
             break;
         }
     }
 
     if (clone_data->xml_obj_child == NULL) {
         pcmk__config_err("%s has nothing to clone", rsc->id);
         return FALSE;
     }
 
     /*
      * Make clones ever so slightly sticky by default
      *
      * This helps ensure clone instances are not shuffled around the cluster
      * for no benefit in situations when pre-allocation is not appropriate
      */
     if (g_hash_table_lookup(rsc->priv->meta,
                             PCMK_META_RESOURCE_STICKINESS) == NULL) {
         pcmk__insert_meta(rsc->priv, PCMK_META_RESOURCE_STICKINESS, "1");
     }
 
     /* This ensures that the PCMK_META_GLOBALLY_UNIQUE value always exists for
      * children to inherit when being unpacked, as well as in resource agents'
      * environment.
      */
     pcmk__insert_meta(rsc->priv, PCMK_META_GLOBALLY_UNIQUE,
                       pcmk__flag_text(rsc->flags, pcmk__rsc_unique));
 
     if (clone_data->clone_max <= 0) {
         /* Create one child instance so that unpack_find_resource() will hook up
          * any orphans up to the parent correctly.
          */
         if (pe__create_clone_child(rsc, scheduler) == NULL) {
             return FALSE;
         }
 
     } else {
         // Create a child instance for each available instance number
         for (lpc = 0; lpc < clone_data->clone_max; lpc++) {
             if (pe__create_clone_child(rsc, scheduler) == NULL) {
                 return FALSE;
             }
         }
     }
 
     pcmk__rsc_trace(rsc, "Added %d children to resource %s...",
                     clone_data->clone_max, rsc->id);
     return TRUE;
 }
 
 gboolean
 clone_active(pcmk_resource_t * rsc, gboolean all)
 {
     for (GList *gIter = rsc->priv->children;
          gIter != NULL; gIter = gIter->next) {
 
         pcmk_resource_t *child_rsc = (pcmk_resource_t *) gIter->data;
         gboolean child_active = child_rsc->priv->fns->active(child_rsc, all);
 
         if (all == FALSE && child_active) {
             return TRUE;
         } else if (all && child_active == FALSE) {
             return FALSE;
         }
     }
 
     if (all) {
         return TRUE;
     } else {
         return FALSE;
     }
 }
 
 static const char *
 configured_role_str(pcmk_resource_t * rsc)
 {
     const char *target_role = g_hash_table_lookup(rsc->priv->meta,
                                                   PCMK_META_TARGET_ROLE);
 
     if ((target_role == NULL) && (rsc->priv->children != NULL)) {
         // Any instance will do
         pcmk_resource_t *instance = rsc->priv->children->data;
 
         target_role = g_hash_table_lookup(instance->priv->meta,
                                           PCMK_META_TARGET_ROLE);
     }
     return target_role;
 }
 
 static enum rsc_role_e
 configured_role(pcmk_resource_t *rsc)
 {
     enum rsc_role_e role = pcmk_role_unknown;
     const char *target_role = configured_role_str(rsc);
 
     if (target_role != NULL) {
         role = pcmk_parse_role(target_role);
         if (role == pcmk_role_unknown) {
             pcmk__config_err("Invalid " PCMK_META_TARGET_ROLE
                              " for resource %s", rsc->id);
         }
     }
     return role;
 }
 
 bool
 is_set_recursive(const pcmk_resource_t *rsc, long long flag, bool any)
 {
     bool all = !any;
 
     if (pcmk_is_set(rsc->flags, flag)) {
         if(any) {
             return TRUE;
         }
     } else if(all) {
         return FALSE;
     }
 
     for (GList *gIter = rsc->priv->children;
          gIter != NULL; gIter = gIter->next) {
 
         if(is_set_recursive(gIter->data, flag, any)) {
             if(any) {
                 return TRUE;
             }
 
         } else if(all) {
             return FALSE;
         }
     }
 
     if(all) {
         return TRUE;
     }
     return FALSE;
 }
 
 PCMK__OUTPUT_ARGS("clone", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__clone_xml(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     GList *all = NULL;
     int rc = pcmk_rc_no_output;
     gboolean printed_header = FALSE;
     gboolean print_everything = TRUE;
 
     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) ||
                        (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));
 
     all = g_list_prepend(all, (gpointer) "*");
 
     for (GList *gIter = rsc->priv->children;
          gIter != NULL; gIter = gIter->next) {
 
         pcmk_resource_t *child_rsc = (pcmk_resource_t *) gIter->data;
 
         if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
             continue;
         }
 
         if (child_rsc->priv->fns->is_filtered(child_rsc, only_rsc,
                                               print_everything)) {
             continue;
         }
 
         if (!printed_header) {
             const char *multi_state = pcmk__flag_text(rsc->flags,
                                                       pcmk__rsc_promotable);
             const char *unique = pcmk__flag_text(rsc->flags, pcmk__rsc_unique);
             const char *maintenance = pcmk__flag_text(rsc->flags,
                                                       pcmk__rsc_maintenance);
             const char *managed = pcmk__flag_text(rsc->flags,
                                                   pcmk__rsc_managed);
             const char *disabled = pcmk__btoa(pe__resource_is_disabled(rsc));
             const char *failed = pcmk__flag_text(rsc->flags, pcmk__rsc_failed);
             const char *ignored = pcmk__flag_text(rsc->flags,
                                                   pcmk__rsc_ignore_failure);
             const char *target_role = configured_role_str(rsc);
             const char *desc = pe__resource_description(rsc, show_opts);
 
             printed_header = TRUE;
 
             rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_CLONE,
                                           PCMK_XA_ID, rsc->id,
                                           PCMK_XA_MULTI_STATE, multi_state,
                                           PCMK_XA_UNIQUE, unique,
                                           PCMK_XA_MAINTENANCE, maintenance,
                                           PCMK_XA_MANAGED, managed,
                                           PCMK_XA_DISABLED, disabled,
                                           PCMK_XA_FAILED, failed,
                                           PCMK_XA_FAILURE_IGNORED, ignored,
                                           PCMK_XA_TARGET_ROLE, target_role,
                                           PCMK_XA_DESCRIPTION, desc,
                                           NULL);
             pcmk__assert(rc == pcmk_rc_ok);
         }
 
         out->message(out, (const char *) child_rsc->priv->xml->name,
                      show_opts, child_rsc, only_node, all);
     }
 
     if (printed_header) {
         pcmk__output_xml_pop_parent(out);
     }
 
     g_list_free(all);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("clone", "uint32_t", "pcmk_resource_t *", "GList *",
                   "GList *")
 int
 pe__clone_default(pcmk__output_t *out, va_list args)
 {
     uint32_t show_opts = va_arg(args, uint32_t);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     GHashTable *stopped = NULL;
 
     GString *list_text = NULL;
 
     GList *promoted_list = NULL;
     GList *started_list = NULL;
     GList *gIter = NULL;
 
     const char *desc = NULL;
 
     clone_variant_data_t *clone_data = NULL;
     int active_instances = 0;
     int rc = pcmk_rc_no_output;
     gboolean print_everything = TRUE;
 
     desc = pe__resource_description(rsc, show_opts);
 
     get_clone_variant_data(clone_data, rsc);
 
     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
         return rc;
     }
 
     print_everything = pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches) ||
                        (strstr(rsc->id, ":") != NULL && pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches));
 
     for (gIter = rsc->priv->children; gIter != NULL; gIter = gIter->next) {
         gboolean print_full = FALSE;
         pcmk_resource_t *child_rsc = (pcmk_resource_t *) gIter->data;
         gboolean partially_active = child_rsc->priv->fns->active(child_rsc,
                                                                  FALSE);
 
         if (pcmk__rsc_filtered_by_node(child_rsc, only_node)) {
             continue;
         }
 
         if (child_rsc->priv->fns->is_filtered(child_rsc, only_rsc,
                                               print_everything)) {
             continue;
         }
 
         if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
             print_full = TRUE;
         }
 
         if (pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
             // Print individual instance when unique (except stopped orphans)
             if (partially_active
                 || !pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
                 print_full = TRUE;
             }
 
         // Everything else in this block is for anonymous clones
 
         } else if (pcmk_is_set(show_opts, pcmk_show_pending)
                    && (child_rsc->priv->pending_action != NULL)
                    && (strcmp(child_rsc->priv->pending_action,
                               "probe") != 0)) {
             // Print individual instance when non-probe action is pending
             print_full = TRUE;
 
         } else if (partially_active == FALSE) {
             // List stopped instances when requested (except orphans)
             if (!pcmk_is_set(child_rsc->flags, pcmk__rsc_removed)
                 && !pcmk_is_set(show_opts, pcmk_show_clone_detail)
                 && pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
                 if (stopped == NULL) {
                     stopped = pcmk__strkey_table(free, free);
                 }
                 pcmk__insert_dup(stopped, child_rsc->id, "Stopped");
             }
 
         } else if (is_set_recursive(child_rsc, pcmk__rsc_removed, TRUE)
                    || !is_set_recursive(child_rsc, pcmk__rsc_managed, FALSE)
                    || is_set_recursive(child_rsc, pcmk__rsc_failed, TRUE)) {
 
             // Print individual instance when active orphaned/unmanaged/failed
             print_full = TRUE;
 
         } else if (child_rsc->priv->fns->active(child_rsc, TRUE)) {
             // Instance of fully active anonymous clone
 
             pcmk_node_t *location = NULL;
 
             location = child_rsc->priv->fns->location(child_rsc, NULL,
                                                       pcmk__rsc_node_current);
             if (location) {
                 // Instance is active on a single node
 
                 enum rsc_role_e a_role;
 
                 a_role = child_rsc->priv->fns->state(child_rsc, TRUE);
 
                 if (location->details->online == FALSE && location->details->unclean) {
                     print_full = TRUE;
 
                 } else if (a_role > pcmk_role_unpromoted) {
                     promoted_list = g_list_append(promoted_list, location);
 
                 } else {
                     started_list = g_list_append(started_list, location);
                 }
 
             } else {
                 /* uncolocated group - bleh */
                 print_full = TRUE;
             }
 
         } else {
             // Instance of partially active anonymous clone
             print_full = TRUE;
         }
 
         if (print_full) {
             GList *all = NULL;
 
             clone_header(out, &rc, rsc, clone_data, desc);
 
             /* Print every resource that's a child of this clone. */
             all = g_list_prepend(all, (gpointer) "*");
             out->message(out, (const char *) child_rsc->priv->xml->name,
                          show_opts, child_rsc, only_node, all);
             g_list_free(all);
         }
     }
 
     if (pcmk_is_set(show_opts, pcmk_show_clone_detail)) {
         PCMK__OUTPUT_LIST_FOOTER(out, rc);
         return pcmk_rc_ok;
     }
 
     /* Promoted */
     promoted_list = g_list_sort(promoted_list, pe__cmp_node_name);
     for (gIter = promoted_list; gIter; gIter = gIter->next) {
         pcmk_node_t *host = gIter->data;
 
         if (!pcmk__str_in_list(host->priv->name, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         pcmk__add_word(&list_text, 1024, host->priv->name);
         active_instances++;
     }
     g_list_free(promoted_list);
 
     if ((list_text != NULL) && (list_text->len > 0)) {
         clone_header(out, &rc, rsc, clone_data, desc);
 
         out->list_item(out, NULL, PCMK_ROLE_PROMOTED ": [ %s ]",
                        (const char *) list_text->str);
         g_string_truncate(list_text, 0);
     }
 
     /* Started/Unpromoted */
     started_list = g_list_sort(started_list, pe__cmp_node_name);
     for (gIter = started_list; gIter; gIter = gIter->next) {
         pcmk_node_t *host = gIter->data;
 
         if (!pcmk__str_in_list(host->priv->name, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         pcmk__add_word(&list_text, 1024, host->priv->name);
         active_instances++;
     }
     g_list_free(started_list);
 
     if ((list_text != NULL) && (list_text->len > 0)) {
         clone_header(out, &rc, rsc, clone_data, desc);
 
         if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
             enum rsc_role_e role = configured_role(rsc);
 
             if (role == pcmk_role_unpromoted) {
                 out->list_item(out, NULL,
                                PCMK_ROLE_UNPROMOTED
                                " (" PCMK_META_TARGET_ROLE "): [ %s ]",
                                (const char *) list_text->str);
             } else {
                 out->list_item(out, NULL, PCMK_ROLE_UNPROMOTED ": [ %s ]",
                                (const char *) list_text->str);
             }
 
         } else {
             out->list_item(out, NULL, "Started: [ %s ]",
                            (const char *) list_text->str);
         }
     }
 
     if (list_text != NULL) {
         g_string_free(list_text, TRUE);
     }
 
     if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
         if (!pcmk_is_set(rsc->flags, pcmk__rsc_unique)
             && (clone_data->clone_max > active_instances)) {
 
             GList *nIter;
             GList *list = g_hash_table_get_values(rsc->priv->allowed_nodes);
 
             /* Custom stopped table for non-unique clones */
             if (stopped != NULL) {
                 g_hash_table_destroy(stopped);
                 stopped = NULL;
             }
 
             if (list == NULL) {
                 /* Clusters with PCMK_OPT_SYMMETRIC_CLUSTER=false haven't
                  * calculated allowed nodes yet. If we've not probed for them
                  * yet, the Stopped list will be empty.
                  */
                 list = g_hash_table_get_values(rsc->priv->probed_nodes);
             }
 
             list = g_list_sort(list, pe__cmp_node_name);
             for (nIter = list; nIter != NULL; nIter = nIter->next) {
                 pcmk_node_t *node = (pcmk_node_t *) nIter->data;
 
                 if ((pcmk__find_node_in_list(rsc->priv->active_nodes,
                                              node->priv->name) == NULL)
                     && pcmk__str_in_list(node->priv->name, only_node,
                                          pcmk__str_star_matches|pcmk__str_casei)) {
 
                     xmlNode *probe_op = NULL;
                     const char *state = "Stopped";
 
                     if (configured_role(rsc) == pcmk_role_stopped) {
                         state = "Stopped (disabled)";
                     }
 
                     if (stopped == NULL) {
                         stopped = pcmk__strkey_table(free, free);
                     }
 
                     probe_op = pe__failed_probe_for_rsc(rsc,
                                                         node->priv->name);
                     if (probe_op != NULL) {
                         int rc;
 
                         pcmk__scan_min_int(crm_element_value(probe_op,
                                                              PCMK__XA_RC_CODE),
                                            &rc, 0);
                         g_hash_table_insert(stopped, strdup(node->priv->name),
                                             crm_strdup_printf("Stopped (%s)",
                                                               crm_exit_str(rc)));
                     } else {
                         pcmk__insert_dup(stopped, node->priv->name, state);
                     }
                 }
             }
             g_list_free(list);
         }
 
         if (stopped != NULL) {
             GList *list = sorted_hash_table_values(stopped);
 
             clone_header(out, &rc, rsc, clone_data, desc);
 
             for (GList *status_iter = list; status_iter != NULL; status_iter = status_iter->next) {
                 const char *status = status_iter->data;
                 GList *nodes = nodes_with_status(stopped, status);
                 GString *nodes_str = node_list_to_str(nodes);
 
                 if (nodes_str != NULL) {
                     if (nodes_str->len > 0) {
                         out->list_item(out, NULL, "%s: [ %s ]", status,
                                        (const char *) nodes_str->str);
                     }
                     g_string_free(nodes_str, TRUE);
                 }
 
                 g_list_free(nodes);
             }
 
             g_list_free(list);
             g_hash_table_destroy(stopped);
 
         /* If there are no instances of this clone (perhaps because there are no
          * nodes configured), simply output the clone header by itself.  This can
          * come up in PCS testing.
          */
         } else if (active_instances == 0) {
             clone_header(out, &rc, rsc, clone_data, desc);
             PCMK__OUTPUT_LIST_FOOTER(out, rc);
             return rc;
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 void
 clone_free(pcmk_resource_t * rsc)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
 
     pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
 
     for (GList *gIter = rsc->priv->children;
          gIter != NULL; gIter = gIter->next) {
 
         pcmk_resource_t *child_rsc = (pcmk_resource_t *) gIter->data;
 
         pcmk__assert(child_rsc != NULL);
         pcmk__rsc_trace(child_rsc, "Freeing child %s", child_rsc->id);
         pcmk__xml_free(child_rsc->priv->xml);
         child_rsc->priv->xml = NULL;
         /* There could be a saved unexpanded xml */
         pcmk__xml_free(child_rsc->priv->orig_xml);
         child_rsc->priv->orig_xml = NULL;
         child_rsc->priv->fns->free(child_rsc);
     }
 
     g_list_free(rsc->priv->children);
 
     if (clone_data) {
         pcmk__assert((clone_data->demote_notify == NULL)
                      && (clone_data->stop_notify == NULL)
                      && (clone_data->start_notify == NULL)
                      && (clone_data->promote_notify == NULL));
     }
 
     common_free(rsc);
 }
 
 enum rsc_role_e
 clone_resource_state(const pcmk_resource_t * rsc, gboolean current)
 {
     enum rsc_role_e clone_role = pcmk_role_unknown;
 
     for (GList *gIter = rsc->priv->children;
          gIter != NULL; gIter = gIter->next) {
 
         pcmk_resource_t *child_rsc = (pcmk_resource_t *) gIter->data;
         enum rsc_role_e a_role = child_rsc->priv->fns->state(child_rsc,
                                                              current);
 
         if (a_role > clone_role) {
             clone_role = a_role;
         }
     }
 
     pcmk__rsc_trace(rsc, "%s role: %s", rsc->id, pcmk_role_text(clone_role));
     return clone_role;
 }
 
 /*!
  * \internal
  * \brief Check whether a clone has an instance for every node
  *
  * \param[in] rsc        Clone to check
  * \param[in] scheduler  Scheduler data
  */
 bool
 pe__is_universal_clone(const pcmk_resource_t *rsc,
                        const pcmk_scheduler_t *scheduler)
 {
     if (pcmk__is_clone(rsc)) {
         clone_variant_data_t *clone_data = rsc->priv->variant_opaque;
 
         if (clone_data->clone_max == g_list_length(scheduler->nodes)) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 gboolean
 pe__clone_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
                       gboolean check_parent)
 {
     gboolean passes = FALSE;
     clone_variant_data_t *clone_data = NULL;
 
     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
         passes = TRUE;
     } else {
         get_clone_variant_data(clone_data, rsc);
         passes = pcmk__str_in_list(pcmk__xe_id(clone_data->xml_obj_child),
                                    only_rsc, pcmk__str_star_matches);
 
         if (!passes) {
             for (const GList *iter = rsc->priv->children;
                  iter != NULL; iter = iter->next) {
 
                 const pcmk_resource_t *child_rsc = NULL;
 
                 child_rsc = (const pcmk_resource_t *) iter->data;
                 if (!child_rsc->priv->fns->is_filtered(child_rsc, only_rsc,
                                                        FALSE)) {
                     passes = TRUE;
                     break;
                 }
             }
         }
     }
     return !passes;
 }
 
 const char *
 pe__clone_child_id(const pcmk_resource_t *rsc)
 {
     clone_variant_data_t *clone_data = NULL;
     get_clone_variant_data(clone_data, rsc);
     return pcmk__xe_id(clone_data->xml_obj_child);
 }
 
 /*!
  * \internal
  * \brief Check whether a clone is ordered
  *
  * \param[in] clone  Clone resource to check
  *
  * \return true if clone is ordered, otherwise false
  */
 bool
 pe__clone_is_ordered(const pcmk_resource_t *clone)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, clone);
     return pcmk_is_set(clone_data->flags, pcmk__clone_ordered);
 }
 
 /*!
  * \internal
  * \brief Set a clone flag
  *
  * \param[in,out] clone  Clone resource to set flag for
  * \param[in]     flag   Clone flag to set
  *
  * \return Standard Pacemaker return code (either pcmk_rc_ok if flag was not
  *         already set or pcmk_rc_already if it was)
  */
 int
 pe__set_clone_flag(pcmk_resource_t *clone, enum pcmk__clone_flags flag)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, clone);
     if (pcmk_is_set(clone_data->flags, flag)) {
         return pcmk_rc_already;
     }
     clone_data->flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,
                                            "Clone", clone->id,
                                            clone_data->flags, flag, "flag");
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Check whether a clone flag is set
  *
  * \param[in] group  Clone resource to check
  * \param[in] flags  Flag or flags to check
  *
  * \return \c true if all \p flags are set for \p clone, otherwise \c false
  */
 bool
 pe__clone_flag_is_set(const pcmk_resource_t *clone, uint32_t flags)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, clone);
     pcmk__assert(clone_data != NULL);
 
     return pcmk_all_flags_set(clone_data->flags, flags);
 }
 
 /*!
  * \internal
  * \brief Create pseudo-actions needed for promotable clones
  *
  * \param[in,out] clone          Promotable clone to create actions for
  * \param[in]     any_promoting  Whether any instances will be promoted
  * \param[in]     any_demoting   Whether any instance will be demoted
  */
 void
 pe__create_promotable_pseudo_ops(pcmk_resource_t *clone, bool any_promoting,
                                  bool any_demoting)
 {
     pcmk_action_t *action = NULL;
     pcmk_action_t *action_complete = NULL;
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, clone);
 
     // Create a "promote" action for the clone itself
     action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTE,
                                        !any_promoting, true);
 
     // Create a "promoted" action for when all promotions are done
     action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_PROMOTED,
                                                 !any_promoting, true);
     action_complete->priority = PCMK_SCORE_INFINITY;
 
     // Create notification pseudo-actions for promotion
     if (clone_data->promote_notify == NULL) {
         clone_data->promote_notify = pe__action_notif_pseudo_ops(clone,
                                                                  PCMK_ACTION_PROMOTE,
                                                                  action,
                                                                  action_complete);
     }
 
     // Create a "demote" action for the clone itself
     action = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTE,
                                        !any_demoting, true);
 
     // Create a "demoted" action for when all demotions are done
     action_complete = pe__new_rsc_pseudo_action(clone, PCMK_ACTION_DEMOTED,
                                                 !any_demoting, true);
     action_complete->priority = PCMK_SCORE_INFINITY;
 
     // Create notification pseudo-actions for demotion
     if (clone_data->demote_notify == NULL) {
         clone_data->demote_notify = pe__action_notif_pseudo_ops(clone,
                                                                 PCMK_ACTION_DEMOTE,
                                                                 action,
                                                                 action_complete);
 
         if (clone_data->promote_notify != NULL) {
             order_actions(clone_data->stop_notify->post_done,
                           clone_data->promote_notify->pre, pcmk__ar_ordered);
             order_actions(clone_data->start_notify->post_done,
                           clone_data->promote_notify->pre, pcmk__ar_ordered);
             order_actions(clone_data->demote_notify->post_done,
                           clone_data->promote_notify->pre, pcmk__ar_ordered);
             order_actions(clone_data->demote_notify->post_done,
                           clone_data->start_notify->pre, pcmk__ar_ordered);
             order_actions(clone_data->demote_notify->post_done,
                           clone_data->stop_notify->pre, pcmk__ar_ordered);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Create all notification data and actions for a clone
  *
  * \param[in,out] clone  Clone to create notifications for
  */
 void
 pe__create_clone_notifications(pcmk_resource_t *clone)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, clone);
 
     pe__create_action_notifications(clone, clone_data->start_notify);
     pe__create_action_notifications(clone, clone_data->stop_notify);
     pe__create_action_notifications(clone, clone_data->promote_notify);
     pe__create_action_notifications(clone, clone_data->demote_notify);
 }
 
 /*!
  * \internal
  * \brief Free all notification data for a clone
  *
  * \param[in,out] clone  Clone to free notification data for
  */
 void
 pe__free_clone_notification_data(pcmk_resource_t *clone)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, clone);
 
     pe__free_action_notification_data(clone_data->demote_notify);
     clone_data->demote_notify = NULL;
 
     pe__free_action_notification_data(clone_data->stop_notify);
     clone_data->stop_notify = NULL;
 
     pe__free_action_notification_data(clone_data->start_notify);
     clone_data->start_notify = NULL;
 
     pe__free_action_notification_data(clone_data->promote_notify);
     clone_data->promote_notify = NULL;
 }
 
 /*!
  * \internal
  * \brief Create pseudo-actions for clone start/stop notifications
  *
  * \param[in,out] clone    Clone to create pseudo-actions for
  * \param[in,out] start    Start action for \p clone
  * \param[in,out] stop     Stop action for \p clone
  * \param[in,out] started  Started action for \p clone
  * \param[in,out] stopped  Stopped action for \p clone
  */
 void
 pe__create_clone_notif_pseudo_ops(pcmk_resource_t *clone,
                                   pcmk_action_t *start, pcmk_action_t *started,
                                   pcmk_action_t *stop, pcmk_action_t *stopped)
 {
     clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, clone);
 
     if (clone_data->start_notify == NULL) {
         clone_data->start_notify = pe__action_notif_pseudo_ops(clone,
                                                                PCMK_ACTION_START,
                                                                start, started);
     }
 
     if (clone_data->stop_notify == NULL) {
         clone_data->stop_notify = pe__action_notif_pseudo_ops(clone,
                                                               PCMK_ACTION_STOP,
                                                               stop, stopped);
         if ((clone_data->start_notify != NULL)
             && (clone_data->stop_notify != NULL)) {
             order_actions(clone_data->stop_notify->post_done,
                           clone_data->start_notify->pre, pcmk__ar_ordered);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Get maximum clone resource instances per node
  *
  * \param[in] rsc  Clone resource to check
  *
  * \return Maximum number of \p rsc instances that can be active on one node
  */
 unsigned int
 pe__clone_max_per_node(const pcmk_resource_t *rsc)
 {
     const clone_variant_data_t *clone_data = NULL;
 
     get_clone_variant_data(clone_data, rsc);
     return clone_data->clone_node_max;
 }
diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
index 7bc65dcad1..3497e56779 100644
--- a/lib/pengine/pe_actions.c
+++ b/lib/pengine/pe_actions.c
@@ -1,1770 +1,1774 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 #include <stdbool.h>
 
 #include <crm/crm.h>
 #include <crm/common/xml.h>
 #include <crm/common/scheduler_internal.h>
 #include <crm/pengine/internal.h>
 #include <crm/common/xml_internal.h>
 #include "pe_status_private.h"
 
 static void unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj,
                              guint interval_ms);
 
 static void
 add_singleton(pcmk_scheduler_t *scheduler, pcmk_action_t *action)
 {
     if (scheduler->priv->singletons == NULL) {
         scheduler->priv->singletons = pcmk__strkey_table(NULL, NULL);
     }
     g_hash_table_insert(scheduler->priv->singletons, action->uuid, action);
 }
 
 static pcmk_action_t *
 lookup_singleton(pcmk_scheduler_t *scheduler, const char *action_uuid)
 {
     /* @TODO This is the only use of the pcmk_scheduler_t:singletons hash table.
      * Compare the performance of this approach to keeping the
      * pcmk_scheduler_t:actions list sorted by action key and just searching
      * that instead.
      */
     if (scheduler->priv->singletons == NULL) {
         return NULL;
     }
     return g_hash_table_lookup(scheduler->priv->singletons, action_uuid);
 }
 
 /*!
  * \internal
  * \brief Find an existing action that matches arguments
  *
  * \param[in] key        Action key to match
  * \param[in] rsc        Resource to match (if any)
  * \param[in] node       Node to match (if any)
  * \param[in] scheduler  Scheduler data
  *
  * \return Existing action that matches arguments (or NULL if none)
  */
 static pcmk_action_t *
 find_existing_action(const char *key, const pcmk_resource_t *rsc,
                      const pcmk_node_t *node, const pcmk_scheduler_t *scheduler)
 {
     /* When rsc is NULL, it would be quicker to check
      * scheduler->priv->singletons, but checking all scheduler->priv->actions
      * takes the node into account.
      */
     GList *actions = (rsc == NULL)? scheduler->priv->actions : rsc->priv->actions;
     GList *matches = find_actions(actions, key, node);
     pcmk_action_t *action = NULL;
 
     if (matches == NULL) {
         return NULL;
     }
     CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));
 
     action = matches->data;
     g_list_free(matches);
     return action;
 }
 
 /*!
  * \internal
  * \brief Find the XML configuration corresponding to a specific action key
  *
  * \param[in] rsc               Resource to find action configuration for
  * \param[in] key               "RSC_ACTION_INTERVAL" of action to find
  * \param[in] include_disabled  If false, do not return disabled actions
  *
  * \return XML configuration of desired action if any, otherwise NULL
  */
 static xmlNode *
 find_exact_action_config(const pcmk_resource_t *rsc, const char *action_name,
                          guint interval_ms, bool include_disabled)
 {
     for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
                                                    PCMK_XE_OP, NULL, NULL);
          operation != NULL; operation = pcmk__xe_next(operation, PCMK_XE_OP)) {
 
         bool enabled = false;
         const char *config_name = NULL;
         const char *interval_spec = NULL;
         guint tmp_ms = 0U;
 
         // @TODO This does not consider meta-attributes, rules, defaults, etc.
         if (!include_disabled
             && (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
                                        &enabled) == pcmk_rc_ok) && !enabled) {
             continue;
         }
 
         interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
         pcmk_parse_interval_spec(interval_spec, &tmp_ms);
         if (tmp_ms != interval_ms) {
             continue;
         }
 
         config_name = crm_element_value(operation, PCMK_XA_NAME);
         if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) {
             return operation;
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Find the XML configuration of a resource action
  *
  * \param[in] rsc               Resource to find action configuration for
  * \param[in] action_name       Action name to search for
  * \param[in] interval_ms       Action interval (in milliseconds) to search for
  * \param[in] include_disabled  If false, do not return disabled actions
  *
  * \return XML configuration of desired action if any, otherwise NULL
  */
 xmlNode *
 pcmk__find_action_config(const pcmk_resource_t *rsc, const char *action_name,
                          guint interval_ms, bool include_disabled)
 {
     xmlNode *action_config = NULL;
 
     // Try requested action first
     action_config = find_exact_action_config(rsc, action_name, interval_ms,
                                              include_disabled);
 
     // For migrate_to and migrate_from actions, retry with "migrate"
     // @TODO This should be either documented or deprecated
     if ((action_config == NULL)
         && pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO,
                             PCMK_ACTION_MIGRATE_FROM, NULL)) {
         action_config = find_exact_action_config(rsc, "migrate", 0,
                                                  include_disabled);
     }
 
     return action_config;
 }
 
 /*!
  * \internal
  * \brief Create a new action object
  *
  * \param[in]     key        Action key
  * \param[in]     task       Action name
  * \param[in,out] rsc        Resource that action is for (if any)
  * \param[in]     node       Node that action is on (if any)
  * \param[in]     optional   Whether action should be considered optional
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Newly allocated action
  * \note This function takes ownership of \p key. It is the caller's
  *       responsibility to free the return value with pe_free_action().
  */
 static pcmk_action_t *
 new_action(char *key, const char *task, pcmk_resource_t *rsc,
            const pcmk_node_t *node, bool optional, pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *action = pcmk__assert_alloc(1, sizeof(pcmk_action_t));
 
     action->rsc = rsc;
     action->task = pcmk__str_copy(task);
     action->uuid = key;
     action->scheduler = scheduler;
 
     if (node) {
         action->node = pe__copy_node(node);
     }
 
     if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
         // Resource history deletion for a node can be done on the DC
         pcmk__set_action_flags(action, pcmk__action_on_dc);
     }
 
     pcmk__set_action_flags(action, pcmk__action_runnable);
     if (optional) {
         pcmk__set_action_flags(action, pcmk__action_optional);
     } else {
         pcmk__clear_action_flags(action, pcmk__action_optional);
     }
 
     if (rsc == NULL) {
         action->meta = pcmk__strkey_table(free, free);
     } else {
         guint interval_ms = 0;
 
         parse_op_key(key, NULL, NULL, &interval_ms);
         action->op_entry = pcmk__find_action_config(rsc, task, interval_ms,
                                                     true);
 
         /* If the given key is for one of the many notification pseudo-actions
          * (pre_notify_promote, etc.), the actual action name is "notify"
          */
         if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) {
             action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY,
                                                         0, true);
         }
 
         unpack_operation(action, action->op_entry, interval_ms);
     }
 
     pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
                     (optional? "optional" : "required"),
                     scheduler->priv->next_action_id, key, task,
                     ((rsc == NULL)? "no resource" : rsc->id),
                     pcmk__node_name(node));
     action->id = scheduler->priv->next_action_id++;
 
     scheduler->priv->actions = g_list_prepend(scheduler->priv->actions, action);
     if (rsc == NULL) {
         add_singleton(scheduler, action);
     } else {
         rsc->priv->actions = g_list_prepend(rsc->priv->actions, action);
     }
     return action;
 }
 
 /*!
  * \internal
  * \brief Unpack a resource's action-specific instance parameters
  *
  * \param[in]     action_xml  XML of action's configuration in CIB (if any)
  * \param[in,out] node_attrs  Table of node attributes (for rule evaluation)
  * \param[in,out] scheduler   Cluster working set (for rule evaluation)
  *
  * \return Newly allocated hash table of action-specific instance parameters
  */
 GHashTable *
 pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
                                GHashTable *node_attrs,
                                pcmk_scheduler_t *scheduler)
 {
     GHashTable *params = pcmk__strkey_table(free, free);
 
     const pcmk_rule_input_t rule_input = {
         .now = scheduler->priv->now,
         .node_attrs = node_attrs,
     };
 
     pe__unpack_dataset_nvpairs(action_xml, PCMK_XE_INSTANCE_ATTRIBUTES,
                                &rule_input, params, NULL, scheduler);
     return params;
 }
 
 /*!
  * \internal
  * \brief Update an action's optional flag
  *
  * \param[in,out] action    Action to update
  * \param[in]     optional  Requested optional status
  */
 static void
 update_action_optional(pcmk_action_t *action, gboolean optional)
 {
     // Force a non-recurring action to be optional if its resource is unmanaged
     if ((action->rsc != NULL) && (action->node != NULL)
         && !pcmk_is_set(action->flags, pcmk__action_pseudo)
         && !pcmk_is_set(action->rsc->flags, pcmk__rsc_managed)
         && (g_hash_table_lookup(action->meta, PCMK_META_INTERVAL) == NULL)) {
             pcmk__rsc_debug(action->rsc,
                             "%s on %s is optional (%s is unmanaged)",
                             action->uuid, pcmk__node_name(action->node),
                             action->rsc->id);
             pcmk__set_action_flags(action, pcmk__action_optional);
             // We shouldn't clear runnable here because ... something
 
     // Otherwise require the action if requested
     } else if (!optional) {
         pcmk__clear_action_flags(action, pcmk__action_optional);
     }
 }
 
 static enum pe_quorum_policy
 effective_quorum_policy(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     enum pe_quorum_policy policy = scheduler->no_quorum_policy;
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
         policy = pcmk_no_quorum_ignore;
 
     } else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) {
         switch (rsc->priv->orig_role) {
             case pcmk_role_promoted:
             case pcmk_role_unpromoted:
                 if (rsc->priv->next_role > pcmk_role_unpromoted) {
                     pe__set_next_role(rsc, pcmk_role_unpromoted,
                                       PCMK_OPT_NO_QUORUM_POLICY "=demote");
                 }
                 policy = pcmk_no_quorum_ignore;
                 break;
             default:
                 policy = pcmk_no_quorum_stop;
                 break;
         }
     }
     return policy;
 }
 
 /*!
  * \internal
  * \brief Update a resource action's runnable flag
  *
  * \param[in,out] action     Action to update
  * \param[in,out] scheduler  Scheduler data
  *
  * \note This may also schedule fencing if a stop is unrunnable.
  */
 static void
 update_resource_action_runnable(pcmk_action_t *action,
                                 pcmk_scheduler_t *scheduler)
 {
     pcmk_resource_t *rsc = action->rsc;
 
     if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
         return;
     }
 
     if (action->node == NULL) {
         pcmk__rsc_trace(rsc, "%s is unrunnable (unallocated)", action->uuid);
         pcmk__clear_action_flags(action, pcmk__action_runnable);
 
     } else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
                && !(action->node->details->online)
                && (!pcmk__is_guest_or_bundle_node(action->node)
                    || pcmk_is_set(action->node->priv->flags,
                                   pcmk__node_remote_reset))) {
         pcmk__clear_action_flags(action, pcmk__action_runnable);
         do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)",
                    action->uuid, pcmk__node_name(action->node));
         if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)
             && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
             && !(action->node->details->unclean)) {
             pe_fence_node(scheduler, action->node, "stop is unrunnable", false);
         }
 
     } else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
                && action->node->details->pending) {
         pcmk__clear_action_flags(action, pcmk__action_runnable);
         do_crm_log(LOG_WARNING,
                    "Action %s on %s is unrunnable (node is pending)",
                    action->uuid, pcmk__node_name(action->node));
 
     } else if (action->needs == pcmk__requires_nothing) {
         pe_action_set_reason(action, NULL, TRUE);
         if (pcmk__is_guest_or_bundle_node(action->node)
             && !pe_can_fence(scheduler, action->node)) {
             /* An action that requires nothing usually does not require any
              * fencing in order to be runnable. However, there is an exception:
              * such an action cannot be completed if it is on a guest node whose
              * host is unclean and cannot be fenced.
              */
             pcmk__rsc_debug(rsc,
                             "%s on %s is unrunnable "
                             "(node's host cannot be fenced)",
                             action->uuid, pcmk__node_name(action->node));
             pcmk__clear_action_flags(action, pcmk__action_runnable);
         } else {
             pcmk__rsc_trace(rsc,
                             "%s on %s does not require fencing or quorum",
                             action->uuid, pcmk__node_name(action->node));
             pcmk__set_action_flags(action, pcmk__action_runnable);
         }
 
     } else {
         switch (effective_quorum_policy(rsc, scheduler)) {
             case pcmk_no_quorum_stop:
                 pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
                                 action->uuid, pcmk__node_name(action->node));
                 pcmk__clear_action_flags(action, pcmk__action_runnable);
                 pe_action_set_reason(action, "no quorum", true);
                 break;
 
             case pcmk_no_quorum_freeze:
                 if (!rsc->priv->fns->active(rsc, TRUE)
                     || (rsc->priv->next_role > rsc->priv->orig_role)) {
                     pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
                                     action->uuid,
                                     pcmk__node_name(action->node));
                     pcmk__clear_action_flags(action, pcmk__action_runnable);
                     pe_action_set_reason(action, "quorum freeze", true);
                 }
                 break;
 
             default:
                 //pe_action_set_reason(action, NULL, TRUE);
                 pcmk__set_action_flags(action, pcmk__action_runnable);
                 break;
         }
     }
 }
 
 static bool
 valid_stop_on_fail(const char *value)
 {
     return !pcmk__strcase_any_of(value,
                                  PCMK_VALUE_STANDBY, PCMK_VALUE_DEMOTE,
                                  PCMK_VALUE_STOP, NULL);
 }
 
 /*!
  * \internal
  * \brief Validate (and possibly reset) resource action's on_fail meta-attribute
  *
  * \param[in]     rsc            Resource that action is for
  * \param[in]     action_name    Action name
  * \param[in]     action_config  Action configuration XML from CIB (if any)
  * \param[in,out] meta           Table of action meta-attributes
  */
 static void
 validate_on_fail(const pcmk_resource_t *rsc, const char *action_name,
                  const xmlNode *action_config, GHashTable *meta)
 {
     const char *name = NULL;
     const char *role = NULL;
     const char *interval_spec = NULL;
     const char *value = g_hash_table_lookup(meta, PCMK_META_ON_FAIL);
     guint interval_ms = 0U;
 
     // Stop actions can only use certain on-fail values
     if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
         && !valid_stop_on_fail(value)) {
 
         pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s stop "
                          "action to default value because '%s' is not "
                          "allowed for stop", rsc->id, value);
         g_hash_table_remove(meta, PCMK_META_ON_FAIL);
         return;
     }
 
     /* Demote actions default on-fail to the on-fail value for the first
      * recurring monitor for the promoted role (if any).
      */
     if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
         && (value == NULL)) {
 
         /* @TODO This does not consider promote options set in a meta-attribute
          * block (which may have rules that need to be evaluated) rather than
          * XML properties.
          */
         for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
                                                        PCMK_XE_OP, NULL, NULL);
              operation != NULL;
              operation = pcmk__xe_next(operation, PCMK_XE_OP)) {
 
             bool enabled = false;
             const char *promote_on_fail = NULL;
 
             /* We only care about explicit on-fail (if promote uses default, so
              * can demote)
              */
             promote_on_fail = crm_element_value(operation, PCMK_META_ON_FAIL);
             if (promote_on_fail == NULL) {
                 continue;
             }
 
             // We only care about recurring monitors for the promoted role
             name = crm_element_value(operation, PCMK_XA_NAME);
             role = crm_element_value(operation, PCMK_XA_ROLE);
             if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
                 || !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
                                          PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
                 continue;
             }
             interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
             pcmk_parse_interval_spec(interval_spec, &interval_ms);
             if (interval_ms == 0U) {
                 continue;
             }
 
             // We only care about enabled monitors
             if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
                                         &enabled) == pcmk_rc_ok) && !enabled) {
                 continue;
             }
 
             /* Demote actions can't default to
              * PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE
              */
             if (pcmk__str_eq(promote_on_fail, PCMK_VALUE_DEMOTE,
                              pcmk__str_casei)) {
                 continue;
             }
 
             // Use value from first applicable promote action found
             pcmk__insert_dup(meta, PCMK_META_ON_FAIL, promote_on_fail);
         }
         return;
     }
 
     if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
         && !pcmk__str_eq(value, PCMK_VALUE_IGNORE, pcmk__str_casei)) {
 
         pcmk__insert_dup(meta, PCMK_META_ON_FAIL, PCMK_VALUE_IGNORE);
         return;
     }
 
     // PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE is allowed only for certain actions
     if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
         name = crm_element_value(action_config, PCMK_XA_NAME);
         role = crm_element_value(action_config, PCMK_XA_ROLE);
         interval_spec = crm_element_value(action_config, PCMK_META_INTERVAL);
         pcmk_parse_interval_spec(interval_spec, &interval_ms);
 
         if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
             && ((interval_ms == 0U)
                 || !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
                 || !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
                                          PCMK__ROLE_PROMOTED_LEGACY, NULL))) {
 
             pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s %s "
                              "action to default value because 'demote' is not "
                              "allowed for it", rsc->id, name);
             g_hash_table_remove(meta, PCMK_META_ON_FAIL);
             return;
         }
     }
 }
 
 static int
 unpack_timeout(const char *value)
 {
     long long timeout_ms = crm_get_msec(value);
 
     if (timeout_ms <= 0) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
     return (int) QB_MIN(timeout_ms, INT_MAX);
 }
 
 // true if value contains valid, non-NULL interval origin for recurring op
 static bool
 unpack_interval_origin(const char *value, const xmlNode *xml_obj,
                        guint interval_ms, const crm_time_t *now,
                        long long *start_delay)
 {
     long long result = 0;
     guint interval_sec = pcmk__timeout_ms2s(interval_ms);
     crm_time_t *origin = NULL;
 
     // Ignore unspecified values and non-recurring operations
     if ((value == NULL) || (interval_ms == 0) || (now == NULL)) {
         return false;
     }
 
     // Parse interval origin from text
     origin = crm_time_new(value);
     if (origin == NULL) {
         pcmk__config_err("Ignoring '" PCMK_META_INTERVAL_ORIGIN "' for "
                          "operation '%s' because '%s' is not valid",
                          pcmk__s(pcmk__xe_id(xml_obj), "(missing ID)"), value);
         return false;
     }
 
     // Get seconds since origin (negative if origin is in the future)
     result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
     crm_time_free(origin);
 
     // Calculate seconds from closest interval to now
     result = result % interval_sec;
 
     // Calculate seconds remaining until next interval
     result = ((result <= 0)? 0 : interval_sec) - result;
     crm_info("Calculated a start delay of %llds for operation '%s'",
              result, pcmk__s(pcmk__xe_id(xml_obj), "(unspecified)"));
 
     if (start_delay != NULL) {
         *start_delay = result * 1000; // milliseconds
     }
     return true;
 }
 
 static int
 unpack_start_delay(const char *value, GHashTable *meta)
 {
     long long start_delay_ms = 0;
 
     if (value == NULL) {
         return 0;
     }
 
     start_delay_ms = crm_get_msec(value);
     start_delay_ms = QB_MIN(start_delay_ms, INT_MAX);
     if (start_delay_ms < 0) {
         start_delay_ms = 0;
     }
 
     if (meta != NULL) {
         g_hash_table_replace(meta, strdup(PCMK_META_START_DELAY),
                              pcmk__itoa(start_delay_ms));
     }
 
     return (int) start_delay_ms;
 }
 
 /*!
  * \internal
  * \brief Find a resource's most frequent recurring monitor
  *
  * \param[in] rsc  Resource to check
  *
  * \return Operation XML configured for most frequent recurring monitor for
  *         \p rsc (if any)
  */
 static xmlNode *
 most_frequent_monitor(const pcmk_resource_t *rsc)
 {
     guint min_interval_ms = G_MAXUINT;
     xmlNode *op = NULL;
 
     for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
                                                    PCMK_XE_OP, NULL, NULL);
          operation != NULL; operation = pcmk__xe_next(operation, PCMK_XE_OP)) {
 
         bool enabled = false;
         guint interval_ms = 0U;
         const char *interval_spec = crm_element_value(operation,
                                                       PCMK_META_INTERVAL);
 
         // We only care about enabled recurring monitors
         if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME),
                           PCMK_ACTION_MONITOR, pcmk__str_none)) {
             continue;
         }
 
         pcmk_parse_interval_spec(interval_spec, &interval_ms);
         if (interval_ms == 0U) {
             continue;
         }
 
         // @TODO This does not consider meta-attributes, rules, defaults, etc.
         if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
                                     &enabled) == pcmk_rc_ok) && !enabled) {
             continue;
         }
 
         if (interval_ms < min_interval_ms) {
             min_interval_ms = interval_ms;
             op = operation;
         }
     }
     return op;
 }
 
 /*!
  * \internal
  * \brief Unpack action meta-attributes
  *
  * \param[in,out] rsc            Resource that action is for
  * \param[in]     node           Node that action is on
  * \param[in]     action_name    Action name
  * \param[in]     interval_ms    Action interval (in milliseconds)
  * \param[in]     action_config  Action XML configuration from CIB (if any)
  *
  * Unpack a resource action's meta-attributes (normalizing the interval,
  * timeout, and start delay values as integer milliseconds) from its CIB XML
  * configuration (including defaults).
  *
  * \return Newly allocated hash table with normalized action meta-attributes
  */
 GHashTable *
 pcmk__unpack_action_meta(pcmk_resource_t *rsc, const pcmk_node_t *node,
                          const char *action_name, guint interval_ms,
                          const xmlNode *action_config)
 {
     GHashTable *meta = NULL;
     const char *timeout_spec = NULL;
     const char *str = NULL;
 
     const pcmk_rule_input_t rule_input = {
         /* Node attributes are not set because node expressions are not allowed
          * for meta-attributes
          */
         .now = rsc->priv->scheduler->priv->now,
         .rsc_standard = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS),
         .rsc_provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER),
         .rsc_agent = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE),
         .op_name = action_name,
         .op_interval_ms = interval_ms,
     };
 
     meta = pcmk__strkey_table(free, free);
 
     if (action_config != NULL) {
         // <op> <meta_attributes> take precedence over defaults
         pe__unpack_dataset_nvpairs(action_config, PCMK_XE_META_ATTRIBUTES,
                                    &rule_input, meta, NULL,
                                    rsc->priv->scheduler);
 
         /* Anything set as an <op> XML property has highest precedence.
          * This ensures we use the name and interval from the <op> tag.
          * (See below for the only exception, fence device start/probe timeout.)
          */
         for (xmlAttrPtr attr = action_config->properties;
              attr != NULL; attr = attr->next) {
             pcmk__insert_dup(meta, (const char *) attr->name,
                              pcmk__xml_attr_value(attr));
         }
     }
 
     // Derive default timeout for probes from recurring monitor timeouts
     if (pcmk_is_probe(action_name, interval_ms)
         && (g_hash_table_lookup(meta, PCMK_META_TIMEOUT) == NULL)) {
 
         xmlNode *min_interval_mon = most_frequent_monitor(rsc);
 
         if (min_interval_mon != NULL) {
             /* @TODO This does not consider timeouts set in
              * PCMK_XE_META_ATTRIBUTES blocks (which may also have rules that
              * need to be evaluated).
              */
             timeout_spec = crm_element_value(min_interval_mon,
                                              PCMK_META_TIMEOUT);
             if (timeout_spec != NULL) {
                 pcmk__rsc_trace(rsc,
                                 "Setting default timeout for %s probe to "
                                 "most frequent monitor's timeout '%s'",
                                 rsc->id, timeout_spec);
                 pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
             }
         }
     }
 
     // Cluster-wide <op_defaults> <meta_attributes>
     pe__unpack_dataset_nvpairs(rsc->priv->scheduler->priv->op_defaults,
                                PCMK_XE_META_ATTRIBUTES, &rule_input, meta, NULL,
                                rsc->priv->scheduler);
 
     g_hash_table_remove(meta, PCMK_XA_ID);
 
     // Normalize interval to milliseconds
     if (interval_ms > 0) {
         g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_INTERVAL),
                             crm_strdup_printf("%u", interval_ms));
     } else {
         g_hash_table_remove(meta, PCMK_META_INTERVAL);
     }
 
     /* Timeout order of precedence (highest to lowest):
      *   1. pcmk_monitor_timeout resource parameter (only for starts and probes
      *      when rsc has pcmk_ra_cap_fence_params; this gets used for recurring
      *      monitors via the executor instead)
      *   2. timeout configured in <op> (with <op timeout> taking precedence over
      *      <op> <meta_attributes>)
      *   3. timeout configured in <op_defaults> <meta_attributes>
      *   4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
      */
 
     // Check for pcmk_monitor_timeout
     if (pcmk_is_set(pcmk_get_ra_caps(rule_input.rsc_standard),
                     pcmk_ra_cap_fence_params)
         && (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)
             || pcmk_is_probe(action_name, interval_ms))) {
 
         GHashTable *params = pe_rsc_params(rsc, node, rsc->priv->scheduler);
 
         timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout");
         if (timeout_spec != NULL) {
             pcmk__rsc_trace(rsc,
                             "Setting timeout for %s %s to "
                             "pcmk_monitor_timeout (%s)",
                             rsc->id, action_name, timeout_spec);
             pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
         }
     }
 
     // Normalize timeout to positive milliseconds
     timeout_spec = g_hash_table_lookup(meta, PCMK_META_TIMEOUT);
     g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_TIMEOUT),
                         pcmk__itoa(unpack_timeout(timeout_spec)));
 
     // Ensure on-fail has a valid value
     validate_on_fail(rsc, action_name, action_config, meta);
 
     // Normalize PCMK_META_START_DELAY
     str = g_hash_table_lookup(meta, PCMK_META_START_DELAY);
     if (str != NULL) {
         unpack_start_delay(str, meta);
     } else {
         long long start_delay = 0;
 
         str = g_hash_table_lookup(meta, PCMK_META_INTERVAL_ORIGIN);
         if (unpack_interval_origin(str, action_config, interval_ms,
                                    rsc->priv->scheduler->priv->now,
                                    &start_delay)) {
             g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_START_DELAY),
                                 crm_strdup_printf("%lld", start_delay));
         }
     }
     return meta;
 }
 
 /*!
  * \internal
  * \brief Determine an action's quorum and fencing dependency
  *
  * \param[in] rsc          Resource that action is for
  * \param[in] action_name  Name of action being unpacked
  *
  * \return Quorum and fencing dependency appropriate to action
  */
 enum pcmk__requires
 pcmk__action_requires(const pcmk_resource_t *rsc, const char *action_name)
 {
     const char *value = NULL;
     enum pcmk__requires requires = pcmk__requires_nothing;
 
     CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires);
 
     if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START,
                               PCMK_ACTION_PROMOTE, NULL)) {
         value = "nothing (not start or promote)";
 
     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
         requires = pcmk__requires_fencing;
         value = "fencing";
 
     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_quorum)) {
         requires = pcmk__requires_quorum;
         value = "quorum";
 
     } else {
         value = "nothing";
     }
     pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value);
     return requires;
 }
 
 /*!
  * \internal
  * \brief Parse action failure response from a user-provided string
  *
  * \param[in] rsc          Resource that action is for
  * \param[in] action_name  Name of action
  * \param[in] interval_ms  Action interval (in milliseconds)
  * \param[in] value        User-provided configuration value for on-fail
  *
  * \return Action failure response parsed from \p text
  */
 enum pcmk__on_fail
 pcmk__parse_on_fail(const pcmk_resource_t *rsc, const char *action_name,
                     guint interval_ms, const char *value)
 {
     const char *desc = NULL;
     bool needs_remote_reset = false;
     enum pcmk__on_fail on_fail = pcmk__on_fail_ignore;
     const pcmk_scheduler_t *scheduler = NULL;
 
     // There's no enum value for unknown or invalid, so assert
     pcmk__assert((rsc != NULL) && (action_name != NULL));
     scheduler = rsc->priv->scheduler;
 
     if (value == NULL) {
         // Use default
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_BLOCK, pcmk__str_casei)) {
         on_fail = pcmk__on_fail_block;
         desc = "block";
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_FENCE, pcmk__str_casei)) {
         if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
             on_fail = pcmk__on_fail_fence_node;
             desc = "node fencing";
         } else {
             pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for "
                              "%s of %s to 'stop' because 'fence' is not "
                              "valid when fencing is disabled",
                              action_name, rsc->id);
+            /* @TODO This should probably do
+            g_hash_table_remove(meta, PCMK_META_ON_FAIL);
+            like the other "Resetting" spots, to avoid repeating the message
+            */
             on_fail = pcmk__on_fail_stop;
             desc = "stop resource";
         }
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_STANDBY, pcmk__str_casei)) {
         on_fail = pcmk__on_fail_standby_node;
         desc = "node standby";
 
     } else if (pcmk__strcase_any_of(value,
                                     PCMK_VALUE_IGNORE, PCMK_VALUE_NOTHING,
                                     NULL)) {
         desc = "ignore";
 
     } else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
         on_fail = pcmk__on_fail_ban;
         desc = "force migration";
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_STOP, pcmk__str_casei)) {
         on_fail = pcmk__on_fail_stop;
         desc = "stop resource";
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_RESTART, pcmk__str_casei)) {
         on_fail = pcmk__on_fail_restart;
         desc = "restart (and possibly migrate)";
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_RESTART_CONTAINER,
                             pcmk__str_casei)) {
         if (rsc->priv->launcher == NULL) {
             pcmk__rsc_debug(rsc,
                             "Using default " PCMK_META_ON_FAIL " for %s "
                             "of %s because it does not have a launcher",
                             action_name, rsc->id);
         } else {
             on_fail = pcmk__on_fail_restart_container;
             desc = "restart container (and possibly migrate)";
         }
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
         on_fail = pcmk__on_fail_demote;
         desc = "demote instance";
 
     } else {
         pcmk__config_err("Using default '" PCMK_META_ON_FAIL "' for "
                          "%s of %s because '%s' is not valid",
                          action_name, rsc->id, value);
     }
 
     /* Remote node connections are handled specially. Failures that result
      * in dropping an active connection must result in fencing. The only
      * failures that don't are probes and starts. The user can explicitly set
      * PCMK_META_ON_FAIL=PCMK_VALUE_FENCE to fence after start failures.
      */
     if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)
         && pcmk__is_remote_node(pcmk_find_node(scheduler, rsc->id))
         && !pcmk_is_probe(action_name, interval_ms)
         && !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
         needs_remote_reset = true;
         if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
             desc = NULL; // Force default for unmanaged connections
         }
     }
 
     if (desc != NULL) {
         // Explicit value used, default not needed
 
     } else if (rsc->priv->launcher != NULL) {
         on_fail = pcmk__on_fail_restart_container;
         desc = "restart container (and possibly migrate) (default)";
 
     } else if (needs_remote_reset) {
         if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
             if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
                 desc = "fence remote node (default)";
             } else {
                 desc = "recover remote node connection (default)";
             }
             on_fail = pcmk__on_fail_reset_remote;
         } else {
             on_fail = pcmk__on_fail_stop;
             desc = "stop unmanaged remote node (enforcing default)";
         }
 
     } else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
         if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
             on_fail = pcmk__on_fail_fence_node;
             desc = "resource fence (default)";
         } else {
             on_fail = pcmk__on_fail_block;
             desc = "resource block (default)";
         }
 
     } else {
         on_fail = pcmk__on_fail_restart;
         desc = "restart (and possibly migrate) (default)";
     }
 
     pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s",
                     pcmk__readable_interval(interval_ms), action_name,
                     rsc->id, desc);
     return on_fail;
 }
 
 /*!
  * \internal
  * \brief Determine a resource's role after failure of an action
  *
  * \param[in] rsc          Resource that action is for
  * \param[in] action_name  Action name
  * \param[in] on_fail      Failure handling for action
  * \param[in] meta         Unpacked action meta-attributes
  *
  * \return Resource role that results from failure of action
  */
 enum rsc_role_e
 pcmk__role_after_failure(const pcmk_resource_t *rsc, const char *action_name,
                          enum pcmk__on_fail on_fail, GHashTable *meta)
 {
     enum rsc_role_e role = pcmk_role_unknown;
 
     // Set default for role after failure specially in certain circumstances
     switch (on_fail) {
         case pcmk__on_fail_stop:
             role = pcmk_role_stopped;
             break;
 
         case pcmk__on_fail_reset_remote:
             if (rsc->priv->remote_reconnect_ms != 0U) {
                 role = pcmk_role_stopped;
             }
             break;
 
         default:
             break;
     }
 
     if (role == pcmk_role_unknown) {
         // Use default
         if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
             role = pcmk_role_unpromoted;
         } else {
             role = pcmk_role_started;
         }
     }
     pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s",
                     rsc->id, action_name, pcmk_role_text(role));
     return role;
 }
 
 /*!
  * \internal
  * \brief Unpack action configuration
  *
  * Unpack a resource action's meta-attributes (normalizing the interval,
  * timeout, and start delay values as integer milliseconds), requirements, and
  * failure policy from its CIB XML configuration (including defaults).
  *
  * \param[in,out] action       Resource action to unpack into
  * \param[in]     xml_obj      Action configuration XML (NULL for defaults only)
  * \param[in]     interval_ms  How frequently to perform the operation
  */
 static void
 unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj,
                  guint interval_ms)
 {
     const char *value = NULL;
 
     action->meta = pcmk__unpack_action_meta(action->rsc, action->node,
                                             action->task, interval_ms, xml_obj);
     action->needs = pcmk__action_requires(action->rsc, action->task);
 
     value = g_hash_table_lookup(action->meta, PCMK_META_ON_FAIL);
     action->on_fail = pcmk__parse_on_fail(action->rsc, action->task,
                                           interval_ms, value);
 
     action->fail_role = pcmk__role_after_failure(action->rsc, action->task,
                                                  action->on_fail, action->meta);
 }
 
 /*!
  * \brief Create or update an action object
  *
  * \param[in,out] rsc          Resource that action is for (if any)
  * \param[in,out] key          Action key (must be non-NULL)
  * \param[in]     task         Action name (must be non-NULL)
  * \param[in]     on_node      Node that action is on (if any)
  * \param[in]     optional     Whether action should be considered optional
  * \param[in,out] scheduler    Scheduler data
  *
  * \return Action object corresponding to arguments (guaranteed not to be
  *         \c NULL)
  * \note This function takes ownership of (and might free) \p key, and
  *       \p scheduler takes ownership of the returned action (the caller should
  *       not free it).
  */
 pcmk_action_t *
 custom_action(pcmk_resource_t *rsc, char *key, const char *task,
               const pcmk_node_t *on_node, gboolean optional,
               pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *action = NULL;
 
     pcmk__assert((key != NULL) && (task != NULL) && (scheduler != NULL));
 
     action = find_existing_action(key, rsc, on_node, scheduler);
     if (action == NULL) {
         action = new_action(key, task, rsc, on_node, optional, scheduler);
     } else {
         free(key);
     }
 
     update_action_optional(action, optional);
 
     if (rsc != NULL) {
         /* An action can be initially created with a NULL node, and later have
          * the node added via find_existing_action() (above) -> find_actions().
          * That is why the extra parameters are unpacked here rather than in
          * new_action().
          */
         if ((action->node != NULL) && (action->op_entry != NULL)
             && !pcmk_is_set(action->flags, pcmk__action_attrs_evaluated)) {
 
             GHashTable *attrs = action->node->priv->attrs;
 
             if (action->extra != NULL) {
                 g_hash_table_destroy(action->extra);
             }
             action->extra = pcmk__unpack_action_rsc_params(action->op_entry,
                                                            attrs, scheduler);
             pcmk__set_action_flags(action, pcmk__action_attrs_evaluated);
         }
 
         update_resource_action_runnable(action, scheduler);
     }
 
     if (action->extra == NULL) {
         action->extra = pcmk__strkey_table(free, free);
     }
 
     return action;
 }
 
 pcmk_action_t *
 get_pseudo_op(const char *name, pcmk_scheduler_t *scheduler)
 {
     pcmk_action_t *op = lookup_singleton(scheduler, name);
 
     if (op == NULL) {
         op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler);
         pcmk__set_action_flags(op, pcmk__action_pseudo|pcmk__action_runnable);
     }
     return op;
 }
 
 static GList *
 find_unfencing_devices(GList *candidates, GList *matches) 
 {
     for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
         pcmk_resource_t *candidate = gIter->data;
 
         if (candidate->priv->children != NULL) {
             matches = find_unfencing_devices(candidate->priv->children,
                                              matches);
 
         } else if (!pcmk_is_set(candidate->flags, pcmk__rsc_fence_device)) {
             continue;
 
         } else if (pcmk_is_set(candidate->flags, pcmk__rsc_needs_unfencing)) {
             matches = g_list_prepend(matches, candidate);
 
         } else if (pcmk__str_eq(g_hash_table_lookup(candidate->priv->meta,
                                                     PCMK_STONITH_PROVIDES),
                                 PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
             matches = g_list_prepend(matches, candidate);
         }
     }
     return matches;
 }
 
 static int
 node_priority_fencing_delay(const pcmk_node_t *node,
                             const pcmk_scheduler_t *scheduler)
 {
     int member_count = 0;
     int online_count = 0;
     int top_priority = 0;
     int lowest_priority = 0;
     GList *gIter = NULL;
 
     // PCMK_OPT_PRIORITY_FENCING_DELAY is disabled
     if (scheduler->priv->priority_fencing_ms == 0U) {
         return 0;
     }
 
     /* No need to request a delay if the fencing target is not a normal cluster
      * member, for example if it's a remote node or a guest node. */
     if (node->priv->variant != pcmk__node_variant_cluster) {
         return 0;
     }
 
     // No need to request a delay if the fencing target is in our partition
     if (node->details->online) {
         return 0;
     }
 
     for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *n = gIter->data;
 
         if (n->priv->variant != pcmk__node_variant_cluster) {
             continue;
         }
 
         member_count ++;
 
         if (n->details->online) {
             online_count++;
         }
 
         if (member_count == 1
             || n->priv->priority > top_priority) {
             top_priority = n->priv->priority;
         }
 
         if (member_count == 1
             || n->priv->priority < lowest_priority) {
             lowest_priority = n->priv->priority;
         }
     }
 
     // No need to delay if we have more than half of the cluster members
     if (online_count > member_count / 2) {
         return 0;
     }
 
     /* All the nodes have equal priority.
      * Any configured corresponding `pcmk_delay_base/max` will be applied. */
     if (lowest_priority == top_priority) {
         return 0;
     }
 
     if (node->priv->priority < top_priority) {
         return 0;
     }
 
     return pcmk__timeout_ms2s(scheduler->priv->priority_fencing_ms);
 }
 
 pcmk_action_t *
 pe_fence_op(pcmk_node_t *node, const char *op, bool optional,
             const char *reason, bool priority_delay,
             pcmk_scheduler_t *scheduler)
 {
     char *op_key = NULL;
     pcmk_action_t *stonith_op = NULL;
 
     if(op == NULL) {
         op = scheduler->priv->fence_action;
     }
 
     op_key = crm_strdup_printf("%s-%s-%s",
                                PCMK_ACTION_STONITH, node->priv->name, op);
 
     stonith_op = lookup_singleton(scheduler, op_key);
     if(stonith_op == NULL) {
         stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
                                    TRUE, scheduler);
 
         pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE, node->priv->name);
         pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE_UUID,
                           node->priv->id);
         pcmk__insert_meta(stonith_op, PCMK__META_STONITH_ACTION, op);
 
         if (pcmk_is_set(scheduler->flags, pcmk__sched_enable_unfencing)) {
             /* Extra work to detect device changes
              */
             GString *digests_all = g_string_sized_new(1024);
             GString *digests_secure = g_string_sized_new(1024);
 
             GList *matches = find_unfencing_devices(scheduler->priv->resources,
                                                     NULL);
 
             for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
                 pcmk_resource_t *match = gIter->data;
                 const char *agent = g_hash_table_lookup(match->priv->meta,
                                                         PCMK_XA_TYPE);
                 pcmk__op_digest_t *data = NULL;
 
                 data = pe__compare_fencing_digest(match, agent, node,
                                                   scheduler);
                 if (data->rc == pcmk__digest_mismatch) {
                     optional = FALSE;
                     crm_notice("Unfencing node %s because the definition of "
                                "%s changed", pcmk__node_name(node), match->id);
                     if (!pcmk__is_daemon && (scheduler->priv->out != NULL)) {
                         pcmk__output_t *out = scheduler->priv->out;
 
                         out->info(out,
                                   "notice: Unfencing node %s because the "
                                   "definition of %s changed",
                                   pcmk__node_name(node), match->id);
                     }
                 }
 
                 pcmk__g_strcat(digests_all,
                                match->id, ":", agent, ":",
                                data->digest_all_calc, ",", NULL);
                 pcmk__g_strcat(digests_secure,
                                match->id, ":", agent, ":",
                                data->digest_secure_calc, ",", NULL);
             }
             pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_ALL,
                              digests_all->str);
             g_string_free(digests_all, TRUE);
 
             pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_SECURE,
                              digests_secure->str);
             g_string_free(digests_secure, TRUE);
 
             g_list_free(matches);
         }
 
     } else {
         free(op_key);
     }
 
     if ((scheduler->priv->priority_fencing_ms > 0U)
 
             /* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY
              * applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as
              * an indicator.
              */
         && (priority_delay
 
             /* The priority delay needs to be recalculated if this function has
              * been called by schedule_fencing_and_shutdowns() after node
              * priority has already been calculated by native_add_running().
              */
             || g_hash_table_lookup(stonith_op->meta,
                                    PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) {
 
             /* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if
              * it's 0 for the targeting node. So that it takes precedence over
              * any possible `pcmk_delay_base/max`.
              */
             char *delay_s = pcmk__itoa(node_priority_fencing_delay(node,
                                                                    scheduler));
 
             g_hash_table_insert(stonith_op->meta,
                                 strdup(PCMK_OPT_PRIORITY_FENCING_DELAY),
                                 delay_s);
     }
 
     if(optional == FALSE && pe_can_fence(scheduler, node)) {
         pcmk__clear_action_flags(stonith_op, pcmk__action_optional);
         pe_action_set_reason(stonith_op, reason, false);
 
     } else if(reason && stonith_op->reason == NULL) {
         stonith_op->reason = strdup(reason);
     }
 
     return stonith_op;
 }
 
 void
 pe_free_action(pcmk_action_t *action)
 {
     if (action == NULL) {
         return;
     }
     g_list_free_full(action->actions_before, free);
     g_list_free_full(action->actions_after, free);
     if (action->extra) {
         g_hash_table_destroy(action->extra);
     }
     if (action->meta) {
         g_hash_table_destroy(action->meta);
     }
     pcmk__free_node_copy(action->node);
     free(action->cancel_task);
     free(action->reason);
     free(action->task);
     free(action->uuid);
     free(action);
 }
 
 enum pcmk__action_type
 get_complex_task(const pcmk_resource_t *rsc, const char *name)
 {
     enum pcmk__action_type task = pcmk__parse_action(name);
 
     if (pcmk__is_primitive(rsc)) {
         switch (task) {
             case pcmk__action_stopped:
             case pcmk__action_started:
             case pcmk__action_demoted:
             case pcmk__action_promoted:
                 crm_trace("Folding %s back into its atomic counterpart for %s",
                           name, rsc->id);
                 --task;
                 break;
             default:
                 break;
         }
     }
     return task;
 }
 
 /*!
  * \internal
  * \brief Find first matching action in a list
  *
  * \param[in] input    List of actions to search
  * \param[in] uuid     If not NULL, action must have this UUID
  * \param[in] task     If not NULL, action must have this action name
  * \param[in] on_node  If not NULL, action must be on this node
  *
  * \return First action in list that matches criteria, or NULL if none
  */
 pcmk_action_t *
 find_first_action(const GList *input, const char *uuid, const char *task,
                   const pcmk_node_t *on_node)
 {
     CRM_CHECK(uuid || task, return NULL);
 
     for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
 
         if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
             continue;
 
         } else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
             continue;
 
         } else if (on_node == NULL) {
             return action;
 
         } else if (action->node == NULL) {
             continue;
 
         } else if (pcmk__same_node(on_node, action->node)) {
             return action;
         }
     }
 
     return NULL;
 }
 
 GList *
 find_actions(GList *input, const char *key, const pcmk_node_t *on_node)
 {
     GList *gIter = input;
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
 
         if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
             continue;
 
         } else if (on_node == NULL) {
             crm_trace("Action %s matches (ignoring node)", key);
             result = g_list_prepend(result, action);
 
         } else if (action->node == NULL) {
             crm_trace("Action %s matches (unallocated, assigning to %s)",
                       key, pcmk__node_name(on_node));
 
             action->node = pe__copy_node(on_node);
             result = g_list_prepend(result, action);
 
         } else if (pcmk__same_node(on_node, action->node)) {
             crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 GList *
 find_actions_exact(GList *input, const char *key, const pcmk_node_t *on_node)
 {
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     if (on_node == NULL) {
         return NULL;
     }
 
     for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
 
         if ((action->node != NULL)
             && pcmk__str_eq(key, action->uuid, pcmk__str_casei)
             && pcmk__same_node(on_node, action->node)) {
 
             crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 /*!
  * \brief Find all actions of given type for a resource
  *
  * \param[in] rsc           Resource to search
  * \param[in] node          Find only actions scheduled on this node
  * \param[in] task          Action name to search for
  * \param[in] require_node  If TRUE, NULL node or action node will not match
  *
  * \return List of actions found (or NULL if none)
  * \note If node is not NULL and require_node is FALSE, matching actions
  *       without a node will be assigned to node.
  */
 GList *
 pe__resource_actions(const pcmk_resource_t *rsc, const pcmk_node_t *node,
                      const char *task, bool require_node)
 {
     GList *result = NULL;
     char *key = pcmk__op_key(rsc->id, task, 0);
 
     if (require_node) {
         result = find_actions_exact(rsc->priv->actions, key, node);
     } else {
         result = find_actions(rsc->priv->actions, key, node);
     }
     free(key);
     return result;
 }
 
 /*!
  * \internal
  * \brief Create an action reason string based on the action itself
  *
  * \param[in] action  Action to create reason string for
  * \param[in] flag    Action flag that was cleared
  *
  * \return Newly allocated string suitable for use as action reason
  * \note It is the caller's responsibility to free() the result.
  */
 char *
 pe__action2reason(const pcmk_action_t *action, enum pcmk__action_flags flag)
 {
     const char *change = NULL;
 
     switch (flag) {
         case pcmk__action_runnable:
             change = "unrunnable";
             break;
         case pcmk__action_migratable:
             change = "unmigrateable";
             break;
         case pcmk__action_optional:
             change = "required";
             break;
         default:
             // Bug: caller passed unsupported flag
             CRM_CHECK(change != NULL, change = "");
             break;
     }
     return crm_strdup_printf("%s%s%s %s", change,
                              (action->rsc == NULL)? "" : " ",
                              (action->rsc == NULL)? "" : action->rsc->id,
                              action->task);
 }
 
 void pe_action_set_reason(pcmk_action_t *action, const char *reason,
                           bool overwrite)
 {
     if (action->reason != NULL && overwrite) {
         pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
                         action->uuid, action->reason,
                         pcmk__s(reason, "(none)"));
     } else if (action->reason == NULL) {
         pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'",
                         action->uuid, pcmk__s(reason, "(none)"));
     } else {
         // crm_assert(action->reason != NULL && !overwrite);
         return;
     }
 
     pcmk__str_update(&action->reason, reason);
 }
 
 /*!
  * \internal
  * \brief Create an action to clear a resource's history from CIB
  *
  * \param[in,out] rsc       Resource to clear
  * \param[in]     node      Node to clear history on
  */
 void
 pe__clear_resource_history(pcmk_resource_t *rsc, const pcmk_node_t *node)
 {
     pcmk__assert((rsc != NULL) && (node != NULL));
 
     custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
                   PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->priv->scheduler);
 }
 
 #define sort_return(an_int, why) do {					\
 	free(a_uuid);						\
 	free(b_uuid);						\
 	crm_trace("%s (%d) %c %s (%d) : %s",				\
 		  a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=',	\
 		  b_xml_id, b_call_id, why);				\
 	return an_int;							\
     } while(0)
 
 int
 pe__is_newer_op(const xmlNode *xml_a, const xmlNode *xml_b)
 {
     int a_call_id = -1;
     int b_call_id = -1;
 
     char *a_uuid = NULL;
     char *b_uuid = NULL;
 
     const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID);
     const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID);
 
     const char *a_node = crm_element_value(xml_a, PCMK__META_ON_NODE);
     const char *b_node = crm_element_value(xml_b, PCMK__META_ON_NODE);
     bool same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);
 
     if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
         /* We have duplicate PCMK__XE_LRM_RSC_OP entries in the status
          * section which is unlikely to be a good thing
          *    - we can handle it easily enough, but we need to get
          *    to the bottom of why it's happening.
          */
         pcmk__config_err("Duplicate " PCMK__XE_LRM_RSC_OP " entries named %s",
                          a_xml_id);
         sort_return(0, "duplicate");
     }
 
     crm_element_value_int(xml_a, PCMK__XA_CALL_ID, &a_call_id);
     crm_element_value_int(xml_b, PCMK__XA_CALL_ID, &b_call_id);
 
     if (a_call_id == -1 && b_call_id == -1) {
         /* both are pending ops so it doesn't matter since
          *   stops are never pending
          */
         sort_return(0, "pending");
 
     } else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
         sort_return(-1, "call id");
 
     } else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
         sort_return(1, "call id");
 
     } else if (a_call_id >= 0 && b_call_id >= 0
                && (!same_node || a_call_id == b_call_id)) {
         /* The op and last_failed_op are the same. Order on
          * PCMK_XA_LAST_RC_CHANGE.
          */
         time_t last_a = -1;
         time_t last_b = -1;
 
         crm_element_value_epoch(xml_a, PCMK_XA_LAST_RC_CHANGE, &last_a);
         crm_element_value_epoch(xml_b, PCMK_XA_LAST_RC_CHANGE, &last_b);
 
         crm_trace("rc-change: %lld vs %lld",
                   (long long) last_a, (long long) last_b);
         if (last_a >= 0 && last_a < last_b) {
             sort_return(-1, "rc-change");
 
         } else if (last_b >= 0 && last_a > last_b) {
             sort_return(1, "rc-change");
         }
         sort_return(0, "rc-change");
 
     } else {
         /* One of the inputs is a pending operation.
          * Attempt to use PCMK__XA_TRANSITION_MAGIC to determine its age relative
          * to the other.
          */
 
         int a_id = -1;
         int b_id = -1;
 
         const char *a_magic = crm_element_value(xml_a,
                                                 PCMK__XA_TRANSITION_MAGIC);
         const char *b_magic = crm_element_value(xml_b,
                                                 PCMK__XA_TRANSITION_MAGIC);
 
         CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
         if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic a");
         }
         if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic b");
         }
         /* try to determine the relative age of the operation...
          * some pending operations (e.g. a start) may have been superseded
          *   by a subsequent stop
          *
          * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last
          */
         if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) || a_id == b_id) {
             /*
              * some of the logic in here may be redundant...
              *
              * if the UUID from the TE doesn't match then one better
              *   be a pending operation.
              * pending operations don't survive between elections and joins
              *   because we query the LRM directly
              */
 
             if (b_call_id == -1) {
                 sort_return(-1, "transition + call");
 
             } else if (a_call_id == -1) {
                 sort_return(1, "transition + call");
             }
 
         } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
             sort_return(-1, "transition");
 
         } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
             sort_return(1, "transition");
         }
     }
 
     /* we should never end up here */
     CRM_CHECK(FALSE, sort_return(0, "default"));
 }
 
 gint
 sort_op_by_callid(gconstpointer a, gconstpointer b)
 {
     return pe__is_newer_op((const xmlNode *) a, (const xmlNode *) b);
 }
 
 /*!
  * \internal
  * \brief Create a new pseudo-action for a resource
  *
  * \param[in,out] rsc       Resource to create action for
  * \param[in]     task      Action name
  * \param[in]     optional  Whether action should be considered optional
  * \param[in]     runnable  Whethe action should be considered runnable
  *
  * \return New action object corresponding to arguments
  */
 pcmk_action_t *
 pe__new_rsc_pseudo_action(pcmk_resource_t *rsc, const char *task, bool optional,
                           bool runnable)
 {
     pcmk_action_t *action = NULL;
 
     pcmk__assert((rsc != NULL) && (task != NULL));
 
     action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
                            optional, rsc->priv->scheduler);
     pcmk__set_action_flags(action, pcmk__action_pseudo);
     if (runnable) {
         pcmk__set_action_flags(action, pcmk__action_runnable);
     }
     return action;
 }
 
 /*!
  * \internal
  * \brief Add the expected result to an action
  *
  * \param[in,out] action           Action to add expected result to
  * \param[in]     expected_result  Expected result to add
  *
  * \note This is more efficient than calling pcmk__insert_meta().
  */
 void
 pe__add_action_expected_result(pcmk_action_t *action, int expected_result)
 {
     pcmk__assert((action != NULL) && (action->meta != NULL));
 
     g_hash_table_insert(action->meta, pcmk__str_copy(PCMK__META_OP_TARGET_RC),
                         pcmk__itoa(expected_result));
 }
diff --git a/lib/pengine/pe_notif.c b/lib/pengine/pe_notif.c
index 4e17cdd5c2..89d678a712 100644
--- a/lib/pengine/pe_notif.c
+++ b/lib/pengine/pe_notif.c
@@ -1,1016 +1,1027 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/common/xml.h>
 
 #include <crm/pengine/internal.h>
 #include <pacemaker-internal.h>
 
 #include "pe_status_private.h"
 
 typedef struct notify_entry_s {
     const pcmk_resource_t *rsc;
     const pcmk_node_t *node;
 } notify_entry_t;
 
 /*!
  * \internal
  * \brief Compare two notification entries
  *
  * Compare two notification entries, where the one with the alphabetically first
  * resource name (or if equal, node ID) sorts as first, with NULL sorting as
  * less than non-NULL.
  *
  * \param[in] a  First notification entry to compare
  * \param[in] b  Second notification entry to compare
  *
  * \return -1 if \p a sorts before \p b, 0 if they are equal, otherwise 1
  */
 static gint
 compare_notify_entries(gconstpointer a, gconstpointer b)
 {
     int tmp;
     const notify_entry_t *entry_a = a;
     const notify_entry_t *entry_b = b;
 
     // NULL a or b is not actually possible
     if ((entry_a == NULL) && (entry_b == NULL)) {
         return 0;
     }
     if (entry_a == NULL) {
         return 1;
     }
     if (entry_b == NULL) {
         return -1;
     }
 
     // NULL resources sort first
     if ((entry_a->rsc == NULL) && (entry_b->rsc == NULL)) {
         return 0;
     }
     if (entry_a->rsc == NULL) {
         return 1;
     }
     if (entry_b->rsc == NULL) {
         return -1;
     }
 
     // Compare resource names
     tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
     if (tmp != 0) {
         return tmp;
     }
 
     // Otherwise NULL nodes sort first
     if ((entry_a->node == NULL) && (entry_b->node == NULL)) {
         return 0;
     }
     if (entry_a->node == NULL) {
         return 1;
     }
     if (entry_b->node == NULL) {
         return -1;
     }
 
     // Finally, compare node IDs
     return strcmp(entry_a->node->priv->id, entry_b->node->priv->id);
 }
 
 /*!
  * \internal
  * \brief Duplicate a notification entry
  *
  * \param[in] entry  Entry to duplicate
  *
  * \return Newly allocated duplicate of \p entry
  * \note It is the caller's responsibility to free the return value.
  */
 static notify_entry_t *
 dup_notify_entry(const notify_entry_t *entry)
 {
     notify_entry_t *dup = pcmk__assert_alloc(1, sizeof(notify_entry_t));
 
     dup->rsc = entry->rsc;
     dup->node = entry->node;
     return dup;
 }
 
 /*!
  * \internal
  * \brief Given a list of nodes, create strings with node names
  *
  * \param[in]  list             List of nodes (as pcmk_node_t *)
  * \param[out] all_node_names   If not NULL, will be set to space-separated list
  *                              of the names of all nodes in \p list
  * \param[out] host_node_names  Same as \p all_node_names, except active
  *                              guest nodes will list the name of their host
  *
  * \note The caller is responsible for freeing the output argument values using
  *       \p g_string_free().
  */
 static void
 get_node_names(const GList *list, GString **all_node_names,
                GString **host_node_names)
 {
     if (all_node_names != NULL) {
         *all_node_names = NULL;
     }
     if (host_node_names != NULL) {
         *host_node_names = NULL;
     }
 
     for (const GList *iter = list; iter != NULL; iter = iter->next) {
         const pcmk_node_t *node = (const pcmk_node_t *) iter->data;
 
         if (node->priv->name == NULL) {
+            /* @TODO This breaks the comparability of the various notification
+             * variables and thus any agent relying on it. Maybe add "UNKNOWN"
+             * or something like that.
+             */
             continue;
         }
 
         // Always add to list of all node names
         if (all_node_names != NULL) {
             pcmk__add_word(all_node_names, 1024, node->priv->name);
         }
 
         // Add to host node name list if appropriate
         if (host_node_names != NULL) {
             if (pcmk__is_guest_or_bundle_node(node)) {
                 const pcmk_resource_t *launcher = NULL;
 
                 launcher = node->priv->remote->priv->launcher;
                 if (launcher->priv->active_nodes != NULL) {
                     node = pcmk__current_node(launcher);
                     if (node->priv->name == NULL) {
                         continue;
                     }
                 }
             }
             pcmk__add_word(host_node_names, 1024, node->priv->name);
         }
     }
 
     if ((all_node_names != NULL) && (*all_node_names == NULL)) {
         *all_node_names = g_string_new(" ");
     }
     if ((host_node_names != NULL) && (*host_node_names == NULL)) {
         *host_node_names = g_string_new(" ");
     }
 }
 
 /*!
  * \internal
  * \brief Create strings of instance and node names from notification entries
  *
  * \param[in,out] list        List of notification entries (will be sorted here)
  * \param[out]    rsc_names   If not NULL, will be set to space-separated list
  *                            of clone instances from \p list
  * \param[out]    node_names  If not NULL, will be set to space-separated list
  *                            of node names from \p list
  *
  * \return (Possibly new) head of sorted \p list
  * \note The caller is responsible for freeing the output argument values using
  *       \p g_list_free_full() and \p g_string_free().
  */
 static GList *
 notify_entries_to_strings(GList *list, GString **rsc_names,
                           GString **node_names)
 {
     const char *last_rsc_id = NULL;
 
     // Initialize output lists to NULL
     if (rsc_names != NULL) {
         *rsc_names = NULL;
     }
     if (node_names != NULL) {
         *node_names = NULL;
     }
 
     // Sort input list for user-friendliness (and ease of filtering duplicates)
     list = g_list_sort(list, compare_notify_entries);
 
     for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         notify_entry_t *entry = (notify_entry_t *) gIter->data;
 
         // Entry must have a resource (with ID)
         CRM_LOG_ASSERT((entry != NULL) && (entry->rsc != NULL)
                        && (entry->rsc->id != NULL));
         if ((entry == NULL) || (entry->rsc == NULL)
             || (entry->rsc->id == NULL)) {
             continue;
         }
 
         // Entry must have a node unless listing inactive resources
         CRM_LOG_ASSERT((node_names == NULL) || (entry->node != NULL));
         if ((node_names != NULL) && (entry->node == NULL)) {
             continue;
         }
 
         // Don't add duplicates of a particular clone instance
         if (pcmk__str_eq(entry->rsc->id, last_rsc_id, pcmk__str_none)) {
             continue;
         }
         last_rsc_id = entry->rsc->id;
 
         if (rsc_names != NULL) {
             pcmk__add_word(rsc_names, 1024, entry->rsc->id);
         }
         if ((node_names != NULL) && (entry->node->priv->name != NULL)) {
             pcmk__add_word(node_names, 1024, entry->node->priv->name);
         }
     }
 
     // If there are no entries, return "empty" lists
     if ((rsc_names != NULL) && (*rsc_names == NULL)) {
         *rsc_names = g_string_new(" ");
     }
     if ((node_names != NULL) && (*node_names == NULL)) {
         *node_names = g_string_new(" ");
     }
 
     return list;
 }
 
 /*!
  * \internal
  * \brief Copy a meta-attribute into a notify action
  *
  * \param[in]     key        Name of meta-attribute to copy
  * \param[in]     value      Value of meta-attribute to copy
  * \param[in,out] user_data  Notify action to copy into
  */
 static void
 copy_meta_to_notify(gpointer key, gpointer value, gpointer user_data)
 {
     pcmk_action_t *notify = (pcmk_action_t *) user_data;
 
     /* Any existing meta-attributes (for example, the action timeout) are for
      * the notify action itself, so don't override those.
      */
     if (g_hash_table_lookup(notify->meta, (const char *) key) != NULL) {
         return;
     }
 
     pcmk__insert_dup(notify->meta, (const char *) key, (const char *) value);
 }
 
 static void
 add_notify_data_to_action_meta(const notify_data_t *n_data,
                                pcmk_action_t *action)
 {
     for (const GSList *item = n_data->keys; item; item = item->next) {
         const pcmk_nvpair_t *nvpair = (const pcmk_nvpair_t *) item->data;
 
         pcmk__insert_meta(action, nvpair->name, nvpair->value);
     }
 }
 
 /*!
  * \internal
  * \brief Create a new notify pseudo-action for a clone resource
  *
  * \param[in,out] rsc           Clone resource that notification is for
  * \param[in]     action        Action to use in notify action key
  * \param[in]     notif_action  PCMK_ACTION_NOTIFY or PCMK_ACTION_NOTIFIED
  * \param[in]     notif_type    "pre", "post", "confirmed-pre", "confirmed-post"
  *
  * \return Newly created notify pseudo-action
  */
 static pcmk_action_t *
 new_notify_pseudo_action(pcmk_resource_t *rsc, const pcmk_action_t *action,
                          const char *notif_action, const char *notif_type)
 {
     pcmk_action_t *notify = NULL;
 
     notify = custom_action(rsc,
                            pcmk__notify_key(rsc->id, notif_type, action->task),
                            notif_action, NULL,
                            pcmk_is_set(action->flags, pcmk__action_optional),
                            rsc->priv->scheduler);
     pcmk__set_action_flags(notify, pcmk__action_pseudo);
     pcmk__insert_meta(notify, "notify_key_type", notif_type);
     pcmk__insert_meta(notify, "notify_key_operation", action->task);
     return notify;
 }
 
 /*!
  * \internal
  * \brief Create a new notify action for a clone instance
  *
  * \param[in,out] rsc          Clone instance that notification is for
  * \param[in]     node         Node that notification is for
  * \param[in,out] op           Action that notification is for
  * \param[in,out] notify_done  Parent pseudo-action for notifications complete
  * \param[in]     n_data       Notification values to add to action meta-data
  *
  * \return Newly created notify action
  */
 static pcmk_action_t *
 new_notify_action(pcmk_resource_t *rsc, const pcmk_node_t *node,
                   pcmk_action_t *op, pcmk_action_t *notify_done,
                   const notify_data_t *n_data)
 {
     char *key = NULL;
     pcmk_action_t *notify_action = NULL;
     const char *value = NULL;
     const char *task = NULL;
     const char *skip_reason = NULL;
 
     CRM_CHECK((rsc != NULL) && (node != NULL), return NULL);
 
     // Ensure we have all the info we need
     if (op == NULL) {
         skip_reason = "no action";
     } else if (notify_done == NULL) {
         skip_reason = "no parent notification";
     } else if (!node->details->online) {
         skip_reason = "node offline";
     } else if (!pcmk_is_set(op->flags, pcmk__action_runnable)) {
         skip_reason = "original action not runnable";
     }
     if (skip_reason != NULL) {
         pcmk__rsc_trace(rsc, "Skipping notify action for %s on %s: %s",
                         rsc->id, pcmk__node_name(node), skip_reason);
         return NULL;
     }
 
     value = g_hash_table_lookup(op->meta, "notify_type");     // "pre" or "post"
     task = g_hash_table_lookup(op->meta, "notify_operation"); // original action
 
     pcmk__rsc_trace(rsc, "Creating notify action for %s on %s (%s-%s)",
                     rsc->id, pcmk__node_name(node), value, task);
 
     // Create the notify action
     key = pcmk__notify_key(rsc->id, value, task);
     notify_action = custom_action(rsc, key, op->task, node,
                                   pcmk_is_set(op->flags, pcmk__action_optional),
                                   rsc->priv->scheduler);
 
     // Add meta-data to notify action
     g_hash_table_foreach(op->meta, copy_meta_to_notify, notify_action);
     add_notify_data_to_action_meta(n_data, notify_action);
 
     // Order notify after original action and before parent notification
     order_actions(op, notify_action, pcmk__ar_ordered);
     order_actions(notify_action, notify_done, pcmk__ar_ordered);
     return notify_action;
 }
 
 /*!
  * \internal
  * \brief Create a new "post-" notify action for a clone instance
  *
  * \param[in,out] rsc     Clone instance that notification is for
  * \param[in]     node    Node that notification is for
  * \param[in,out] n_data  Notification values to add to action meta-data
  */
 static void
 new_post_notify_action(pcmk_resource_t *rsc, const pcmk_node_t *node,
                        notify_data_t *n_data)
 {
     pcmk_action_t *notify = NULL;
 
     pcmk__assert(n_data != NULL);
 
     // Create the "post-" notify action for specified instance
     notify = new_notify_action(rsc, node, n_data->post, n_data->post_done,
                                n_data);
     if (notify != NULL) {
         notify->priority = PCMK_SCORE_INFINITY;
     }
 
     // Order recurring monitors after all "post-" notifications complete
     if (n_data->post_done == NULL) {
         return;
     }
     for (GList *iter = rsc->priv->actions; iter != NULL; iter = iter->next) {
         pcmk_action_t *mon = (pcmk_action_t *) iter->data;
         const char *interval_ms_s = NULL;
 
         interval_ms_s = g_hash_table_lookup(mon->meta, PCMK_META_INTERVAL);
         if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)
             || pcmk__str_eq(mon->task, PCMK_ACTION_CANCEL, pcmk__str_none)) {
             continue; // Not a recurring monitor
         }
         order_actions(n_data->post_done, mon, pcmk__ar_ordered);
     }
 }
 
 /*!
  * \internal
  * \brief Create and order notification pseudo-actions for a clone action
  *
  * In addition to the actual notify actions needed for each clone instance,
  * clone notifications also require pseudo-actions to provide ordering points
  * in the notification process. This creates the notification data, along with
  * appropriate pseudo-actions and their orderings.
  *
  * For example, the ordering sequence for starting a clone is:
  *
  *     "pre-" notify pseudo-action for clone
  *     -> "pre-" notify actions for each clone instance
  *     -> "pre-" notifications complete pseudo-action for clone
  *     -> start actions for each clone instance
  *     -> "started" pseudo-action for clone
  *     -> "post-" notify pseudo-action for clone
  *     -> "post-" notify actions for each clone instance
  *     -> "post-" notifications complete pseudo-action for clone
  *
  * \param[in,out] rsc       Clone that notifications are for
  * \param[in]     task      Name of action that notifications are for
  * \param[in,out] action    If not NULL, create a "pre-" pseudo-action ordered
  *                          before a "pre-" complete pseudo-action, ordered
  *                          before this action
  * \param[in,out] complete  If not NULL, create a "post-" pseudo-action ordered
  *                          after this action, and a "post-" complete
  *                          pseudo-action ordered after that
  *
  * \return Newly created notification data
  */
 notify_data_t *
 pe__action_notif_pseudo_ops(pcmk_resource_t *rsc, const char *task,
                             pcmk_action_t *action, pcmk_action_t *complete)
 {
     notify_data_t *n_data = NULL;
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_notify)) {
         return NULL;
     }
 
     n_data = pcmk__assert_alloc(1, sizeof(notify_data_t));
 
     n_data->action = task;
 
     if (action != NULL) { // Need "pre-" pseudo-actions
 
         // Create "pre-" notify pseudo-action for clone
         n_data->pre = new_notify_pseudo_action(rsc, action, PCMK_ACTION_NOTIFY,
                                                "pre");
         pcmk__set_action_flags(n_data->pre, pcmk__action_runnable);
         pcmk__insert_meta(n_data->pre, "notify_type", "pre");
         pcmk__insert_meta(n_data->pre, "notify_operation", n_data->action);
 
         // Create "pre-" notifications complete pseudo-action for clone
         n_data->pre_done = new_notify_pseudo_action(rsc, action,
                                                     PCMK_ACTION_NOTIFIED,
                                                     "confirmed-pre");
         pcmk__set_action_flags(n_data->pre_done, pcmk__action_runnable);
         pcmk__insert_meta(n_data->pre_done, "notify_type", "pre");
         pcmk__insert_meta(n_data->pre_done, "notify_operation", n_data->action);
 
         // Order "pre-" -> "pre-" complete -> original action
         order_actions(n_data->pre, n_data->pre_done, pcmk__ar_ordered);
         order_actions(n_data->pre_done, action, pcmk__ar_ordered);
     }
 
     if (complete != NULL) { // Need "post-" pseudo-actions
 
         // Create "post-" notify pseudo-action for clone
         n_data->post = new_notify_pseudo_action(rsc, complete,
                                                 PCMK_ACTION_NOTIFY, "post");
         n_data->post->priority = PCMK_SCORE_INFINITY;
         if (pcmk_is_set(complete->flags, pcmk__action_runnable)) {
             pcmk__set_action_flags(n_data->post, pcmk__action_runnable);
         } else {
             pcmk__clear_action_flags(n_data->post, pcmk__action_runnable);
         }
         pcmk__insert_meta(n_data->post, "notify_type", "post");
         pcmk__insert_meta(n_data->post, "notify_operation", n_data->action);
 
         // Create "post-" notifications complete pseudo-action for clone
         n_data->post_done = new_notify_pseudo_action(rsc, complete,
                                                      PCMK_ACTION_NOTIFIED,
                                                      "confirmed-post");
         n_data->post_done->priority = PCMK_SCORE_INFINITY;
         if (pcmk_is_set(complete->flags, pcmk__action_runnable)) {
             pcmk__set_action_flags(n_data->post_done, pcmk__action_runnable);
         } else {
             pcmk__clear_action_flags(n_data->post_done, pcmk__action_runnable);
         }
         pcmk__insert_meta(n_data->post_done, "notify_type", "post");
         pcmk__insert_meta(n_data->post_done,
                           "notify_operation", n_data->action);
 
-        // Order original action complete -> "post-" -> "post-" complete
+        /* Order original action complete -> "post-" -> "post-" complete
+         *
+         * @TODO Should we add |pcmk__ar_unrunnable_first_blocks to these?
+         * Otherwise we might get an invalid transition due to unresolved
+         * dependencies when "complete" is a fencing op (which can happen at
+         * least for bundles) but that op is unrunnable (due to lack of quorum,
+         * for example).
+         */
         order_actions(complete, n_data->post, pcmk__ar_first_implies_then);
         order_actions(n_data->post, n_data->post_done,
                       pcmk__ar_first_implies_then);
     }
 
     // If we created both, order "pre-" complete -> "post-"
     if ((action != NULL) && (complete != NULL)) {
         order_actions(n_data->pre_done, n_data->post, pcmk__ar_ordered);
     }
     return n_data;
 }
 
 /*!
  * \internal
  * \brief Create a new notification entry
  *
  * \param[in] rsc   Resource for notification
  * \param[in] node  Node for notification
  *
  * \return Newly allocated notification entry
  * \note The caller is responsible for freeing the return value.
  */
 static notify_entry_t *
 new_notify_entry(const pcmk_resource_t *rsc, const pcmk_node_t *node)
 {
     notify_entry_t *entry = pcmk__assert_alloc(1, sizeof(notify_entry_t));
 
     entry->rsc = rsc;
     entry->node = node;
     return entry;
 }
 
 /*!
  * \internal
  * \brief Add notification data for resource state and optionally actions
  *
  * \param[in,out] rsc       Clone or clone instance being notified
  * \param[in]     activity  Whether to add notification entries for actions
  * \param[in,out] n_data    Notification data for clone
  */
 static void
 collect_resource_data(pcmk_resource_t *rsc, bool activity,
                       notify_data_t *n_data)
 {
     const GList *iter = NULL;
     notify_entry_t *entry = NULL;
     const pcmk_node_t *node = NULL;
 
     if (n_data == NULL) {
         return;
     }
 
     if (n_data->allowed_nodes == NULL) {
         n_data->allowed_nodes = rsc->priv->allowed_nodes;
     }
 
     // If this is a clone, call recursively for each instance
     if (rsc->priv->children != NULL) {
         for (iter = rsc->priv->children; iter != NULL; iter = iter->next) {
             pcmk_resource_t *child = iter->data;
 
             collect_resource_data(child, activity, n_data);
         }
         return;
     }
 
     // This is a notification for a single clone instance
 
     if (rsc->priv->active_nodes != NULL) {
         node = rsc->priv->active_nodes->data; // First is sufficient
     }
     entry = new_notify_entry(rsc, node);
 
     // Add notification indicating the resource state
     switch (rsc->priv->orig_role) {
         case pcmk_role_stopped:
             n_data->inactive = g_list_prepend(n_data->inactive, entry);
             break;
 
         case pcmk_role_started:
             n_data->active = g_list_prepend(n_data->active, entry);
             break;
 
         case pcmk_role_unpromoted:
             n_data->unpromoted = g_list_prepend(n_data->unpromoted, entry);
             n_data->active = g_list_prepend(n_data->active,
                                             dup_notify_entry(entry));
             break;
 
         case pcmk_role_promoted:
             n_data->promoted = g_list_prepend(n_data->promoted, entry);
             n_data->active = g_list_prepend(n_data->active,
                                             dup_notify_entry(entry));
             break;
 
         default:
             pcmk__sched_err(rsc->priv->scheduler,
                             "Resource %s role on %s (%s) is not supported for "
                             "notifications (bug?)",
                             rsc->id, pcmk__node_name(node),
                             pcmk_role_text(rsc->priv->orig_role));
             free(entry);
             break;
     }
 
     if (!activity) {
         return;
     }
 
     // Add notification entries for each of the resource's actions
     for (iter = rsc->priv->actions; iter != NULL; iter = iter->next) {
         const pcmk_action_t *op = (const pcmk_action_t *) iter->data;
 
         if (!pcmk_is_set(op->flags, pcmk__action_optional)
             && (op->node != NULL)) {
             enum pcmk__action_type task = pcmk__parse_action(op->task);
 
             if ((task == pcmk__action_stop) && op->node->details->unclean) {
                 // Create anyway (additional noise if node can't be fenced)
             } else if (!pcmk_is_set(op->flags, pcmk__action_runnable)) {
                 continue;
             }
 
             entry = new_notify_entry(rsc, op->node);
 
             switch (task) {
                 case pcmk__action_start:
                     n_data->start = g_list_prepend(n_data->start, entry);
                     break;
                 case pcmk__action_stop:
                     n_data->stop = g_list_prepend(n_data->stop, entry);
                     break;
                 case pcmk__action_promote:
                     n_data->promote = g_list_prepend(n_data->promote, entry);
                     break;
                 case pcmk__action_demote:
                     n_data->demote = g_list_prepend(n_data->demote, entry);
                     break;
                 default:
                     free(entry);
                     break;
             }
         }
     }
 }
 
 // For (char *) value
 #define add_notify_env(n_data, key, value) do {                         \
          n_data->keys = pcmk_prepend_nvpair(n_data->keys, key, value);  \
     } while (0)
 
 // For (GString *) value
 #define add_notify_env_gs(n_data, key, value) do {                      \
          n_data->keys = pcmk_prepend_nvpair(n_data->keys, key,          \
                                             (const char *) value->str); \
     } while (0)
 
 // For (GString *) value
 #define add_notify_env_free_gs(n_data, key, value) do {                 \
          n_data->keys = pcmk_prepend_nvpair(n_data->keys, key,          \
                                             (const char *) value->str); \
          g_string_free(value, TRUE); value = NULL;                      \
     } while (0)
 
 /*!
  * \internal
  * \brief Create notification name/value pairs from structured data
  *
  * \param[in]     rsc       Resource that notification is for
  * \param[in,out] n_data    Notification data
  */
 static void
 add_notif_keys(const pcmk_resource_t *rsc, notify_data_t *n_data)
 {
     bool required = false; // Whether to make notify actions required
     GString *rsc_list = NULL;
     GString *node_list = NULL;
     GString *metal_list = NULL;
     const char *source = NULL;
     GList *nodes = NULL;
 
     n_data->stop = notify_entries_to_strings(n_data->stop,
                                              &rsc_list, &node_list);
     if ((strcmp(" ", (const char *) rsc_list->str) != 0)
         && pcmk__str_eq(n_data->action, PCMK_ACTION_STOP, pcmk__str_none)) {
         required = true;
     }
     add_notify_env_free_gs(n_data, "notify_stop_resource", rsc_list);
     add_notify_env_free_gs(n_data, "notify_stop_uname", node_list);
 
     if ((n_data->start != NULL)
         && pcmk__str_eq(n_data->action, PCMK_ACTION_START, pcmk__str_none)) {
         required = true;
     }
     n_data->start = notify_entries_to_strings(n_data->start,
                                               &rsc_list, &node_list);
     add_notify_env_free_gs(n_data, "notify_start_resource", rsc_list);
     add_notify_env_free_gs(n_data, "notify_start_uname", node_list);
 
     if ((n_data->demote != NULL)
         && pcmk__str_eq(n_data->action, PCMK_ACTION_DEMOTE, pcmk__str_none)) {
         required = true;
     }
     n_data->demote = notify_entries_to_strings(n_data->demote,
                                                &rsc_list, &node_list);
     add_notify_env_free_gs(n_data, "notify_demote_resource", rsc_list);
     add_notify_env_free_gs(n_data, "notify_demote_uname", node_list);
 
     if ((n_data->promote != NULL)
         && pcmk__str_eq(n_data->action, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
         required = true;
     }
     n_data->promote = notify_entries_to_strings(n_data->promote,
                                                 &rsc_list, &node_list);
     add_notify_env_free_gs(n_data, "notify_promote_resource", rsc_list);
     add_notify_env_free_gs(n_data, "notify_promote_uname", node_list);
 
     n_data->active = notify_entries_to_strings(n_data->active,
                                                &rsc_list, &node_list);
     add_notify_env_free_gs(n_data, "notify_active_resource", rsc_list);
     add_notify_env_free_gs(n_data, "notify_active_uname", node_list);
 
     n_data->unpromoted = notify_entries_to_strings(n_data->unpromoted,
                                                    &rsc_list, &node_list);
     add_notify_env_gs(n_data, "notify_unpromoted_resource", rsc_list);
     add_notify_env_gs(n_data, "notify_unpromoted_uname", node_list);
 
     // Deprecated: kept for backward compatibility with older resource agents
     add_notify_env_free_gs(n_data, "notify_slave_resource", rsc_list);
     add_notify_env_free_gs(n_data, "notify_slave_uname", node_list);
 
     n_data->promoted = notify_entries_to_strings(n_data->promoted,
                                                  &rsc_list, &node_list);
     add_notify_env_gs(n_data, "notify_promoted_resource", rsc_list);
     add_notify_env_gs(n_data, "notify_promoted_uname", node_list);
 
     // Deprecated: kept for backward compatibility with older resource agents
     add_notify_env_free_gs(n_data, "notify_master_resource", rsc_list);
     add_notify_env_free_gs(n_data, "notify_master_uname", node_list);
 
     n_data->inactive = notify_entries_to_strings(n_data->inactive,
                                                  &rsc_list, NULL);
     add_notify_env_free_gs(n_data, "notify_inactive_resource", rsc_list);
 
     nodes = g_hash_table_get_values(n_data->allowed_nodes);
     if (!pcmk__is_daemon) {
         /* For display purposes, sort the node list, for consistent
          * regression test output (while avoiding the performance hit
          * for the live cluster).
          */
         nodes = g_list_sort(nodes, pe__cmp_node_name);
     }
     get_node_names(nodes, &node_list, NULL);
     add_notify_env_free_gs(n_data, "notify_available_uname", node_list);
     g_list_free(nodes);
 
     source = g_hash_table_lookup(rsc->priv->meta,
                                  PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
     if (pcmk__str_eq(PCMK_VALUE_HOST, source, pcmk__str_none)) {
         get_node_names(rsc->priv->scheduler->nodes, &node_list, &metal_list);
         add_notify_env_free_gs(n_data, "notify_all_hosts", metal_list);
     } else {
         get_node_names(rsc->priv->scheduler->nodes, &node_list, NULL);
     }
     add_notify_env_free_gs(n_data, "notify_all_uname", node_list);
 
     if (required && (n_data->pre != NULL)) {
         pcmk__clear_action_flags(n_data->pre, pcmk__action_optional);
         pcmk__clear_action_flags(n_data->pre_done, pcmk__action_optional);
     }
 
     if (required && (n_data->post != NULL)) {
         pcmk__clear_action_flags(n_data->post, pcmk__action_optional);
         pcmk__clear_action_flags(n_data->post_done, pcmk__action_optional);
     }
 }
 
 /*
  * \internal
  * \brief Find any remote connection start relevant to an action
  *
  * \param[in] action  Action to check
  *
  * \return If action is behind a remote connection, connection's start
  */
 static pcmk_action_t *
 find_remote_start(pcmk_action_t *action)
 {
     if ((action != NULL) && (action->node != NULL)) {
         pcmk_resource_t *remote_rsc = action->node->priv->remote;
 
         if (remote_rsc != NULL) {
             return find_first_action(remote_rsc->priv->actions, NULL,
                                      PCMK_ACTION_START,
                                      NULL);
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Create notify actions, and add notify data to original actions
  *
  * \param[in,out] rsc     Clone or clone instance that notification is for
  * \param[in,out] n_data  Clone notification data for some action
  */
 static void
 create_notify_actions(pcmk_resource_t *rsc, notify_data_t *n_data)
 {
     GList *iter = NULL;
     pcmk_action_t *stop = NULL;
     pcmk_action_t *start = NULL;
     enum pcmk__action_type task = pcmk__parse_action(n_data->action);
 
     // If this is a clone, call recursively for each instance
     if (rsc->priv->children != NULL) {
         g_list_foreach(rsc->priv->children, (GFunc) create_notify_actions,
                        n_data);
         return;
     }
 
     // Add notification meta-attributes to original actions
     for (iter = rsc->priv->actions; iter != NULL; iter = iter->next) {
         pcmk_action_t *op = (pcmk_action_t *) iter->data;
 
         if (!pcmk_is_set(op->flags, pcmk__action_optional)
             && (op->node != NULL)) {
             switch (pcmk__parse_action(op->task)) {
                 case pcmk__action_start:
                 case pcmk__action_stop:
                 case pcmk__action_promote:
                 case pcmk__action_demote:
                     add_notify_data_to_action_meta(n_data, op);
                     break;
                 default:
                     break;
             }
         }
     }
 
     // Skip notify action itself if original action was not needed
     switch (task) {
         case pcmk__action_start:
             if (n_data->start == NULL) {
                 pcmk__rsc_trace(rsc, "No notify action needed for %s %s",
                                 rsc->id, n_data->action);
                 return;
             }
             break;
 
         case pcmk__action_promote:
             if (n_data->promote == NULL) {
                 pcmk__rsc_trace(rsc, "No notify action needed for %s %s",
                                 rsc->id, n_data->action);
                 return;
             }
             break;
 
         case pcmk__action_demote:
             if (n_data->demote == NULL) {
                 pcmk__rsc_trace(rsc, "No notify action needed for %s %s",
                                 rsc->id, n_data->action);
                 return;
             }
             break;
 
         default:
             // We cannot do same for stop because it might be implied by fencing
             break;
     }
 
     pcmk__rsc_trace(rsc, "Creating notify actions for %s %s",
                     rsc->id, n_data->action);
 
     // Create notify actions for stop or demote
     if ((rsc->priv->orig_role != pcmk_role_stopped)
         && ((task == pcmk__action_stop) || (task == pcmk__action_demote))) {
 
         stop = find_first_action(rsc->priv->actions, NULL, PCMK_ACTION_STOP,
                                  NULL);
 
         for (iter = rsc->priv->active_nodes;
              iter != NULL; iter = iter->next) {
 
             pcmk_node_t *current_node = (pcmk_node_t *) iter->data;
 
             /* If a stop is a pseudo-action implied by fencing, don't try to
              * notify the node getting fenced.
              */
             if ((stop != NULL)
                 && pcmk_is_set(stop->flags, pcmk__action_pseudo)
                 && (current_node->details->unclean
                     || pcmk_is_set(current_node->priv->flags,
                                    pcmk__node_remote_reset))) {
                 continue;
             }
 
             new_notify_action(rsc, current_node, n_data->pre,
                               n_data->pre_done, n_data);
 
             if ((task == pcmk__action_demote) || (stop == NULL)
                 || pcmk_is_set(stop->flags, pcmk__action_optional)) {
                 new_post_notify_action(rsc, current_node, n_data);
             }
         }
     }
 
     // Create notify actions for start or promote
     if ((rsc->priv->next_role != pcmk_role_stopped)
         && ((task == pcmk__action_start) || (task == pcmk__action_promote))) {
 
         start = find_first_action(rsc->priv->actions, NULL,
                                   PCMK_ACTION_START, NULL);
         if (start != NULL) {
             pcmk_action_t *remote_start = find_remote_start(start);
 
             if ((remote_start != NULL)
                 && !pcmk_is_set(remote_start->flags, pcmk__action_runnable)) {
                 /* Start and promote actions for a clone instance behind
                  * a Pacemaker Remote connection happen after the
                  * connection starts. If the connection start is blocked, do
                  * not schedule notifications for these actions.
                  */
                 return;
             }
         }
         if (rsc->priv->assigned_node == NULL) {
             pcmk__sched_err(rsc->priv->scheduler,
                             "Next role '%s' but %s is not allocated",
                             pcmk_role_text(rsc->priv->next_role), rsc->id);
             return;
         }
         if ((task != pcmk__action_start) || (start == NULL)
             || pcmk_is_set(start->flags, pcmk__action_optional)) {
 
             new_notify_action(rsc, rsc->priv->assigned_node, n_data->pre,
                               n_data->pre_done, n_data);
         }
         new_post_notify_action(rsc, rsc->priv->assigned_node, n_data);
     }
 }
 
 /*!
  * \internal
  * \brief Create notification data and actions for one clone action
  *
  * \param[in,out] rsc     Clone resource that notification is for
  * \param[in,out] n_data  Clone notification data for some action
  */
 void
 pe__create_action_notifications(pcmk_resource_t *rsc, notify_data_t *n_data)
 {
     if ((rsc == NULL) || (n_data == NULL)) {
         return;
     }
     collect_resource_data(rsc, true, n_data);
     add_notif_keys(rsc, n_data);
     create_notify_actions(rsc, n_data);
 }
 
 /*!
  * \internal
  * \brief Free notification data for one action
  *
  * \param[in,out] n_data  Notification data to free
  */
 void
 pe__free_action_notification_data(notify_data_t *n_data)
 {
     if (n_data == NULL) {
         return;
     }
     g_list_free_full(n_data->stop, free);
     g_list_free_full(n_data->start, free);
     g_list_free_full(n_data->demote, free);
     g_list_free_full(n_data->promote, free);
     g_list_free_full(n_data->promoted, free);
     g_list_free_full(n_data->unpromoted, free);
     g_list_free_full(n_data->active, free);
     g_list_free_full(n_data->inactive, free);
     pcmk_free_nvpairs(n_data->keys);
     free(n_data);
 }
 
 /*!
  * \internal
  * \brief Order clone "notifications complete" pseudo-action after fencing
  *
  * If a stop action is implied by fencing, the usual notification pseudo-actions
  * will not be sufficient to order things properly, or even create all needed
  * notifications if the clone is also stopping on another node, and another
  * clone is ordered after it. This function creates new notification
  * pseudo-actions relative to the fencing to ensure everything works properly.
  *
  * \param[in]     stop        Stop action implied by fencing
  * \param[in,out] rsc         Clone resource that notification is for
  * \param[in,out] stonith_op  Fencing action that implies \p stop
  */
 void
 pe__order_notifs_after_fencing(const pcmk_action_t *stop, pcmk_resource_t *rsc,
                                pcmk_action_t *stonith_op)
 {
     notify_data_t *n_data;
 
     crm_info("Ordering notifications for implied %s after fencing", stop->uuid);
     n_data = pe__action_notif_pseudo_ops(rsc, PCMK_ACTION_STOP, NULL,
                                          stonith_op);
 
     if (n_data != NULL) {
         collect_resource_data(rsc, false, n_data);
         add_notify_env(n_data, "notify_stop_resource", rsc->id);
         add_notify_env(n_data, "notify_stop_uname", stop->node->priv->name);
         create_notify_actions(uber_parent(rsc), n_data);
         pe__free_action_notification_data(n_data);
     }
 }
diff --git a/lib/pengine/pe_output.c b/lib/pengine/pe_output.c
index 3d87bdbca4..04857dfba5 100644
--- a/lib/pengine/pe_output.c
+++ b/lib/pengine/pe_output.c
@@ -1,3472 +1,3473 @@
 /*
  * Copyright 2019-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdint.h>
 
 #include <crm/common/xml_internal.h>
 #include <crm/common/output.h>
 #include <crm/common/scheduler_internal.h>
 #include <crm/cib/util.h>
 #include <crm/common/xml.h>
 #include <crm/pengine/internal.h>
 
 const char *
 pe__resource_description(const pcmk_resource_t *rsc, uint32_t show_opts)
 {
     const char * desc = NULL;
 
     // User-supplied description
     if (pcmk_any_flags_set(show_opts, pcmk_show_rsc_only|pcmk_show_description)) {
         desc = crm_element_value(rsc->priv->xml, PCMK_XA_DESCRIPTION);
     }
     return desc;
 }
 
 /* Never display node attributes whose name starts with one of these prefixes */
 #define FILTER_STR { PCMK__FAIL_COUNT_PREFIX, PCMK__LAST_FAILURE_PREFIX,    \
                      PCMK__NODE_ATTR_SHUTDOWN, PCMK_NODE_ATTR_TERMINATE,    \
                      PCMK_NODE_ATTR_STANDBY, "#", NULL }
 
 static int
 compare_attribute(gconstpointer a, gconstpointer b)
 {
     int rc;
 
     rc = strcmp((const char *)a, (const char *)b);
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Determine whether extended information about an attribute should be added.
  *
  * \param[in]     node            Node that ran this resource
  * \param[in,out] rsc_list        List of resources for this node
  * \param[in,out] scheduler       Scheduler data
  * \param[in]     attrname        Attribute to find
  * \param[out]    expected_score  Expected value for this attribute
  *
  * \return true if extended information should be printed, false otherwise
  * \note Currently, extended information is only supported for ping/pingd
  *       resources, for which a message will be printed if connectivity is lost
  *       or degraded.
  */
 static bool
 add_extra_info(const pcmk_node_t *node, GList *rsc_list,
                pcmk_scheduler_t *scheduler, const char *attrname,
                int *expected_score)
 {
     GList *gIter = NULL;
 
     for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
         pcmk_resource_t *rsc = (pcmk_resource_t *) gIter->data;
         const char *type = g_hash_table_lookup(rsc->priv->meta,
                                                PCMK_XA_TYPE);
         const char *name = NULL;
         GHashTable *params = NULL;
 
         if (rsc->priv->children != NULL) {
             if (add_extra_info(node, rsc->priv->children, scheduler,
                                attrname, expected_score)) {
                 return true;
             }
         }
 
         if (!pcmk__strcase_any_of(type, "ping", "pingd", NULL)) {
             continue;
         }
 
         params = pe_rsc_params(rsc, node, scheduler);
         name = g_hash_table_lookup(params, PCMK_XA_NAME);
 
         if (name == NULL) {
             name = "pingd";
         }
 
         /* To identify the resource with the attribute name. */
         if (pcmk__str_eq(name, attrname, pcmk__str_casei)) {
             int host_list_num = 0;
             const char *hosts = g_hash_table_lookup(params, "host_list");
             const char *multiplier = g_hash_table_lookup(params, "multiplier");
             int multiplier_i;
 
             if (hosts) {
                 char **host_list = g_strsplit(hosts, " ", 0);
                 host_list_num = g_strv_length(host_list);
                 g_strfreev(host_list);
             }
 
             if ((multiplier == NULL)
                 || (pcmk__scan_min_int(multiplier, &multiplier_i,
                                        INT_MIN) != pcmk_rc_ok)) {
                 /* The ocf:pacemaker:ping resource agent defaults multiplier to
                  * 1. The agent currently does not handle invalid text, but it
                  * should, and this would be a reasonable choice ...
                  */
                 multiplier_i = 1;
             }
             *expected_score = host_list_num * multiplier_i;
 
             return true;
         }
     }
     return false;
 }
 
 static GList *
 filter_attr_list(GList *attr_list, char *name)
 {
     int i;
     const char *filt_str[] = FILTER_STR;
 
     CRM_CHECK(name != NULL, return attr_list);
 
     /* filtering automatic attributes */
     for (i = 0; filt_str[i] != NULL; i++) {
         if (g_str_has_prefix(name, filt_str[i])) {
             return attr_list;
         }
     }
 
     return g_list_insert_sorted(attr_list, name, compare_attribute);
 }
 
 static GList *
 get_operation_list(xmlNode *rsc_entry) {
     GList *op_list = NULL;
     xmlNode *rsc_op = NULL;
 
     for (rsc_op = pcmk__xe_first_child(rsc_entry, PCMK__XE_LRM_RSC_OP, NULL,
                                        NULL);
          rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op, PCMK__XE_LRM_RSC_OP)) {
 
         const char *task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
 
         if (pcmk__str_eq(task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
             continue; // Ignore notify actions
         } else {
             int exit_status;
 
             pcmk__scan_min_int(crm_element_value(rsc_op, PCMK__XA_RC_CODE),
                                &exit_status, 0);
             if ((exit_status == CRM_EX_NOT_RUNNING)
                 && pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_none)
                 && pcmk__str_eq(crm_element_value(rsc_op, PCMK_META_INTERVAL),
                                 "0", pcmk__str_null_matches)) {
                 continue; // Ignore probes that found the resource not running
             }
         }
 
         op_list = g_list_append(op_list, rsc_op);
     }
 
     op_list = g_list_sort(op_list, sort_op_by_callid);
     return op_list;
 }
 
 static void
 add_dump_node(gpointer key, gpointer value, gpointer user_data)
 {
     xmlNodePtr node = user_data;
 
     node = pcmk__xe_create(node, (const char *) key);
     pcmk__xe_set_content(node, "%s", (const char *) value);
 }
 
 static void
 append_dump_text(gpointer key, gpointer value, gpointer user_data)
 {
     char **dump_text = user_data;
     char *new_text = crm_strdup_printf("%s %s=%s",
                                        *dump_text, (char *)key, (char *)value);
 
     free(*dump_text);
     *dump_text = new_text;
 }
 
 #define XPATH_STACK "//" PCMK_XE_NVPAIR     \
                     "[@" PCMK_XA_NAME "='"  \
                         PCMK_OPT_CLUSTER_INFRASTRUCTURE "']"
 
 static const char *
 get_cluster_stack(pcmk_scheduler_t *scheduler)
 {
     xmlNode *stack = get_xpath_object(XPATH_STACK, scheduler->input, LOG_DEBUG);
 
     if (stack != NULL) {
         return crm_element_value(stack, PCMK_XA_VALUE);
     }
     return PCMK_VALUE_UNKNOWN;
 }
 
 static char *
 last_changed_string(const char *last_written, const char *user,
                     const char *client, const char *origin) {
     if (last_written != NULL || user != NULL || client != NULL || origin != NULL) {
         return crm_strdup_printf("%s%s%s%s%s%s%s",
                                  last_written ? last_written : "",
                                  user ? " by " : "",
                                  user ? user : "",
                                  client ? " via " : "",
                                  client ? client : "",
                                  origin ? " on " : "",
                                  origin ? origin : "");
     } else {
         return strdup("");
     }
 }
 
 static char *
 op_history_string(xmlNode *xml_op, const char *task, const char *interval_ms_s,
                   int rc, bool print_timing) {
     const char *call = crm_element_value(xml_op, PCMK__XA_CALL_ID);
     char *interval_str = NULL;
     char *buf = NULL;
 
     if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
         char *pair = pcmk__format_nvpair(PCMK_XA_INTERVAL, interval_ms_s, "ms");
         interval_str = crm_strdup_printf(" %s", pair);
         free(pair);
     }
 
     if (print_timing) {
         char *last_change_str = NULL;
         char *exec_str = NULL;
         char *queue_str = NULL;
 
         const char *value = NULL;
 
         time_t epoch = 0;
 
         if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                      &epoch) == pcmk_ok)
             && (epoch > 0)) {
             char *epoch_str = pcmk__epoch2str(&epoch, 0);
 
             last_change_str = crm_strdup_printf(" %s=\"%s\"",
                                                 PCMK_XA_LAST_RC_CHANGE,
                                                 pcmk__s(epoch_str, ""));
             free(epoch_str);
         }
 
         value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
         if (value) {
             char *pair = pcmk__format_nvpair(PCMK_XA_EXEC_TIME, value, "ms");
             exec_str = crm_strdup_printf(" %s", pair);
             free(pair);
         }
 
         value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
         if (value) {
             char *pair = pcmk__format_nvpair(PCMK_XA_QUEUE_TIME, value, "ms");
             queue_str = crm_strdup_printf(" %s", pair);
             free(pair);
         }
 
         buf = crm_strdup_printf("(%s) %s:%s%s%s%s rc=%d (%s)", call, task,
                                 interval_str ? interval_str : "",
                                 last_change_str ? last_change_str : "",
                                 exec_str ? exec_str : "",
                                 queue_str ? queue_str : "",
                                 rc, crm_exit_str(rc));
 
         if (last_change_str) {
             free(last_change_str);
         }
 
         if (exec_str) {
             free(exec_str);
         }
 
         if (queue_str) {
             free(queue_str);
         }
     } else {
         buf = crm_strdup_printf("(%s) %s%s%s", call, task,
                                 interval_str ? ":" : "",
                                 interval_str ? interval_str : "");
     }
 
     if (interval_str) {
         free(interval_str);
     }
 
     return buf;
 }
 
 static char *
 resource_history_string(pcmk_resource_t *rsc, const char *rsc_id, bool all,
                         int failcount, time_t last_failure) {
     char *buf = NULL;
 
     if (rsc == NULL) {
         buf = crm_strdup_printf("%s: orphan", rsc_id);
     } else if (all || failcount || last_failure > 0) {
         char *failcount_s = NULL;
         char *lastfail_s = NULL;
 
         if (failcount > 0) {
             failcount_s = crm_strdup_printf(" %s=%d",
                                             PCMK_XA_FAIL_COUNT, failcount);
         } else {
             failcount_s = strdup("");
         }
         if (last_failure > 0) {
             buf = pcmk__epoch2str(&last_failure, 0);
             lastfail_s = crm_strdup_printf(" %s='%s'",
                                            PCMK_XA_LAST_FAILURE, buf);
             free(buf);
         }
 
         buf = crm_strdup_printf("%s: " PCMK_META_MIGRATION_THRESHOLD "=%d%s%s",
                                 rsc_id, rsc->priv->ban_after_failures,
                                 failcount_s, pcmk__s(lastfail_s, ""));
         free(failcount_s);
         free(lastfail_s);
     } else {
         buf = crm_strdup_printf("%s:", rsc_id);
     }
 
     return buf;
 }
 
 /*!
  * \internal
  * \brief Get a node's feature set for status display purposes
  *
  * \param[in] node  Node to check
  *
  * \return String representation of feature set if the node is fully up (using
  *         "<3.15.1" for older nodes that don't set the #feature-set attribute),
  *         otherwise NULL
  */
 static const char *
 get_node_feature_set(const pcmk_node_t *node)
 {
     if (node->details->online
         && pcmk_is_set(node->priv->flags, pcmk__node_expected_up)
         && !pcmk__is_pacemaker_remote_node(node)) {
 
         const char *feature_set = g_hash_table_lookup(node->priv->attrs,
                                                       CRM_ATTR_FEATURE_SET);
 
         /* The feature set attribute is present since 3.15.1. If it is missing,
          * then the node must be running an earlier version.
          */
         return pcmk__s(feature_set, "<3.15.1");
     }
     return NULL;
 }
 
 static bool
 is_mixed_version(pcmk_scheduler_t *scheduler)
 {
     const char *feature_set = NULL;
     for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = gIter->data;
         const char *node_feature_set = get_node_feature_set(node);
         if (node_feature_set != NULL) {
             if (feature_set == NULL) {
                 feature_set = node_feature_set;
             } else if (strcmp(feature_set, node_feature_set) != 0) {
                 return true;
             }
         }
     }
     return false;
 }
 
 static void
 formatted_xml_buf(const pcmk_resource_t *rsc, GString *xml_buf, bool raw)
 {
     if (raw && (rsc->priv->orig_xml != NULL)) {
         pcmk__xml_string(rsc->priv->orig_xml, pcmk__xml_fmt_pretty, xml_buf,
                          0);
     } else {
         pcmk__xml_string(rsc->priv->xml, pcmk__xml_fmt_pretty, xml_buf, 0);
     }
 }
 
 #define XPATH_DC_VERSION "//" PCMK_XE_NVPAIR    \
                          "[@" PCMK_XA_NAME "='" PCMK_OPT_DC_VERSION "']"
 
 PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
                   "enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
 static int
 cluster_summary(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     int rc = pcmk_rc_no_output;
     const char *stack_s = get_cluster_stack(scheduler);
 
     if (pcmk_is_set(section_opts, pcmk_section_stack)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-stack", stack_s, pcmkd_state);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_dc)) {
         xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
                                                scheduler->input, LOG_DEBUG);
         const char *dc_version_s = dc_version?
                                    crm_element_value(dc_version, PCMK_XA_VALUE)
                                    : NULL;
         const char *quorum = crm_element_value(scheduler->input,
                                                PCMK_XA_HAVE_QUORUM);
         char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
         bool mixed_version = is_mixed_version(scheduler);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-dc", scheduler->dc_node, quorum,
                      dc_version_s, dc_name, mixed_version);
         free(dc_name);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_times)) {
         const char *last_written = crm_element_value(scheduler->input,
                                                      PCMK_XA_CIB_LAST_WRITTEN);
         const char *user = crm_element_value(scheduler->input,
                                              PCMK_XA_UPDATE_USER);
         const char *client = crm_element_value(scheduler->input,
                                                PCMK_XA_UPDATE_CLIENT);
         const char *origin = crm_element_value(scheduler->input,
                                                PCMK_XA_UPDATE_ORIGIN);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-times", scheduler->priv->local_node_name,
                      last_written, user, client, origin);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_counts)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
                      scheduler->priv->ninstances,
                      scheduler->priv->disabled_resources,
                      scheduler->priv->blocked_resources);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_options)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-options", scheduler);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
 
     if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
         if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("cluster-summary", "pcmk_scheduler_t *",
                   "enum pcmk_pacemakerd_state", "uint32_t", "uint32_t")
 static int
 cluster_summary_html(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     int rc = pcmk_rc_no_output;
     const char *stack_s = get_cluster_stack(scheduler);
 
     if (pcmk_is_set(section_opts, pcmk_section_stack)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-stack", stack_s, pcmkd_state);
     }
 
     /* Always print DC if none, even if not requested */
     if ((scheduler->dc_node == NULL)
         || pcmk_is_set(section_opts, pcmk_section_dc)) {
         xmlNode *dc_version = get_xpath_object(XPATH_DC_VERSION,
                                                scheduler->input, LOG_DEBUG);
         const char *dc_version_s = dc_version?
                                    crm_element_value(dc_version, PCMK_XA_VALUE)
                                    : NULL;
         const char *quorum = crm_element_value(scheduler->input,
                                                PCMK_XA_HAVE_QUORUM);
         char *dc_name = scheduler->dc_node? pe__node_display_name(scheduler->dc_node, pcmk_is_set(show_opts, pcmk_show_node_id)) : NULL;
         bool mixed_version = is_mixed_version(scheduler);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-dc", scheduler->dc_node, quorum,
                      dc_version_s, dc_name, mixed_version);
         free(dc_name);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_times)) {
         const char *last_written = crm_element_value(scheduler->input,
                                                      PCMK_XA_CIB_LAST_WRITTEN);
         const char *user = crm_element_value(scheduler->input,
                                              PCMK_XA_UPDATE_USER);
         const char *client = crm_element_value(scheduler->input,
                                                PCMK_XA_UPDATE_CLIENT);
         const char *origin = crm_element_value(scheduler->input,
                                                PCMK_XA_UPDATE_ORIGIN);
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-times", scheduler->priv->local_node_name,
                      last_written, user, client, origin);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_counts)) {
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Cluster Summary");
         out->message(out, "cluster-counts", g_list_length(scheduler->nodes),
                      scheduler->priv->ninstances,
                      scheduler->priv->disabled_resources,
                      scheduler->priv->blocked_resources);
     }
 
     if (pcmk_is_set(section_opts, pcmk_section_options)) {
         /* Kind of a hack - close the list we may have opened earlier in this
          * function so we can put all the options into their own list.  We
          * only want to do this on HTML output, though.
          */
         PCMK__OUTPUT_LIST_FOOTER(out, rc);
 
         out->begin_list(out, NULL, NULL, "Config Options");
         out->message(out, "cluster-options", scheduler);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
 
     if (pcmk_is_set(section_opts, pcmk_section_maint_mode)) {
         if (out->message(out, "maint-mode", scheduler->flags) == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     return rc;
 }
 
 char *
 pe__node_display_name(pcmk_node_t *node, bool print_detail)
 {
     char *node_name;
     const char *node_host = NULL;
     const char *node_id = NULL;
     int name_len;
 
     pcmk__assert((node != NULL) && (node->priv->name != NULL));
 
     /* Host is displayed only if this is a guest node and detail is requested */
     if (print_detail && pcmk__is_guest_or_bundle_node(node)) {
         const pcmk_resource_t *launcher = NULL;
         const pcmk_node_t *host_node = NULL;
 
         launcher = node->priv->remote->priv->launcher;
         host_node = pcmk__current_node(launcher);
 
         if (host_node && host_node->details) {
             node_host = host_node->priv->name;
         }
         if (node_host == NULL) {
             node_host = ""; /* so we at least get "uname@" to indicate guest */
         }
     }
 
     /* Node ID is displayed if different from uname and detail is requested */
     if (print_detail
         && !pcmk__str_eq(node->priv->name, node->priv->id,
                          pcmk__str_casei)) {
         node_id = node->priv->id;
     }
 
     /* Determine name length */
     name_len = strlen(node->priv->name) + 1;
     if (node_host) {
         name_len += strlen(node_host) + 1; /* "@node_host" */
     }
     if (node_id) {
         name_len += strlen(node_id) + 3; /* + " (node_id)" */
     }
 
     /* Allocate and populate display name */
     node_name = pcmk__assert_alloc(name_len, sizeof(char));
     strcpy(node_name, node->priv->name);
     if (node_host) {
         strcat(node_name, "@");
         strcat(node_name, node_host);
     }
     if (node_id) {
         strcat(node_name, " (");
         strcat(node_name, node_id);
         strcat(node_name, ")");
     }
     return node_name;
 }
 
 int
 pe__name_and_nvpairs_xml(pcmk__output_t *out, bool is_list, const char *tag_name,
                          ...)
 {
     xmlNodePtr xml_node = NULL;
     va_list pairs;
 
     pcmk__assert(tag_name != NULL);
 
     xml_node = pcmk__output_xml_peek_parent(out);
     pcmk__assert(xml_node != NULL);
     xml_node = pcmk__xe_create(xml_node, tag_name);
 
     va_start(pairs, tag_name);
     pcmk__xe_set_propv(xml_node, pairs);
     va_end(pairs);
 
     if (is_list) {
         pcmk__output_xml_push_parent(out, xml_node);
     }
     return pcmk_rc_ok;
 }
 
 static const char *
 role_desc(enum rsc_role_e role)
 {
     if (role == pcmk_role_promoted) {
         return "in " PCMK_ROLE_PROMOTED " role ";
     }
     return "";
 }
 
 PCMK__OUTPUT_ARGS("ban", "pcmk_node_t *", "pcmk__location_t *", "uint32_t")
 static int
 ban_html(pcmk__output_t *out, va_list args) {
     pcmk_node_t *pe_node = va_arg(args, pcmk_node_t *);
     pcmk__location_t *location = va_arg(args, pcmk__location_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     char *node_name = pe__node_display_name(pe_node,
                                             pcmk_is_set(show_opts, pcmk_show_node_id));
     char *buf = crm_strdup_printf("%s\tprevents %s from running %son %s",
                                   location->id, location->rsc->id,
                                   role_desc(location->role_filter), node_name);
 
     pcmk__output_create_html_node(out, "li", NULL, NULL, buf);
 
     free(node_name);
     free(buf);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ban", "pcmk_node_t *", "pcmk__location_t *", "uint32_t")
 static int
 ban_text(pcmk__output_t *out, va_list args) {
     pcmk_node_t *pe_node = va_arg(args, pcmk_node_t *);
     pcmk__location_t *location = va_arg(args, pcmk__location_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     char *node_name = pe__node_display_name(pe_node,
                                             pcmk_is_set(show_opts, pcmk_show_node_id));
     out->list_item(out, NULL, "%s\tprevents %s from running %son %s",
                    location->id, location->rsc->id,
                    role_desc(location->role_filter), node_name);
 
     free(node_name);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ban", "pcmk_node_t *", "pcmk__location_t *", "uint32_t")
 static int
 ban_xml(pcmk__output_t *out, va_list args) {
     pcmk_node_t *pe_node = va_arg(args, pcmk_node_t *);
     pcmk__location_t *location = va_arg(args, pcmk__location_t *);
     uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
 
     const char *promoted_only = pcmk__btoa(location->role_filter == pcmk_role_promoted);
     char *weight_s = pcmk__itoa(pe_node->assign->score);
 
     pcmk__output_create_xml_node(out, PCMK_XE_BAN,
                                  PCMK_XA_ID, location->id,
                                  PCMK_XA_RESOURCE, location->rsc->id,
                                  PCMK_XA_NODE, pe_node->priv->name,
                                  PCMK_XA_WEIGHT, weight_s,
                                  PCMK_XA_PROMOTED_ONLY, promoted_only,
                                  /* This is a deprecated alias for
                                   * promoted_only. Removing it will break
                                   * backward compatibility of the API schema,
                                   * which will require an API schema major
                                   * version bump.
                                   */
                                  PCMK__XA_PROMOTED_ONLY_LEGACY, promoted_only,
                                  NULL);
 
     free(weight_s);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ban-list", "pcmk_scheduler_t *", "const char *", "GList *",
                   "uint32_t", "bool")
 static int
 ban_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     const char *prefix = va_arg(args, const char *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     GList *gIter, *gIter2;
     int rc = pcmk_rc_no_output;
 
     /* Print each ban */
     for (gIter = scheduler->priv->location_constraints;
          gIter != NULL; gIter = gIter->next) {
         pcmk__location_t *location = gIter->data;
         const pcmk_resource_t *rsc = location->rsc;
 
         if (prefix != NULL && !g_str_has_prefix(location->id, prefix)) {
             continue;
         }
 
         if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
                                pcmk__str_star_matches)
             && !pcmk__str_in_list(rsc_printable_id(pe__const_top_resource(rsc, false)),
                                   only_rsc, pcmk__str_star_matches)) {
             continue;
         }
 
         for (gIter2 = location->nodes; gIter2 != NULL; gIter2 = gIter2->next) {
             pcmk_node_t *node = (pcmk_node_t *) gIter2->data;
 
             if (node->assign->score < 0) {
                 PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Negative Location Constraints");
                 out->message(out, "ban", node, location, show_opts);
             }
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
 static int
 cluster_counts_html(pcmk__output_t *out, va_list args) {
     unsigned int nnodes = va_arg(args, unsigned int);
     int nresources = va_arg(args, int);
     int ndisabled = va_arg(args, int);
     int nblocked = va_arg(args, int);
 
     xmlNodePtr nodes_node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNodePtr resources_node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNode *child = NULL;
 
     child = pcmk__html_create(nodes_node, PCMK__XE_SPAN, NULL, NULL);
     pcmk__xe_set_content(child, "%d node%s configured",
                          nnodes, pcmk__plural_s(nnodes));
 
     if (ndisabled && nblocked) {
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
                              nresources, pcmk__plural_s(nresources), ndisabled);
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_BOLD);
         pcmk__xe_set_content(child, "DISABLED");
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, ", %d ", nblocked);
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_BOLD);
         pcmk__xe_set_content(child, "BLOCKED");
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, " from further action due to failure)");
 
     } else if (ndisabled && !nblocked) {
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
                              nresources, pcmk__plural_s(nresources),
                              ndisabled);
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_BOLD);
         pcmk__xe_set_content(child, "DISABLED");
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, ")");
 
     } else if (!ndisabled && nblocked) {
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%d resource instance%s configured (%d ",
                              nresources, pcmk__plural_s(nresources),
                              nblocked);
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_BOLD);
         pcmk__xe_set_content(child, "BLOCKED");
 
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, " from further action due to failure)");
 
     } else {
         child = pcmk__html_create(resources_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%d resource instance%s configured",
                              nresources, pcmk__plural_s(nresources));
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
 static int
 cluster_counts_text(pcmk__output_t *out, va_list args) {
     unsigned int nnodes = va_arg(args, unsigned int);
     int nresources = va_arg(args, int);
     int ndisabled = va_arg(args, int);
     int nblocked = va_arg(args, int);
 
     out->list_item(out, NULL, "%d node%s configured",
                    nnodes, pcmk__plural_s(nnodes));
 
     if (ndisabled && nblocked) {
         out->list_item(out, NULL, "%d resource instance%s configured "
                                   "(%d DISABLED, %d BLOCKED from "
                                   "further action due to failure)",
                        nresources, pcmk__plural_s(nresources), ndisabled,
                        nblocked);
     } else if (ndisabled && !nblocked) {
         out->list_item(out, NULL, "%d resource instance%s configured "
                                   "(%d DISABLED)",
                        nresources, pcmk__plural_s(nresources), ndisabled);
     } else if (!ndisabled && nblocked) {
         out->list_item(out, NULL, "%d resource instance%s configured "
                                   "(%d BLOCKED from further action "
                                   "due to failure)",
                        nresources, pcmk__plural_s(nresources), nblocked);
     } else {
         out->list_item(out, NULL, "%d resource instance%s configured",
                        nresources, pcmk__plural_s(nresources));
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-counts", "unsigned int", "int", "int", "int")
 static int
 cluster_counts_xml(pcmk__output_t *out, va_list args) {
     unsigned int nnodes = va_arg(args, unsigned int);
     int nresources = va_arg(args, int);
     int ndisabled = va_arg(args, int);
     int nblocked = va_arg(args, int);
 
     xmlNodePtr nodes_node = NULL;
     xmlNodePtr resources_node = NULL;
     char *s = NULL;
 
     nodes_node = pcmk__output_create_xml_node(out, PCMK_XE_NODES_CONFIGURED,
                                               NULL);
     resources_node = pcmk__output_create_xml_node(out,
                                                   PCMK_XE_RESOURCES_CONFIGURED,
                                                   NULL);
 
     s = pcmk__itoa(nnodes);
     crm_xml_add(nodes_node, PCMK_XA_NUMBER, s);
     free(s);
 
     s = pcmk__itoa(nresources);
     crm_xml_add(resources_node, PCMK_XA_NUMBER, s);
     free(s);
 
     s = pcmk__itoa(ndisabled);
     crm_xml_add(resources_node, PCMK_XA_DISABLED, s);
     free(s);
 
     s = pcmk__itoa(nblocked);
     crm_xml_add(resources_node, PCMK_XA_BLOCKED, s);
     free(s);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t *", "const char *", "const char *",
                   "char *", "int")
 static int
 cluster_dc_html(pcmk__output_t *out, va_list args) {
     pcmk_node_t *dc = va_arg(args, pcmk_node_t *);
     const char *quorum = va_arg(args, const char *);
     const char *dc_version_s = va_arg(args, const char *);
     char *dc_name = va_arg(args, char *);
     bool mixed_version = va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNode *child = NULL;
 
     child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
     pcmk__xe_set_content(child, "Current DC: ");
 
     if (dc) {
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%s (version %s) -",
                              dc_name, pcmk__s(dc_version_s, "unknown"));
 
         if (mixed_version) {
             child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
                                       PCMK__VALUE_WARNING);
             pcmk__xe_set_content(child, " MIXED-VERSION");
         }
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, " partition");
 
         if (crm_is_true(quorum)) {
             child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
             pcmk__xe_set_content(child, " with");
 
         } else {
             child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
                                       PCMK__VALUE_WARNING);
             pcmk__xe_set_content(child, " WITHOUT");
         }
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, " quorum");
 
     } else {
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_WARNING);
         pcmk__xe_set_content(child, "NONE");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t *", "const char *", "const char *",
                   "char *", "int")
 static int
 cluster_dc_text(pcmk__output_t *out, va_list args) {
     pcmk_node_t *dc = va_arg(args, pcmk_node_t *);
     const char *quorum = va_arg(args, const char *);
     const char *dc_version_s = va_arg(args, const char *);
     char *dc_name = va_arg(args, char *);
     bool mixed_version = va_arg(args, int);
 
     if (dc) {
         out->list_item(out, "Current DC",
                        "%s (version %s) - %spartition %s quorum",
                        dc_name, dc_version_s ? dc_version_s : "unknown",
                        mixed_version ? "MIXED-VERSION " : "",
                        crm_is_true(quorum) ? "with" : "WITHOUT");
     } else {
         out->list_item(out, "Current DC", "NONE");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-dc", "pcmk_node_t *", "const char *", "const char *",
                   "char *", "int")
 static int
 cluster_dc_xml(pcmk__output_t *out, va_list args) {
     pcmk_node_t *dc = va_arg(args, pcmk_node_t *);
     const char *quorum = va_arg(args, const char *);
     const char *dc_version_s = va_arg(args, const char *);
     char *dc_name G_GNUC_UNUSED = va_arg(args, char *);
     bool mixed_version = va_arg(args, int);
 
     if (dc) {
         const char *with_quorum = pcmk__btoa(crm_is_true(quorum));
         const char *mixed_version_s = pcmk__btoa(mixed_version);
 
         pcmk__output_create_xml_node(out, PCMK_XE_CURRENT_DC,
                                      PCMK_XA_PRESENT, PCMK_VALUE_TRUE,
                                      PCMK_XA_VERSION, pcmk__s(dc_version_s, ""),
                                      PCMK_XA_NAME, dc->priv->name,
                                      PCMK_XA_ID, dc->priv->id,
                                      PCMK_XA_WITH_QUORUM, with_quorum,
                                      PCMK_XA_MIXED_VERSION, mixed_version_s,
                                      NULL);
     } else {
         pcmk__output_create_xml_node(out, PCMK_XE_CURRENT_DC,
                                      PCMK_XA_PRESENT, PCMK_VALUE_FALSE,
                                      NULL);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("maint-mode", "uint64_t")
 static int
 cluster_maint_mode_text(pcmk__output_t *out, va_list args) {
     uint64_t flags = va_arg(args, uint64_t);
 
     if (pcmk_is_set(flags, pcmk__sched_in_maintenance)) {
         pcmk__formatted_printf(out, "\n              *** Resource management is DISABLED ***\n");
         pcmk__formatted_printf(out, "  The cluster will not attempt to start, stop or recover services\n");
         return pcmk_rc_ok;
     } else if (pcmk_is_set(flags, pcmk__sched_stop_all)) {
         pcmk__formatted_printf(out, "\n    *** Resource management is DISABLED ***\n");
         pcmk__formatted_printf(out, "  The cluster will keep all resources stopped\n");
         return pcmk_rc_ok;
     } else {
         return pcmk_rc_no_output;
     }
 }
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_html(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
         out->list_item(out, NULL, "STONITH of failed nodes enabled");
     } else {
         out->list_item(out, NULL, "STONITH of failed nodes disabled");
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_symmetric_cluster)) {
         out->list_item(out, NULL, "Cluster is symmetric");
     } else {
         out->list_item(out, NULL, "Cluster is asymmetric");
     }
 
     switch (scheduler->no_quorum_policy) {
         case pcmk_no_quorum_freeze:
             out->list_item(out, NULL, "No quorum policy: Freeze resources");
             break;
 
         case pcmk_no_quorum_stop:
             out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
             break;
 
         case pcmk_no_quorum_demote:
             out->list_item(out, NULL, "No quorum policy: Demote promotable "
                            "resources and stop all other resources");
             break;
 
         case pcmk_no_quorum_ignore:
             out->list_item(out, NULL, "No quorum policy: Ignore");
             break;
 
         case pcmk_no_quorum_fence:
             out->list_item(out, NULL,
                            "No quorum policy: Fence nodes in partition");
             break;
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_in_maintenance)) {
         xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
         xmlNode *child = NULL;
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "Resource management: ");
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
         pcmk__xe_set_content(child, "DISABLED");
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child,
                              " (the cluster will not attempt to start, stop,"
                              " or recover services)");
 
     } else if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_all)) {
         xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
         xmlNode *child = NULL;
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "Resource management: ");
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
         pcmk__xe_set_content(child, "STOPPED");
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child,
                              " (the cluster will keep all resources stopped)");
 
     } else {
         out->list_item(out, NULL, "Resource management: enabled");
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_log(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_in_maintenance)) {
         return out->info(out, "Resource management is DISABLED.  The cluster will not attempt to start, stop or recover services.");
     } else if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_all)) {
         return out->info(out, "Resource management is DISABLED.  The cluster has stopped all resources.");
     } else {
         return pcmk_rc_no_output;
     }
 }
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_text(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
         out->list_item(out, NULL, "STONITH of failed nodes enabled");
     } else {
         out->list_item(out, NULL, "STONITH of failed nodes disabled");
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_symmetric_cluster)) {
         out->list_item(out, NULL, "Cluster is symmetric");
     } else {
         out->list_item(out, NULL, "Cluster is asymmetric");
     }
 
     switch (scheduler->no_quorum_policy) {
         case pcmk_no_quorum_freeze:
             out->list_item(out, NULL, "No quorum policy: Freeze resources");
             break;
 
         case pcmk_no_quorum_stop:
             out->list_item(out, NULL, "No quorum policy: Stop ALL resources");
             break;
 
         case pcmk_no_quorum_demote:
             out->list_item(out, NULL, "No quorum policy: Demote promotable "
                            "resources and stop all other resources");
             break;
 
         case pcmk_no_quorum_ignore:
             out->list_item(out, NULL, "No quorum policy: Ignore");
             break;
 
         case pcmk_no_quorum_fence:
             out->list_item(out, NULL,
                            "No quorum policy: Fence nodes in partition");
             break;
     }
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Get readable string representation of a no-quorum policy
  *
  * \param[in] policy  No-quorum policy
  *
  * \return String representation of \p policy
  */
 static const char *
 no_quorum_policy_text(enum pe_quorum_policy policy)
 {
     switch (policy) {
         case pcmk_no_quorum_freeze:
             return PCMK_VALUE_FREEZE;
 
         case pcmk_no_quorum_stop:
             return PCMK_VALUE_STOP;
 
         case pcmk_no_quorum_demote:
             return PCMK_VALUE_DEMOTE;
 
         case pcmk_no_quorum_ignore:
             return PCMK_VALUE_IGNORE;
 
         case pcmk_no_quorum_fence:
             return PCMK_VALUE_FENCE;
 
         default:
             return PCMK_VALUE_UNKNOWN;
     }
 }
 
 PCMK__OUTPUT_ARGS("cluster-options", "pcmk_scheduler_t *")
 static int
 cluster_options_xml(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
 
     const char *stonith_enabled = pcmk__flag_text(scheduler->flags,
                                                   pcmk__sched_fencing_enabled);
     const char *symmetric_cluster =
         pcmk__flag_text(scheduler->flags, pcmk__sched_symmetric_cluster);
     const char *no_quorum_policy =
         no_quorum_policy_text(scheduler->no_quorum_policy);
     const char *maintenance_mode = pcmk__flag_text(scheduler->flags,
                                                    pcmk__sched_in_maintenance);
     const char *stop_all_resources = pcmk__flag_text(scheduler->flags,
                                                      pcmk__sched_stop_all);
     char *stonith_timeout_ms_s =
         crm_strdup_printf("%u", scheduler->priv->fence_timeout_ms);
 
     char *priority_fencing_delay_ms_s =
         crm_strdup_printf("%u", scheduler->priv->priority_fencing_ms);
 
     pcmk__output_create_xml_node(out, PCMK_XE_CLUSTER_OPTIONS,
                                  PCMK_XA_STONITH_ENABLED, stonith_enabled,
                                  PCMK_XA_SYMMETRIC_CLUSTER, symmetric_cluster,
                                  PCMK_XA_NO_QUORUM_POLICY, no_quorum_policy,
                                  PCMK_XA_MAINTENANCE_MODE, maintenance_mode,
                                  PCMK_XA_STOP_ALL_RESOURCES, stop_all_resources,
                                  PCMK_XA_STONITH_TIMEOUT_MS,
                                      stonith_timeout_ms_s,
                                  PCMK_XA_PRIORITY_FENCING_DELAY_MS,
                                      priority_fencing_delay_ms_s,
                                  NULL);
     free(stonith_timeout_ms_s);
     free(priority_fencing_delay_ms_s);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
 static int
 cluster_stack_html(pcmk__output_t *out, va_list args) {
     const char *stack_s = va_arg(args, const char *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNode *child = NULL;
 
     child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, PCMK__VALUE_BOLD);
     pcmk__xe_set_content(child, "Stack: ");
 
     child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
     pcmk__xe_set_content(child, "%s", stack_s);
 
     if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, " (");
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%s",
                              pcmk__pcmkd_state_enum2friendly(pcmkd_state));
 
         child = pcmk__html_create(node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, ")");
     }
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
 static int
 cluster_stack_text(pcmk__output_t *out, va_list args) {
     const char *stack_s = va_arg(args, const char *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
 
     if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
         out->list_item(out, "Stack", "%s (%s)",
                        stack_s, pcmk__pcmkd_state_enum2friendly(pcmkd_state));
     } else {
         out->list_item(out, "Stack", "%s", stack_s);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-stack", "const char *", "enum pcmk_pacemakerd_state")
 static int
 cluster_stack_xml(pcmk__output_t *out, va_list args) {
     const char *stack_s = va_arg(args, const char *);
     enum pcmk_pacemakerd_state pcmkd_state =
         (enum pcmk_pacemakerd_state) va_arg(args, int);
 
     const char *state_s = NULL;
 
     if (pcmkd_state != pcmk_pacemakerd_state_invalid) {
         state_s = pcmk_pacemakerd_api_daemon_state_enum2text(pcmkd_state);
     }
 
     pcmk__output_create_xml_node(out, PCMK_XE_STACK,
                                  PCMK_XA_TYPE, stack_s,
                                  PCMK_XA_PACEMAKERD_STATE, state_s,
                                  NULL);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *",
                   "const char *", "const char *", "const char *")
 static int
 cluster_times_html(pcmk__output_t *out, va_list args) {
     const char *our_nodename = va_arg(args, const char *);
     const char *last_written = va_arg(args, const char *);
     const char *user = va_arg(args, const char *);
     const char *client = va_arg(args, const char *);
     const char *origin = va_arg(args, const char *);
 
     xmlNodePtr updated_node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNodePtr changed_node = pcmk__output_create_xml_node(out, "li", NULL);
     xmlNode *child = NULL;
 
     char *time_s = NULL;
 
     child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL,
                               PCMK__VALUE_BOLD);
     pcmk__xe_set_content(child, "Last updated: ");
 
     child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
     time_s = pcmk__epoch2str(NULL, 0);
     pcmk__xe_set_content(child, "%s", time_s);
     free(time_s);
 
     if (our_nodename != NULL) {
         child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, " on ");
 
         child = pcmk__html_create(updated_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%s", our_nodename);
     }
 
     child = pcmk__html_create(changed_node, PCMK__XE_SPAN, NULL,
                               PCMK__VALUE_BOLD);
     pcmk__xe_set_content(child, "Last change: ");
 
     child = pcmk__html_create(changed_node, PCMK__XE_SPAN, NULL, NULL);
     time_s = last_changed_string(last_written, user, client, origin);
     pcmk__xe_set_content(child, "%s", time_s);
     free(time_s);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *",
                   "const char *", "const char *", "const char *")
 static int
 cluster_times_xml(pcmk__output_t *out, va_list args) {
     const char *our_nodename = va_arg(args, const char *);
     const char *last_written = va_arg(args, const char *);
     const char *user = va_arg(args, const char *);
     const char *client = va_arg(args, const char *);
     const char *origin = va_arg(args, const char *);
 
     char *time_s = pcmk__epoch2str(NULL, 0);
 
     pcmk__output_create_xml_node(out, PCMK_XE_LAST_UPDATE,
                                  PCMK_XA_TIME, time_s,
                                  PCMK_XA_ORIGIN, our_nodename,
                                  NULL);
 
     pcmk__output_create_xml_node(out, PCMK_XE_LAST_CHANGE,
                                  PCMK_XA_TIME, pcmk__s(last_written, ""),
                                  PCMK_XA_USER, pcmk__s(user, ""),
                                  PCMK_XA_CLIENT, pcmk__s(client, ""),
                                  PCMK_XA_ORIGIN, pcmk__s(origin, ""),
                                  NULL);
 
     free(time_s);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("cluster-times", "const char *", "const char *",
                   "const char *", "const char *", "const char *")
 static int
 cluster_times_text(pcmk__output_t *out, va_list args) {
     const char *our_nodename = va_arg(args, const char *);
     const char *last_written = va_arg(args, const char *);
     const char *user = va_arg(args, const char *);
     const char *client = va_arg(args, const char *);
     const char *origin = va_arg(args, const char *);
 
     char *time_s = pcmk__epoch2str(NULL, 0);
 
     out->list_item(out, "Last updated", "%s%s%s",
                    time_s, (our_nodename != NULL)? " on " : "",
                    pcmk__s(our_nodename, ""));
 
     free(time_s);
     time_s = last_changed_string(last_written, user, client, origin);
 
     out->list_item(out, "Last change", " %s", time_s);
 
     free(time_s);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Display a failed action in less-technical natural language
  *
  * \param[in,out] out          Output object to use for display
  * \param[in]     xml_op       XML containing failed action
  * \param[in]     op_key       Operation key of failed action
  * \param[in]     node_name    Where failed action occurred
  * \param[in]     rc           OCF exit code of failed action
  * \param[in]     status       Execution status of failed action
  * \param[in]     exit_reason  Exit reason given for failed action
  * \param[in]     exec_time    String containing execution time in milliseconds
  */
 static void
 failed_action_friendly(pcmk__output_t *out, const xmlNode *xml_op,
                        const char *op_key, const char *node_name, int rc,
                        int status, const char *exit_reason,
                        const char *exec_time)
 {
     char *rsc_id = NULL;
     char *task = NULL;
     guint interval_ms = 0;
     time_t last_change_epoch = 0;
     GString *str = NULL;
 
     if (pcmk__str_empty(op_key)
         || !parse_op_key(op_key, &rsc_id, &task, &interval_ms)) {
 
         pcmk__str_update(&rsc_id, "unknown resource");
         pcmk__str_update(&task, "unknown action");
         interval_ms = 0;
     }
     pcmk__assert((rsc_id != NULL) && (task != NULL));
 
     str = g_string_sized_new(256); // Should be sufficient for most messages
 
     pcmk__g_strcat(str, rsc_id, " ", NULL);
 
     if (interval_ms != 0) {
         pcmk__g_strcat(str, pcmk__readable_interval(interval_ms), "-interval ",
                        NULL);
     }
     pcmk__g_strcat(str, pcmk__readable_action(task, interval_ms), " on ",
                    node_name, NULL);
 
     if (status == PCMK_EXEC_DONE) {
         pcmk__g_strcat(str, " returned '", crm_exit_str(rc), "'", NULL);
         if (!pcmk__str_empty(exit_reason)) {
             pcmk__g_strcat(str, " (", exit_reason, ")", NULL);
         }
 
     } else {
         pcmk__g_strcat(str, " could not be executed (",
                        pcmk_exec_status_str(status), NULL);
         if (!pcmk__str_empty(exit_reason)) {
             pcmk__g_strcat(str, ": ", exit_reason, NULL);
         }
         g_string_append_c(str, ')');
     }
 
 
     if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                 &last_change_epoch) == pcmk_ok) {
         char *s = pcmk__epoch2str(&last_change_epoch, 0);
 
         pcmk__g_strcat(str, " at ", s, NULL);
         free(s);
     }
     if (!pcmk__str_empty(exec_time)) {
         int exec_time_ms = 0;
 
         if ((pcmk__scan_min_int(exec_time, &exec_time_ms, 0) == pcmk_rc_ok)
             && (exec_time_ms > 0)) {
 
             pcmk__g_strcat(str, " after ",
                            pcmk__readable_interval(exec_time_ms), NULL);
         }
     }
 
     out->list_item(out, NULL, "%s", str->str);
     g_string_free(str, TRUE);
     free(rsc_id);
     free(task);
 }
 
 /*!
  * \internal
  * \brief Display a failed action with technical details
  *
  * \param[in,out] out          Output object to use for display
  * \param[in]     xml_op       XML containing failed action
  * \param[in]     op_key       Operation key of failed action
  * \param[in]     node_name    Where failed action occurred
  * \param[in]     rc           OCF exit code of failed action
  * \param[in]     status       Execution status of failed action
  * \param[in]     exit_reason  Exit reason given for failed action
  * \param[in]     exec_time    String containing execution time in milliseconds
  */
 static void
 failed_action_technical(pcmk__output_t *out, const xmlNode *xml_op,
                         const char *op_key, const char *node_name, int rc,
                         int status, const char *exit_reason,
                         const char *exec_time)
 {
     const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
     const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
     const char *exit_status = crm_exit_str(rc);
     const char *lrm_status = pcmk_exec_status_str(status);
     time_t last_change_epoch = 0;
     GString *str = NULL;
 
     if (pcmk__str_empty(op_key)) {
         op_key = "unknown operation";
     }
     if (pcmk__str_empty(exit_status)) {
         exit_status = "unknown exit status";
     }
     if (pcmk__str_empty(call_id)) {
         call_id = "unknown";
     }
 
     str = g_string_sized_new(256);
 
     g_string_append_printf(str, "%s on %s '%s' (%d): call=%s, status='%s'",
                            op_key, node_name, exit_status, rc, call_id,
                            lrm_status);
 
     if (!pcmk__str_empty(exit_reason)) {
         pcmk__g_strcat(str, ", exitreason='", exit_reason, "'", NULL);
     }
 
     if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                 &last_change_epoch) == pcmk_ok) {
         char *last_change_str = pcmk__epoch2str(&last_change_epoch, 0);
 
         pcmk__g_strcat(str,
                        ", " PCMK_XA_LAST_RC_CHANGE "="
                        "'", last_change_str, "'", NULL);
         free(last_change_str);
     }
     if (!pcmk__str_empty(queue_time)) {
         pcmk__g_strcat(str, ", queued=", queue_time, "ms", NULL);
     }
     if (!pcmk__str_empty(exec_time)) {
         pcmk__g_strcat(str, ", exec=", exec_time, "ms", NULL);
     }
 
     out->list_item(out, NULL, "%s", str->str);
     g_string_free(str, TRUE);
 }
 
 PCMK__OUTPUT_ARGS("failed-action", "xmlNode *", "uint32_t")
 static int
 failed_action_default(pcmk__output_t *out, va_list args)
 {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     const char *op_key = pcmk__xe_history_key(xml_op);
     const char *node_name = crm_element_value(xml_op, PCMK_XA_UNAME);
     const char *exit_reason = crm_element_value(xml_op, PCMK_XA_EXIT_REASON);
     const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
 
     int rc;
     int status;
 
     pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);
 
     pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
                        0);
 
     if (pcmk__str_empty(node_name)) {
         node_name = "unknown node";
     }
 
     if (pcmk_is_set(show_opts, pcmk_show_failed_detail)) {
         failed_action_technical(out, xml_op, op_key, node_name, rc, status,
                                 exit_reason, exec_time);
     } else {
         failed_action_friendly(out, xml_op, op_key, node_name, rc, status,
                                exit_reason, exec_time);
     }
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("failed-action", "xmlNode *", "uint32_t")
 static int
 failed_action_xml(pcmk__output_t *out, va_list args) {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
 
     const char *op_key = pcmk__xe_history_key(xml_op);
     const char *op_key_name = PCMK_XA_OP_KEY;
     int rc;
     int status;
     const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
     const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
     const char *exitstatus = NULL;
     const char *exit_reason = pcmk__s(crm_element_value(xml_op,
                                                         PCMK_XA_EXIT_REASON),
                                       "none");
     const char *status_s = NULL;
 
     time_t epoch = 0;
     gchar *exit_reason_esc = NULL;
     char *rc_s = NULL;
     xmlNodePtr node = NULL;
 
     if (pcmk__xml_needs_escape(exit_reason, pcmk__xml_escape_attr)) {
         exit_reason_esc = pcmk__xml_escape(exit_reason, pcmk__xml_escape_attr);
         exit_reason = exit_reason_esc;
     }
     pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_RC_CODE), &rc, 0);
     pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
                        0);
 
     if (crm_element_value(xml_op, PCMK__XA_OPERATION_KEY) == NULL) {
         op_key_name = PCMK_XA_ID;
     }
     exitstatus = crm_exit_str(rc);
     rc_s = pcmk__itoa(rc);
     status_s = pcmk_exec_status_str(status);
     node = pcmk__output_create_xml_node(out, PCMK_XE_FAILURE,
                                         op_key_name, op_key,
                                         PCMK_XA_NODE, uname,
                                         PCMK_XA_EXITSTATUS, exitstatus,
                                         PCMK_XA_EXITREASON, exit_reason,
                                         PCMK_XA_EXITCODE, rc_s,
                                         PCMK_XA_CALL, call_id,
                                         PCMK_XA_STATUS, status_s,
                                         NULL);
     free(rc_s);
 
     if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                  &epoch) == pcmk_ok) && (epoch > 0)) {
 
         const char *queue_time = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
         const char *exec = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
         const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
         guint interval_ms = 0;
         char *interval_ms_s = NULL;
         char *rc_change = pcmk__epoch2str(&epoch,
                                           crm_time_log_date
                                           |crm_time_log_timeofday
                                           |crm_time_log_with_timezone);
 
         crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &interval_ms);
         interval_ms_s = crm_strdup_printf("%u", interval_ms);
 
         pcmk__xe_set_props(node,
                            PCMK_XA_LAST_RC_CHANGE, rc_change,
                            PCMK_XA_QUEUED, queue_time,
                            PCMK_XA_EXEC, exec,
                            PCMK_XA_INTERVAL, interval_ms_s,
                            PCMK_XA_TASK, task,
                            NULL);
 
         free(interval_ms_s);
         free(rc_change);
     }
 
     g_free(exit_reason_esc);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("failed-action-list", "pcmk_scheduler_t *", "GList *",
                   "GList *", "uint32_t", "bool")
 static int
 failed_action_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     xmlNode *xml_op = NULL;
     int rc = pcmk_rc_no_output;
 
     if (xmlChildElementCount(scheduler->priv->failed) == 0) {
         return rc;
     }
 
     for (xml_op = pcmk__xe_first_child(scheduler->priv->failed, NULL, NULL,
                                        NULL);
          xml_op != NULL; xml_op = pcmk__xe_next(xml_op, NULL)) {
 
         char *rsc = NULL;
 
         if (!pcmk__str_in_list(crm_element_value(xml_op, PCMK_XA_UNAME),
                                only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         if (pcmk_xe_mask_probe_failure(xml_op)) {
             continue;
         }
 
         if (!parse_op_key(pcmk__xe_history_key(xml_op), &rsc, NULL, NULL)) {
             continue;
         }
 
         if (!pcmk__str_in_list(rsc, only_rsc, pcmk__str_star_matches)) {
             free(rsc);
             continue;
         }
 
         free(rsc);
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Failed Resource Actions");
         out->message(out, "failed-action", xml_op, show_opts);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 static void
 status_node(pcmk_node_t *node, xmlNodePtr parent, uint32_t show_opts)
 {
     int health = pe__node_health(node);
     xmlNode *child = NULL;
 
     // Cluster membership
     if (node->details->online) {
         child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
                                   PCMK_VALUE_ONLINE);
         pcmk__xe_set_content(child, " online");
 
     } else {
         child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
                                   PCMK_VALUE_OFFLINE);
         pcmk__xe_set_content(child, " OFFLINE");
     }
 
     // Standby mode
     if (pcmk_is_set(node->priv->flags, pcmk__node_fail_standby)) {
         child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
                                   PCMK_VALUE_STANDBY);
         if (node->details->running_rsc == NULL) {
             pcmk__xe_set_content(child,
                                  " (in standby due to " PCMK_META_ON_FAIL ")");
         } else {
             pcmk__xe_set_content(child,
                                  " (in standby due to " PCMK_META_ON_FAIL ","
                                  " with active resources)");
         }
 
     } else if (pcmk_is_set(node->priv->flags, pcmk__node_standby)) {
         child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
                                   PCMK_VALUE_STANDBY);
         if (node->details->running_rsc == NULL) {
             pcmk__xe_set_content(child, " (in standby)");
         } else {
             pcmk__xe_set_content(child, " (in standby, with active resources)");
         }
     }
 
     // Maintenance mode
     if (node->details->maintenance) {
         child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_MAINT);
         pcmk__xe_set_content(child, " (in maintenance mode)");
     }
 
     // Node health
     if (health < 0) {
         child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_HEALTH_RED);
         pcmk__xe_set_content(child, " (health is RED)");
 
     } else if (health == 0) {
         child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL,
                                   PCMK__VALUE_HEALTH_YELLOW);
         pcmk__xe_set_content(child, " (health is YELLOW)");
     }
 
     // Feature set
     if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
         const char *feature_set = get_node_feature_set(node);
         if (feature_set != NULL) {
             child = pcmk__html_create(parent, PCMK__XE_SPAN, NULL, NULL);
             pcmk__xe_set_content(child, ", feature set %s", feature_set);
         }
     }
 }
 
 PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool",
                   "GList *", "GList *")
 static int
 node_html(pcmk__output_t *out, va_list args) {
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool full = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
 
     if (full) {
         xmlNode *item_node = NULL;
         xmlNode *child = NULL;
 
         if (pcmk_all_flags_set(show_opts, pcmk_show_brief | pcmk_show_rscs_by_node)) {
             GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);
 
             out->begin_list(out, NULL, NULL, "%s:", node_name);
             item_node = pcmk__output_xml_create_parent(out, "li", NULL);
             child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
             pcmk__xe_set_content(child, "Status:");
             status_node(node, item_node, show_opts);
 
             if (rscs != NULL) {
                 uint32_t new_show_opts = (show_opts | pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
                 out->begin_list(out, NULL, NULL, "Resources");
                 pe__rscs_brief_output(out, rscs, new_show_opts);
                 out->end_list(out);
             }
 
             pcmk__output_xml_pop_parent(out);
             out->end_list(out);
 
         } else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             GList *lpc2 = NULL;
             int rc = pcmk_rc_no_output;
 
             out->begin_list(out, NULL, NULL, "%s:", node_name);
             item_node = pcmk__output_xml_create_parent(out, "li", NULL);
             child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
             pcmk__xe_set_content(child, "Status:");
             status_node(node, item_node, show_opts);
 
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 pcmk_resource_t *rsc = (pcmk_resource_t *) lpc2->data;
 
                 PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Resources");
 
                 show_opts |= pcmk_show_rsc_only;
                 out->message(out, (const char *) rsc->priv->xml->name,
                              show_opts, rsc, only_node, only_rsc);
             }
 
             PCMK__OUTPUT_LIST_FOOTER(out, rc);
             pcmk__output_xml_pop_parent(out);
             out->end_list(out);
 
         } else {
             item_node = pcmk__output_create_xml_node(out, "li", NULL);
             child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
                                       PCMK__VALUE_BOLD);
             pcmk__xe_set_content(child, "%s:", node_name);
             status_node(node, item_node, show_opts);
         }
     } else {
         out->begin_list(out, NULL, NULL, "%s:", node_name);
     }
 
     free(node_name);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Get a human-friendly textual description of a node's status
  *
  * \param[in] node  Node to check
  *
  * \return String representation of node's status
  */
 static const char *
 node_text_status(const pcmk_node_t *node)
 {
     if (node->details->unclean) {
         if (node->details->online) {
             return "UNCLEAN (online)";
 
         } else if (node->details->pending) {
             return "UNCLEAN (pending)";
 
         } else {
             return "UNCLEAN (offline)";
         }
 
     } else if (node->details->pending) {
         return "pending";
 
     } else if (pcmk_is_set(node->priv->flags, pcmk__node_fail_standby)
                && node->details->online) {
         return "standby (" PCMK_META_ON_FAIL ")";
 
     } else if (pcmk_is_set(node->priv->flags, pcmk__node_standby)) {
         if (!node->details->online) {
             return "OFFLINE (standby)";
         } else if (node->details->running_rsc == NULL) {
             return "standby";
         } else {
             return "standby (with active resources)";
         }
 
     } else if (node->details->maintenance) {
         if (node->details->online) {
             return "maintenance";
         } else {
             return "OFFLINE (maintenance)";
         }
 
     } else if (node->details->online) {
         return "online";
     }
 
     return "OFFLINE";
 }
 
 PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool", "GList *",
                   "GList *")
 static int
 node_text(pcmk__output_t *out, va_list args) {
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool full = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     if (full) {
         char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
         GString *str = g_string_sized_new(64);
         int health = pe__node_health(node);
 
         // Create a summary line with node type, name, and status
         if (pcmk__is_guest_or_bundle_node(node)) {
             g_string_append(str, "GuestNode");
         } else if (pcmk__is_remote_node(node)) {
             g_string_append(str, "RemoteNode");
         } else {
             g_string_append(str, "Node");
         }
         pcmk__g_strcat(str, " ", node_name, ": ", node_text_status(node), NULL);
 
         if (health < 0) {
             g_string_append(str, " (health is RED)");
         } else if (health == 0) {
             g_string_append(str, " (health is YELLOW)");
         }
         if (pcmk_is_set(show_opts, pcmk_show_feature_set)) {
             const char *feature_set = get_node_feature_set(node);
             if (feature_set != NULL) {
                 pcmk__g_strcat(str, ", feature set ", feature_set, NULL);
             }
         }
 
         /* If we're grouping by node, print its resources */
         if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             if (pcmk_is_set(show_opts, pcmk_show_brief)) {
                 GList *rscs = pe__filter_rsc_list(node->details->running_rsc, only_rsc);
 
                 if (rscs != NULL) {
                     uint32_t new_show_opts = (show_opts | pcmk_show_rsc_only) & ~pcmk_show_inactive_rscs;
                     out->begin_list(out, NULL, NULL, "%s", str->str);
                     out->begin_list(out, NULL, NULL, "Resources");
 
                     pe__rscs_brief_output(out, rscs, new_show_opts);
 
                     out->end_list(out);
                     out->end_list(out);
 
                     g_list_free(rscs);
                 }
 
             } else {
                 GList *gIter2 = NULL;
 
                 out->begin_list(out, NULL, NULL, "%s", str->str);
                 out->begin_list(out, NULL, NULL, "Resources");
 
                 for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
                     pcmk_resource_t *rsc = (pcmk_resource_t *) gIter2->data;
 
                     show_opts |= pcmk_show_rsc_only;
                     out->message(out, (const char *) rsc->priv->xml->name,
                                  show_opts, rsc, only_node, only_rsc);
                 }
 
                 out->end_list(out);
                 out->end_list(out);
             }
         } else {
             out->list_item(out, NULL, "%s", str->str);
         }
 
         g_string_free(str, TRUE);
         free(node_name);
     } else {
         char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
         out->begin_list(out, NULL, NULL, "Node: %s", node_name);
         free(node_name);
     }
 
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Convert an integer health value to a string representation
  *
  * \param[in] health  Integer health value
  *
  * \retval \c PCMK_VALUE_RED if \p health is less than 0
  * \retval \c PCMK_VALUE_YELLOW if \p health is equal to 0
  * \retval \c PCMK_VALUE_GREEN if \p health is greater than 0
  */
 static const char *
 health_text(int health)
 {
     if (health < 0) {
         return PCMK_VALUE_RED;
     } else if (health == 0) {
         return PCMK_VALUE_YELLOW;
     } else {
         return PCMK_VALUE_GREEN;
     }
 }
 
 /*!
  * \internal
  * \brief Convert a node variant to a string representation
  *
  * \param[in] variant  Node variant
  *
  * \retval \c PCMK_VALUE_MEMBER if \p node_type is \c pcmk__node_variant_cluster
  * \retval \c PCMK_VALUE_REMOTE if \p node_type is \c pcmk__node_variant_remote
  * \retval \c PCMK_VALUE_UNKNOWN otherwise
  */
 static const char *
 node_variant_text(enum pcmk__node_variant variant)
 {
     switch (variant) {
         case pcmk__node_variant_cluster:
             return PCMK_VALUE_MEMBER;
         case pcmk__node_variant_remote:
             return PCMK_VALUE_REMOTE;
         default:
             return PCMK_VALUE_UNKNOWN;
     }
 }
 
 PCMK__OUTPUT_ARGS("node", "pcmk_node_t *", "uint32_t", "bool", "GList *",
                   "GList *")
 static int
 node_xml(pcmk__output_t *out, va_list args) {
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     uint32_t show_opts G_GNUC_UNUSED = va_arg(args, uint32_t);
     bool full = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     if (full) {
         const char *online = pcmk__btoa(node->details->online);
         const char *standby = pcmk__flag_text(node->priv->flags,
                                               pcmk__node_standby);
         const char *standby_onfail = pcmk__flag_text(node->priv->flags,
                                                      pcmk__node_fail_standby);
         const char *maintenance = pcmk__btoa(node->details->maintenance);
         const char *pending = pcmk__btoa(node->details->pending);
         const char *unclean = pcmk__btoa(node->details->unclean);
         const char *health = health_text(pe__node_health(node));
         const char *feature_set = get_node_feature_set(node);
         const char *shutdown = pcmk__btoa(node->details->shutdown);
         const char *expected_up = pcmk__flag_text(node->priv->flags,
                                                   pcmk__node_expected_up);
         const bool is_dc = pcmk__same_node(node,
                                            node->priv->scheduler->dc_node);
         int length = g_list_length(node->details->running_rsc);
         char *resources_running = pcmk__itoa(length);
         const char *node_type = node_variant_text(node->priv->variant);
 
         int rc = pcmk_rc_ok;
 
         rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_NODE,
                                       PCMK_XA_NAME, node->priv->name,
                                       PCMK_XA_ID, node->priv->id,
                                       PCMK_XA_ONLINE, online,
                                       PCMK_XA_STANDBY, standby,
                                       PCMK_XA_STANDBY_ONFAIL, standby_onfail,
                                       PCMK_XA_MAINTENANCE, maintenance,
                                       PCMK_XA_PENDING, pending,
                                       PCMK_XA_UNCLEAN, unclean,
                                       PCMK_XA_HEALTH, health,
                                       PCMK_XA_FEATURE_SET, feature_set,
                                       PCMK_XA_SHUTDOWN, shutdown,
                                       PCMK_XA_EXPECTED_UP, expected_up,
                                       PCMK_XA_IS_DC, pcmk__btoa(is_dc),
                                       PCMK_XA_RESOURCES_RUNNING, resources_running,
                                       PCMK_XA_TYPE, node_type,
                                       NULL);
 
         free(resources_running);
         pcmk__assert(rc == pcmk_rc_ok);
 
         if (pcmk__is_guest_or_bundle_node(node)) {
             xmlNodePtr xml_node = pcmk__output_xml_peek_parent(out);
             crm_xml_add(xml_node, PCMK_XA_ID_AS_RESOURCE,
                         node->priv->remote->priv->launcher->id);
         }
 
         if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             GList *lpc = NULL;
 
             for (lpc = node->details->running_rsc; lpc != NULL; lpc = lpc->next) {
                 pcmk_resource_t *rsc = (pcmk_resource_t *) lpc->data;
 
                 show_opts |= pcmk_show_rsc_only;
                 out->message(out, (const char *) rsc->priv->xml->name,
                              show_opts, rsc, only_node, only_rsc);
             }
         }
 
         out->end_list(out);
     } else {
         pcmk__output_xml_create_parent(out, PCMK_XE_NODE,
                                        PCMK_XA_NAME, node->priv->name,
                                        NULL);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "bool", "int")
 static int
 node_attribute_text(pcmk__output_t *out, va_list args) {
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
     bool add_extra = va_arg(args, int);
     int expected_score = va_arg(args, int);
 
     if (add_extra) {
         int v;
 
         if (value == NULL) {
             v = 0;
         } else {
             pcmk__scan_min_int(value, &v, INT_MIN);
         }
         if (v <= 0) {
             out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is lost", name, value);
         } else if (v < expected_score) {
             out->list_item(out, NULL, "%-32s\t: %-10s\t: Connectivity is degraded (Expected=%d)", name, value, expected_score);
         } else {
             out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
         }
     } else {
         out->list_item(out, NULL, "%-32s\t: %-10s", name, value);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "bool", "int")
 static int
 node_attribute_html(pcmk__output_t *out, va_list args) {
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
     bool add_extra = va_arg(args, int);
     int expected_score = va_arg(args, int);
 
     if (add_extra) {
         int v = 0;
         xmlNodePtr item_node = pcmk__output_create_xml_node(out, "li", NULL);
         xmlNode *child = NULL;
 
         if (value != NULL) {
             pcmk__scan_min_int(value, &v, INT_MIN);
         }
 
         child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL, NULL);
         pcmk__xe_set_content(child, "%s: %s", name, value);
 
         if (v <= 0) {
             child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
                                       PCMK__VALUE_BOLD);
             pcmk__xe_set_content(child, "(connectivity is lost)");
 
         } else if (v < expected_score) {
             child = pcmk__html_create(item_node, PCMK__XE_SPAN, NULL,
                                       PCMK__VALUE_BOLD);
             pcmk__xe_set_content(child,
                                  "(connectivity is degraded -- expected %d)",
                                  expected_score);
         }
     } else {
         out->list_item(out, NULL, "%s: %s", name, value);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNode *")
 static int
 node_and_op(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
 
     pcmk_resource_t *rsc = NULL;
     gchar *node_str = NULL;
     char *last_change_str = NULL;
 
     const char *op_rsc = crm_element_value(xml_op, PCMK_XA_RESOURCE);
     int status;
     time_t last_change = 0;
 
     pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS), &status,
                        PCMK_EXEC_UNKNOWN);
 
     rsc = pe_find_resource(scheduler->priv->resources, op_rsc);
 
     if (rsc) {
         const pcmk_node_t *node = pcmk__current_node(rsc);
         const char *target_role = g_hash_table_lookup(rsc->priv->meta,
                                                       PCMK_META_TARGET_ROLE);
         uint32_t show_opts = pcmk_show_rsc_only | pcmk_show_pending;
 
         if (node == NULL) {
             node = rsc->priv->pending_node;
         }
 
         node_str = pcmk__native_output_string(rsc, rsc_printable_id(rsc), node,
                                               show_opts, target_role, false);
     } else {
         node_str = crm_strdup_printf("Unknown resource %s", op_rsc);
     }
 
     if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                 &last_change) == pcmk_ok) {
         const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
 
         last_change_str = crm_strdup_printf(", %s='%s', exec=%sms",
                                             PCMK_XA_LAST_RC_CHANGE,
                                             pcmk__trim(ctime(&last_change)),
                                             exec_time);
     }
 
     out->list_item(out, NULL, "%s: %s (node=%s, call=%s, rc=%s%s): %s",
                    node_str, pcmk__xe_history_key(xml_op),
                    crm_element_value(xml_op, PCMK_XA_UNAME),
                    crm_element_value(xml_op, PCMK__XA_CALL_ID),
                    crm_element_value(xml_op, PCMK__XA_RC_CODE),
                    last_change_str ? last_change_str : "",
                    pcmk_exec_status_str(status));
 
     g_free(node_str);
     free(last_change_str);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-and-op", "pcmk_scheduler_t *", "xmlNode *")
 static int
 node_and_op_xml(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
 
     pcmk_resource_t *rsc = NULL;
     const char *uname = crm_element_value(xml_op, PCMK_XA_UNAME);
     const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
     const char *rc_s = crm_element_value(xml_op, PCMK__XA_RC_CODE);
     const char *status_s = NULL;
     const char *op_rsc = crm_element_value(xml_op, PCMK_XA_RESOURCE);
     int status;
     time_t last_change = 0;
     xmlNode *node = NULL;
 
     pcmk__scan_min_int(crm_element_value(xml_op, PCMK__XA_OP_STATUS),
                        &status, PCMK_EXEC_UNKNOWN);
     status_s = pcmk_exec_status_str(status);
 
     node = pcmk__output_create_xml_node(out, PCMK_XE_OPERATION,
                                         PCMK_XA_OP, pcmk__xe_history_key(xml_op),
                                         PCMK_XA_NODE, uname,
                                         PCMK_XA_CALL, call_id,
                                         PCMK_XA_RC, rc_s,
                                         PCMK_XA_STATUS, status_s,
                                         NULL);
 
     rsc = pe_find_resource(scheduler->priv->resources, op_rsc);
 
     if (rsc) {
         const char *class = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
         const char *provider = crm_element_value(rsc->priv->xml,
                                                  PCMK_XA_PROVIDER);
         const char *kind = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE);
         bool has_provider = pcmk_is_set(pcmk_get_ra_caps(class),
                                         pcmk_ra_cap_provider);
 
         char *agent_tuple = crm_strdup_printf("%s:%s:%s",
                                               class,
                                               (has_provider? provider : ""),
                                               kind);
 
         pcmk__xe_set_props(node,
                            PCMK_XA_RSC, rsc_printable_id(rsc),
                            PCMK_XA_AGENT, agent_tuple,
                            NULL);
         free(agent_tuple);
     }
 
     if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                 &last_change) == pcmk_ok) {
         const char *last_rc_change = pcmk__trim(ctime(&last_change));
         const char *exec_time = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
 
         pcmk__xe_set_props(node,
                            PCMK_XA_LAST_RC_CHANGE, last_rc_change,
                            PCMK_XA_EXEC_TIME, exec_time,
                            NULL);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute", "const char *", "const char *", "bool", "int")
 static int
 node_attribute_xml(pcmk__output_t *out, va_list args) {
     const char *name = va_arg(args, const char *);
     const char *value = va_arg(args, const char *);
     bool add_extra = va_arg(args, int);
     int expected_score = va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_ATTRIBUTE,
                                                    PCMK_XA_NAME, name,
                                                    PCMK_XA_VALUE, value,
                                                    NULL);
 
     if (add_extra) {
         char *buf = pcmk__itoa(expected_score);
         crm_xml_add(node, PCMK_XA_EXPECTED, buf);
         free(buf);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-attribute-list", "pcmk_scheduler_t *", "uint32_t",
                   "bool", "GList *", "GList *")
 static int
 node_attribute_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
 
     int rc = pcmk_rc_no_output;
 
     /* Display each node's attributes */
     for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = gIter->data;
 
         GList *attr_list = NULL;
         GHashTableIter iter;
         gpointer key;
 
         if (!node || !node->details || !node->details->online) {
             continue;
         }
 
+        // @TODO Maybe skip filtering for XML output
         g_hash_table_iter_init(&iter, node->priv->attrs);
         while (g_hash_table_iter_next (&iter, &key, NULL)) {
             attr_list = filter_attr_list(attr_list, key);
         }
 
         if (attr_list == NULL) {
             continue;
         }
 
         if (!pcmk__str_in_list(node->priv->name, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             g_list_free(attr_list);
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node Attributes");
 
         out->message(out, "node", node, show_opts, false, only_node, only_rsc);
 
         for (GList *aIter = attr_list; aIter != NULL; aIter = aIter->next) {
             const char *name = aIter->data;
             const char *value = NULL;
             int expected_score = 0;
             bool add_extra = false;
 
             value = pcmk__node_attr(node, name, NULL, pcmk__rsc_node_current);
 
             add_extra = add_extra_info(node, node->details->running_rsc,
                                        scheduler, name, &expected_score);
 
             /* Print attribute name and value */
             out->message(out, "node-attribute", name, value, add_extra,
                          expected_score);
         }
 
         g_list_free(attr_list);
         out->end_list(out);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t *", "const char *")
 static int
 node_capacity(pcmk__output_t *out, va_list args)
 {
     const pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *comment = va_arg(args, const char *);
 
     char *dump_text = crm_strdup_printf("%s: %s capacity:",
                                         comment, pcmk__node_name(node));
 
     g_hash_table_foreach(node->priv->utilization, append_dump_text,
                          &dump_text);
     out->list_item(out, NULL, "%s", dump_text);
     free(dump_text);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-capacity", "const pcmk_node_t *", "const char *")
 static int
 node_capacity_xml(pcmk__output_t *out, va_list args)
 {
     const pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *uname = node->priv->name;
     const char *comment = va_arg(args, const char *);
 
     xmlNodePtr xml_node = pcmk__output_create_xml_node(out, PCMK_XE_CAPACITY,
                                                        PCMK_XA_NODE, uname,
                                                        PCMK_XA_COMMENT, comment,
                                                        NULL);
     g_hash_table_foreach(node->priv->utilization, add_dump_node, xml_node);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-history-list", "pcmk_scheduler_t *", "pcmk_node_t *",
                   "xmlNode *", "GList *", "GList *", "uint32_t", "uint32_t")
 static int
 node_history_list(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     xmlNode *node_state = va_arg(args, xmlNode *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     xmlNode *lrm_rsc = NULL;
     xmlNode *rsc_entry = NULL;
     int rc = pcmk_rc_no_output;
 
     lrm_rsc = pcmk__xe_first_child(node_state, PCMK__XE_LRM, NULL, NULL);
     lrm_rsc = pcmk__xe_first_child(lrm_rsc, PCMK__XE_LRM_RESOURCES, NULL, NULL);
 
     /* Print history of each of the node's resources */
     for (rsc_entry = pcmk__xe_first_child(lrm_rsc, PCMK__XE_LRM_RESOURCE, NULL,
                                           NULL);
          rsc_entry != NULL;
          rsc_entry = pcmk__xe_next(rsc_entry, PCMK__XE_LRM_RESOURCE)) {
 
         const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
         pcmk_resource_t *rsc = NULL;
         const pcmk_resource_t *parent = NULL;
 
         if (rsc_id == NULL) {
             continue; // Malformed entry
         }
 
         rsc = pe_find_resource(scheduler->priv->resources, rsc_id);
         if (rsc == NULL) {
             continue; // Resource was removed from configuration
         }
 
         /* We can't use is_filtered here to filter group resources.  For is_filtered,
          * we have to decide whether to check the parent or not.  If we check the
          * parent, all elements of a group will always be printed because that's how
          * is_filtered works for groups.  If we do not check the parent, sometimes
          * this will filter everything out.
          *
          * For other resource types, is_filtered is okay.
          */
         parent = pe__const_top_resource(rsc, false);
         if (pcmk__is_group(parent)) {
             if (!pcmk__str_in_list(rsc_printable_id(rsc), only_rsc,
                                    pcmk__str_star_matches)
                 && !pcmk__str_in_list(rsc_printable_id(parent), only_rsc,
                                       pcmk__str_star_matches)) {
                 continue;
             }
         } else if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
             continue;
         }
 
         if (!pcmk_is_set(section_opts, pcmk_section_operations)) {
             time_t last_failure = 0;
             int failcount = pe_get_failcount(node, rsc, &last_failure,
                                              pcmk__fc_default, NULL);
 
             if (failcount <= 0) {
                 continue;
             }
 
             if (rc == pcmk_rc_no_output) {
                 rc = pcmk_rc_ok;
                 out->message(out, "node", node, show_opts, false, only_node,
                              only_rsc);
             }
 
             out->message(out, "resource-history", rsc, rsc_id, false,
                          failcount, last_failure, false);
         } else {
             GList *op_list = get_operation_list(rsc_entry);
             pcmk_resource_t *rsc = NULL;
 
             if (op_list == NULL) {
                 continue;
             }
 
             rsc = pe_find_resource(scheduler->priv->resources,
                                    crm_element_value(rsc_entry, PCMK_XA_ID));
 
             if (rc == pcmk_rc_no_output) {
                 rc = pcmk_rc_ok;
                 out->message(out, "node", node, show_opts, false, only_node,
                              only_rsc);
             }
 
             out->message(out, "resource-operation-list", scheduler, rsc, node,
                          op_list, show_opts);
         }
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-list", "GList *", "GList *", "GList *", "uint32_t", "bool")
 static int
 node_list_html(pcmk__output_t *out, va_list args) {
     GList *nodes = va_arg(args, GList *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer G_GNUC_UNUSED = va_arg(args, int);
 
     int rc = pcmk_rc_no_output;
 
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) gIter->data;
 
         if (!pcmk__str_in_list(node->priv->name, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, false, rc, "Node List");
 
         out->message(out, "node", node, show_opts, true, only_node, only_rsc);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-list", "GList *", "GList *", "GList *", "uint32_t", "bool")
 static int
 node_list_text(pcmk__output_t *out, va_list args) {
     GList *nodes = va_arg(args, GList *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     /* space-separated lists of node names */
     GString *online_nodes = NULL;
     GString *online_remote_nodes = NULL;
     GString *online_guest_nodes = NULL;
     GString *offline_nodes = NULL;
     GString *offline_remote_nodes = NULL;
 
     int rc = pcmk_rc_no_output;
 
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) gIter->data;
         char *node_name = pe__node_display_name(node, pcmk_is_set(show_opts, pcmk_show_node_id));
 
         if (!pcmk__str_in_list(node->priv->name, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             free(node_name);
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc, "Node List");
 
         // Determine whether to display node individually or in a list
         if (node->details->unclean || node->details->pending
             || (pcmk_is_set(node->priv->flags, pcmk__node_fail_standby)
                 && node->details->online)
             || pcmk_is_set(node->priv->flags, pcmk__node_standby)
             || node->details->maintenance
             || pcmk_is_set(show_opts, pcmk_show_rscs_by_node)
             || pcmk_is_set(show_opts, pcmk_show_feature_set)
             || (pe__node_health(node) <= 0)) {
             // Display node individually
 
         } else if (node->details->online) {
             // Display online node in a list
             if (pcmk__is_guest_or_bundle_node(node)) {
                 pcmk__add_word(&online_guest_nodes, 1024, node_name);
 
             } else if (pcmk__is_remote_node(node)) {
                 pcmk__add_word(&online_remote_nodes, 1024, node_name);
 
             } else {
                 pcmk__add_word(&online_nodes, 1024, node_name);
             }
             free(node_name);
             continue;
 
         } else {
             // Display offline node in a list
             if (pcmk__is_remote_node(node)) {
                 pcmk__add_word(&offline_remote_nodes, 1024, node_name);
 
             } else if (pcmk__is_guest_or_bundle_node(node)) {
                 /* ignore offline guest nodes */
 
             } else {
                 pcmk__add_word(&offline_nodes, 1024, node_name);
             }
             free(node_name);
             continue;
         }
 
         /* If we get here, node is in bad state, or we're grouping by node */
         out->message(out, "node", node, show_opts, true, only_node, only_rsc);
         free(node_name);
     }
 
     /* If we're not grouping by node, summarize nodes by status */
     if (online_nodes != NULL) {
         out->list_item(out, "Online", "[ %s ]",
                        (const char *) online_nodes->str);
         g_string_free(online_nodes, TRUE);
     }
     if (offline_nodes != NULL) {
         out->list_item(out, "OFFLINE", "[ %s ]",
                        (const char *) offline_nodes->str);
         g_string_free(offline_nodes, TRUE);
     }
     if (online_remote_nodes) {
         out->list_item(out, "RemoteOnline", "[ %s ]",
                        (const char *) online_remote_nodes->str);
         g_string_free(online_remote_nodes, TRUE);
     }
     if (offline_remote_nodes) {
         out->list_item(out, "RemoteOFFLINE", "[ %s ]",
                        (const char *) offline_remote_nodes->str);
         g_string_free(offline_remote_nodes, TRUE);
     }
     if (online_guest_nodes != NULL) {
         out->list_item(out, "GuestOnline", "[ %s ]",
                        (const char *) online_guest_nodes->str);
         g_string_free(online_guest_nodes, TRUE);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-list", "GList *", "GList *", "GList *", "uint32_t", "bool")
 static int
 node_list_xml(pcmk__output_t *out, va_list args) {
     GList *nodes = va_arg(args, GList *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer G_GNUC_UNUSED = va_arg(args, int);
 
     /* PCMK_XE_NODES acts as the list's element name for CLI tools that use
      * pcmk__output_enable_list_element.  Otherwise PCMK_XE_NODES is the
      * value of the list's PCMK_XA_NAME attribute.
      */
     out->begin_list(out, NULL, NULL, PCMK_XE_NODES);
     for (GList *gIter = nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *node = (pcmk_node_t *) gIter->data;
 
         if (!pcmk__str_in_list(node->priv->name, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         out->message(out, "node", node, show_opts, true, only_node, only_rsc);
     }
     out->end_list(out);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-summary", "pcmk_scheduler_t *", "GList *", "GList *",
                   "uint32_t", "uint32_t", "bool")
 static int
 node_summary(pcmk__output_t *out, va_list args) {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     uint32_t section_opts = va_arg(args, uint32_t);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_spacer = va_arg(args, int);
 
     xmlNode *node_state = NULL;
     xmlNode *cib_status = pcmk_find_cib_element(scheduler->input,
                                                 PCMK_XE_STATUS);
     int rc = pcmk_rc_no_output;
 
     if (xmlChildElementCount(cib_status) == 0) {
         return rc;
     }
 
     for (node_state = pcmk__xe_first_child(cib_status, PCMK__XE_NODE_STATE,
                                            NULL, NULL);
          node_state != NULL;
          node_state = pcmk__xe_next(node_state, PCMK__XE_NODE_STATE)) {
 
         pcmk_node_t *node = pe_find_node_id(scheduler->nodes,
                                             pcmk__xe_id(node_state));
 
         if (!node || !node->details || !node->details->online) {
             continue;
         }
 
         if (!pcmk__str_in_list(node->priv->name, only_node,
                                pcmk__str_star_matches|pcmk__str_casei)) {
             continue;
         }
 
         PCMK__OUTPUT_LIST_HEADER(out, print_spacer, rc,
                                  pcmk_is_set(section_opts, pcmk_section_operations) ? "Operations" : "Migration Summary");
 
         out->message(out, "node-history-list", scheduler, node, node_state,
                      only_node, only_rsc, section_opts, show_opts);
     }
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t *", "const char *",
                   "const char *", "const char *")
 static int
 node_weight(pcmk__output_t *out, va_list args)
 {
     const pcmk_resource_t *rsc = va_arg(args, const pcmk_resource_t *);
     const char *prefix = va_arg(args, const char *);
     const char *uname = va_arg(args, const char *);
     const char *score = va_arg(args, const char *);
 
     if (rsc) {
         out->list_item(out, NULL, "%s: %s allocation score on %s: %s",
                        prefix, rsc->id, uname, score);
     } else {
         out->list_item(out, NULL, "%s: %s = %s", prefix, uname, score);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("node-weight", "const pcmk_resource_t *", "const char *",
                   "const char *", "const char *")
 static int
 node_weight_xml(pcmk__output_t *out, va_list args)
 {
     const pcmk_resource_t *rsc = va_arg(args, const pcmk_resource_t *);
     const char *prefix = va_arg(args, const char *);
     const char *uname = va_arg(args, const char *);
     const char *score = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_NODE_WEIGHT,
                                                    PCMK_XA_FUNCTION, prefix,
                                                    PCMK_XA_NODE, uname,
                                                    PCMK_XA_SCORE, score,
                                                    NULL);
 
     if (rsc) {
         crm_xml_add(node, PCMK_XA_ID, rsc->id);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("op-history", "xmlNode *", "const char *", "const char *", "int", "uint32_t")
 static int
 op_history_text(pcmk__output_t *out, va_list args) {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     const char *task = va_arg(args, const char *);
     const char *interval_ms_s = va_arg(args, const char *);
     int rc = va_arg(args, int);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     char *buf = op_history_string(xml_op, task, interval_ms_s, rc,
                                   pcmk_is_set(show_opts, pcmk_show_timing));
 
     out->list_item(out, NULL, "%s", buf);
 
     free(buf);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("op-history", "xmlNode *", "const char *", "const char *", "int", "uint32_t")
 static int
 op_history_xml(pcmk__output_t *out, va_list args) {
     xmlNodePtr xml_op = va_arg(args, xmlNodePtr);
     const char *task = va_arg(args, const char *);
     const char *interval_ms_s = va_arg(args, const char *);
     int rc = va_arg(args, int);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     const char *call_id = crm_element_value(xml_op, PCMK__XA_CALL_ID);
     char *rc_s = pcmk__itoa(rc);
     const char *rc_text = crm_exit_str(rc);
     xmlNodePtr node = NULL;
 
     node = pcmk__output_create_xml_node(out, PCMK_XE_OPERATION_HISTORY,
                                         PCMK_XA_CALL, call_id,
                                         PCMK_XA_TASK, task,
                                         PCMK_XA_RC, rc_s,
                                         PCMK_XA_RC_TEXT, rc_text,
                                         NULL);
     free(rc_s);
 
     if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
         char *s = crm_strdup_printf("%sms", interval_ms_s);
         crm_xml_add(node, PCMK_XA_INTERVAL, s);
         free(s);
     }
 
     if (pcmk_is_set(show_opts, pcmk_show_timing)) {
         const char *value = NULL;
         time_t epoch = 0;
 
         if ((crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                      &epoch) == pcmk_ok) && (epoch > 0)) {
             char *s = pcmk__epoch2str(&epoch, 0);
             crm_xml_add(node, PCMK_XA_LAST_RC_CHANGE, s);
             free(s);
         }
 
         value = crm_element_value(xml_op, PCMK_XA_EXEC_TIME);
         if (value) {
             char *s = crm_strdup_printf("%sms", value);
             crm_xml_add(node, PCMK_XA_EXEC_TIME, s);
             free(s);
         }
         value = crm_element_value(xml_op, PCMK_XA_QUEUE_TIME);
         if (value) {
             char *s = crm_strdup_printf("%sms", value);
             crm_xml_add(node, PCMK_XA_QUEUE_TIME, s);
             free(s);
         }
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 promotion_score(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *child_rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *chosen = va_arg(args, pcmk_node_t *);
     const char *score = va_arg(args, const char *);
 
     if (chosen == NULL) {
         out->list_item(out, NULL, "%s promotion score (inactive): %s",
                        child_rsc->id, score);
     } else {
         out->list_item(out, NULL, "%s promotion score on %s: %s",
                        child_rsc->id, pcmk__node_name(chosen), score);
     }
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("promotion-score", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 promotion_score_xml(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *child_rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *chosen = va_arg(args, pcmk_node_t *);
     const char *score = va_arg(args, const char *);
 
     xmlNodePtr node = pcmk__output_create_xml_node(out, PCMK_XE_PROMOTION_SCORE,
                                                    PCMK_XA_ID, child_rsc->id,
                                                    PCMK_XA_SCORE, score,
                                                    NULL);
 
     if (chosen) {
         crm_xml_add(node, PCMK_XA_NODE, chosen->priv->name);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
 static int
 resource_config(pcmk__output_t *out, va_list args) {
     const pcmk_resource_t *rsc = va_arg(args, const pcmk_resource_t *);
     GString *xml_buf = g_string_sized_new(1024);
     bool raw = va_arg(args, int);
 
     formatted_xml_buf(rsc, xml_buf, raw);
 
     out->output_xml(out, PCMK_XE_XML, xml_buf->str);
 
     g_string_free(xml_buf, TRUE);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-config", "const pcmk_resource_t *", "bool")
 static int
 resource_config_text(pcmk__output_t *out, va_list args) {
     pcmk__formatted_printf(out, "Resource XML:\n");
     return resource_config(out, args);
 }
 
 PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t *", "const char *",
                   "bool", "int", "time_t", "bool")
 static int
 resource_history_text(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *rsc_id = va_arg(args, const char *);
     bool all = va_arg(args, int);
     int failcount = va_arg(args, int);
     time_t last_failure = va_arg(args, time_t);
     bool as_header = va_arg(args, int);
 
     char *buf = resource_history_string(rsc, rsc_id, all, failcount, last_failure);
 
     if (as_header) {
         out->begin_list(out, NULL, NULL, "%s", buf);
     } else {
         out->list_item(out, NULL, "%s", buf);
     }
 
     free(buf);
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-history", "pcmk_resource_t *", "const char *",
                   "bool", "int", "time_t", "bool")
 static int
 resource_history_xml(pcmk__output_t *out, va_list args) {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     const char *rsc_id = va_arg(args, const char *);
     bool all = va_arg(args, int);
     int failcount = va_arg(args, int);
     time_t last_failure = va_arg(args, time_t);
     bool as_header = va_arg(args, int);
 
     xmlNodePtr node = pcmk__output_xml_create_parent(out,
                                                      PCMK_XE_RESOURCE_HISTORY,
                                                      PCMK_XA_ID, rsc_id,
                                                      NULL);
 
     if (rsc == NULL) {
         pcmk__xe_set_bool_attr(node, PCMK_XA_ORPHAN, true);
     } else if (all || failcount || last_failure > 0) {
         char *migration_s = pcmk__itoa(rsc->priv->ban_after_failures);
 
         pcmk__xe_set_props(node,
                            PCMK_XA_ORPHAN, PCMK_VALUE_FALSE,
                            PCMK_META_MIGRATION_THRESHOLD, migration_s,
                            NULL);
         free(migration_s);
 
         if (failcount > 0) {
             char *s = pcmk__itoa(failcount);
 
             crm_xml_add(node, PCMK_XA_FAIL_COUNT, s);
             free(s);
         }
 
         if (last_failure > 0) {
             char *s = pcmk__epoch2str(&last_failure, 0);
 
             crm_xml_add(node, PCMK_XA_LAST_FAILURE, s);
             free(s);
         }
     }
 
     if (!as_header) {
         pcmk__output_xml_pop_parent(out);
     }
 
     return pcmk_rc_ok;
 }
 
 static void
 print_resource_header(pcmk__output_t *out, uint32_t show_opts)
 {
     if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
         /* Active resources have already been printed by node */
         out->begin_list(out, NULL, NULL, "Inactive Resources");
     } else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
         out->begin_list(out, NULL, NULL, "Full List of Resources");
     } else {
         out->begin_list(out, NULL, NULL, "Active Resources");
     }
 }
 
 
 PCMK__OUTPUT_ARGS("resource-list", "pcmk_scheduler_t *", "uint32_t", "bool",
                   "GList *", "GList *", "bool")
 static int
 resource_list(pcmk__output_t *out, va_list args)
 {
     pcmk_scheduler_t *scheduler = va_arg(args, pcmk_scheduler_t *);
     uint32_t show_opts = va_arg(args, uint32_t);
     bool print_summary = va_arg(args, int);
     GList *only_node = va_arg(args, GList *);
     GList *only_rsc = va_arg(args, GList *);
     bool print_spacer = va_arg(args, int);
 
     GList *rsc_iter;
     int rc = pcmk_rc_no_output;
     bool printed_header = false;
 
     /* If we already showed active resources by node, and
      * we're not showing inactive resources, we have nothing to do
      */
     if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node) &&
         !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
         return rc;
     }
 
     /* If we haven't already printed resources grouped by node,
      * and brief output was requested, print resource summary */
     if (pcmk_is_set(show_opts, pcmk_show_brief)
         && !pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
         GList *rscs = pe__filter_rsc_list(scheduler->priv->resources, only_rsc);
 
         PCMK__OUTPUT_SPACER_IF(out, print_spacer);
         print_resource_header(out, show_opts);
         printed_header = true;
 
         rc = pe__rscs_brief_output(out, rscs, show_opts);
         g_list_free(rscs);
     }
 
     /* For each resource, display it if appropriate */
     for (rsc_iter = scheduler->priv->resources;
          rsc_iter != NULL; rsc_iter = rsc_iter->next) {
 
         pcmk_resource_t *rsc = (pcmk_resource_t *) rsc_iter->data;
         int x;
 
         /* Complex resources may have some sub-resources active and some inactive */
         gboolean is_active = rsc->priv->fns->active(rsc, TRUE);
         gboolean partially_active = rsc->priv->fns->active(rsc, FALSE);
 
         /* Skip inactive orphans (deleted but still in CIB) */
         if (pcmk_is_set(rsc->flags, pcmk__rsc_removed) && !is_active) {
             continue;
 
         /* Skip active resources if we already displayed them by node */
         } else if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             if (is_active) {
                 continue;
             }
 
         /* Skip primitives already counted in a brief summary */
         } else if (pcmk_is_set(show_opts, pcmk_show_brief)
                    && pcmk__is_primitive(rsc)) {
             continue;
 
         /* Skip resources that aren't at least partially active,
          * unless we're displaying inactive resources
          */
         } else if (!partially_active && !pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
             continue;
 
         } else if (partially_active && !pe__rsc_running_on_any(rsc, only_node)) {
             continue;
         }
 
         if (!printed_header) {
             PCMK__OUTPUT_SPACER_IF(out, print_spacer);
             print_resource_header(out, show_opts);
             printed_header = true;
         }
 
         /* Print this resource */
         x = out->message(out, (const char *) rsc->priv->xml->name,
                          show_opts, rsc, only_node, only_rsc);
         if (x == pcmk_rc_ok) {
             rc = pcmk_rc_ok;
         }
     }
 
     if (print_summary && rc != pcmk_rc_ok) {
         if (!printed_header) {
             PCMK__OUTPUT_SPACER_IF(out, print_spacer);
             print_resource_header(out, show_opts);
             printed_header = true;
         }
 
         if (pcmk_is_set(show_opts, pcmk_show_rscs_by_node)) {
             out->list_item(out, NULL, "No inactive resources");
         } else if (pcmk_is_set(show_opts, pcmk_show_inactive_rscs)) {
             out->list_item(out, NULL, "No resources");
         } else {
             out->list_item(out, NULL, "No active resources");
         }
     }
 
     if (printed_header) {
         out->end_list(out);
     }
 
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("resource-operation-list", "pcmk_scheduler_t *",
                   "pcmk_resource_t *", "pcmk_node_t *", "GList *", "uint32_t")
 static int
 resource_operation_list(pcmk__output_t *out, va_list args)
 {
     pcmk_scheduler_t *scheduler G_GNUC_UNUSED = va_arg(args,
                                                        pcmk_scheduler_t *);
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     GList *op_list = va_arg(args, GList *);
     uint32_t show_opts = va_arg(args, uint32_t);
 
     GList *gIter = NULL;
     int rc = pcmk_rc_no_output;
 
     /* Print each operation */
     for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *xml_op = (xmlNode *) gIter->data;
         const char *task = crm_element_value(xml_op, PCMK_XA_OPERATION);
         const char *interval_ms_s = crm_element_value(xml_op,
                                                       PCMK_META_INTERVAL);
         const char *op_rc = crm_element_value(xml_op, PCMK__XA_RC_CODE);
         int op_rc_i;
 
         pcmk__scan_min_int(op_rc, &op_rc_i, 0);
 
         /* Display 0-interval monitors as "probe" */
         if (pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)
             && pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches | pcmk__str_casei)) {
             task = "probe";
         }
 
         /* If this is the first printed operation, print heading for resource */
         if (rc == pcmk_rc_no_output) {
             time_t last_failure = 0;
             int failcount = pe_get_failcount(node, rsc, &last_failure,
                                              pcmk__fc_default, NULL);
 
             out->message(out, "resource-history", rsc, rsc_printable_id(rsc), true,
                          failcount, last_failure, true);
             rc = pcmk_rc_ok;
         }
 
         /* Print the operation */
         out->message(out, "op-history", xml_op, task, interval_ms_s,
                      op_rc_i, show_opts);
     }
 
     /* Free the list we created (no need to free the individual items) */
     g_list_free(op_list);
 
     PCMK__OUTPUT_LIST_FOOTER(out, rc);
     return rc;
 }
 
 PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 resource_util(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *fn = va_arg(args, const char *);
 
     char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
                                         fn, rsc->id, pcmk__node_name(node));
 
     g_hash_table_foreach(rsc->priv->utilization, append_dump_text,
                          &dump_text);
     out->list_item(out, NULL, "%s", dump_text);
     free(dump_text);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("resource-util", "pcmk_resource_t *", "pcmk_node_t *",
                   "const char *")
 static int
 resource_util_xml(pcmk__output_t *out, va_list args)
 {
     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
     pcmk_node_t *node = va_arg(args, pcmk_node_t *);
     const char *uname = node->priv->name;
     const char *fn = va_arg(args, const char *);
 
     xmlNodePtr xml_node = NULL;
 
     xml_node = pcmk__output_create_xml_node(out, PCMK_XE_UTILIZATION,
                                             PCMK_XA_RESOURCE, rsc->id,
                                             PCMK_XA_NODE, uname,
                                             PCMK_XA_FUNCTION, fn,
                                             NULL);
     g_hash_table_foreach(rsc->priv->utilization, add_dump_node, xml_node);
 
     return pcmk_rc_ok;
 }
 
 static inline const char *
 ticket_status(pcmk__ticket_t *ticket)
 {
     if (pcmk_is_set(ticket->flags, pcmk__ticket_granted)) {
         return PCMK_VALUE_GRANTED;
     }
     return PCMK_VALUE_REVOKED;
 }
 
 static inline const char *
 ticket_standby_text(pcmk__ticket_t *ticket)
 {
     return pcmk_is_set(ticket->flags, pcmk__ticket_standby)? " [standby]" : "";
 }
 
 PCMK__OUTPUT_ARGS("ticket", "pcmk__ticket_t *", "bool", "bool")
 static int
 ticket_default(pcmk__output_t *out, va_list args) {
     pcmk__ticket_t *ticket = va_arg(args, pcmk__ticket_t *);
     bool raw = va_arg(args, int);
     bool details = va_arg(args, int);
 
     GString *detail_str = NULL;
 
     if (raw) {
         out->list_item(out, ticket->id, "%s", ticket->id);
         return pcmk_rc_ok;
     }
 
     if (details && g_hash_table_size(ticket->state) > 0) {
         GHashTableIter iter;
         const char *name = NULL;
         const char *value = NULL;
         bool already_added = false;
 
         detail_str = g_string_sized_new(100);
         pcmk__g_strcat(detail_str, "\t(", NULL);
 
         g_hash_table_iter_init(&iter, ticket->state);
         while (g_hash_table_iter_next(&iter, (void **) &name, (void **) &value)) {
             if (already_added) {
                 g_string_append_printf(detail_str, ", %s=", name);
             } else {
                 g_string_append_printf(detail_str, "%s=", name);
                 already_added = true;
             }
 
             if (pcmk__str_any_of(name, PCMK_XA_LAST_GRANTED, "expires", NULL)) {
                 char *epoch_str = NULL;
                 long long time_ll;
 
                 (void) pcmk__scan_ll(value, &time_ll, 0);
                 epoch_str = pcmk__epoch2str((const time_t *) &time_ll, 0);
                 pcmk__g_strcat(detail_str, epoch_str, NULL);
                 free(epoch_str);
             } else {
                 pcmk__g_strcat(detail_str, value, NULL);
             }
         }
 
         pcmk__g_strcat(detail_str, ")", NULL);
     }
 
     if (ticket->last_granted > -1) {
         /* Prior to the introduction of the details & raw arguments to this
          * function, last-granted would always be added in this block.  We need
          * to preserve that behavior.  At the same time, we also need to preserve
          * the existing behavior from crm_ticket, which would include last-granted
          * as part of the (...) detail string.
          *
          * Luckily we can check detail_str - if it's NULL, either there were no
          * details, or we are preserving the previous behavior of this function.
          * If it's not NULL, we are either preserving the previous behavior of
          * crm_ticket or we were given details=true as an argument.
          */
         if (detail_str == NULL) {
             char *epoch_str = pcmk__epoch2str(&(ticket->last_granted), 0);
 
             out->list_item(out, NULL, "%s\t%s%s last-granted=\"%s\"",
                            ticket->id, ticket_status(ticket),
                            ticket_standby_text(ticket), pcmk__s(epoch_str, ""));
             free(epoch_str);
         } else {
             out->list_item(out, NULL, "%s\t%s%s %s",
                            ticket->id, ticket_status(ticket),
                            ticket_standby_text(ticket), detail_str->str);
         }
     } else {
         out->list_item(out, NULL, "%s\t%s%s%s", ticket->id,
                        ticket_status(ticket),
                        ticket_standby_text(ticket),
                        detail_str != NULL ? detail_str->str : "");
     }
 
     if (detail_str != NULL) {
         g_string_free(detail_str, TRUE);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ticket", "pcmk__ticket_t *", "bool", "bool")
 static int
 ticket_xml(pcmk__output_t *out, va_list args) {
     pcmk__ticket_t *ticket = va_arg(args, pcmk__ticket_t *);
     bool raw G_GNUC_UNUSED = va_arg(args, int);
     bool details G_GNUC_UNUSED = va_arg(args, int);
 
     const char *standby = pcmk__flag_text(ticket->flags, pcmk__ticket_standby);
 
     xmlNodePtr node = NULL;
     GHashTableIter iter;
     const char *name = NULL;
     const char *value = NULL;
 
     node = pcmk__output_create_xml_node(out, PCMK_XE_TICKET,
                                         PCMK_XA_ID, ticket->id,
                                         PCMK_XA_STATUS, ticket_status(ticket),
                                         PCMK_XA_STANDBY, standby,
                                         NULL);
 
     if (ticket->last_granted > -1) {
         char *buf = pcmk__epoch2str(&ticket->last_granted, 0);
 
         crm_xml_add(node, PCMK_XA_LAST_GRANTED, buf);
         free(buf);
     }
 
     g_hash_table_iter_init(&iter, ticket->state);
     while (g_hash_table_iter_next(&iter, (void **) &name, (void **) &value)) {
         /* PCMK_XA_LAST_GRANTED and "expires" are already added by the check
          * for ticket->last_granted above.
          */
         if (pcmk__str_any_of(name, PCMK_XA_LAST_GRANTED, PCMK_XA_EXPIRES,
                              NULL)) {
             continue;
         }
 
         crm_xml_add(node, name, value);
     }
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("ticket-list", "GHashTable *", "bool", "bool", "bool")
 static int
 ticket_list(pcmk__output_t *out, va_list args) {
     GHashTable *tickets = va_arg(args, GHashTable *);
     bool print_spacer = va_arg(args, int);
     bool raw = va_arg(args, int);
     bool details = va_arg(args, int);
 
     GHashTableIter iter;
     gpointer value;
 
     if (g_hash_table_size(tickets) == 0) {
         return pcmk_rc_no_output;
     }
 
     PCMK__OUTPUT_SPACER_IF(out, print_spacer);
 
     /* Print section heading */
     out->begin_list(out, NULL, NULL, "Tickets");
 
     /* Print each ticket */
     g_hash_table_iter_init(&iter, tickets);
     while (g_hash_table_iter_next(&iter, NULL, &value)) {
         pcmk__ticket_t *ticket = (pcmk__ticket_t *) value;
         out->message(out, "ticket", ticket, raw, details);
     }
 
     /* Close section */
     out->end_list(out);
     return pcmk_rc_ok;
 }
 
 static pcmk__message_entry_t fmt_functions[] = {
     { "ban", "default", ban_text },
     { "ban", "html", ban_html },
     { "ban", "xml", ban_xml },
     { "ban-list", "default", ban_list },
     { "bundle", "default", pe__bundle_text },
     { "bundle", "xml",  pe__bundle_xml },
     { "bundle", "html",  pe__bundle_html },
     { "clone", "default", pe__clone_default },
     { "clone", "xml",  pe__clone_xml },
     { "cluster-counts", "default", cluster_counts_text },
     { "cluster-counts", "html", cluster_counts_html },
     { "cluster-counts", "xml", cluster_counts_xml },
     { "cluster-dc", "default", cluster_dc_text },
     { "cluster-dc", "html", cluster_dc_html },
     { "cluster-dc", "xml", cluster_dc_xml },
     { "cluster-options", "default", cluster_options_text },
     { "cluster-options", "html", cluster_options_html },
     { "cluster-options", "log", cluster_options_log },
     { "cluster-options", "xml", cluster_options_xml },
     { "cluster-summary", "default", cluster_summary },
     { "cluster-summary", "html", cluster_summary_html },
     { "cluster-stack", "default", cluster_stack_text },
     { "cluster-stack", "html", cluster_stack_html },
     { "cluster-stack", "xml", cluster_stack_xml },
     { "cluster-times", "default", cluster_times_text },
     { "cluster-times", "html", cluster_times_html },
     { "cluster-times", "xml", cluster_times_xml },
     { "failed-action", "default", failed_action_default },
     { "failed-action", "xml", failed_action_xml },
     { "failed-action-list", "default", failed_action_list },
     { "group", "default",  pe__group_default},
     { "group", "xml",  pe__group_xml },
     { "maint-mode", "text", cluster_maint_mode_text },
     { "node", "default", node_text },
     { "node", "html", node_html },
     { "node", "xml", node_xml },
     { "node-and-op", "default", node_and_op },
     { "node-and-op", "xml", node_and_op_xml },
     { "node-capacity", "default", node_capacity },
     { "node-capacity", "xml", node_capacity_xml },
     { "node-history-list", "default", node_history_list },
     { "node-list", "default", node_list_text },
     { "node-list", "html", node_list_html },
     { "node-list", "xml", node_list_xml },
     { "node-weight", "default", node_weight },
     { "node-weight", "xml", node_weight_xml },
     { "node-attribute", "default", node_attribute_text },
     { "node-attribute", "html", node_attribute_html },
     { "node-attribute", "xml", node_attribute_xml },
     { "node-attribute-list", "default", node_attribute_list },
     { "node-summary", "default", node_summary },
     { "op-history", "default", op_history_text },
     { "op-history", "xml", op_history_xml },
     { "primitive", "default",  pe__resource_text },
     { "primitive", "xml",  pe__resource_xml },
     { "primitive", "html",  pe__resource_html },
     { "promotion-score", "default", promotion_score },
     { "promotion-score", "xml", promotion_score_xml },
     { "resource-config", "default", resource_config },
     { "resource-config", "text", resource_config_text },
     { "resource-history", "default", resource_history_text },
     { "resource-history", "xml", resource_history_xml },
     { "resource-list", "default", resource_list },
     { "resource-operation-list", "default", resource_operation_list },
     { "resource-util", "default", resource_util },
     { "resource-util", "xml", resource_util_xml },
     { "ticket", "default", ticket_default },
     { "ticket", "xml", ticket_xml },
     { "ticket-list", "default", ticket_list },
 
     { NULL, NULL, NULL }
 };
 
 void
 pe__register_messages(pcmk__output_t *out) {
     pcmk__register_messages(out, fmt_functions);
 }
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 1fe58ab9c2..973e94bdc0 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,5099 +1,5109 @@
 /*
  * Copyright 2004-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <string.h>
 #include <glib.h>
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <crm/common/util.h>
 #include <crm/pengine/internal.h>
 #include <pe_status_private.h>
 
 CRM_TRACE_INIT_DATA(pe_status);
 
 // A (parsed) resource action history entry
 struct action_history {
     pcmk_resource_t *rsc;       // Resource that history is for
     pcmk_node_t *node;        // Node that history is for
     xmlNode *xml;             // History entry XML
 
     // Parsed from entry XML
     const char *id;           // XML ID of history entry
     const char *key;          // Operation key of action
     const char *task;         // Action name
     const char *exit_reason;  // Exit reason given for result
     guint interval_ms;        // Action interval
     int call_id;              // Call ID of action
     int expected_exit_status; // Expected exit status of action
     int exit_status;          // Actual exit status of action
     int execution_status;     // Execution status of action
 };
 
 /* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
  * use pcmk__set_scheduler_flags()/pcmk__clear_scheduler_flags() so that the
  * flag is stringified more readably in log messages.
  */
 #define set_config_flag(scheduler, option, flag) do {                         \
         GHashTable *config_hash = (scheduler)->priv->options;                 \
         const char *scf_value = pcmk__cluster_option(config_hash, (option));  \
                                                                               \
         if (scf_value != NULL) {                                              \
             if (crm_is_true(scf_value)) {                                     \
                 (scheduler)->flags = pcmk__set_flags_as(__func__, __LINE__,   \
                                     LOG_TRACE, "Scheduler",                   \
                                     crm_system_name, (scheduler)->flags,      \
                                     (flag), #flag);                           \
             } else {                                                          \
                 (scheduler)->flags = pcmk__clear_flags_as(__func__, __LINE__, \
                                     LOG_TRACE, "Scheduler",                   \
                                     crm_system_name, (scheduler)->flags,      \
                                     (flag), #flag);                           \
             }                                                                 \
         }                                                                     \
     } while(0)
 
 static void unpack_rsc_op(pcmk_resource_t *rsc, pcmk_node_t *node,
                           xmlNode *xml_op, xmlNode **last_failure,
                           enum pcmk__on_fail *failed);
 static void determine_remote_online_status(pcmk_scheduler_t *scheduler,
                                            pcmk_node_t *this_node);
 static void add_node_attrs(const xmlNode *xml_obj, pcmk_node_t *node,
                            bool overwrite, pcmk_scheduler_t *scheduler);
 static void determine_online_status(const xmlNode *node_state,
                                     pcmk_node_t *this_node,
                                     pcmk_scheduler_t *scheduler);
 
 static void unpack_node_lrm(pcmk_node_t *node, const xmlNode *xml,
                             pcmk_scheduler_t *scheduler);
 
 
 /*!
  * \internal
  * \brief Check whether a node is a dangling guest node
  *
  * \param[in] node  Node to check
  *
  * \return true if \p node had a Pacemaker Remote connection resource with a
  *         launcher that was removed from the CIB, otherwise false.
  */
 static bool
 is_dangling_guest_node(pcmk_node_t *node)
 {
     return pcmk__is_pacemaker_remote_node(node)
            && (node->priv->remote != NULL)
            && (node->priv->remote->priv->launcher == NULL)
            && pcmk_is_set(node->priv->remote->flags,
                           pcmk__rsc_removed_launched);
 }
 
 /*!
  * \brief Schedule a fence action for a node
  *
  * \param[in,out] scheduler       Scheduler data
  * \param[in,out] node            Node to fence
  * \param[in]     reason          Text description of why fencing is needed
  * \param[in]     priority_delay  Whether to consider
  *                                \c PCMK_OPT_PRIORITY_FENCING_DELAY
  */
 void
 pe_fence_node(pcmk_scheduler_t *scheduler, pcmk_node_t *node,
               const char *reason, bool priority_delay)
 {
     CRM_CHECK(node, return);
 
     if (pcmk__is_guest_or_bundle_node(node)) {
         // Fence a guest or bundle node by marking its launcher as failed
         pcmk_resource_t *rsc = node->priv->remote->priv->launcher;
 
         if (!pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
             if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
                 crm_notice("Not fencing guest node %s "
                            "(otherwise would because %s): "
                            "its guest resource %s is unmanaged",
                            pcmk__node_name(node), reason, rsc->id);
             } else {
                 pcmk__sched_warn(scheduler,
                                  "Guest node %s will be fenced "
                                  "(by recovering its guest resource %s): %s",
                                  pcmk__node_name(node), rsc->id, reason);
 
                 /* We don't mark the node as unclean because that would prevent the
                  * node from running resources. We want to allow it to run resources
                  * in this transition if the recovery succeeds.
                  */
                 pcmk__set_node_flags(node, pcmk__node_remote_reset);
                 pcmk__set_rsc_flags(rsc,
                                     pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
             }
         }
 
     } else if (is_dangling_guest_node(node)) {
         crm_info("Cleaning up dangling connection for guest node %s: "
                  "fencing was already done because %s, "
                  "and guest resource no longer exists",
                  pcmk__node_name(node), reason);
         pcmk__set_rsc_flags(node->priv->remote,
                             pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
 
     } else if (pcmk__is_remote_node(node)) {
         pcmk_resource_t *rsc = node->priv->remote;
 
         if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
             crm_notice("Not fencing remote node %s "
                        "(otherwise would because %s): connection is unmanaged",
                        pcmk__node_name(node), reason);
         } else if (!pcmk_is_set(node->priv->flags, pcmk__node_remote_reset)) {
             pcmk__set_node_flags(node, pcmk__node_remote_reset);
             pcmk__sched_warn(scheduler, "Remote node %s %s: %s",
                              pcmk__node_name(node),
                              pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
                              reason);
         }
         node->details->unclean = TRUE;
         // No need to apply PCMK_OPT_PRIORITY_FENCING_DELAY for remote nodes
         pe_fence_op(node, NULL, TRUE, reason, FALSE, scheduler);
 
     } else if (node->details->unclean) {
         crm_trace("Cluster node %s %s because %s",
                   pcmk__node_name(node),
                   pe_can_fence(scheduler, node)? "would also be fenced" : "also is unclean",
                   reason);
 
     } else {
         pcmk__sched_warn(scheduler, "Cluster node %s %s: %s",
                          pcmk__node_name(node),
                          pe_can_fence(scheduler, node)? "will be fenced" : "is unclean",
                          reason);
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, priority_delay, scheduler);
     }
 }
 
 // @TODO xpaths can't handle templates, rules, or id-refs
 
 // nvpair with provides or requires set to unfencing
 #define XPATH_UNFENCING_NVPAIR PCMK_XE_NVPAIR           \
     "[(@" PCMK_XA_NAME "='" PCMK_STONITH_PROVIDES "'"   \
     "or @" PCMK_XA_NAME "='" PCMK_META_REQUIRES "') "   \
     "and @" PCMK_XA_VALUE "='" PCMK_VALUE_UNFENCING "']"
 
 // unfencing in rsc_defaults or any resource
 #define XPATH_ENABLE_UNFENCING \
     "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RESOURCES     \
     "//" PCMK_XE_META_ATTRIBUTES "/" XPATH_UNFENCING_NVPAIR             \
     "|/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION "/" PCMK_XE_RSC_DEFAULTS \
     "/" PCMK_XE_META_ATTRIBUTES "/" XPATH_UNFENCING_NVPAIR
 
 static void
 set_if_xpath(uint64_t flag, const char *xpath, pcmk_scheduler_t *scheduler)
 {
     xmlXPathObjectPtr result = NULL;
 
     if (!pcmk_is_set(scheduler->flags, flag)) {
         result = xpath_search(scheduler->input, xpath);
         if (result && (numXpathResults(result) > 0)) {
             pcmk__set_scheduler_flags(scheduler, flag);
         }
         freeXpathObject(result);
     }
 }
 
 gboolean
 unpack_config(xmlNode *config, pcmk_scheduler_t *scheduler)
 {
     const char *value = NULL;
     GHashTable *config_hash = pcmk__strkey_table(free, free);
 
     const pcmk_rule_input_t rule_input = {
         .now = scheduler->priv->now,
     };
 
     scheduler->priv->options = config_hash;
 
     pe__unpack_dataset_nvpairs(config, PCMK_XE_CLUSTER_PROPERTY_SET,
                                &rule_input, config_hash,
                                PCMK_VALUE_CIB_BOOTSTRAP_OPTIONS, scheduler);
 
     pcmk__validate_cluster_options(config_hash);
 
     set_config_flag(scheduler, PCMK_OPT_ENABLE_STARTUP_PROBES,
                     pcmk__sched_probe_resources);
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_probe_resources)) {
         crm_info("Startup probes: disabled (dangerous)");
     }
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_HAVE_WATCHDOG);
     if (value && crm_is_true(value)) {
         crm_info("Watchdog-based self-fencing will be performed via SBD if "
                  "fencing is required and " PCMK_OPT_STONITH_WATCHDOG_TIMEOUT
                  " is nonzero");
         pcmk__set_scheduler_flags(scheduler, pcmk__sched_have_fencing);
     }
 
     /* Set certain flags via xpath here, so they can be used before the relevant
      * configuration sections are unpacked.
      */
     set_if_xpath(pcmk__sched_enable_unfencing, XPATH_ENABLE_UNFENCING,
                  scheduler);
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_STONITH_TIMEOUT);
     pcmk_parse_interval_spec(value, &(scheduler->priv->fence_timeout_ms));
 
     crm_debug("Default fencing action timeout: %s",
               pcmk__readable_interval(scheduler->priv->fence_timeout_ms));
 
     set_config_flag(scheduler, PCMK_OPT_STONITH_ENABLED,
                     pcmk__sched_fencing_enabled);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
         crm_debug("STONITH of failed nodes is enabled");
     } else {
         crm_debug("STONITH of failed nodes is disabled");
     }
 
     scheduler->priv->fence_action =
         pcmk__cluster_option(config_hash, PCMK_OPT_STONITH_ACTION);
     crm_trace("STONITH will %s nodes", scheduler->priv->fence_action);
 
     set_config_flag(scheduler, PCMK_OPT_CONCURRENT_FENCING,
                     pcmk__sched_concurrent_fencing);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_concurrent_fencing)) {
         crm_debug("Concurrent fencing is enabled");
     } else {
         crm_debug("Concurrent fencing is disabled");
     }
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_PRIORITY_FENCING_DELAY);
     if (value) {
         pcmk_parse_interval_spec(value,
                                  &(scheduler->priv->priority_fencing_ms));
         crm_trace("Priority fencing delay is %s",
                   pcmk__readable_interval(scheduler->priv->priority_fencing_ms));
     }
 
     set_config_flag(scheduler, PCMK_OPT_STOP_ALL_RESOURCES,
                     pcmk__sched_stop_all);
     crm_debug("Stop all active resources: %s",
               pcmk__flag_text(scheduler->flags, pcmk__sched_stop_all));
 
     set_config_flag(scheduler, PCMK_OPT_SYMMETRIC_CLUSTER,
                     pcmk__sched_symmetric_cluster);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_symmetric_cluster)) {
         crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
     }
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_NO_QUORUM_POLICY);
 
     if (pcmk__str_eq(value, PCMK_VALUE_IGNORE, pcmk__str_casei)) {
         scheduler->no_quorum_policy = pcmk_no_quorum_ignore;
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_FREEZE, pcmk__str_casei)) {
         scheduler->no_quorum_policy = pcmk_no_quorum_freeze;
 
     } else if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
         scheduler->no_quorum_policy = pcmk_no_quorum_demote;
 
     } else if (pcmk__strcase_any_of(value, PCMK_VALUE_FENCE,
                                     PCMK_VALUE_FENCE_LEGACY, NULL)) {
         if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
             int do_panic = 0;
 
             crm_element_value_int(scheduler->input, PCMK_XA_NO_QUORUM_PANIC,
                                   &do_panic);
             if (do_panic
                 || pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
                 scheduler->no_quorum_policy = pcmk_no_quorum_fence;
             } else {
                 crm_notice("Resetting " PCMK_OPT_NO_QUORUM_POLICY
                            " to 'stop': cluster has never had quorum");
                 scheduler->no_quorum_policy = pcmk_no_quorum_stop;
             }
         } else {
             pcmk__config_err("Resetting " PCMK_OPT_NO_QUORUM_POLICY
                              " to 'stop' because fencing is disabled");
             scheduler->no_quorum_policy = pcmk_no_quorum_stop;
         }
 
     } else {
         scheduler->no_quorum_policy = pcmk_no_quorum_stop;
     }
 
     switch (scheduler->no_quorum_policy) {
         case pcmk_no_quorum_freeze:
             crm_debug("On loss of quorum: Freeze resources");
             break;
         case pcmk_no_quorum_stop:
             crm_debug("On loss of quorum: Stop ALL resources");
             break;
         case pcmk_no_quorum_demote:
             crm_debug("On loss of quorum: "
                       "Demote promotable resources and stop other resources");
             break;
         case pcmk_no_quorum_fence:
             crm_notice("On loss of quorum: Fence all remaining nodes");
             break;
         case pcmk_no_quorum_ignore:
             crm_notice("On loss of quorum: Ignore");
             break;
     }
 
     set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_RESOURCES,
                     pcmk__sched_stop_removed_resources);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_removed_resources)) {
         crm_trace("Orphan resources are stopped");
     } else {
         crm_trace("Orphan resources are ignored");
     }
 
     set_config_flag(scheduler, PCMK_OPT_STOP_ORPHAN_ACTIONS,
                     pcmk__sched_cancel_removed_actions);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_cancel_removed_actions)) {
         crm_trace("Orphan resource actions are stopped");
     } else {
         crm_trace("Orphan resource actions are ignored");
     }
 
     set_config_flag(scheduler, PCMK_OPT_MAINTENANCE_MODE,
                     pcmk__sched_in_maintenance);
     crm_trace("Maintenance mode: %s",
               pcmk__flag_text(scheduler->flags, pcmk__sched_in_maintenance));
 
     set_config_flag(scheduler, PCMK_OPT_START_FAILURE_IS_FATAL,
                     pcmk__sched_start_failure_fatal);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_start_failure_fatal)) {
         crm_trace("Start failures are always fatal");
     } else {
         crm_trace("Start failures are handled by failcount");
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
         set_config_flag(scheduler, PCMK_OPT_STARTUP_FENCING,
                         pcmk__sched_startup_fencing);
     }
     if (pcmk_is_set(scheduler->flags, pcmk__sched_startup_fencing)) {
         crm_trace("Unseen nodes will be fenced");
     } else {
         pcmk__warn_once(pcmk__wo_blind,
                         "Blind faith: not fencing unseen nodes");
     }
 
     pe__unpack_node_health_scores(scheduler);
 
     scheduler->priv->placement_strategy =
         pcmk__cluster_option(config_hash, PCMK_OPT_PLACEMENT_STRATEGY);
     crm_trace("Placement strategy: %s", scheduler->priv->placement_strategy);
 
     set_config_flag(scheduler, PCMK_OPT_SHUTDOWN_LOCK,
                     pcmk__sched_shutdown_lock);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_shutdown_lock)) {
         value = pcmk__cluster_option(config_hash, PCMK_OPT_SHUTDOWN_LOCK_LIMIT);
         pcmk_parse_interval_spec(value, &(scheduler->priv->shutdown_lock_ms));
         crm_trace("Resources will be locked to nodes that were cleanly "
                   "shut down (locks expire after %s)",
                   pcmk__readable_interval(scheduler->priv->shutdown_lock_ms));
     } else {
         crm_trace("Resources will not be locked to nodes that were cleanly "
                   "shut down");
     }
 
     value = pcmk__cluster_option(config_hash, PCMK_OPT_NODE_PENDING_TIMEOUT);
     pcmk_parse_interval_spec(value, &(scheduler->priv->node_pending_ms));
     if (scheduler->priv->node_pending_ms == 0U) {
         crm_trace("Do not fence pending nodes");
     } else {
         crm_trace("Fence pending nodes after %s",
                   pcmk__readable_interval(scheduler->priv->node_pending_ms));
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Create a new node object in scheduler data
  *
  * \param[in]     id         ID of new node
  * \param[in]     uname      Name of new node
  * \param[in]     type       Type of new node
  * \param[in]     score      Score of new node
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Newly created node object
  * \note The returned object is part of the scheduler data and should not be
  *       freed separately.
  */
 pcmk_node_t *
 pe_create_node(const char *id, const char *uname, const char *type,
                int score, pcmk_scheduler_t *scheduler)
 {
     enum pcmk__node_variant variant = pcmk__node_variant_cluster;
     pcmk_node_t *new_node = NULL;
 
     if (pcmk_find_node(scheduler, uname) != NULL) {
         pcmk__config_warn("More than one node entry has name '%s'", uname);
     }
 
     if (pcmk__str_eq(type, PCMK_VALUE_MEMBER,
                      pcmk__str_null_matches|pcmk__str_casei)) {
         variant = pcmk__node_variant_cluster;
 
     } else if (pcmk__str_eq(type, PCMK_VALUE_REMOTE, pcmk__str_casei)) {
         variant = pcmk__node_variant_remote;
 
     } else {
         pcmk__config_err("Ignoring node %s with unrecognized type '%s'",
                          pcmk__s(uname, "without name"), type);
         return NULL;
     }
 
     new_node = calloc(1, sizeof(pcmk_node_t));
     if (new_node == NULL) {
         pcmk__sched_err(scheduler, "Could not allocate memory for node %s",
                         uname);
         return NULL;
     }
 
     new_node->assign = calloc(1, sizeof(struct pcmk__node_assignment));
     new_node->details = calloc(1, sizeof(struct pcmk__node_details));
     new_node->priv = calloc(1, sizeof(pcmk__node_private_t));
     if ((new_node->assign == NULL) || (new_node->details == NULL)
         || (new_node->priv == NULL)) {
         free(new_node->assign);
         free(new_node->details);
         free(new_node->priv);
         free(new_node);
         pcmk__sched_err(scheduler, "Could not allocate memory for node %s",
                         uname);
         return NULL;
     }
 
     crm_trace("Creating node for entry %s/%s", uname, id);
     new_node->assign->score = score;
     new_node->priv->id = id;
     new_node->priv->name = uname;
     new_node->priv->flags = pcmk__node_probes_allowed;
     new_node->details->online = FALSE;
     new_node->details->shutdown = FALSE;
     new_node->details->running_rsc = NULL;
     new_node->priv->scheduler = scheduler;
     new_node->priv->variant = variant;
     new_node->priv->attrs = pcmk__strkey_table(free, free);
     new_node->priv->utilization = pcmk__strkey_table(free, free);
     new_node->priv->digest_cache = pcmk__strkey_table(free, pe__free_digests);
 
     if (pcmk__is_pacemaker_remote_node(new_node)) {
         pcmk__insert_dup(new_node->priv->attrs, CRM_ATTR_KIND, "remote");
         pcmk__set_scheduler_flags(scheduler, pcmk__sched_have_remote_nodes);
     } else {
         pcmk__insert_dup(new_node->priv->attrs, CRM_ATTR_KIND, "cluster");
     }
 
     scheduler->nodes = g_list_insert_sorted(scheduler->nodes, new_node,
                                             pe__cmp_node_name);
     return new_node;
 }
 
 static const char *
 expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pcmk_scheduler_t *data)
 {
     xmlNode *attr_set = NULL;
     xmlNode *attr = NULL;
 
     const char *container_id = pcmk__xe_id(xml_obj);
     const char *remote_name = NULL;
     const char *remote_server = NULL;
     const char *remote_port = NULL;
     const char *connect_timeout = "60s";
     const char *remote_allow_migrate=NULL;
     const char *is_managed = NULL;
 
+    // @TODO This doesn't handle rules or id-ref
     for (attr_set = pcmk__xe_first_child(xml_obj, PCMK_XE_META_ATTRIBUTES,
                                          NULL, NULL);
          attr_set != NULL;
          attr_set = pcmk__xe_next(attr_set, PCMK_XE_META_ATTRIBUTES)) {
 
         for (attr = pcmk__xe_first_child(attr_set, NULL, NULL, NULL);
              attr != NULL; attr = pcmk__xe_next(attr, NULL)) {
 
             const char *value = crm_element_value(attr, PCMK_XA_VALUE);
             const char *name = crm_element_value(attr, PCMK_XA_NAME);
 
             if (name == NULL) { // Sanity
                 continue;
             }
 
             if (strcmp(name, PCMK_META_REMOTE_NODE) == 0) {
                 remote_name = value;
 
             } else if (strcmp(name, PCMK_META_REMOTE_ADDR) == 0) {
                 remote_server = value;
 
             } else if (strcmp(name, PCMK_META_REMOTE_PORT) == 0) {
                 remote_port = value;
 
             } else if (strcmp(name, PCMK_META_REMOTE_CONNECT_TIMEOUT) == 0) {
                 connect_timeout = value;
 
             } else if (strcmp(name, PCMK_META_REMOTE_ALLOW_MIGRATE) == 0) {
                 remote_allow_migrate = value;
 
             } else if (strcmp(name, PCMK_META_IS_MANAGED) == 0) {
                 is_managed = value;
             }
         }
     }
 
     if (remote_name == NULL) {
         return NULL;
     }
 
     if (pe_find_resource(data->priv->resources, remote_name) != NULL) {
         return NULL;
     }
 
     pe_create_remote_xml(parent, remote_name, container_id,
                          remote_allow_migrate, is_managed,
                          connect_timeout, remote_server, remote_port);
     return remote_name;
 }
 
 static void
 handle_startup_fencing(pcmk_scheduler_t *scheduler, pcmk_node_t *new_node)
 {
     if ((new_node->priv->variant == pcmk__node_variant_remote)
         && (new_node->priv->remote == NULL)) {
         /* Ignore fencing for remote nodes that don't have a connection resource
          * associated with them. This happens when remote node entries get left
          * in the nodes section after the connection resource is removed.
          */
         return;
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_startup_fencing)) {
         // All nodes are unclean until we've seen their status entry
         new_node->details->unclean = TRUE;
 
     } else {
         // Blind faith ...
         new_node->details->unclean = FALSE;
     }
 }
 
 gboolean
 unpack_nodes(xmlNode *xml_nodes, pcmk_scheduler_t *scheduler)
 {
     xmlNode *xml_obj = NULL;
     pcmk_node_t *new_node = NULL;
     const char *id = NULL;
     const char *uname = NULL;
     const char *type = NULL;
 
     for (xml_obj = pcmk__xe_first_child(xml_nodes, PCMK_XE_NODE, NULL, NULL);
          xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj, PCMK_XE_NODE)) {
 
         int score = 0;
         int rc = pcmk__xe_get_score(xml_obj, PCMK_XA_SCORE, &score, 0);
 
         new_node = NULL;
 
         id = crm_element_value(xml_obj, PCMK_XA_ID);
         uname = crm_element_value(xml_obj, PCMK_XA_UNAME);
         type = crm_element_value(xml_obj, PCMK_XA_TYPE);
         crm_trace("Processing node %s/%s", uname, id);
 
         if (id == NULL) {
             pcmk__config_err("Ignoring <" PCMK_XE_NODE
                              "> entry in configuration without id");
             continue;
         }
         if (rc != pcmk_rc_ok) {
             // Not possible with schema validation enabled
             pcmk__config_warn("Using 0 as score for node %s "
                               "because '%s' is not a valid score: %s",
                               pcmk__s(uname, "without name"),
                               crm_element_value(xml_obj, PCMK_XA_SCORE),
                               pcmk_rc_str(rc));
         }
         new_node = pe_create_node(id, uname, type, score, scheduler);
 
         if (new_node == NULL) {
             return FALSE;
         }
 
         handle_startup_fencing(scheduler, new_node);
 
         add_node_attrs(xml_obj, new_node, FALSE, scheduler);
 
         crm_trace("Done with node %s",
                   crm_element_value(xml_obj, PCMK_XA_UNAME));
     }
 
     return TRUE;
 }
 
 static void
 unpack_launcher(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
 {
     const char *launcher_id = NULL;
 
     if (rsc->priv->children != NULL) {
         g_list_foreach(rsc->priv->children, (GFunc) unpack_launcher,
                        scheduler);
         return;
     }
 
     launcher_id = g_hash_table_lookup(rsc->priv->meta, PCMK__META_CONTAINER);
     if ((launcher_id != NULL)
         && !pcmk__str_eq(launcher_id, rsc->id, pcmk__str_none)) {
         pcmk_resource_t *launcher = pe_find_resource(scheduler->priv->resources,
                                                      launcher_id);
 
         if (launcher != NULL) {
             rsc->priv->launcher = launcher;
             launcher->priv->launched =
                 g_list_append(launcher->priv->launched, rsc);
             pcmk__rsc_trace(rsc, "Resource %s's launcher is %s",
                             rsc->id, launcher_id);
         } else {
             pcmk__config_err("Resource %s: Unknown " PCMK__META_CONTAINER " %s",
                              rsc->id, launcher_id);
         }
     }
 }
 
 gboolean
 unpack_remote_nodes(xmlNode *xml_resources, pcmk_scheduler_t *scheduler)
 {
     xmlNode *xml_obj = NULL;
 
     /* Create remote nodes and guest nodes from the resource configuration
      * before unpacking resources.
      */
     for (xml_obj = pcmk__xe_first_child(xml_resources, NULL, NULL, NULL);
          xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj, NULL)) {
 
         const char *new_node_id = NULL;
 
         /* Check for remote nodes, which are defined by ocf:pacemaker:remote
          * primitives.
          */
         if (xml_contains_remote_node(xml_obj)) {
             new_node_id = pcmk__xe_id(xml_obj);
             /* The pcmk_find_node() check ensures we don't iterate over an
              * expanded node that has already been added to the node list
              */
             if (new_node_id
                 && (pcmk_find_node(scheduler, new_node_id) == NULL)) {
                 crm_trace("Found remote node %s defined by resource %s",
                           new_node_id, pcmk__xe_id(xml_obj));
                 pe_create_node(new_node_id, new_node_id, PCMK_VALUE_REMOTE,
                                0, scheduler);
             }
             continue;
         }
 
         /* Check for guest nodes, which are defined by special meta-attributes
          * of a primitive of any type (for example, VirtualDomain or Xen).
          */
         if (pcmk__xe_is(xml_obj, PCMK_XE_PRIMITIVE)) {
             /* This will add an ocf:pacemaker:remote primitive to the
              * configuration for the guest node's connection, to be unpacked
              * later.
              */
             new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources,
                                                  scheduler);
             if (new_node_id
                 && (pcmk_find_node(scheduler, new_node_id) == NULL)) {
                 crm_trace("Found guest node %s in resource %s",
                           new_node_id, pcmk__xe_id(xml_obj));
                 pe_create_node(new_node_id, new_node_id, PCMK_VALUE_REMOTE,
                                0, scheduler);
             }
             continue;
         }
 
         /* Check for guest nodes inside a group. Clones are currently not
          * supported as guest nodes.
          */
         if (pcmk__xe_is(xml_obj, PCMK_XE_GROUP)) {
             xmlNode *xml_obj2 = NULL;
             for (xml_obj2 = pcmk__xe_first_child(xml_obj, NULL, NULL, NULL);
                  xml_obj2 != NULL; xml_obj2 = pcmk__xe_next(xml_obj2, NULL)) {
 
                 new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources,
                                                      scheduler);
 
                 if (new_node_id
                     && (pcmk_find_node(scheduler, new_node_id) == NULL)) {
                     crm_trace("Found guest node %s in resource %s inside group %s",
                               new_node_id, pcmk__xe_id(xml_obj2),
                               pcmk__xe_id(xml_obj));
                     pe_create_node(new_node_id, new_node_id, PCMK_VALUE_REMOTE,
                                    0, scheduler);
                 }
             }
         }
     }
     return TRUE;
 }
 
 /* Call this after all the nodes and resources have been
  * unpacked, but before the status section is read.
  *
  * A remote node's online status is reflected by the state
  * of the remote node's connection resource. We need to link
  * the remote node to this connection resource so we can have
  * easy access to the connection resource during the scheduler calculations.
  */
 static void
 link_rsc2remotenode(pcmk_scheduler_t *scheduler, pcmk_resource_t *new_rsc)
 {
     pcmk_node_t *remote_node = NULL;
 
     if (!pcmk_is_set(new_rsc->flags, pcmk__rsc_is_remote_connection)) {
         return;
     }
 
     if (pcmk_is_set(scheduler->flags, pcmk__sched_location_only)) {
         /* remote_nodes and remote_resources are not linked in quick location calculations */
         return;
     }
 
     remote_node = pcmk_find_node(scheduler, new_rsc->id);
     CRM_CHECK(remote_node != NULL, return);
 
     pcmk__rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
                     new_rsc->id, pcmk__node_name(remote_node));
     remote_node->priv->remote = new_rsc;
 
     if (new_rsc->priv->launcher == NULL) {
         /* Handle start-up fencing for remote nodes (as opposed to guest nodes)
          * the same as is done for cluster nodes.
          */
         handle_startup_fencing(scheduler, remote_node);
 
     } else {
         /* pe_create_node() marks the new node as "remote" or "cluster"; now
          * that we know the node is a guest node, update it correctly.
          */
         pcmk__insert_dup(remote_node->priv->attrs,
                          CRM_ATTR_KIND, "container");
     }
 }
 
 /*!
  * \internal
  * \brief Parse configuration XML for resource information
  *
  * \param[in]     xml_resources  Top of resource configuration XML
  * \param[in,out] scheduler      Scheduler data
  *
  * \return TRUE
  *
  * \note unpack_remote_nodes() MUST be called before this, so that the nodes can
  *       be used when pe__unpack_resource() calls resource_location()
  */
 gboolean
 unpack_resources(const xmlNode *xml_resources, pcmk_scheduler_t *scheduler)
 {
     xmlNode *xml_obj = NULL;
     GList *gIter = NULL;
 
     scheduler->priv->templates = pcmk__strkey_table(free, pcmk__free_idref);
 
     for (xml_obj = pcmk__xe_first_child(xml_resources, NULL, NULL, NULL);
          xml_obj != NULL; xml_obj = pcmk__xe_next(xml_obj, NULL)) {
 
         pcmk_resource_t *new_rsc = NULL;
         const char *id = pcmk__xe_id(xml_obj);
 
         if (pcmk__str_empty(id)) {
             pcmk__config_err("Ignoring <%s> resource without ID",
                              xml_obj->name);
             continue;
         }
 
         if (pcmk__xe_is(xml_obj, PCMK_XE_TEMPLATE)) {
             if (g_hash_table_lookup_extended(scheduler->priv->templates, id,
                                              NULL, NULL) == FALSE) {
                 /* Record the template's ID for the knowledge of its existence anyway. */
                 pcmk__insert_dup(scheduler->priv->templates, id, NULL);
             }
             continue;
         }
 
         crm_trace("Unpacking <%s " PCMK_XA_ID "='%s'>", xml_obj->name, id);
         if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
                                 scheduler) == pcmk_rc_ok) {
             scheduler->priv->resources =
                 g_list_append(scheduler->priv->resources, new_rsc);
             pcmk__rsc_trace(new_rsc, "Added resource %s", new_rsc->id);
 
         } else {
             pcmk__config_err("Ignoring <%s> resource '%s' "
                              "because configuration is invalid",
                              xml_obj->name, id);
         }
     }
 
     for (gIter = scheduler->priv->resources;
          gIter != NULL; gIter = gIter->next) {
 
         pcmk_resource_t *rsc = (pcmk_resource_t *) gIter->data;
 
         unpack_launcher(rsc, scheduler);
         link_rsc2remotenode(scheduler, rsc);
     }
 
     scheduler->priv->resources = g_list_sort(scheduler->priv->resources,
                                              pe__cmp_rsc_priority);
     if (pcmk_is_set(scheduler->flags, pcmk__sched_location_only)) {
         /* Ignore */
 
     } else if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)
                && !pcmk_is_set(scheduler->flags, pcmk__sched_have_fencing)) {
 
         pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
         pcmk__config_err("Either configure some or disable STONITH with the "
                          PCMK_OPT_STONITH_ENABLED " option");
         pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Validate the levels in a fencing topology
  *
  * \param[in] xml  \c PCMK_XE_FENCING_TOPOLOGY element
  */
 void
 pcmk__validate_fencing_topology(const xmlNode *xml)
 {
     if (xml == NULL) {
         return;
     }
 
     CRM_CHECK(pcmk__xe_is(xml, PCMK_XE_FENCING_TOPOLOGY), return);
 
     for (const xmlNode *level = pcmk__xe_first_child(xml, PCMK_XE_FENCING_LEVEL,
                                                      NULL, NULL);
          level != NULL; level = pcmk__xe_next(level, PCMK_XE_FENCING_LEVEL)) {
 
         const char *id = pcmk__xe_id(level);
         int index = 0;
 
         if (pcmk__str_empty(id)) {
             pcmk__config_err("Ignoring fencing level without ID");
             continue;
         }
 
         if (crm_element_value_int(level, PCMK_XA_INDEX, &index) != 0) {
             pcmk__config_err("Ignoring fencing level %s with invalid index",
                              id);
             continue;
         }
 
         if ((index < ST__LEVEL_MIN) || (index > ST__LEVEL_MAX)) {
             pcmk__config_err("Ignoring fencing level %s with out-of-range "
                              "index %d",
                              id, index);
         }
     }
 }
 
 gboolean
 unpack_tags(xmlNode *xml_tags, pcmk_scheduler_t *scheduler)
 {
     xmlNode *xml_tag = NULL;
 
     scheduler->priv->tags = pcmk__strkey_table(free, pcmk__free_idref);
 
     for (xml_tag = pcmk__xe_first_child(xml_tags, PCMK_XE_TAG, NULL, NULL);
          xml_tag != NULL; xml_tag = pcmk__xe_next(xml_tag, PCMK_XE_TAG)) {
 
         xmlNode *xml_obj_ref = NULL;
         const char *tag_id = pcmk__xe_id(xml_tag);
 
         if (tag_id == NULL) {
             pcmk__config_err("Ignoring <%s> without " PCMK_XA_ID,
                              (const char *) xml_tag->name);
             continue;
         }
 
         for (xml_obj_ref = pcmk__xe_first_child(xml_tag, PCMK_XE_OBJ_REF,
                                                 NULL, NULL);
              xml_obj_ref != NULL;
              xml_obj_ref = pcmk__xe_next(xml_obj_ref, PCMK_XE_OBJ_REF)) {
 
             const char *obj_ref = pcmk__xe_id(xml_obj_ref);
 
             if (obj_ref == NULL) {
                 pcmk__config_err("Ignoring <%s> for tag '%s' without " PCMK_XA_ID,
                                  xml_obj_ref->name, tag_id);
                 continue;
             }
 
             pcmk__add_idref(scheduler->priv->tags, tag_id, obj_ref);
         }
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Unpack a ticket state entry
  *
  * \param[in]     xml_ticket  XML ticket state to unpack
  * \param[in,out] userdata    Scheduler data
  *
  * \return pcmk_rc_ok (to always continue unpacking further entries)
  */
 static int
 unpack_ticket_state(xmlNode *xml_ticket, void *userdata)
 {
     pcmk_scheduler_t *scheduler = userdata;
 
     const char *ticket_id = NULL;
     const char *granted = NULL;
     const char *last_granted = NULL;
     const char *standby = NULL;
     xmlAttrPtr xIter = NULL;
 
     pcmk__ticket_t *ticket = NULL;
 
     ticket_id = pcmk__xe_id(xml_ticket);
     if (pcmk__str_empty(ticket_id)) {
         return pcmk_rc_ok;
     }
 
     crm_trace("Processing ticket state for %s", ticket_id);
 
     ticket = g_hash_table_lookup(scheduler->priv->ticket_constraints,
                                  ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, scheduler);
         if (ticket == NULL) {
             return pcmk_rc_ok;
         }
     }
 
     for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         const char *prop_value = pcmk__xml_attr_value(xIter);
 
         if (pcmk__str_eq(prop_name, PCMK_XA_ID, pcmk__str_none)) {
             continue;
         }
         pcmk__insert_dup(ticket->state, prop_name, prop_value);
     }
 
     granted = g_hash_table_lookup(ticket->state, PCMK__XA_GRANTED);
     if (granted && crm_is_true(granted)) {
         pcmk__set_ticket_flags(ticket, pcmk__ticket_granted);
         crm_info("We have ticket '%s'", ticket->id);
     } else {
         pcmk__clear_ticket_flags(ticket, pcmk__ticket_granted);
         crm_info("We do not have ticket '%s'", ticket->id);
     }
 
     last_granted = g_hash_table_lookup(ticket->state, PCMK_XA_LAST_GRANTED);
     if (last_granted) {
         long long last_granted_ll = 0LL;
         int rc = pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
 
         if (rc != pcmk_rc_ok) {
             crm_warn("Using %lld instead of invalid " PCMK_XA_LAST_GRANTED
                      " value '%s' in state for ticket %s: %s",
                      last_granted_ll, last_granted, ticket->id,
                      pcmk_rc_str(rc));
         }
         ticket->last_granted = (time_t) last_granted_ll;
     }
 
     standby = g_hash_table_lookup(ticket->state, PCMK_XA_STANDBY);
     if (standby && crm_is_true(standby)) {
         pcmk__set_ticket_flags(ticket, pcmk__ticket_standby);
         if (pcmk_is_set(ticket->flags, pcmk__ticket_granted)) {
             crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
         }
     } else {
         pcmk__clear_ticket_flags(ticket, pcmk__ticket_standby);
     }
 
     crm_trace("Done with ticket state for %s", ticket_id);
 
     return pcmk_rc_ok;
 }
 
 static void
 unpack_handle_remote_attrs(pcmk_node_t *this_node, const xmlNode *state,
                            pcmk_scheduler_t *scheduler)
 {
     const char *discovery = NULL;
     const xmlNode *attrs = NULL;
     pcmk_resource_t *rsc = NULL;
     int maint = 0;
 
     if (!pcmk__xe_is(state, PCMK__XE_NODE_STATE)) {
         return;
     }
 
     if ((this_node == NULL) || !pcmk__is_pacemaker_remote_node(this_node)) {
         return;
     }
     crm_trace("Processing Pacemaker Remote node %s",
               pcmk__node_name(this_node));
 
     pcmk__scan_min_int(crm_element_value(state, PCMK__XA_NODE_IN_MAINTENANCE),
                        &maint, 0);
     if (maint) {
         pcmk__set_node_flags(this_node, pcmk__node_remote_maint);
     } else {
         pcmk__clear_node_flags(this_node, pcmk__node_remote_maint);
     }
 
     rsc = this_node->priv->remote;
     if (!pcmk_is_set(this_node->priv->flags, pcmk__node_remote_reset)) {
         this_node->details->unclean = FALSE;
         pcmk__set_node_flags(this_node, pcmk__node_seen);
     }
     attrs = pcmk__xe_first_child(state, PCMK__XE_TRANSIENT_ATTRIBUTES, NULL,
                                  NULL);
     add_node_attrs(attrs, this_node, TRUE, scheduler);
 
     if (pe__shutdown_requested(this_node)) {
         crm_info("%s is shutting down", pcmk__node_name(this_node));
         this_node->details->shutdown = TRUE;
     }
 
     if (crm_is_true(pcmk__node_attr(this_node, PCMK_NODE_ATTR_STANDBY, NULL,
                                     pcmk__rsc_node_current))) {
         crm_info("%s is in standby mode", pcmk__node_name(this_node));
         pcmk__set_node_flags(this_node, pcmk__node_standby);
     }
 
     if (crm_is_true(pcmk__node_attr(this_node, PCMK_NODE_ATTR_MAINTENANCE, NULL,
                                     pcmk__rsc_node_current))
         || ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk__rsc_managed))) {
         crm_info("%s is in maintenance mode", pcmk__node_name(this_node));
         this_node->details->maintenance = TRUE;
     }
 
     discovery = pcmk__node_attr(this_node,
                                 PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED,
                                 NULL, pcmk__rsc_node_current);
     if ((discovery != NULL) && !crm_is_true(discovery)) {
         pcmk__warn_once(pcmk__wo_rdisc_enabled,
                         "Support for the "
                         PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED
                         " node attribute is deprecated and will be removed"
                         " (and behave as 'true') in a future release.");
 
         if (pcmk__is_remote_node(this_node)
             && !pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
             pcmk__config_warn("Ignoring "
                               PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED
                               " attribute on Pacemaker Remote node %s"
                               " because fencing is disabled",
                               pcmk__node_name(this_node));
         } else {
             /* This is either a remote node with fencing enabled, or a guest
              * node. We don't care whether fencing is enabled when fencing guest
              * nodes, because they are "fenced" by recovering their containing
              * resource.
              */
             crm_info("%s has resource discovery disabled",
                      pcmk__node_name(this_node));
             pcmk__clear_node_flags(this_node, pcmk__node_probes_allowed);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Unpack a cluster node's transient attributes
  *
  * \param[in]     state      CIB node state XML
  * \param[in,out] node       Cluster node whose attributes are being unpacked
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 unpack_transient_attributes(const xmlNode *state, pcmk_node_t *node,
                             pcmk_scheduler_t *scheduler)
 {
     const char *discovery = NULL;
     const xmlNode *attrs = pcmk__xe_first_child(state,
                                                 PCMK__XE_TRANSIENT_ATTRIBUTES,
                                                 NULL, NULL);
 
     add_node_attrs(attrs, node, TRUE, scheduler);
 
     if (crm_is_true(pcmk__node_attr(node, PCMK_NODE_ATTR_STANDBY, NULL,
                                     pcmk__rsc_node_current))) {
         crm_info("%s is in standby mode", pcmk__node_name(node));
         pcmk__set_node_flags(node, pcmk__node_standby);
     }
 
     if (crm_is_true(pcmk__node_attr(node, PCMK_NODE_ATTR_MAINTENANCE, NULL,
                                     pcmk__rsc_node_current))) {
         crm_info("%s is in maintenance mode", pcmk__node_name(node));
         node->details->maintenance = TRUE;
     }
 
     discovery = pcmk__node_attr(node,
                                 PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED,
                                 NULL, pcmk__rsc_node_current);
     if ((discovery != NULL) && !crm_is_true(discovery)) {
         pcmk__config_warn("Ignoring "
                           PCMK__NODE_ATTR_RESOURCE_DISCOVERY_ENABLED
                           " attribute for %s because disabling resource"
                           " discovery is not allowed for cluster nodes",
                           pcmk__node_name(node));
     }
 }
 
 /*!
  * \internal
  * \brief Unpack a node state entry (first pass)
  *
  * Unpack one node state entry from status. This unpacks information from the
  * \C PCMK__XE_NODE_STATE element itself and node attributes inside it, but not
  * the resource history inside it. Multiple passes through the status are needed
  * to fully unpack everything.
  *
  * \param[in]     state      CIB node state XML
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 unpack_node_state(const xmlNode *state, pcmk_scheduler_t *scheduler)
 {
     const char *id = NULL;
     const char *uname = NULL;
     pcmk_node_t *this_node = NULL;
 
     id = crm_element_value(state, PCMK_XA_ID);
     if (id == NULL) {
         pcmk__config_err("Ignoring invalid " PCMK__XE_NODE_STATE " entry without "
                          PCMK_XA_ID);
         crm_log_xml_info(state, "missing-id");
         return;
     }
 
     uname = crm_element_value(state, PCMK_XA_UNAME);
     if (uname == NULL) {
         /* If a joining peer makes the cluster acquire the quorum from Corosync
          * but has not joined the controller CPG membership yet, it's possible
          * that the created PCMK__XE_NODE_STATE entry doesn't have a
          * PCMK_XA_UNAME yet. Recognize the node as pending and wait for it to
          * join CPG.
          */
         crm_trace("Handling " PCMK__XE_NODE_STATE " entry with id=\"%s\" "
                   "without " PCMK_XA_UNAME,
                   id);
     }
 
     this_node = pe_find_node_any(scheduler->nodes, id, uname);
     if (this_node == NULL) {
         crm_notice("Ignoring recorded state for removed node with name %s and "
                    PCMK_XA_ID " %s", pcmk__s(uname, "unknown"), id);
         return;
     }
 
     if (pcmk__is_pacemaker_remote_node(this_node)) {
         int remote_fenced = 0;
 
         /* We can't determine the online status of Pacemaker Remote nodes until
          * after all resource history has been unpacked. In this first pass, we
          * do need to mark whether the node has been fenced, as this plays a
          * role during unpacking cluster node resource state.
          */
         pcmk__scan_min_int(crm_element_value(state, PCMK__XA_NODE_FENCED),
                            &remote_fenced, 0);
         if (remote_fenced) {
             pcmk__set_node_flags(this_node, pcmk__node_remote_fenced);
         } else {
             pcmk__clear_node_flags(this_node, pcmk__node_remote_fenced);
         }
         return;
     }
 
     unpack_transient_attributes(state, this_node, scheduler);
 
     /* Provisionally mark this cluster node as clean. We have at least seen it
      * in the current cluster's lifetime.
      */
     this_node->details->unclean = FALSE;
     pcmk__set_node_flags(this_node, pcmk__node_seen);
 
     crm_trace("Determining online status of cluster node %s (id %s)",
               pcmk__node_name(this_node), id);
     determine_online_status(state, this_node, scheduler);
 
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_quorate)
         && this_node->details->online
         && (scheduler->no_quorum_policy == pcmk_no_quorum_fence)) {
         /* Everything else should flow from this automatically
          * (at least until the scheduler becomes able to migrate off
          * healthy resources)
          */
         pe_fence_node(scheduler, this_node, "cluster does not have quorum",
                       FALSE);
     }
 }
 
 /*!
  * \internal
  * \brief Unpack nodes' resource history as much as possible
  *
  * Unpack as many nodes' resource history as possible in one pass through the
  * status. We need to process Pacemaker Remote nodes' connections/containers
  * before unpacking their history; the connection/container history will be
  * in another node's history, so it might take multiple passes to unpack
  * everything.
  *
  * \param[in]     status     CIB XML status section
  * \param[in]     fence      If true, treat any not-yet-unpacked nodes as unseen
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
  *         or EAGAIN if more unpacking remains to be done)
  */
 static int
 unpack_node_history(const xmlNode *status, bool fence,
                     pcmk_scheduler_t *scheduler)
 {
     int rc = pcmk_rc_ok;
 
     // Loop through all PCMK__XE_NODE_STATE entries in CIB status
     for (const xmlNode *state = pcmk__xe_first_child(status,
                                                      PCMK__XE_NODE_STATE, NULL,
                                                      NULL);
          state != NULL; state = pcmk__xe_next(state, PCMK__XE_NODE_STATE)) {
 
         const char *id = pcmk__xe_id(state);
         const char *uname = crm_element_value(state, PCMK_XA_UNAME);
         pcmk_node_t *this_node = NULL;
 
         if ((id == NULL) || (uname == NULL)) {
             // Warning already logged in first pass through status section
             crm_trace("Not unpacking resource history from malformed "
                       PCMK__XE_NODE_STATE " without id and/or uname");
             continue;
         }
 
         this_node = pe_find_node_any(scheduler->nodes, id, uname);
         if (this_node == NULL) {
             // Warning already logged in first pass through status section
             crm_trace("Not unpacking resource history for node %s because "
                       "no longer in configuration", id);
             continue;
         }
 
         if (pcmk_is_set(this_node->priv->flags, pcmk__node_unpacked)) {
             crm_trace("Not unpacking resource history for node %s because "
                       "already unpacked", id);
             continue;
         }
 
         if (fence) {
             // We're processing all remaining nodes
 
         } else if (pcmk__is_guest_or_bundle_node(this_node)) {
             /* We can unpack a guest node's history only after we've unpacked
              * other resource history to the point that we know that the node's
              * connection and containing resource are both up.
              */
             const pcmk_resource_t *remote = this_node->priv->remote;
             const pcmk_resource_t *launcher = remote->priv->launcher;
 
             if ((remote->priv->orig_role != pcmk_role_started)
                 || (launcher->priv->orig_role != pcmk_role_started)) {
                 crm_trace("Not unpacking resource history for guest node %s "
                           "because launcher and connection are not known to "
                           "be up", id);
                 continue;
             }
 
         } else if (pcmk__is_remote_node(this_node)) {
             /* We can unpack a remote node's history only after we've unpacked
              * other resource history to the point that we know that the node's
              * connection is up, with the exception of when shutdown locks are
              * in use.
              */
             pcmk_resource_t *rsc = this_node->priv->remote;
 
             if ((rsc == NULL)
                 || (!pcmk_is_set(scheduler->flags, pcmk__sched_shutdown_lock)
                     && (rsc->priv->orig_role != pcmk_role_started))) {
                 crm_trace("Not unpacking resource history for remote node %s "
                           "because connection is not known to be up", id);
                 continue;
             }
 
         /* If fencing and shutdown locks are disabled and we're not processing
          * unseen nodes, then we don't want to unpack offline nodes until online
          * nodes have been unpacked. This allows us to number active clone
          * instances first.
          */
         } else if (!pcmk_any_flags_set(scheduler->flags,
                                        pcmk__sched_fencing_enabled
                                        |pcmk__sched_shutdown_lock)
                    && !this_node->details->online) {
             crm_trace("Not unpacking resource history for offline "
                       "cluster node %s", id);
             continue;
         }
 
         if (pcmk__is_pacemaker_remote_node(this_node)) {
             determine_remote_online_status(scheduler, this_node);
             unpack_handle_remote_attrs(this_node, state, scheduler);
         }
 
         crm_trace("Unpacking resource history for %snode %s",
                   (fence? "unseen " : ""), id);
 
         pcmk__set_node_flags(this_node, pcmk__node_unpacked);
         unpack_node_lrm(this_node, state, scheduler);
 
         rc = EAGAIN; // Other node histories might depend on this one
     }
     return rc;
 }
 
 /* remove nodes that are down, stopping */
 /* create positive rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode *status, pcmk_scheduler_t *scheduler)
 {
     xmlNode *state = NULL;
 
     crm_trace("Beginning unpack");
 
     if (scheduler->priv->ticket_constraints == NULL) {
         scheduler->priv->ticket_constraints =
             pcmk__strkey_table(free, destroy_ticket);
     }
 
     for (state = pcmk__xe_first_child(status, NULL, NULL, NULL); state != NULL;
          state = pcmk__xe_next(state, NULL)) {
 
         if (pcmk__xe_is(state, PCMK_XE_TICKETS)) {
             pcmk__xe_foreach_child(state, PCMK__XE_TICKET_STATE,
                                    unpack_ticket_state, scheduler);
 
         } else if (pcmk__xe_is(state, PCMK__XE_NODE_STATE)) {
             unpack_node_state(state, scheduler);
         }
     }
 
     while (unpack_node_history(status, FALSE, scheduler) == EAGAIN) {
         crm_trace("Another pass through node resource histories is needed");
     }
 
     // Now catch any nodes we didn't see
     unpack_node_history(status,
                         pcmk_is_set(scheduler->flags,
                                     pcmk__sched_fencing_enabled),
                         scheduler);
 
     /* Now that we know where resources are, we can schedule stops of containers
      * with failed bundle connections
      */
     if (scheduler->priv->stop_needed != NULL) {
         for (GList *item = scheduler->priv->stop_needed;
              item != NULL; item = item->next) {
 
             pcmk_resource_t *container = item->data;
             pcmk_node_t *node = pcmk__current_node(container);
 
             if (node) {
                 stop_action(container, node, FALSE);
             }
         }
         g_list_free(scheduler->priv->stop_needed);
         scheduler->priv->stop_needed = NULL;
     }
 
     /* Now that we know status of all Pacemaker Remote connections and nodes,
      * we can stop connections for node shutdowns, and check the online status
      * of remote/guest nodes that didn't have any node history to unpack.
      */
     for (GList *gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
         pcmk_node_t *this_node = gIter->data;
 
         if (!pcmk__is_pacemaker_remote_node(this_node)) {
             continue;
         }
         if (this_node->details->shutdown
             && (this_node->priv->remote != NULL)) {
             pe__set_next_role(this_node->priv->remote, pcmk_role_stopped,
                               "remote shutdown");
         }
         if (!pcmk_is_set(this_node->priv->flags, pcmk__node_unpacked)) {
             determine_remote_online_status(scheduler, this_node);
         }
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Unpack node's time when it became a member at the cluster layer
  *
  * \param[in]     node_state  Node's \c PCMK__XE_NODE_STATE entry
  * \param[in,out] scheduler   Scheduler data
  *
  * \return Epoch time when node became a cluster member
  *         (or scheduler effective time for legacy entries) if a member,
  *         0 if not a member, or -1 if no valid information available
  */
 static long long
 unpack_node_member(const xmlNode *node_state, pcmk_scheduler_t *scheduler)
 {
     const char *member_time = crm_element_value(node_state, PCMK__XA_IN_CCM);
     int member = 0;
 
     if (member_time == NULL) {
         return -1LL;
 
     } else if (crm_str_to_boolean(member_time, &member) == 1) {
         /* If in_ccm=0, we'll return 0 here. If in_ccm=1, either the entry was
          * recorded as a boolean for a DC < 2.1.7, or the node is pending
          * shutdown and has left the CPG, in which case it was set to 1 to avoid
          * fencing for PCMK_OPT_NODE_PENDING_TIMEOUT.
          *
          * We return the effective time for in_ccm=1 because what's important to
          * avoid fencing is that effective time minus this value is less than
          * the pending node timeout.
          */
         return member? (long long) get_effective_time(scheduler) : 0LL;
 
     } else {
         long long when_member = 0LL;
 
         if ((pcmk__scan_ll(member_time, &when_member,
                            0LL) != pcmk_rc_ok) || (when_member < 0LL)) {
             crm_warn("Unrecognized value '%s' for " PCMK__XA_IN_CCM
                      " in " PCMK__XE_NODE_STATE " entry", member_time);
             return -1LL;
         }
         return when_member;
     }
 }
 
 /*!
  * \internal
  * \brief Unpack node's time when it became online in process group
  *
  * \param[in] node_state  Node's \c PCMK__XE_NODE_STATE entry
  *
  * \return Epoch time when node became online in process group (or 0 if not
  *         online, or 1 for legacy online entries)
  */
 static long long
 unpack_node_online(const xmlNode *node_state)
 {
     const char *peer_time = crm_element_value(node_state, PCMK_XA_CRMD);
 
     // @COMPAT Entries recorded for DCs < 2.1.7 have "online" or "offline"
     if (pcmk__str_eq(peer_time, PCMK_VALUE_OFFLINE,
                      pcmk__str_casei|pcmk__str_null_matches)) {
         return 0LL;
 
     } else if (pcmk__str_eq(peer_time, PCMK_VALUE_ONLINE, pcmk__str_casei)) {
         return 1LL;
 
     } else {
         long long when_online = 0LL;
 
         if ((pcmk__scan_ll(peer_time, &when_online, 0LL) != pcmk_rc_ok)
             || (when_online < 0)) {
             crm_warn("Unrecognized value '%s' for " PCMK_XA_CRMD " in "
                      PCMK__XE_NODE_STATE " entry, assuming offline", peer_time);
             return 0LL;
         }
         return when_online;
     }
 }
 
 /*!
  * \internal
  * \brief Unpack node attribute for user-requested fencing
  *
  * \param[in] node        Node to check
  * \param[in] node_state  Node's \c PCMK__XE_NODE_STATE entry in CIB status
  *
  * \return \c true if fencing has been requested for \p node, otherwise \c false
  */
 static bool
 unpack_node_terminate(const pcmk_node_t *node, const xmlNode *node_state)
 {
     long long value = 0LL;
     int value_i = 0;
     int rc = pcmk_rc_ok;
     const char *value_s = pcmk__node_attr(node, PCMK_NODE_ATTR_TERMINATE,
                                           NULL, pcmk__rsc_node_current);
 
     // Value may be boolean or an epoch time
     if (crm_str_to_boolean(value_s, &value_i) == 1) {
         return (value_i != 0);
     }
     rc = pcmk__scan_ll(value_s, &value, 0LL);
     if (rc == pcmk_rc_ok) {
         return (value > 0);
     }
     crm_warn("Ignoring unrecognized value '%s' for " PCMK_NODE_ATTR_TERMINATE
              "node attribute for %s: %s",
              value_s, pcmk__node_name(node), pcmk_rc_str(rc));
     return false;
 }
 
 static gboolean
 determine_online_status_no_fencing(pcmk_scheduler_t *scheduler,
                                    const xmlNode *node_state,
                                    pcmk_node_t *this_node)
 {
     gboolean online = FALSE;
     const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
     const char *exp_state = crm_element_value(node_state, PCMK_XA_EXPECTED);
     long long when_member = unpack_node_member(node_state, scheduler);
     long long when_online = unpack_node_online(node_state);
 
     if (when_member <= 0) {
         crm_trace("Node %s is %sdown", pcmk__node_name(this_node),
                   ((when_member < 0)? "presumed " : ""));
 
     } else if (when_online > 0) {
         if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
             online = TRUE;
         } else {
             crm_debug("Node %s is not ready to run resources: %s",
                       pcmk__node_name(this_node), join);
         }
 
     } else if (!pcmk_is_set(this_node->priv->flags,
                             pcmk__node_expected_up)) {
         crm_trace("Node %s controller is down: "
                   "member@%lld online@%lld join=%s expected=%s",
                   pcmk__node_name(this_node), when_member, when_online,
                   pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
 
     } else {
         /* mark it unclean */
         pe_fence_node(scheduler, this_node, "peer is unexpectedly down", FALSE);
         crm_info("Node %s member@%lld online@%lld join=%s expected=%s",
                  pcmk__node_name(this_node), when_member, when_online,
                  pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
     }
     return online;
 }
 
 /*!
  * \internal
  * \brief Check whether a node has taken too long to join controller group
  *
  * \param[in,out] scheduler    Scheduler data
  * \param[in]     node         Node to check
  * \param[in]     when_member  Epoch time when node became a cluster member
  * \param[in]     when_online  Epoch time when node joined controller group
  *
  * \return true if node has been pending (on the way up) longer than
  *         \c PCMK_OPT_NODE_PENDING_TIMEOUT, otherwise false
  * \note This will also update the cluster's recheck time if appropriate.
  */
 static inline bool
 pending_too_long(pcmk_scheduler_t *scheduler, const pcmk_node_t *node,
                  long long when_member, long long when_online)
 {
     if ((scheduler->priv->node_pending_ms > 0U)
         && (when_member > 0) && (when_online <= 0)) {
         // There is a timeout on pending nodes, and node is pending
 
         time_t timeout = when_member
                          + pcmk__timeout_ms2s(scheduler->priv->node_pending_ms);
 
         if (get_effective_time(node->priv->scheduler) >= timeout) {
             return true; // Node has timed out
         }
 
         // Node is pending, but still has time
         pe__update_recheck_time(timeout, scheduler, "pending node timeout");
     }
     return false;
 }
 
 static bool
 determine_online_status_fencing(pcmk_scheduler_t *scheduler,
                                 const xmlNode *node_state,
                                 pcmk_node_t *this_node)
 {
     bool termination_requested = unpack_node_terminate(this_node, node_state);
     const char *join = crm_element_value(node_state, PCMK__XA_JOIN);
     const char *exp_state = crm_element_value(node_state, PCMK_XA_EXPECTED);
     long long when_member = unpack_node_member(node_state, scheduler);
     long long when_online = unpack_node_online(node_state);
 
 /*
   - PCMK__XA_JOIN          ::= member|down|pending|banned
   - PCMK_XA_EXPECTED       ::= member|down
 
   @COMPAT with entries recorded for DCs < 2.1.7
   - PCMK__XA_IN_CCM        ::= true|false
   - PCMK_XA_CRMD           ::= online|offline
 
   Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
   - PCMK__XA_IN_CCM        ::= <timestamp>|0
   Since when node has been a cluster member. A value 0 of means the node is not
   a cluster member.
 
   - PCMK_XA_CRMD           ::= <timestamp>|0
   Since when peer has been online in CPG. A value 0 means the peer is offline
   in CPG.
 */
 
     crm_trace("Node %s member@%lld online@%lld join=%s expected=%s%s",
               pcmk__node_name(this_node), when_member, when_online,
               pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"),
               (termination_requested? " (termination requested)" : ""));
 
     if (this_node->details->shutdown) {
         crm_debug("%s is shutting down", pcmk__node_name(this_node));
 
         /* Slightly different criteria since we can't shut down a dead peer */
         return (when_online > 0);
     }
 
     if (when_member < 0) {
         pe_fence_node(scheduler, this_node,
                       "peer has not been seen by the cluster", FALSE);
         return false;
     }
 
     if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_none)) {
         pe_fence_node(scheduler, this_node,
                       "peer failed Pacemaker membership criteria", FALSE);
 
     } else if (termination_requested) {
         if ((when_member <= 0) && (when_online <= 0)
             && pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_none)) {
             crm_info("%s was fenced as requested", pcmk__node_name(this_node));
             return false;
         }
         pe_fence_node(scheduler, this_node, "fencing was requested", false);
 
     } else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN,
                             pcmk__str_null_matches)) {
 
         if (pending_too_long(scheduler, this_node, when_member, when_online)) {
             pe_fence_node(scheduler, this_node,
                           "peer pending timed out on joining the process group",
                           FALSE);
 
         } else if ((when_member > 0) || (when_online > 0)) {
             crm_info("- %s is not ready to run resources",
                      pcmk__node_name(this_node));
             pcmk__set_node_flags(this_node, pcmk__node_standby);
             this_node->details->pending = TRUE;
 
         } else {
             crm_trace("%s is down or still coming up",
                       pcmk__node_name(this_node));
         }
 
     } else if (when_member <= 0) {
         // Consider PCMK_OPT_PRIORITY_FENCING_DELAY for lost nodes
         pe_fence_node(scheduler, this_node,
                       "peer is no longer part of the cluster", TRUE);
 
     } else if (when_online <= 0) {
         pe_fence_node(scheduler, this_node,
                       "peer process is no longer available", FALSE);
 
         /* Everything is running at this point, now check join state */
 
     } else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_none)) {
         crm_info("%s is active", pcmk__node_name(this_node));
 
     } else if (pcmk__str_any_of(join, CRMD_JOINSTATE_PENDING,
                                 CRMD_JOINSTATE_DOWN, NULL)) {
         crm_info("%s is not ready to run resources",
                  pcmk__node_name(this_node));
         pcmk__set_node_flags(this_node, pcmk__node_standby);
         this_node->details->pending = TRUE;
 
     } else {
         pe_fence_node(scheduler, this_node, "peer was in an unknown state",
                       FALSE);
     }
 
     return (when_member > 0);
 }
 
 static void
 determine_remote_online_status(pcmk_scheduler_t *scheduler,
                                pcmk_node_t *this_node)
 {
     pcmk_resource_t *rsc = this_node->priv->remote;
     pcmk_resource_t *launcher = NULL;
     pcmk_node_t *host = NULL;
     const char *node_type = "Remote";
 
     if (rsc == NULL) {
         /* This is a leftover node state entry for a former Pacemaker Remote
          * node whose connection resource was removed. Consider it offline.
          */
         crm_trace("Pacemaker Remote node %s is considered OFFLINE because "
                   "its connection resource has been removed from the CIB",
                   this_node->priv->id);
         this_node->details->online = FALSE;
         return;
     }
 
     launcher = rsc->priv->launcher;
     if (launcher != NULL) {
         node_type = "Guest";
         if (pcmk__list_of_1(rsc->priv->active_nodes)) {
             host = rsc->priv->active_nodes->data;
         }
     }
 
     /* If the resource is currently started, mark it online. */
     if (rsc->priv->orig_role == pcmk_role_started) {
         this_node->details->online = TRUE;
     }
 
     /* consider this node shutting down if transitioning start->stop */
     if ((rsc->priv->orig_role == pcmk_role_started)
         && (rsc->priv->next_role == pcmk_role_stopped)) {
 
         crm_trace("%s node %s shutting down because connection resource is stopping",
                   node_type, this_node->priv->id);
         this_node->details->shutdown = TRUE;
     }
 
     /* Now check all the failure conditions. */
     if ((launcher != NULL) && pcmk_is_set(launcher->flags, pcmk__rsc_failed)) {
         crm_trace("Guest node %s UNCLEAN because guest resource failed",
                   this_node->priv->id);
         this_node->details->online = FALSE;
         pcmk__set_node_flags(this_node, pcmk__node_remote_reset);
 
     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
         crm_trace("%s node %s OFFLINE because connection resource failed",
                   node_type, this_node->priv->id);
         this_node->details->online = FALSE;
 
     } else if ((rsc->priv->orig_role == pcmk_role_stopped)
                || ((launcher != NULL)
                    && (launcher->priv->orig_role == pcmk_role_stopped))) {
 
         crm_trace("%s node %s OFFLINE because its resource is stopped",
                   node_type, this_node->priv->id);
         this_node->details->online = FALSE;
         pcmk__clear_node_flags(this_node, pcmk__node_remote_reset);
 
     } else if (host && (host->details->online == FALSE)
                && host->details->unclean) {
         crm_trace("Guest node %s UNCLEAN because host is unclean",
                   this_node->priv->id);
         this_node->details->online = FALSE;
         pcmk__set_node_flags(this_node, pcmk__node_remote_reset);
 
     } else {
         crm_trace("%s node %s is %s",
                   node_type, this_node->priv->id,
                   this_node->details->online? "ONLINE" : "OFFLINE");
     }
 }
 
 static void
 determine_online_status(const xmlNode *node_state, pcmk_node_t *this_node,
                         pcmk_scheduler_t *scheduler)
 {
     gboolean online = FALSE;
     const char *exp_state = crm_element_value(node_state, PCMK_XA_EXPECTED);
 
     CRM_CHECK(this_node != NULL, return);
 
     this_node->details->shutdown = FALSE;
 
     if (pe__shutdown_requested(this_node)) {
         this_node->details->shutdown = TRUE;
 
     } else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
         pcmk__set_node_flags(this_node, pcmk__node_expected_up);
     }
 
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
         online = determine_online_status_no_fencing(scheduler, node_state,
                                                     this_node);
 
     } else {
         online = determine_online_status_fencing(scheduler, node_state,
                                                  this_node);
     }
 
     if (online) {
         this_node->details->online = TRUE;
 
     } else {
         /* remove node from contention */
         this_node->assign->score = -PCMK_SCORE_INFINITY;
     }
 
     if (online && this_node->details->shutdown) {
         /* don't run resources here */
         this_node->assign->score = -PCMK_SCORE_INFINITY;
     }
 
     if (this_node->details->unclean) {
         pcmk__sched_warn(scheduler, "%s is unclean",
                          pcmk__node_name(this_node));
 
     } else if (!this_node->details->online) {
         crm_trace("%s is offline", pcmk__node_name(this_node));
 
     } else if (this_node->details->shutdown) {
         crm_info("%s is shutting down", pcmk__node_name(this_node));
 
     } else if (this_node->details->pending) {
         crm_info("%s is pending", pcmk__node_name(this_node));
 
     } else if (pcmk_is_set(this_node->priv->flags, pcmk__node_standby)) {
         crm_info("%s is in standby", pcmk__node_name(this_node));
 
     } else if (this_node->details->maintenance) {
         crm_info("%s is in maintenance", pcmk__node_name(this_node));
 
     } else {
         crm_info("%s is online", pcmk__node_name(this_node));
     }
 }
 
 /*!
  * \internal
  * \brief Find the end of a resource's name, excluding any clone suffix
  *
  * \param[in] id  Resource ID to check
  *
  * \return Pointer to last character of resource's base name
  */
 const char *
 pe_base_name_end(const char *id)
 {
     if (!pcmk__str_empty(id)) {
         const char *end = id + strlen(id) - 1;
 
         for (const char *s = end; s > id; --s) {
             switch (*s) {
                 case '0':
                 case '1':
                 case '2':
                 case '3':
                 case '4':
                 case '5':
                 case '6':
                 case '7':
                 case '8':
                 case '9':
                     break;
                 case ':':
                     return (s == end)? s : (s - 1);
                 default:
                     return end;
             }
         }
         return end;
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Get a resource name excluding any clone suffix
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_strip(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     char *basename = NULL;
 
     pcmk__assert(end != NULL);
     basename = strndup(last_rsc_id, end - last_rsc_id + 1);
     pcmk__assert(basename != NULL);
     return basename;
 }
 
 /*!
  * \internal
  * \brief Get the name of the first instance of a cloned resource
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name plus :0
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_zero(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     size_t base_name_len = end - last_rsc_id + 1;
     char *zero = NULL;
 
     pcmk__assert(end != NULL);
     zero = pcmk__assert_alloc(base_name_len + 3, sizeof(char));
     memcpy(zero, last_rsc_id, base_name_len);
     zero[base_name_len] = ':';
     zero[base_name_len + 1] = '0';
     return zero;
 }
 
 static pcmk_resource_t *
 create_fake_resource(const char *rsc_id, const xmlNode *rsc_entry,
                      pcmk_scheduler_t *scheduler)
 {
     pcmk_resource_t *rsc = NULL;
     xmlNode *xml_rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);
 
     pcmk__xe_copy_attrs(xml_rsc, rsc_entry, pcmk__xaf_none);
     crm_xml_add(xml_rsc, PCMK_XA_ID, rsc_id);
     crm_log_xml_debug(xml_rsc, "Orphan resource");
 
     if (pe__unpack_resource(xml_rsc, &rsc, NULL, scheduler) != pcmk_rc_ok) {
         return NULL;
     }
 
     if (xml_contains_remote_node(xml_rsc)) {
         pcmk_node_t *node;
 
         crm_debug("Detected orphaned remote node %s", rsc_id);
         node = pcmk_find_node(scheduler, rsc_id);
         if (node == NULL) {
             node = pe_create_node(rsc_id, rsc_id, PCMK_VALUE_REMOTE, 0,
                                   scheduler);
         }
         link_rsc2remotenode(scheduler, rsc);
 
         if (node) {
             crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
             node->details->shutdown = TRUE;
         }
     }
 
     if (crm_element_value(rsc_entry, PCMK__META_CONTAINER)) {
         // This removed resource needs to be mapped to a launcher
         crm_trace("Launched resource %s was removed from the configuration",
                   rsc_id);
         pcmk__set_rsc_flags(rsc, pcmk__rsc_removed_launched);
     }
     pcmk__set_rsc_flags(rsc, pcmk__rsc_removed);
     scheduler->priv->resources = g_list_append(scheduler->priv->resources, rsc);
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Create orphan instance for anonymous clone resource history
  *
  * \param[in,out] parent     Clone resource that orphan will be added to
  * \param[in]     rsc_id     Orphan's resource ID
  * \param[in]     node       Where orphan is active (for logging only)
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Newly added orphaned instance of \p parent
  */
 static pcmk_resource_t *
 create_anonymous_orphan(pcmk_resource_t *parent, const char *rsc_id,
                         const pcmk_node_t *node, pcmk_scheduler_t *scheduler)
 {
     pcmk_resource_t *top = pe__create_clone_child(parent, scheduler);
     pcmk_resource_t *orphan = NULL;
 
     // find_rsc() because we might be a cloned group
     orphan = top->priv->fns->find_rsc(top, rsc_id, NULL,
                                       pcmk_rsc_match_clone_only);
 
     pcmk__rsc_debug(parent, "Created orphan %s for %s: %s on %s",
                     top->id, parent->id, rsc_id, pcmk__node_name(node));
     return orphan;
 }
 
 /*!
  * \internal
  * \brief Check a node for an instance of an anonymous clone
  *
  * Return a child instance of the specified anonymous clone, in order of
  * preference: (1) the instance running on the specified node, if any;
  * (2) an inactive instance (i.e. within the total of \c PCMK_META_CLONE_MAX
  * instances); (3) a newly created orphan (that is, \c PCMK_META_CLONE_MAX
  * instances are already active).
  *
  * \param[in,out] scheduler  Scheduler data
  * \param[in]     node       Node on which to check for instance
  * \param[in,out] parent     Clone to check
  * \param[in]     rsc_id     Name of cloned resource in history (no instance)
  */
 static pcmk_resource_t *
 find_anonymous_clone(pcmk_scheduler_t *scheduler, const pcmk_node_t *node,
                      pcmk_resource_t *parent, const char *rsc_id)
 {
     GList *rIter = NULL;
     pcmk_resource_t *rsc = NULL;
     pcmk_resource_t *inactive_instance = NULL;
     gboolean skip_inactive = FALSE;
 
     pcmk__assert(pcmk__is_anonymous_clone(parent));
 
     // Check for active (or partially active, for cloned groups) instance
     pcmk__rsc_trace(parent, "Looking for %s on %s in %s",
                     rsc_id, pcmk__node_name(node), parent->id);
 
     for (rIter = parent->priv->children;
          (rIter != NULL) && (rsc == NULL); rIter = rIter->next) {
 
         GList *locations = NULL;
         pcmk_resource_t *child = rIter->data;
 
         /* Check whether this instance is already known to be active or pending
          * anywhere, at this stage of unpacking. Because this function is called
          * for a resource before the resource's individual operation history
          * entries are unpacked, locations will generally not contain the
          * desired node.
          *
          * However, there are three exceptions:
          * (1) when child is a cloned group and we have already unpacked the
          *     history of another member of the group on the same node;
          * (2) when we've already unpacked the history of another numbered
          *     instance on the same node (which can happen if
          *     PCMK_META_GLOBALLY_UNIQUE was flipped from true to false); and
          * (3) when we re-run calculations on the same scheduler data as part of
          *     a simulation.
          */
         child->priv->fns->location(child, &locations, pcmk__rsc_node_current
                                                       |pcmk__rsc_node_pending);
         if (locations) {
             /* We should never associate the same numbered anonymous clone
              * instance with multiple nodes, and clone instances can't migrate,
              * so there must be only one location, regardless of history.
              */
             CRM_LOG_ASSERT(locations->next == NULL);
 
             if (pcmk__same_node((pcmk_node_t *) locations->data, node)) {
                 /* This child instance is active on the requested node, so check
                  * for a corresponding configured resource. We use find_rsc()
                  * instead of child because child may be a cloned group, and we
                  * need the particular member corresponding to rsc_id.
                  *
                  * If the history entry is orphaned, rsc will be NULL.
                  */
                 rsc = parent->priv->fns->find_rsc(child, rsc_id, NULL,
                                                   pcmk_rsc_match_clone_only);
                 if (rsc) {
                     /* If there are multiple instance history entries for an
                      * anonymous clone in a single node's history (which can
                      * happen if PCMK_META_GLOBALLY_UNIQUE is switched from true
                      * to false), we want to consider the instances beyond the
                      * first as orphans, even if there are inactive instance
                      * numbers available.
                      */
                     if (rsc->priv->active_nodes != NULL) {
                         crm_notice("Active (now-)anonymous clone %s has "
                                    "multiple (orphan) instance histories on %s",
                                    parent->id, pcmk__node_name(node));
                         skip_inactive = TRUE;
                         rsc = NULL;
                     } else {
                         pcmk__rsc_trace(parent, "Resource %s, active", rsc->id);
                     }
                 }
             }
             g_list_free(locations);
 
         } else {
             pcmk__rsc_trace(parent, "Resource %s, skip inactive", child->id);
             if (!skip_inactive && !inactive_instance
                 && !pcmk_is_set(child->flags, pcmk__rsc_blocked)) {
                 // Remember one inactive instance in case we don't find active
                 inactive_instance =
                     parent->priv->fns->find_rsc(child, rsc_id, NULL,
                                                 pcmk_rsc_match_clone_only);
 
                 /* ... but don't use it if it was already associated with a
                  * pending action on another node
                  */
                 if (inactive_instance != NULL) {
                     const pcmk_node_t *pending_node = NULL;
 
                     pending_node = inactive_instance->priv->pending_node;
                     if ((pending_node != NULL)
                         && !pcmk__same_node(pending_node, node)) {
                         inactive_instance = NULL;
                     }
                 }
             }
         }
     }
 
     if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
         pcmk__rsc_trace(parent, "Resource %s, empty slot",
                         inactive_instance->id);
         rsc = inactive_instance;
     }
 
     /* If the resource has PCMK_META_REQUIRES set to PCMK_VALUE_QUORUM or
      * PCMK_VALUE_NOTHING, and we don't have a clone instance for every node, we
      * don't want to consume a valid instance number for unclean nodes. Such
      * instances may appear to be active according to the history, but should be
      * considered inactive, so we can start an instance elsewhere. Treat such
      * instances as orphans.
      *
      * An exception is instances running on guest nodes -- since guest node
      * "fencing" is actually just a resource stop, requires shouldn't apply.
      *
      * @TODO Ideally, we'd use an inactive instance number if it is not needed
      * for any clean instances. However, we don't know that at this point.
      */
     if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)
         && (!node->details->online || node->details->unclean)
         && !pcmk__is_guest_or_bundle_node(node)
         && !pe__is_universal_clone(parent, scheduler)) {
 
         rsc = NULL;
     }
 
     if (rsc == NULL) {
         rsc = create_anonymous_orphan(parent, rsc_id, node, scheduler);
         pcmk__rsc_trace(parent, "Resource %s, orphan", rsc->id);
     }
     return rsc;
 }
 
 static pcmk_resource_t *
 unpack_find_resource(pcmk_scheduler_t *scheduler, const pcmk_node_t *node,
                      const char *rsc_id)
 {
     pcmk_resource_t *rsc = NULL;
     pcmk_resource_t *parent = NULL;
 
     crm_trace("looking for %s", rsc_id);
     rsc = pe_find_resource(scheduler->priv->resources, rsc_id);
 
     if (rsc == NULL) {
         /* If we didn't find the resource by its name in the operation history,
          * check it again as a clone instance. Even when PCMK_META_CLONE_MAX=0,
          * we create a single :0 orphan to match against here.
          */
         char *clone0_id = clone_zero(rsc_id);
         pcmk_resource_t *clone0 = pe_find_resource(scheduler->priv->resources,
                                                    clone0_id);
 
         if (clone0 && !pcmk_is_set(clone0->flags, pcmk__rsc_unique)) {
             rsc = clone0;
             parent = uber_parent(clone0);
             crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
         } else {
             crm_trace("%s is not known as %s either (orphan)",
                       rsc_id, clone0_id);
         }
         free(clone0_id);
 
     } else if (rsc->priv->variant > pcmk__rsc_variant_primitive) {
         crm_trace("Resource history for %s is orphaned "
                   "because it is no longer primitive", rsc_id);
         return NULL;
 
     } else {
         parent = uber_parent(rsc);
     }
 
     if (pcmk__is_anonymous_clone(parent)) {
 
         if (pcmk__is_bundled(parent)) {
             rsc = pe__find_bundle_replica(parent->priv->parent, node);
         } else {
             char *base = clone_strip(rsc_id);
 
             rsc = find_anonymous_clone(scheduler, node, parent, base);
             free(base);
             pcmk__assert(rsc != NULL);
         }
     }
 
     if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_none)
         && !pcmk__str_eq(rsc_id, rsc->priv->history_id, pcmk__str_none)) {
 
         pcmk__str_update(&(rsc->priv->history_id), rsc_id);
         pcmk__rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
                         rsc_id, pcmk__node_name(node), rsc->id,
                         pcmk_is_set(rsc->flags, pcmk__rsc_removed)? " (ORPHAN)" : "");
     }
     return rsc;
 }
 
 static pcmk_resource_t *
 process_orphan_resource(const xmlNode *rsc_entry, const pcmk_node_t *node,
                         pcmk_scheduler_t *scheduler)
 {
     pcmk_resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
 
     crm_debug("Detected orphan resource %s on %s",
               rsc_id, pcmk__node_name(node));
     rsc = create_fake_resource(rsc_id, rsc_entry, scheduler);
     if (rsc == NULL) {
         return NULL;
     }
 
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_stop_removed_resources)) {
         pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
 
     } else {
         CRM_CHECK(rsc != NULL, return NULL);
         pcmk__rsc_trace(rsc, "Added orphan %s", rsc->id);
         resource_location(rsc, NULL, -PCMK_SCORE_INFINITY,
                           "__orphan_do_not_run__", scheduler);
     }
     return rsc;
 }
 
 static void
 process_rsc_state(pcmk_resource_t *rsc, pcmk_node_t *node,
                   enum pcmk__on_fail on_fail)
 {
     pcmk_node_t *tmpnode = NULL;
     char *reason = NULL;
     enum pcmk__on_fail save_on_fail = pcmk__on_fail_ignore;
     pcmk_scheduler_t *scheduler = NULL;
     bool known_active = false;
 
     pcmk__assert(rsc != NULL);
     scheduler = rsc->priv->scheduler;
     known_active = (rsc->priv->orig_role > pcmk_role_stopped);
     pcmk__rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
                     rsc->id, pcmk_role_text(rsc->priv->orig_role),
                     pcmk__node_name(node), pcmk__on_fail_text(on_fail));
 
     /* process current state */
     if (rsc->priv->orig_role != pcmk_role_unknown) {
         pcmk_resource_t *iter = rsc;
 
         while (iter) {
             if (g_hash_table_lookup(iter->priv->probed_nodes,
                                     node->priv->id) == NULL) {
                 pcmk_node_t *n = pe__copy_node(node);
 
                 pcmk__rsc_trace(rsc, "%s (%s in history) known on %s",
                                 rsc->id,
                                 pcmk__s(rsc->priv->history_id, "the same"),
                                 pcmk__node_name(n));
                 g_hash_table_insert(iter->priv->probed_nodes,
                                     (gpointer) n->priv->id, n);
             }
             if (pcmk_is_set(iter->flags, pcmk__rsc_unique)) {
                 break;
             }
             iter = iter->priv->parent;
         }
     }
 
     /* If a managed resource is believed to be running, but node is down ... */
     if (known_active && !node->details->online && !node->details->maintenance
         && pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
 
         gboolean should_fence = FALSE;
 
         /* If this is a guest node, fence it (regardless of whether fencing is
          * enabled, because guest node fencing is done by recovery of the
          * container resource rather than by the fencer). Mark the resource
          * we're processing as failed. When the guest comes back up, its
          * operation history in the CIB will be cleared, freeing the affected
          * resource to run again once we are sure we know its state.
          */
         if (pcmk__is_guest_or_bundle_node(node)) {
             pcmk__set_rsc_flags(rsc, pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
             should_fence = TRUE;
 
         } else if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
             if (pcmk__is_remote_node(node)
                 && (node->priv->remote != NULL)
                 && !pcmk_is_set(node->priv->remote->flags,
                                 pcmk__rsc_failed)) {
 
                 /* Setting unseen means that fencing of the remote node will
                  * occur only if the connection resource is not going to start
                  * somewhere. This allows connection resources on a failed
                  * cluster node to move to another node without requiring the
                  * remote nodes to be fenced as well.
                  */
                 pcmk__clear_node_flags(node, pcmk__node_seen);
                 reason = crm_strdup_printf("%s is active there (fencing will be"
                                            " revoked if remote connection can "
                                            "be re-established elsewhere)",
                                            rsc->id);
             }
             should_fence = TRUE;
         }
 
         if (should_fence) {
             if (reason == NULL) {
                reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
             }
             pe_fence_node(scheduler, node, reason, FALSE);
         }
         free(reason);
     }
 
     /* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
     save_on_fail = on_fail;
 
     if (node->details->unclean) {
         /* No extra processing needed
          * Also allows resources to be started again after a node is shot
          */
         on_fail = pcmk__on_fail_ignore;
     }
 
     switch (on_fail) {
         case pcmk__on_fail_ignore:
             /* nothing to do */
             break;
 
         case pcmk__on_fail_demote:
             pcmk__set_rsc_flags(rsc, pcmk__rsc_failed);
             demote_action(rsc, node, FALSE);
             break;
 
         case pcmk__on_fail_fence_node:
             /* treat it as if it is still running
              * but also mark the node as unclean
              */
             reason = crm_strdup_printf("%s failed there", rsc->id);
             pe_fence_node(scheduler, node, reason, FALSE);
             free(reason);
             break;
 
         case pcmk__on_fail_standby_node:
             pcmk__set_node_flags(node,
                                  pcmk__node_standby|pcmk__node_fail_standby);
             break;
 
         case pcmk__on_fail_block:
             /* is_managed == FALSE will prevent any
              * actions being sent for the resource
              */
             pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
             pcmk__set_rsc_flags(rsc, pcmk__rsc_blocked);
             break;
 
         case pcmk__on_fail_ban:
             /* make sure it comes up somewhere else
              * or not at all
              */
             resource_location(rsc, node, -PCMK_SCORE_INFINITY,
                               "__action_migration_auto__", scheduler);
             break;
 
         case pcmk__on_fail_stop:
             pe__set_next_role(rsc, pcmk_role_stopped,
                               PCMK_META_ON_FAIL "=" PCMK_VALUE_STOP);
             break;
 
         case pcmk__on_fail_restart:
             if (known_active) {
                 pcmk__set_rsc_flags(rsc,
                                     pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case pcmk__on_fail_restart_container:
             pcmk__set_rsc_flags(rsc, pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
             if ((rsc->priv->launcher != NULL) && pcmk__is_bundled(rsc)) {
                 /* A bundle's remote connection can run on a different node than
                  * the bundle's container. We don't necessarily know where the
                  * container is running yet, so remember it and add a stop
                  * action for it later.
                  */
                 scheduler->priv->stop_needed =
                     g_list_prepend(scheduler->priv->stop_needed,
                                    rsc->priv->launcher);
             } else if (rsc->priv->launcher != NULL) {
                 stop_action(rsc->priv->launcher, node, FALSE);
             } else if (known_active) {
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case pcmk__on_fail_reset_remote:
             pcmk__set_rsc_flags(rsc, pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
             if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
                 tmpnode = NULL;
                 if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
                     tmpnode = pcmk_find_node(scheduler, rsc->id);
                 }
                 if (pcmk__is_remote_node(tmpnode)
                     && !pcmk_is_set(tmpnode->priv->flags,
                                     pcmk__node_remote_fenced)) {
                     /* The remote connection resource failed in a way that
                      * should result in fencing the remote node.
                      */
                     pe_fence_node(scheduler, tmpnode,
                                   "remote connection is unrecoverable", FALSE);
                 }
             }
 
             /* require the stop action regardless if fencing is occurring or not. */
             if (known_active) {
                 stop_action(rsc, node, FALSE);
             }
 
             /* if reconnect delay is in use, prevent the connection from exiting the
              * "STOPPED" role until the failure is cleared by the delay timeout. */
             if (rsc->priv->remote_reconnect_ms > 0U) {
                 pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
             }
             break;
     }
 
     /* Ensure a remote connection failure forces an unclean Pacemaker Remote
      * node to be fenced. By marking the node as seen, the failure will result
      * in a fencing operation regardless if we're going to attempt to reconnect
      * in this transition.
      */
     if (pcmk_all_flags_set(rsc->flags,
                            pcmk__rsc_failed|pcmk__rsc_is_remote_connection)) {
         tmpnode = pcmk_find_node(scheduler, rsc->id);
         if (tmpnode && tmpnode->details->unclean) {
             pcmk__set_node_flags(tmpnode, pcmk__node_seen);
         }
     }
 
     if (known_active) {
         if (pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
             if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
                 crm_notice("Removed resource %s is active on %s and will be "
                            "stopped when possible",
                            rsc->id, pcmk__node_name(node));
             } else {
                 crm_notice("Removed resource %s must be stopped manually on %s "
                            "because " PCMK_OPT_STOP_ORPHAN_RESOURCES
                            " is set to false", rsc->id, pcmk__node_name(node));
             }
         }
 
         native_add_running(rsc, node, scheduler,
                            (save_on_fail != pcmk__on_fail_ignore));
         switch (on_fail) {
             case pcmk__on_fail_ignore:
                 break;
             case pcmk__on_fail_demote:
             case pcmk__on_fail_block:
                 pcmk__set_rsc_flags(rsc, pcmk__rsc_failed);
                 break;
             default:
                 pcmk__set_rsc_flags(rsc,
                                     pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
                 break;
         }
 
     } else if ((rsc->priv->history_id != NULL)
                && (strchr(rsc->priv->history_id, ':') != NULL)) {
-        /* Only do this for older status sections that included instance numbers
-         * Otherwise stopped instances will appear as orphans
+        /* @COMPAT This is for older (<1.1.8) status sections that included
+         * instance numbers, otherwise stopped instances are considered orphans.
+         *
+         * @TODO We should be able to drop this, but some old regression tests
+         * will need to be updated. Double-check that this is not still needed
+         * for unique clones (which may have been later converted to anonymous).
          */
         pcmk__rsc_trace(rsc, "Clearing history ID %s for %s (stopped)",
                         rsc->priv->history_id, rsc->id);
         free(rsc->priv->history_id);
         rsc->priv->history_id = NULL;
 
     } else {
         GList *possible_matches = pe__resource_actions(rsc, node,
                                                        PCMK_ACTION_STOP, FALSE);
         GList *gIter = possible_matches;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pcmk_action_t *stop = (pcmk_action_t *) gIter->data;
 
             pcmk__set_action_flags(stop, pcmk__action_optional);
         }
 
         g_list_free(possible_matches);
     }
 
     /* A successful stop after migrate_to on the migration source doesn't make
      * the partially migrated resource stopped on the migration target.
      */
     if ((rsc->priv->orig_role == pcmk_role_stopped)
         && (rsc->priv->active_nodes != NULL)
         && (rsc->priv->partial_migration_target != NULL)
         && pcmk__same_node(rsc->priv->partial_migration_source, node)) {
 
         rsc->priv->orig_role = pcmk_role_started;
     }
 }
 
 /* create active recurring operations as optional */
 static void
 process_recurring(pcmk_node_t *node, pcmk_resource_t *rsc,
                   int start_index, int stop_index,
                   GList *sorted_op_list, pcmk_scheduler_t *scheduler)
 {
     int counter = -1;
     const char *task = NULL;
     const char *status = NULL;
     GList *gIter = sorted_op_list;
 
     pcmk__assert(rsc != NULL);
     pcmk__rsc_trace(rsc, "%s: Start index %d, stop index = %d",
                     rsc->id, start_index, stop_index);
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         guint interval_ms = 0;
         char *key = NULL;
         const char *id = pcmk__xe_id(rsc_op);
 
         counter++;
 
         if (node->details->online == FALSE) {
             pcmk__rsc_trace(rsc, "Skipping %s on %s: node is offline",
                             rsc->id, pcmk__node_name(node));
             break;
 
             /* Need to check if there's a monitor for role="Stopped" */
         } else if (start_index < stop_index && counter <= stop_index) {
             pcmk__rsc_trace(rsc, "Skipping %s on %s: resource is not active",
                             id, pcmk__node_name(node));
             continue;
 
         } else if (counter < start_index) {
             pcmk__rsc_trace(rsc, "Skipping %s on %s: old %d",
                             id, pcmk__node_name(node), counter);
             continue;
         }
 
         crm_element_value_ms(rsc_op, PCMK_META_INTERVAL, &interval_ms);
         if (interval_ms == 0) {
             pcmk__rsc_trace(rsc, "Skipping %s on %s: non-recurring",
                             id, pcmk__node_name(node));
             continue;
         }
 
         status = crm_element_value(rsc_op, PCMK__XA_OP_STATUS);
         if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
             pcmk__rsc_trace(rsc, "Skipping %s on %s: status",
                             id, pcmk__node_name(node));
             continue;
         }
         task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
         /* create the action */
         key = pcmk__op_key(rsc->id, task, interval_ms);
         pcmk__rsc_trace(rsc, "Creating %s on %s", key, pcmk__node_name(node));
         custom_action(rsc, key, task, node, TRUE, scheduler);
     }
 }
 
 void
 calculate_active_ops(const GList *sorted_op_list, int *start_index,
                      int *stop_index)
 {
     int counter = -1;
     int implied_monitor_start = -1;
     int implied_clone_start = -1;
     const char *task = NULL;
     const char *status = NULL;
 
     *stop_index = -1;
     *start_index = -1;
 
     for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
         const xmlNode *rsc_op = (const xmlNode *) iter->data;
 
         counter++;
 
         task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
         status = crm_element_value(rsc_op, PCMK__XA_OP_STATUS);
 
         if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
             && pcmk__str_eq(status, "0", pcmk__str_casei)) {
             *stop_index = counter;
 
         } else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
                                         PCMK_ACTION_MIGRATE_FROM, NULL)) {
             *start_index = counter;
 
         } else if ((implied_monitor_start <= *stop_index)
                    && pcmk__str_eq(task, PCMK_ACTION_MONITOR,
                                    pcmk__str_casei)) {
             const char *rc = crm_element_value(rsc_op, PCMK__XA_RC_CODE);
 
             if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
                 implied_monitor_start = counter;
             }
         } else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
                                         PCMK_ACTION_DEMOTE, NULL)) {
             implied_clone_start = counter;
         }
     }
 
     if (*start_index == -1) {
         if (implied_clone_start != -1) {
             *start_index = implied_clone_start;
         } else if (implied_monitor_start != -1) {
             *start_index = implied_monitor_start;
         }
     }
 }
 
 // If resource history entry has shutdown lock, remember lock node and time
 static void
 unpack_shutdown_lock(const xmlNode *rsc_entry, pcmk_resource_t *rsc,
                      const pcmk_node_t *node, pcmk_scheduler_t *scheduler)
 {
     time_t lock_time = 0;   // When lock started (i.e. node shutdown time)
 
     if ((crm_element_value_epoch(rsc_entry, PCMK_OPT_SHUTDOWN_LOCK,
                                  &lock_time) == pcmk_ok) && (lock_time != 0)) {
 
         if ((scheduler->priv->shutdown_lock_ms > 0U)
             && (get_effective_time(scheduler)
                 > (lock_time + pcmk__timeout_ms2s(scheduler->priv->shutdown_lock_ms)))) {
             pcmk__rsc_info(rsc, "Shutdown lock for %s on %s expired",
                            rsc->id, pcmk__node_name(node));
             pe__clear_resource_history(rsc, node);
         } else {
             rsc->priv->lock_node = node;
             rsc->priv->lock_time = lock_time;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Unpack one \c PCMK__XE_LRM_RESOURCE entry from a node's CIB status
  *
  * \param[in,out] node       Node whose status is being unpacked
  * \param[in]     rsc_entry  \c PCMK__XE_LRM_RESOURCE XML being unpacked
  * \param[in,out] scheduler  Scheduler data
  *
  * \return Resource corresponding to the entry, or NULL if no operation history
  */
 static pcmk_resource_t *
 unpack_lrm_resource(pcmk_node_t *node, const xmlNode *lrm_resource,
                     pcmk_scheduler_t *scheduler)
 {
     GList *gIter = NULL;
     int stop_index = -1;
     int start_index = -1;
     enum rsc_role_e req_role = pcmk_role_unknown;
 
     const char *rsc_id = pcmk__xe_id(lrm_resource);
 
     pcmk_resource_t *rsc = NULL;
     GList *op_list = NULL;
     GList *sorted_op_list = NULL;
 
     xmlNode *rsc_op = NULL;
     xmlNode *last_failure = NULL;
 
     enum pcmk__on_fail on_fail = pcmk__on_fail_ignore;
     enum rsc_role_e saved_role = pcmk_role_unknown;
 
     if (rsc_id == NULL) {
         pcmk__config_err("Ignoring invalid " PCMK__XE_LRM_RESOURCE
                          " entry: No " PCMK_XA_ID);
         crm_log_xml_info(lrm_resource, "missing-id");
         return NULL;
     }
     crm_trace("Unpacking " PCMK__XE_LRM_RESOURCE " for %s on %s",
               rsc_id, pcmk__node_name(node));
 
     /* Build a list of individual PCMK__XE_LRM_RSC_OP entries, so we can sort
      * them
      */
     for (rsc_op = pcmk__xe_first_child(lrm_resource, PCMK__XE_LRM_RSC_OP, NULL,
                                        NULL);
          rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op, PCMK__XE_LRM_RSC_OP)) {
 
         op_list = g_list_prepend(op_list, rsc_op);
     }
 
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_shutdown_lock)) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         }
     }
 
     /* find the resource */
     rsc = unpack_find_resource(scheduler, node, rsc_id);
     if (rsc == NULL) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         } else {
             rsc = process_orphan_resource(lrm_resource, node, scheduler);
         }
     }
     pcmk__assert(rsc != NULL);
 
     // Check whether the resource is "shutdown-locked" to this node
     if (pcmk_is_set(scheduler->flags, pcmk__sched_shutdown_lock)) {
         unpack_shutdown_lock(lrm_resource, rsc, node, scheduler);
     }
 
     /* process operations */
     saved_role = rsc->priv->orig_role;
     rsc->priv->orig_role = pcmk_role_unknown;
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
     }
 
     /* create active recurring operations as optional */
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
     process_recurring(node, rsc, start_index, stop_index, sorted_op_list,
                       scheduler);
 
     /* no need to free the contents */
     g_list_free(sorted_op_list);
 
     process_rsc_state(rsc, node, on_fail);
 
     if (get_target_role(rsc, &req_role)) {
         if ((rsc->priv->next_role == pcmk_role_unknown)
             || (req_role < rsc->priv->next_role)) {
 
             pe__set_next_role(rsc, req_role, PCMK_META_TARGET_ROLE);
 
         } else if (req_role > rsc->priv->next_role) {
             pcmk__rsc_info(rsc,
                            "%s: Not overwriting calculated next role %s"
                            " with requested next role %s",
                            rsc->id, pcmk_role_text(rsc->priv->next_role),
                            pcmk_role_text(req_role));
         }
     }
 
     if (saved_role > rsc->priv->orig_role) {
         rsc->priv->orig_role = saved_role;
     }
 
     return rsc;
 }
 
 static void
 handle_removed_launched_resources(const xmlNode *lrm_rsc_list,
                                   pcmk_scheduler_t *scheduler)
 {
     for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list,
                                                          PCMK__XE_LRM_RESOURCE,
                                                          NULL, NULL);
          rsc_entry != NULL;
          rsc_entry = pcmk__xe_next(rsc_entry, PCMK__XE_LRM_RESOURCE)) {
 
         pcmk_resource_t *rsc;
         pcmk_resource_t *launcher = NULL;
         const char *rsc_id;
         const char *launcher_id = NULL;
 
         launcher_id = crm_element_value(rsc_entry, PCMK__META_CONTAINER);
         rsc_id = crm_element_value(rsc_entry, PCMK_XA_ID);
         if ((launcher_id == NULL) || (rsc_id == NULL)) {
             continue;
         }
 
         launcher = pe_find_resource(scheduler->priv->resources, launcher_id);
         if (launcher == NULL) {
             continue;
         }
 
         rsc = pe_find_resource(scheduler->priv->resources, rsc_id);
         if ((rsc == NULL) || (rsc->priv->launcher != NULL)
             || !pcmk_is_set(rsc->flags, pcmk__rsc_removed_launched)) {
             continue;
         }
 
         pcmk__rsc_trace(rsc, "Mapped launcher of removed resource %s to %s",
                         rsc->id, launcher_id);
         rsc->priv->launcher = launcher;
         launcher->priv->launched = g_list_append(launcher->priv->launched,
                                                     rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Unpack one node's lrm status section
  *
  * \param[in,out] node       Node whose status is being unpacked
  * \param[in]     xml        CIB node state XML
  * \param[in,out] scheduler  Scheduler data
  */
 static void
 unpack_node_lrm(pcmk_node_t *node, const xmlNode *xml,
                 pcmk_scheduler_t *scheduler)
 {
     bool found_removed_launched_resource = false;
 
     // Drill down to PCMK__XE_LRM_RESOURCES section
     xml = pcmk__xe_first_child(xml, PCMK__XE_LRM, NULL, NULL);
     if (xml == NULL) {
         return;
     }
     xml = pcmk__xe_first_child(xml, PCMK__XE_LRM_RESOURCES, NULL, NULL);
     if (xml == NULL) {
         return;
     }
 
     // Unpack each PCMK__XE_LRM_RESOURCE entry
     for (const xmlNode *rsc_entry = pcmk__xe_first_child(xml,
                                                          PCMK__XE_LRM_RESOURCE,
                                                          NULL, NULL);
          rsc_entry != NULL;
          rsc_entry = pcmk__xe_next(rsc_entry, PCMK__XE_LRM_RESOURCE)) {
 
         pcmk_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, scheduler);
 
         if ((rsc != NULL)
             && pcmk_is_set(rsc->flags, pcmk__rsc_removed_launched)) {
             found_removed_launched_resource = true;
         }
     }
 
     /* Now that all resource state has been unpacked for this node, map any
      * removed launched resources to their launchers.
      */
     if (found_removed_launched_resource) {
         handle_removed_launched_resources(xml, scheduler);
     }
 }
 
 static void
 set_active(pcmk_resource_t *rsc)
 {
     const pcmk_resource_t *top = pe__const_top_resource(rsc, false);
 
     if (top && pcmk_is_set(top->flags, pcmk__rsc_promotable)) {
         rsc->priv->orig_role = pcmk_role_unpromoted;
     } else {
         rsc->priv->orig_role = pcmk_role_started;
     }
 }
 
 static void
 set_node_score(gpointer key, gpointer value, gpointer user_data)
 {
     pcmk_node_t *node = value;
     int *score = user_data;
 
     node->assign->score = *score;
 }
 
 #define XPATH_NODE_STATE "/" PCMK_XE_CIB "/" PCMK_XE_STATUS \
                          "/" PCMK__XE_NODE_STATE
 #define SUB_XPATH_LRM_RESOURCE "/" PCMK__XE_LRM             \
                                "/" PCMK__XE_LRM_RESOURCES   \
                                "/" PCMK__XE_LRM_RESOURCE
 #define SUB_XPATH_LRM_RSC_OP "/" PCMK__XE_LRM_RSC_OP
 
 static xmlNode *
 find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
             int target_rc, pcmk_scheduler_t *scheduler)
 {
     GString *xpath = NULL;
     xmlNode *xml = NULL;
 
     CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
               return NULL);
 
     xpath = g_string_sized_new(256);
     pcmk__g_strcat(xpath,
                    XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node, "']"
                    SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", resource, "']"
                    SUB_XPATH_LRM_RSC_OP "[@" PCMK_XA_OPERATION "='", op, "'",
                    NULL);
 
     /* Need to check against transition_magic too? */
     if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
         pcmk__g_strcat(xpath,
                        " and @" PCMK__META_MIGRATE_TARGET "='", source, "']",
                        NULL);
 
     } else if ((source != NULL)
                && (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
         pcmk__g_strcat(xpath,
                        " and @" PCMK__META_MIGRATE_SOURCE "='", source, "']",
                        NULL);
     } else {
         g_string_append_c(xpath, ']');
     }
 
     xml = get_xpath_object((const char *) xpath->str, scheduler->input,
                            LOG_DEBUG);
     g_string_free(xpath, TRUE);
 
     if (xml && target_rc >= 0) {
         int rc = PCMK_OCF_UNKNOWN_ERROR;
         int status = PCMK_EXEC_ERROR;
 
         crm_element_value_int(xml, PCMK__XA_RC_CODE, &rc);
         crm_element_value_int(xml, PCMK__XA_OP_STATUS, &status);
         if ((rc != target_rc) || (status != PCMK_EXEC_DONE)) {
             return NULL;
         }
     }
     return xml;
 }
 
 static xmlNode *
 find_lrm_resource(const char *rsc_id, const char *node_name,
                   pcmk_scheduler_t *scheduler)
 {
     GString *xpath = NULL;
     xmlNode *xml = NULL;
 
     CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);
 
     xpath = g_string_sized_new(256);
     pcmk__g_strcat(xpath,
                    XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='", node_name, "']"
                    SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='", rsc_id, "']",
                    NULL);
 
     xml = get_xpath_object((const char *) xpath->str, scheduler->input,
                            LOG_DEBUG);
 
     g_string_free(xpath, TRUE);
     return xml;
 }
 
 /*!
  * \internal
  * \brief Check whether a resource has no completed action history on a node
  *
  * \param[in,out] rsc        Resource to check
  * \param[in]     node_name  Node to check
  *
  * \return true if \p rsc_id is unknown on \p node_name, otherwise false
  */
 static bool
 unknown_on_node(pcmk_resource_t *rsc, const char *node_name)
 {
     bool result = false;
     xmlXPathObjectPtr search;
     char *xpath = NULL;
 
     xpath = crm_strdup_printf(XPATH_NODE_STATE "[@" PCMK_XA_UNAME "='%s']"
                               SUB_XPATH_LRM_RESOURCE "[@" PCMK_XA_ID "='%s']"
                               SUB_XPATH_LRM_RSC_OP
                               "[@" PCMK__XA_RC_CODE "!='%d']",
                               node_name, rsc->id, PCMK_OCF_UNKNOWN);
 
     search = xpath_search(rsc->priv->scheduler->input, xpath);
     result = (numXpathResults(search) == 0);
     freeXpathObject(search);
     free(xpath);
     return result;
 }
 
 /*!
  * \internal
  * \brief Check whether a probe/monitor indicating the resource was not running
  *        on a node happened after some event
  *
  * \param[in]     rsc_id     Resource being checked
  * \param[in]     node_name  Node being checked
  * \param[in]     xml_op     Event that monitor is being compared to
  * \param[in,out] scheduler  Scheduler data
  *
  * \return true if such a monitor happened after event, false otherwise
  */
 static bool
 monitor_not_running_after(const char *rsc_id, const char *node_name,
                           const xmlNode *xml_op, pcmk_scheduler_t *scheduler)
 {
     /* Any probe/monitor operation on the node indicating it was not running
      * there
      */
     xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
                                    NULL, PCMK_OCF_NOT_RUNNING, scheduler);
 
     return (monitor != NULL) && (pe__is_newer_op(monitor, xml_op) > 0);
 }
 
 /*!
  * \internal
  * \brief Check whether any non-monitor operation on a node happened after some
  *        event
  *
  * \param[in]     rsc_id     Resource being checked
  * \param[in]     node_name  Node being checked
  * \param[in]     xml_op     Event that non-monitor is being compared to
  * \param[in,out] scheduler  Scheduler data
  *
  * \return true if such a operation happened after event, false otherwise
  */
 static bool
 non_monitor_after(const char *rsc_id, const char *node_name,
                   const xmlNode *xml_op, pcmk_scheduler_t *scheduler)
 {
     xmlNode *lrm_resource = NULL;
 
     lrm_resource = find_lrm_resource(rsc_id, node_name, scheduler);
     if (lrm_resource == NULL) {
         return false;
     }
 
     for (xmlNode *op = pcmk__xe_first_child(lrm_resource, PCMK__XE_LRM_RSC_OP,
                                             NULL, NULL);
          op != NULL; op = pcmk__xe_next(op, PCMK__XE_LRM_RSC_OP)) {
 
         const char * task = NULL;
 
         if (op == xml_op) {
             continue;
         }
 
         task = crm_element_value(op, PCMK_XA_OPERATION);
 
         if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
                              PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
                              NULL)
             && pe__is_newer_op(op, xml_op) > 0) {
             return true;
         }
     }
 
     return false;
 }
 
 /*!
  * \internal
  * \brief Check whether the resource has newer state on a node after a migration
  *        attempt
  *
  * \param[in]     rsc_id        Resource being checked
  * \param[in]     node_name     Node being checked
  * \param[in]     migrate_to    Any migrate_to event that is being compared to
  * \param[in]     migrate_from  Any migrate_from event that is being compared to
  * \param[in,out] scheduler     Scheduler data
  *
  * \return true if such a operation happened after event, false otherwise
  */
 static bool
 newer_state_after_migrate(const char *rsc_id, const char *node_name,
                           const xmlNode *migrate_to,
                           const xmlNode *migrate_from,
                           pcmk_scheduler_t *scheduler)
 {
     const xmlNode *xml_op = (migrate_from != NULL)? migrate_from : migrate_to;
     const char *source = crm_element_value(xml_op, PCMK__META_MIGRATE_SOURCE);
 
     /* It's preferred to compare to the migrate event on the same node if
      * existing, since call ids are more reliable.
      */
     if ((xml_op != migrate_to) && (migrate_to != NULL)
         && pcmk__str_eq(node_name, source, pcmk__str_casei)) {
 
         xml_op = migrate_to;
     }
 
     /* If there's any newer non-monitor operation on the node, or any newer
      * probe/monitor operation on the node indicating it was not running there,
      * the migration events potentially no longer matter for the node.
      */
     return non_monitor_after(rsc_id, node_name, xml_op, scheduler)
            || monitor_not_running_after(rsc_id, node_name, xml_op, scheduler);
 }
 
 /*!
  * \internal
  * \brief Parse migration source and target node names from history entry
  *
  * \param[in]  entry        Resource history entry for a migration action
  * \param[in]  source_node  If not NULL, source must match this node
  * \param[in]  target_node  If not NULL, target must match this node
  * \param[out] source_name  Where to store migration source node name
  * \param[out] target_name  Where to store migration target node name
  *
  * \return Standard Pacemaker return code
  */
 static int
 get_migration_node_names(const xmlNode *entry, const pcmk_node_t *source_node,
                          const pcmk_node_t *target_node,
                          const char **source_name, const char **target_name)
 {
     *source_name = crm_element_value(entry, PCMK__META_MIGRATE_SOURCE);
     *target_name = crm_element_value(entry, PCMK__META_MIGRATE_TARGET);
     if ((*source_name == NULL) || (*target_name == NULL)) {
         pcmk__config_err("Ignoring resource history entry %s without "
                          PCMK__META_MIGRATE_SOURCE " and "
                          PCMK__META_MIGRATE_TARGET, pcmk__xe_id(entry));
         return pcmk_rc_unpack_error;
     }
 
     if ((source_node != NULL)
         && !pcmk__str_eq(*source_name, source_node->priv->name,
                          pcmk__str_casei|pcmk__str_null_matches)) {
         pcmk__config_err("Ignoring resource history entry %s because "
                          PCMK__META_MIGRATE_SOURCE "='%s' does not match %s",
                          pcmk__xe_id(entry), *source_name,
                          pcmk__node_name(source_node));
         return pcmk_rc_unpack_error;
     }
 
     if ((target_node != NULL)
         && !pcmk__str_eq(*target_name, target_node->priv->name,
                          pcmk__str_casei|pcmk__str_null_matches)) {
         pcmk__config_err("Ignoring resource history entry %s because "
                          PCMK__META_MIGRATE_TARGET "='%s' does not match %s",
                          pcmk__xe_id(entry), *target_name,
                          pcmk__node_name(target_node));
         return pcmk_rc_unpack_error;
     }
 
     return pcmk_rc_ok;
 }
 
 /*
  * \internal
  * \brief Add a migration source to a resource's list of dangling migrations
  *
  * If the migrate_to and migrate_from actions in a live migration both
  * succeeded, but there is no stop on the source, the migration is considered
  * "dangling." Add the source to the resource's dangling migration list, which
  * will be used to schedule a stop on the source without affecting the target.
  *
  * \param[in,out] rsc   Resource involved in migration
  * \param[in]     node  Migration source
  */
 static void
 add_dangling_migration(pcmk_resource_t *rsc, const pcmk_node_t *node)
 {
     pcmk__rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
                     rsc->id, pcmk__node_name(node));
     rsc->priv->orig_role = pcmk_role_stopped;
     rsc->priv->dangling_migration_sources =
         g_list_prepend(rsc->priv->dangling_migration_sources,
                        (gpointer) node);
 }
 
 /*!
  * \internal
  * \brief Update resource role etc. after a successful migrate_to action
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 unpack_migrate_to_success(struct action_history *history)
 {
     /* A complete migration sequence is:
      * 1. migrate_to on source node (which succeeded if we get to this function)
      * 2. migrate_from on target node
      * 3. stop on source node
      *
      * If no migrate_from has happened, the migration is considered to be
      * "partial". If the migrate_from succeeded but no stop has happened, the
      * migration is considered to be "dangling".
      *
      * If a successful migrate_to and stop have happened on the source node, we
      * still need to check for a partial migration, due to scenarios (easier to
      * produce with batch-limit=1) like:
      *
      * - A resource is migrating from node1 to node2, and a migrate_to is
      *   initiated for it on node1.
      *
      * - node2 goes into standby mode while the migrate_to is pending, which
      *   aborts the transition.
      *
      * - Upon completion of the migrate_to, a new transition schedules a stop
      *   on both nodes and a start on node1.
      *
      * - If the new transition is aborted for any reason while the resource is
      *   stopping on node1, the transition after that stop completes will see
      *   the migrate_to and stop on the source, but it's still a partial
      *   migration, and the resource must be stopped on node2 because it is
      *   potentially active there due to the migrate_to.
      *
      *   We also need to take into account that either node's history may be
      *   cleared at any point in the migration process.
      */
     int from_rc = PCMK_OCF_OK;
     int from_status = PCMK_EXEC_PENDING;
     pcmk_node_t *target_node = NULL;
     xmlNode *migrate_from = NULL;
     const char *source = NULL;
     const char *target = NULL;
     bool source_newer_op = false;
     bool target_newer_state = false;
     bool active_on_target = false;
     pcmk_scheduler_t *scheduler = history->rsc->priv->scheduler;
 
     // Get source and target node names from XML
     if (get_migration_node_names(history->xml, history->node, NULL, &source,
                                  &target) != pcmk_rc_ok) {
         return;
     }
 
     // Check for newer state on the source
     source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
                                         scheduler);
 
     // Check for a migrate_from action from this source on the target
     migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
                                target, source, -1, scheduler);
     if (migrate_from != NULL) {
         if (source_newer_op) {
             /* There's a newer non-monitor operation on the source and a
              * migrate_from on the target, so this migrate_to is irrelevant to
              * the resource's state.
              */
             return;
         }
         crm_element_value_int(migrate_from, PCMK__XA_RC_CODE, &from_rc);
         crm_element_value_int(migrate_from, PCMK__XA_OP_STATUS, &from_status);
     }
 
     /* If the resource has newer state on both the source and target after the
      * migration events, this migrate_to is irrelevant to the resource's state.
      */
     target_newer_state = newer_state_after_migrate(history->rsc->id, target,
                                                    history->xml, migrate_from,
                                                    scheduler);
     if (source_newer_op && target_newer_state) {
         return;
     }
 
     /* Check for dangling migration (migrate_from succeeded but stop not done).
      * We know there's no stop because we already returned if the target has a
      * migrate_from and the source has any newer non-monitor operation.
      */
     if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
         add_dangling_migration(history->rsc, history->node);
         return;
     }
 
     /* Without newer state, this migrate_to implies the resource is active.
      * (Clones are not allowed to migrate, so role can't be promoted.)
      */
     history->rsc->priv->orig_role = pcmk_role_started;
 
     target_node = pcmk_find_node(scheduler, target);
     active_on_target = !target_newer_state && (target_node != NULL)
                        && target_node->details->online;
 
     if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
         if (active_on_target) {
             native_add_running(history->rsc, target_node, scheduler, TRUE);
         } else {
             // Mark resource as failed, require recovery, and prevent migration
             pcmk__set_rsc_flags(history->rsc,
                                 pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
             pcmk__clear_rsc_flags(history->rsc, pcmk__rsc_migratable);
         }
         return;
     }
 
     // The migrate_from is pending, complete but erased, or to be scheduled
 
     /* If there is no history at all for the resource on an online target, then
      * it was likely cleaned. Just return, and we'll schedule a probe. Once we
      * have the probe result, it will be reflected in target_newer_state.
      */
     if ((target_node != NULL) && target_node->details->online
         && unknown_on_node(history->rsc, target)) {
         return;
     }
 
     if (active_on_target) {
         pcmk_node_t *source_node = pcmk_find_node(scheduler, source);
 
         native_add_running(history->rsc, target_node, scheduler, FALSE);
         if ((source_node != NULL) && source_node->details->online) {
             /* This is a partial migration: the migrate_to completed
              * successfully on the source, but the migrate_from has not
              * completed. Remember the source and target; if the newly
              * chosen target remains the same when we schedule actions
              * later, we may continue with the migration.
              */
             history->rsc->priv->partial_migration_target = target_node;
             history->rsc->priv->partial_migration_source = source_node;
         }
 
     } else if (!source_newer_op) {
         // Mark resource as failed, require recovery, and prevent migration
         pcmk__set_rsc_flags(history->rsc,
                             pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
         pcmk__clear_rsc_flags(history->rsc, pcmk__rsc_migratable);
     }
 }
 
 /*!
  * \internal
  * \brief Update resource role etc. after a failed migrate_to action
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 unpack_migrate_to_failure(struct action_history *history)
 {
     xmlNode *target_migrate_from = NULL;
     const char *source = NULL;
     const char *target = NULL;
     pcmk_scheduler_t *scheduler = history->rsc->priv->scheduler;
 
     // Get source and target node names from XML
     if (get_migration_node_names(history->xml, history->node, NULL, &source,
                                  &target) != pcmk_rc_ok) {
         return;
     }
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be promoted.
      */
     history->rsc->priv->orig_role = pcmk_role_started;
 
     // Check for migrate_from on the target
     target_migrate_from = find_lrm_op(history->rsc->id,
                                       PCMK_ACTION_MIGRATE_FROM, target, source,
                                       PCMK_OCF_OK, scheduler);
 
     if (/* If the resource state is unknown on the target, it will likely be
          * probed there.
          * Don't just consider it running there. We will get back here anyway in
          * case the probe detects it's running there.
          */
         !unknown_on_node(history->rsc, target)
         /* If the resource has newer state on the target after the migration
          * events, this migrate_to no longer matters for the target.
          */
         && !newer_state_after_migrate(history->rsc->id, target, history->xml,
                                       target_migrate_from, scheduler)) {
         /* The resource has no newer state on the target, so assume it's still
          * active there.
          * (if it is up).
          */
         pcmk_node_t *target_node = pcmk_find_node(scheduler, target);
 
         if (target_node && target_node->details->online) {
             native_add_running(history->rsc, target_node, scheduler, FALSE);
         }
 
     } else if (!non_monitor_after(history->rsc->id, source, history->xml,
                                   scheduler)) {
         /* We know the resource has newer state on the target, but this
          * migrate_to still matters for the source as long as there's no newer
          * non-monitor operation there.
          */
 
         // Mark node as having dangling migration so we can force a stop later
         history->rsc->priv->dangling_migration_sources =
             g_list_prepend(history->rsc->priv->dangling_migration_sources,
                            (gpointer) history->node);
     }
 }
 
 /*!
  * \internal
  * \brief Update resource role etc. after a failed migrate_from action
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 unpack_migrate_from_failure(struct action_history *history)
 {
     xmlNode *source_migrate_to = NULL;
     const char *source = NULL;
     const char *target = NULL;
     pcmk_scheduler_t *scheduler = history->rsc->priv->scheduler;
 
     // Get source and target node names from XML
     if (get_migration_node_names(history->xml, NULL, history->node, &source,
                                  &target) != pcmk_rc_ok) {
         return;
     }
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be promoted.
      */
     history->rsc->priv->orig_role = pcmk_role_started;
 
     // Check for a migrate_to on the source
     source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
                                     source, target, PCMK_OCF_OK, scheduler);
 
     if (/* If the resource state is unknown on the source, it will likely be
          * probed there.
          * Don't just consider it running there. We will get back here anyway in
          * case the probe detects it's running there.
          */
         !unknown_on_node(history->rsc, source)
         /* If the resource has newer state on the source after the migration
          * events, this migrate_from no longer matters for the source.
          */
         && !newer_state_after_migrate(history->rsc->id, source,
                                       source_migrate_to, history->xml,
                                       scheduler)) {
         /* The resource has no newer state on the source, so assume it's still
          * active there (if it is up).
          */
         pcmk_node_t *source_node = pcmk_find_node(scheduler, source);
 
         if (source_node && source_node->details->online) {
             native_add_running(history->rsc, source_node, scheduler, TRUE);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Add an action to cluster's list of failed actions
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 record_failed_op(struct action_history *history)
 {
     const pcmk_scheduler_t *scheduler = history->rsc->priv->scheduler;
 
     if (!(history->node->details->online)) {
         return;
     }
 
     for (const xmlNode *xIter = scheduler->priv->failed->children;
          xIter != NULL; xIter = xIter->next) {
 
         const char *key = pcmk__xe_history_key(xIter);
         const char *uname = crm_element_value(xIter, PCMK_XA_UNAME);
 
         if (pcmk__str_eq(history->key, key, pcmk__str_none)
             && pcmk__str_eq(uname, history->node->priv->name,
                             pcmk__str_casei)) {
             crm_trace("Skipping duplicate entry %s on %s",
                       history->key, pcmk__node_name(history->node));
             return;
         }
     }
 
     crm_trace("Adding entry for %s on %s to failed action list",
               history->key, pcmk__node_name(history->node));
     crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->priv->name);
     crm_xml_add(history->xml, PCMK__XA_RSC_ID, history->rsc->id);
     pcmk__xml_copy(scheduler->priv->failed, history->xml);
 }
 
 static char *
 last_change_str(const xmlNode *xml_op)
 {
     time_t when;
     char *result = NULL;
 
     if (crm_element_value_epoch(xml_op, PCMK_XA_LAST_RC_CHANGE,
                                 &when) == pcmk_ok) {
         char *when_s = pcmk__epoch2str(&when, 0);
         const char *p = strchr(when_s, ' ');
 
         // Skip day of week to make message shorter
         if ((p != NULL) && (*(++p) != '\0')) {
             result = pcmk__str_copy(p);
         }
         free(when_s);
     }
 
     if (result == NULL) {
         result = pcmk__str_copy("unknown_time");
     }
 
     return result;
 }
 
 /*!
  * \internal
  * \brief Ban a resource (or its clone if an anonymous instance) from all nodes
  *
  * \param[in,out] rsc  Resource to ban
  */
 static void
 ban_from_all_nodes(pcmk_resource_t *rsc)
 {
     int score = -PCMK_SCORE_INFINITY;
     const pcmk_scheduler_t *scheduler = rsc->priv->scheduler;
 
     if (rsc->priv->parent != NULL) {
         pcmk_resource_t *parent = uber_parent(rsc);
 
         if (pcmk__is_anonymous_clone(parent)) {
             /* For anonymous clones, if an operation with
              * PCMK_META_ON_FAIL=PCMK_VALUE_STOP fails for any instance, the
              * entire clone must stop.
              */
             rsc = parent;
         }
     }
 
     // Ban the resource from all nodes
     crm_notice("%s will not be started under current conditions", rsc->id);
     if (rsc->priv->allowed_nodes != NULL) {
         g_hash_table_destroy(rsc->priv->allowed_nodes);
     }
     rsc->priv->allowed_nodes = pe__node_list2table(scheduler->nodes);
     g_hash_table_foreach(rsc->priv->allowed_nodes, set_node_score, &score);
 }
 
 /*!
  * \internal
  * \brief Get configured failure handling and role after failure for an action
  *
  * \param[in,out] history    Unpacked action history entry
  * \param[out]    on_fail    Where to set configured failure handling
  * \param[out]    fail_role  Where to set to role after failure
  */
 static void
 unpack_failure_handling(struct action_history *history,
                         enum pcmk__on_fail *on_fail,
                         enum rsc_role_e *fail_role)
 {
     xmlNode *config = pcmk__find_action_config(history->rsc, history->task,
                                                history->interval_ms, true);
 
     GHashTable *meta = pcmk__unpack_action_meta(history->rsc, history->node,
                                                 history->task,
                                                 history->interval_ms, config);
 
     const char *on_fail_str = g_hash_table_lookup(meta, PCMK_META_ON_FAIL);
 
     *on_fail = pcmk__parse_on_fail(history->rsc, history->task,
                                    history->interval_ms, on_fail_str);
     *fail_role = pcmk__role_after_failure(history->rsc, history->task, *on_fail,
                                           meta);
     g_hash_table_destroy(meta);
 }
 
 /*!
  * \internal
  * \brief Update resource role, failure handling, etc., after a failed action
  *
  * \param[in,out] history         Parsed action result history
  * \param[in]     config_on_fail  Action failure handling from configuration
  * \param[in]     fail_role       Resource's role after failure of this action
  * \param[out]    last_failure    This will be set to the history XML
  * \param[in,out] on_fail         Actual handling of action result
  */
 static void
 unpack_rsc_op_failure(struct action_history *history,
                       enum pcmk__on_fail config_on_fail,
                       enum rsc_role_e fail_role, xmlNode **last_failure,
                       enum pcmk__on_fail *on_fail)
 {
     bool is_probe = false;
     char *last_change_s = NULL;
     pcmk_scheduler_t *scheduler = history->rsc->priv->scheduler;
 
     *last_failure = history->xml;
 
     is_probe = pcmk_xe_is_probe(history->xml);
     last_change_s = last_change_str(history->xml);
 
     if (!pcmk_is_set(scheduler->flags, pcmk__sched_symmetric_cluster)
         && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
         crm_trace("Unexpected result (%s%s%s) was recorded for "
                   "%s of %s on %s at %s " QB_XS " exit-status=%d id=%s",
                   crm_exit_str(history->exit_status),
                   (pcmk__str_empty(history->exit_reason)? "" : ": "),
                   pcmk__s(history->exit_reason, ""),
                   (is_probe? "probe" : history->task), history->rsc->id,
                   pcmk__node_name(history->node), last_change_s,
                   history->exit_status, history->id);
     } else {
         pcmk__sched_warn(scheduler,
                          "Unexpected result (%s%s%s) was recorded for %s of "
                          "%s on %s at %s " QB_XS " exit-status=%d id=%s",
                          crm_exit_str(history->exit_status),
                          (pcmk__str_empty(history->exit_reason)? "" : ": "),
                          pcmk__s(history->exit_reason, ""),
                          (is_probe? "probe" : history->task), history->rsc->id,
                          pcmk__node_name(history->node), last_change_s,
                          history->exit_status, history->id);
 
         if (is_probe && (history->exit_status != PCMK_OCF_OK)
             && (history->exit_status != PCMK_OCF_NOT_RUNNING)
             && (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {
 
             /* A failed (not just unexpected) probe result could mean the user
              * didn't know resources will be probed even where they can't run.
              */
             crm_notice("If it is not possible for %s to run on %s, see "
                        "the " PCMK_XA_RESOURCE_DISCOVERY " option for location "
                        "constraints",
                        history->rsc->id, pcmk__node_name(history->node));
         }
 
         record_failed_op(history);
     }
 
     free(last_change_s);
 
     if (*on_fail < config_on_fail) {
         pcmk__rsc_trace(history->rsc, "on-fail %s -> %s for %s",
                         pcmk__on_fail_text(*on_fail),
                         pcmk__on_fail_text(config_on_fail), history->key);
         *on_fail = config_on_fail;
     }
 
     if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
         resource_location(history->rsc, history->node, -PCMK_SCORE_INFINITY,
                           "__stop_fail__", scheduler);
 
     } else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
         unpack_migrate_to_failure(history);
 
     } else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
         unpack_migrate_from_failure(history);
 
     } else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
         history->rsc->priv->orig_role = pcmk_role_promoted;
 
     } else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
         if (config_on_fail == pcmk__on_fail_block) {
             history->rsc->priv->orig_role = pcmk_role_promoted;
             pe__set_next_role(history->rsc, pcmk_role_stopped,
                               "demote with " PCMK_META_ON_FAIL "=block");
 
         } else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
             history->rsc->priv->orig_role = pcmk_role_stopped;
 
         } else {
             /* Staying in the promoted role would put the scheduler and
              * controller into a loop. Setting the role to unpromoted is not
              * dangerous because the resource will be stopped as part of
              * recovery, and any promotion will be ordered after that stop.
              */
             history->rsc->priv->orig_role = pcmk_role_unpromoted;
         }
     }
 
     if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
         /* leave stopped */
         pcmk__rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
         history->rsc->priv->orig_role = pcmk_role_stopped;
 
     } else if (history->rsc->priv->orig_role < pcmk_role_started) {
         pcmk__rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
         set_active(history->rsc);
     }
 
     pcmk__rsc_trace(history->rsc,
                     "Resource %s: role=%s unclean=%s on_fail=%s fail_role=%s",
                     history->rsc->id,
                     pcmk_role_text(history->rsc->priv->orig_role),
                     pcmk__btoa(history->node->details->unclean),
                     pcmk__on_fail_text(config_on_fail),
                     pcmk_role_text(fail_role));
 
     if ((fail_role != pcmk_role_started)
         && (history->rsc->priv->next_role < fail_role)) {
         pe__set_next_role(history->rsc, fail_role, "failure");
     }
 
     if (fail_role == pcmk_role_stopped) {
         ban_from_all_nodes(history->rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Block a resource with a failed action if it cannot be recovered
  *
  * If resource action is a failed stop and fencing is not possible, mark the
  * resource as unmanaged and blocked, since recovery cannot be done.
  *
  * \param[in,out] history  Parsed action history entry
  */
 static void
 block_if_unrecoverable(struct action_history *history)
 {
     char *last_change_s = NULL;
 
     if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
         return; // All actions besides stop are always recoverable
     }
     if (pe_can_fence(history->node->priv->scheduler, history->node)) {
         return; // Failed stops are recoverable via fencing
     }
 
     last_change_s = last_change_str(history->xml);
     pcmk__sched_err(history->node->priv->scheduler,
                     "No further recovery can be attempted for %s "
                     "because %s on %s failed (%s%s%s) at %s "
                     QB_XS " rc=%d id=%s",
                     history->rsc->id, history->task,
                     pcmk__node_name(history->node),
                     crm_exit_str(history->exit_status),
                     (pcmk__str_empty(history->exit_reason)? "" : ": "),
                     pcmk__s(history->exit_reason, ""),
                     last_change_s, history->exit_status, history->id);
 
     free(last_change_s);
 
     pcmk__clear_rsc_flags(history->rsc, pcmk__rsc_managed);
     pcmk__set_rsc_flags(history->rsc, pcmk__rsc_blocked);
 }
 
 /*!
  * \internal
  * \brief Update action history's execution status and why
  *
  * \param[in,out] history  Parsed action history entry
  * \param[out]    why      Where to store reason for update
  * \param[in]     value    New value
  * \param[in]     reason   Description of why value was changed
  */
 static inline void
 remap_because(struct action_history *history, const char **why, int value,
               const char *reason)
 {
     if (history->execution_status != value) {
         history->execution_status = value;
         *why = reason;
     }
 }
 
 /*!
  * \internal
  * \brief Remap informational monitor results and operation status
  *
  * For the monitor results, certain OCF codes are for providing extended information
  * to the user about services that aren't yet failed but not entirely healthy either.
  * These must be treated as the "normal" result by Pacemaker.
  *
  * For operation status, the action result can be used to determine an appropriate
  * status for the purposes of responding to the action.  The status provided by the
  * executor is not directly usable since the executor does not know what was expected.
  *
  * \param[in,out] history  Parsed action history entry
  * \param[in,out] on_fail  What should be done about the result
  * \param[in]     expired  Whether result is expired
  *
  * \note If the result is remapped and the node is not shutting down or failed,
  *       the operation will be recorded in the scheduler data's list of failed
  *       operations to highlight it for the user.
  *
  * \note This may update the resource's current and next role.
  */
 static void
 remap_operation(struct action_history *history,
                 enum pcmk__on_fail *on_fail, bool expired)
 {
+    /* @TODO It would probably also be a good idea to map an exit status of
+     * CRM_EX_PROMOTED or CRM_EX_DEGRADED_PROMOTED to CRM_EX_OK for promote
+     * actions
+     */
+
     bool is_probe = false;
     int orig_exit_status = history->exit_status;
     int orig_exec_status = history->execution_status;
     const char *why = NULL;
     const char *task = history->task;
 
     // Remap degraded results to their successful counterparts
     history->exit_status = pcmk__effective_rc(history->exit_status);
     if (history->exit_status != orig_exit_status) {
         why = "degraded result";
         if (!expired && (!history->node->details->shutdown
                          || history->node->details->online)) {
             record_failed_op(history);
         }
     }
 
     if (!pcmk__is_bundled(history->rsc)
         && pcmk_xe_mask_probe_failure(history->xml)
         && ((history->execution_status != PCMK_EXEC_DONE)
             || (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
         history->execution_status = PCMK_EXEC_DONE;
         history->exit_status = PCMK_OCF_NOT_RUNNING;
         why = "equivalent probe result";
     }
 
     /* If the executor reported an execution status of anything but done or
      * error, consider that final. But for done or error, we know better whether
      * it should be treated as a failure or not, because we know the expected
      * result.
      */
     switch (history->execution_status) {
         case PCMK_EXEC_DONE:
         case PCMK_EXEC_ERROR:
             break;
 
         // These should be treated as node-fatal
         case PCMK_EXEC_NO_FENCE_DEVICE:
         case PCMK_EXEC_NO_SECRETS:
             remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
                           "node-fatal error");
             goto remap_done;
 
         default:
             goto remap_done;
     }
 
     is_probe = pcmk_xe_is_probe(history->xml);
     if (is_probe) {
         task = "probe";
     }
 
     if (history->expected_exit_status < 0) {
         /* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
          * Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
          * expected exit status in the transition key, which (along with the
          * similar case of a corrupted transition key in the CIB) will be
          * reported to this function as -1. Pacemaker 2.0+ does not support
          * rolling upgrades from those versions or processing of saved CIB files
          * from those versions, so we do not need to care much about this case.
          */
         remap_because(history, &why, PCMK_EXEC_ERROR,
                       "obsolete history format");
         pcmk__config_warn("Expected result not found for %s on %s "
                           "(corrupt or obsolete CIB?)",
                           history->key, pcmk__node_name(history->node));
 
     } else if (history->exit_status == history->expected_exit_status) {
         remap_because(history, &why, PCMK_EXEC_DONE, "expected result");
 
     } else {
         remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
         pcmk__rsc_debug(history->rsc,
                         "%s on %s: expected %d (%s), got %d (%s%s%s)",
                         history->key, pcmk__node_name(history->node),
                         history->expected_exit_status,
                         crm_exit_str(history->expected_exit_status),
                         history->exit_status,
                         crm_exit_str(history->exit_status),
                         (pcmk__str_empty(history->exit_reason)? "" : ": "),
                         pcmk__s(history->exit_reason, ""));
     }
 
     switch (history->exit_status) {
         case PCMK_OCF_OK:
             if (is_probe
                 && (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
                 char *last_change_s = last_change_str(history->xml);
 
                 remap_because(history, &why, PCMK_EXEC_DONE, "probe");
                 pcmk__rsc_info(history->rsc,
                                "Probe found %s active on %s at %s",
                                history->rsc->id, pcmk__node_name(history->node),
                                last_change_s);
                 free(last_change_s);
             }
             break;
 
         case PCMK_OCF_NOT_RUNNING:
             if (is_probe
                 || (history->expected_exit_status == history->exit_status)
                 || !pcmk_is_set(history->rsc->flags, pcmk__rsc_managed)) {
 
                 /* For probes, recurring monitors for the Stopped role, and
                  * unmanaged resources, "not running" is not considered a
                  * failure.
                  */
                 remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
                 history->rsc->priv->orig_role = pcmk_role_stopped;
                 *on_fail = pcmk__on_fail_ignore;
                 pe__set_next_role(history->rsc, pcmk_role_unknown,
                                   "not running");
             }
             break;
 
         case PCMK_OCF_RUNNING_PROMOTED:
             if (is_probe
                 && (history->exit_status != history->expected_exit_status)) {
                 char *last_change_s = last_change_str(history->xml);
 
                 remap_because(history, &why, PCMK_EXEC_DONE, "probe");
                 pcmk__rsc_info(history->rsc,
                                "Probe found %s active and promoted on %s at %s",
                                 history->rsc->id,
                                 pcmk__node_name(history->node), last_change_s);
                 free(last_change_s);
             }
             if (!expired
                 || (history->exit_status == history->expected_exit_status)) {
                 history->rsc->priv->orig_role = pcmk_role_promoted;
             }
             break;
 
         case PCMK_OCF_FAILED_PROMOTED:
             if (!expired) {
                 history->rsc->priv->orig_role = pcmk_role_promoted;
             }
             remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
             break;
 
         case PCMK_OCF_NOT_CONFIGURED:
             remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
             break;
 
         case PCMK_OCF_UNIMPLEMENT_FEATURE:
             {
                 guint interval_ms = 0;
                 crm_element_value_ms(history->xml, PCMK_META_INTERVAL,
                                      &interval_ms);
 
                 if (interval_ms == 0) {
                     if (!expired) {
                         block_if_unrecoverable(history);
                     }
                     remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
                                   "exit status");
                 } else {
                     remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
                                   "exit status");
                 }
             }
             break;
 
         case PCMK_OCF_NOT_INSTALLED:
         case PCMK_OCF_INVALID_PARAM:
         case PCMK_OCF_INSUFFICIENT_PRIV:
             if (!expired) {
                 block_if_unrecoverable(history);
             }
             remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
             break;
 
         default:
             if (history->execution_status == PCMK_EXEC_DONE) {
                 char *last_change_s = last_change_str(history->xml);
 
                 crm_info("Treating unknown exit status %d from %s of %s "
                          "on %s at %s as failure",
                          history->exit_status, task, history->rsc->id,
                          pcmk__node_name(history->node), last_change_s);
                 remap_because(history, &why, PCMK_EXEC_ERROR,
                               "unknown exit status");
                 free(last_change_s);
             }
             break;
     }
 
 remap_done:
     if (why != NULL) {
         pcmk__rsc_trace(history->rsc,
                         "Remapped %s result from [%s: %s] to [%s: %s] "
                         "because of %s",
                         history->key, pcmk_exec_status_str(orig_exec_status),
                         crm_exit_str(orig_exit_status),
                         pcmk_exec_status_str(history->execution_status),
                         crm_exit_str(history->exit_status), why);
     }
 }
 
 // return TRUE if start or monitor last failure but parameters changed
 static bool
 should_clear_for_param_change(const xmlNode *xml_op, const char *task,
                               pcmk_resource_t *rsc, pcmk_node_t *node)
 {
     if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
         if (pe__bundle_needs_remote_name(rsc)) {
             /* We haven't allocated resources yet, so we can't reliably
              * substitute addr parameters for the REMOTE_CONTAINER_HACK.
              * When that's needed, defer the check until later.
              */
             pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
                                 rsc->priv->scheduler);
 
         } else {
             pcmk__op_digest_t *digest_data = NULL;
 
             digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
                                                 rsc->priv->scheduler);
             switch (digest_data->rc) {
                 case pcmk__digest_unknown:
                     crm_trace("Resource %s history entry %s on %s"
                               " has no digest to compare",
                               rsc->id, pcmk__xe_history_key(xml_op),
                               node->priv->id);
                     break;
                 case pcmk__digest_match:
                     break;
                 default:
                     return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 // Order action after fencing of remote node, given connection rsc
 static void
 order_after_remote_fencing(pcmk_action_t *action, pcmk_resource_t *remote_conn,
                            pcmk_scheduler_t *scheduler)
 {
     pcmk_node_t *remote_node = pcmk_find_node(scheduler, remote_conn->id);
 
     if (remote_node) {
         pcmk_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
                                            FALSE, scheduler);
 
         order_actions(fence, action, pcmk__ar_first_implies_then);
     }
 }
 
 static bool
 should_ignore_failure_timeout(const pcmk_resource_t *rsc, const char *task,
                               guint interval_ms, bool is_last_failure)
 {
     /* Clearing failures of recurring monitors has special concerns. The
      * executor reports only changes in the monitor result, so if the
      * monitor is still active and still getting the same failure result,
      * that will go undetected after the failure is cleared.
      *
      * Also, the operation history will have the time when the recurring
      * monitor result changed to the given code, not the time when the
      * result last happened.
      *
      * @TODO We probably should clear such failures only when the failure
      * timeout has passed since the last occurrence of the failed result.
      * However we don't record that information. We could maybe approximate
      * that by clearing only if there is a more recent successful monitor or
      * stop result, but we don't even have that information at this point
      * since we are still unpacking the resource's operation history.
      *
      * This is especially important for remote connection resources with a
      * reconnect interval, so in that case, we skip clearing failures
      * if the remote node hasn't been fenced.
      */
     if ((rsc->priv->remote_reconnect_ms > 0U)
         && pcmk_is_set(rsc->priv->scheduler->flags,
                        pcmk__sched_fencing_enabled)
         && (interval_ms != 0)
         && pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
 
         pcmk_node_t *remote_node = pcmk_find_node(rsc->priv->scheduler,
                                                   rsc->id);
 
         if (remote_node && !pcmk_is_set(remote_node->priv->flags,
                                         pcmk__node_remote_fenced)) {
             if (is_last_failure) {
                 crm_info("Waiting to clear monitor failure for remote node %s"
                          " until fencing has occurred", rsc->id);
             }
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Check operation age and schedule failure clearing when appropriate
  *
  * This function has two distinct purposes. The first is to check whether an
  * operation history entry is expired (i.e. the resource has a failure timeout,
  * the entry is older than the timeout, and the resource either has no fail
  * count or its fail count is entirely older than the timeout). The second is to
  * schedule fail count clearing when appropriate (i.e. the operation is expired
  * and either the resource has an expired fail count or the operation is a
  * last_failure for a remote connection resource with a reconnect interval,
  * or the operation is a last_failure for a start or monitor operation and the
  * resource's parameters have changed since the operation).
  *
  * \param[in,out] history  Parsed action result history
  *
  * \return true if operation history entry is expired, otherwise false
  */
 static bool
 check_operation_expiry(struct action_history *history)
 {
     bool expired = false;
     bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
     time_t last_run = 0;
     int unexpired_fail_count = 0;
     const char *clear_reason = NULL;
     const guint expiration_sec =
         pcmk__timeout_ms2s(history->rsc->priv->failure_expiration_ms);
     pcmk_scheduler_t *scheduler = history->rsc->priv->scheduler;
 
     if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
         pcmk__rsc_trace(history->rsc,
                         "Resource history entry %s on %s is not expired: "
                         "Not Installed does not expire",
                         history->id, pcmk__node_name(history->node));
         return false; // "Not installed" must always be cleared manually
     }
 
     if ((expiration_sec > 0)
         && (crm_element_value_epoch(history->xml, PCMK_XA_LAST_RC_CHANGE,
                                     &last_run) == 0)) {
 
         /* Resource has a PCMK_META_FAILURE_TIMEOUT and history entry has a
          * timestamp
          */
 
         time_t now = get_effective_time(scheduler);
         time_t last_failure = 0;
 
         // Is this particular operation history older than the failure timeout?
         if ((now >= (last_run + expiration_sec))
             && !should_ignore_failure_timeout(history->rsc, history->task,
                                               history->interval_ms,
                                               is_last_failure)) {
             expired = true;
         }
 
         // Does the resource as a whole have an unexpired fail count?
         unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
                                                 &last_failure,
                                                 pcmk__fc_effective,
                                                 history->xml);
 
         // Update scheduler recheck time according to *last* failure
         crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d "
                   "expiration=%s last-failure@%lld",
                   history->id, (long long) last_run, (expired? "" : "not "),
                   (long long) now, unexpired_fail_count,
                   pcmk__readable_interval(expiration_sec * 1000),
                   (long long) last_failure);
         last_failure += expiration_sec + 1;
         if (unexpired_fail_count && (now < last_failure)) {
             pe__update_recheck_time(last_failure, scheduler,
                                     "fail count expiration");
         }
     }
 
     if (expired) {
         if (pe_get_failcount(history->node, history->rsc, NULL,
                              pcmk__fc_default, history->xml)) {
             // There is a fail count ignoring timeout
 
             if (unexpired_fail_count == 0) {
                 // There is no fail count considering timeout
                 clear_reason = "it expired";
 
             } else {
                 /* This operation is old, but there is an unexpired fail count.
                  * In a properly functioning cluster, this should only be
                  * possible if this operation is not a failure (otherwise the
                  * fail count should be expired too), so this is really just a
                  * failsafe.
                  */
                 pcmk__rsc_trace(history->rsc,
                                 "Resource history entry %s on %s is not "
                                 "expired: Unexpired fail count",
                                 history->id, pcmk__node_name(history->node));
                 expired = false;
             }
 
         } else if (is_last_failure
                    && (history->rsc->priv->remote_reconnect_ms > 0U)) {
             /* Clear any expired last failure when reconnect interval is set,
              * even if there is no fail count.
              */
             clear_reason = "reconnect interval is set";
         }
     }
 
     if (!expired && is_last_failure
         && should_clear_for_param_change(history->xml, history->task,
                                          history->rsc, history->node)) {
         clear_reason = "resource parameters have changed";
     }
 
     if (clear_reason != NULL) {
         pcmk_action_t *clear_op = NULL;
 
         // Schedule clearing of the fail count
         clear_op = pe__clear_failcount(history->rsc, history->node,
                                        clear_reason, scheduler);
 
         if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)
             && (history->rsc->priv->remote_reconnect_ms > 0)) {
             /* If we're clearing a remote connection due to a reconnect
              * interval, we want to wait until any scheduled fencing
              * completes.
              *
              * We could limit this to remote_node->details->unclean, but at
              * this point, that's always true (it won't be reliable until
              * after unpack_node_history() is done).
              */
             crm_info("Clearing %s failure will wait until any scheduled "
                      "fencing of %s completes",
                      history->task, history->rsc->id);
             order_after_remote_fencing(clear_op, history->rsc, scheduler);
         }
     }
 
     if (expired && (history->interval_ms == 0)
         && pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
         switch (history->exit_status) {
             case PCMK_OCF_OK:
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_PROMOTED:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_PROMOTED:
                 // Don't expire probes that return these values
                 pcmk__rsc_trace(history->rsc,
                                 "Resource history entry %s on %s is not "
                                 "expired: Probe result",
                              history->id, pcmk__node_name(history->node));
                 expired = false;
                 break;
         }
     }
 
     return expired;
 }
 
 int
 pe__target_rc_from_xml(const xmlNode *xml_op)
 {
     int target_rc = 0;
     const char *key = crm_element_value(xml_op, PCMK__XA_TRANSITION_KEY);
 
     if (key == NULL) {
         return -1;
     }
     decode_transition_key(key, NULL, NULL, NULL, &target_rc);
     return target_rc;
 }
 
 /*!
  * \internal
  * \brief Update a resource's state for an action result
  *
  * \param[in,out] history       Parsed action history entry
  * \param[in]     exit_status   Exit status to base new state on
  * \param[in]     last_failure  Resource's last_failure entry, if known
  * \param[in,out] on_fail       Resource's current failure handling
  */
 static void
 update_resource_state(struct action_history *history, int exit_status,
                       const xmlNode *last_failure,
                       enum pcmk__on_fail *on_fail)
 {
     bool clear_past_failure = false;
 
     if ((exit_status == PCMK_OCF_NOT_INSTALLED)
         || (!pcmk__is_bundled(history->rsc)
             && pcmk_xe_mask_probe_failure(history->xml))) {
         history->rsc->priv->orig_role = pcmk_role_stopped;
 
     } else if (exit_status == PCMK_OCF_NOT_RUNNING) {
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
                             pcmk__str_none)) {
         if ((last_failure != NULL)
             && pcmk__str_eq(history->key, pcmk__xe_history_key(last_failure),
                             pcmk__str_none)) {
             clear_past_failure = true;
         }
         if (history->rsc->priv->orig_role < pcmk_role_started) {
             set_active(history->rsc);
         }
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
         history->rsc->priv->orig_role = pcmk_role_started;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
         history->rsc->priv->orig_role = pcmk_role_stopped;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
                             pcmk__str_none)) {
         history->rsc->priv->orig_role = pcmk_role_promoted;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
                             pcmk__str_none)) {
         if (*on_fail == pcmk__on_fail_demote) {
             /* Demote clears an error only if
              * PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE
              */
             clear_past_failure = true;
         }
         history->rsc->priv->orig_role = pcmk_role_unpromoted;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
                             pcmk__str_none)) {
         history->rsc->priv->orig_role = pcmk_role_started;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
                             pcmk__str_none)) {
         unpack_migrate_to_success(history);
 
     } else if (history->rsc->priv->orig_role < pcmk_role_started) {
         pcmk__rsc_trace(history->rsc, "%s active on %s",
                         history->rsc->id, pcmk__node_name(history->node));
         set_active(history->rsc);
     }
 
     if (!clear_past_failure) {
         return;
     }
 
     switch (*on_fail) {
         case pcmk__on_fail_stop:
         case pcmk__on_fail_ban:
         case pcmk__on_fail_standby_node:
         case pcmk__on_fail_fence_node:
             pcmk__rsc_trace(history->rsc,
                             "%s (%s) is not cleared by a completed %s",
                             history->rsc->id, pcmk__on_fail_text(*on_fail),
                             history->task);
             break;
 
         case pcmk__on_fail_block:
         case pcmk__on_fail_ignore:
         case pcmk__on_fail_demote:
         case pcmk__on_fail_restart:
         case pcmk__on_fail_restart_container:
             *on_fail = pcmk__on_fail_ignore;
             pe__set_next_role(history->rsc, pcmk_role_unknown,
                               "clear past failures");
             break;
 
         case pcmk__on_fail_reset_remote:
             if (history->rsc->priv->remote_reconnect_ms == 0U) {
                 /* With no reconnect interval, the connection is allowed to
                  * start again after the remote node is fenced and
                  * completely stopped. (With a reconnect interval, we wait
                  * for the failure to be cleared entirely before attempting
                  * to reconnect.)
                  */
                 *on_fail = pcmk__on_fail_ignore;
                 pe__set_next_role(history->rsc, pcmk_role_unknown,
                                   "clear past failures and reset remote");
             }
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a given history entry matters for resource state
  *
  * \param[in] history  Parsed action history entry
  *
  * \return true if action can affect resource state, otherwise false
  */
 static inline bool
 can_affect_state(struct action_history *history)
 {
      return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
                              PCMK_ACTION_START, PCMK_ACTION_STOP,
                              PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
                              PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
                              "asyncmon", NULL);
 }
 
 /*!
  * \internal
  * \brief Unpack execution/exit status and exit reason from a history entry
  *
  * \param[in,out] history  Action history entry to unpack
  *
  * \return Standard Pacemaker return code
  */
 static int
 unpack_action_result(struct action_history *history)
 {
     if ((crm_element_value_int(history->xml, PCMK__XA_OP_STATUS,
                                &(history->execution_status)) < 0)
         || (history->execution_status < PCMK_EXEC_PENDING)
         || (history->execution_status > PCMK_EXEC_MAX)
         || (history->execution_status == PCMK_EXEC_CANCELLED)) {
         pcmk__config_err("Ignoring resource history entry %s for %s on %s "
                          "with invalid " PCMK__XA_OP_STATUS " '%s'",
                          history->id, history->rsc->id,
                          pcmk__node_name(history->node),
                          pcmk__s(crm_element_value(history->xml,
                                                    PCMK__XA_OP_STATUS),
                                  ""));
         return pcmk_rc_unpack_error;
     }
     if ((crm_element_value_int(history->xml, PCMK__XA_RC_CODE,
                                &(history->exit_status)) < 0)
         || (history->exit_status < 0) || (history->exit_status > CRM_EX_MAX)) {
         pcmk__config_err("Ignoring resource history entry %s for %s on %s "
                          "with invalid " PCMK__XA_RC_CODE " '%s'",
                          history->id, history->rsc->id,
                          pcmk__node_name(history->node),
                          pcmk__s(crm_element_value(history->xml,
                                                    PCMK__XA_RC_CODE),
                                  ""));
         return pcmk_rc_unpack_error;
     }
     history->exit_reason = crm_element_value(history->xml, PCMK_XA_EXIT_REASON);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Process an action history entry whose result expired
  *
  * \param[in,out] history           Parsed action history entry
  * \param[in]     orig_exit_status  Action exit status before remapping
  *
  * \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
  *         entry needs no further processing)
  */
 static int
 process_expired_result(struct action_history *history, int orig_exit_status)
 {
     if (!pcmk__is_bundled(history->rsc)
         && pcmk_xe_mask_probe_failure(history->xml)
         && (orig_exit_status != history->expected_exit_status)) {
 
         if (history->rsc->priv->orig_role <= pcmk_role_stopped) {
             history->rsc->priv->orig_role = pcmk_role_unknown;
         }
         crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
                   "Masked failure expired",
                   history->id, history->rsc->id,
                   pcmk__node_name(history->node));
         return pcmk_rc_ok;
     }
 
     if (history->exit_status == history->expected_exit_status) {
         return pcmk_rc_undetermined; // Only failures expire
     }
 
     if (history->interval_ms == 0) {
         crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
                    "Expired failure",
                    history->id, history->task, history->rsc->id,
                    pcmk__node_name(history->node));
         return pcmk_rc_ok;
     }
 
     if (history->node->details->online && !history->node->details->unclean) {
         /* Reschedule the recurring action. schedule_cancel() won't work at
          * this stage, so as a hacky workaround, forcibly change the restart
          * digest so pcmk__check_action_config() does what we want later.
          *
          * @TODO We should skip this if there is a newer successful monitor.
          *       Also, this causes rescheduling only if the history entry
          *       has a PCMK__XA_OP_DIGEST (which the expire-non-blocked-failure
          *       scheduler regression test doesn't, but that may not be a
          *       realistic scenario in production).
          */
         crm_notice("Rescheduling %s-interval %s of %s on %s "
                    "after failure expired",
                    pcmk__readable_interval(history->interval_ms), history->task,
                    history->rsc->id, pcmk__node_name(history->node));
         crm_xml_add(history->xml, PCMK__XA_OP_RESTART_DIGEST,
                     "calculated-failure-timeout");
         return pcmk_rc_ok;
     }
 
     return pcmk_rc_undetermined;
 }
 
 /*!
  * \internal
  * \brief Process a masked probe failure
  *
  * \param[in,out] history           Parsed action history entry
  * \param[in]     orig_exit_status  Action exit status before remapping
  * \param[in]     last_failure      Resource's last_failure entry, if known
  * \param[in,out] on_fail           Resource's current failure handling
  */
 static void
 mask_probe_failure(struct action_history *history, int orig_exit_status,
                    const xmlNode *last_failure,
                    enum pcmk__on_fail *on_fail)
 {
     pcmk_resource_t *ban_rsc = history->rsc;
 
     if (!pcmk_is_set(history->rsc->flags, pcmk__rsc_unique)) {
         ban_rsc = uber_parent(history->rsc);
     }
 
     crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
                crm_exit_str(orig_exit_status), history->rsc->id,
                pcmk__node_name(history->node));
     update_resource_state(history, history->expected_exit_status, last_failure,
                           on_fail);
     crm_xml_add(history->xml, PCMK_XA_UNAME, history->node->priv->name);
 
     record_failed_op(history);
     resource_location(ban_rsc, history->node, -PCMK_SCORE_INFINITY,
                       "masked-probe-failure", ban_rsc->priv->scheduler);
 }
 
 /*!
  * \internal Check whether a given failure is for a given pending action
  *
  * \param[in] history       Parsed history entry for pending action
  * \param[in] last_failure  Resource's last_failure entry, if known
  *
  * \return true if \p last_failure is failure of pending action in \p history,
  *         otherwise false
  * \note Both \p history and \p last_failure must come from the same
  *       \c PCMK__XE_LRM_RESOURCE block, as node and resource are assumed to be
  *       the same.
  */
 static bool
 failure_is_newer(const struct action_history *history,
                  const xmlNode *last_failure)
 {
     guint failure_interval_ms = 0U;
     long long failure_change = 0LL;
     long long this_change = 0LL;
 
     if (last_failure == NULL) {
         return false; // Resource has no last_failure entry
     }
 
     if (!pcmk__str_eq(history->task,
                       crm_element_value(last_failure, PCMK_XA_OPERATION),
                       pcmk__str_none)) {
         return false; // last_failure is for different action
     }
 
     if ((crm_element_value_ms(last_failure, PCMK_META_INTERVAL,
                               &failure_interval_ms) != pcmk_ok)
         || (history->interval_ms != failure_interval_ms)) {
         return false; // last_failure is for action with different interval
     }
 
     if ((pcmk__scan_ll(crm_element_value(history->xml, PCMK_XA_LAST_RC_CHANGE),
                        &this_change, 0LL) != pcmk_rc_ok)
         || (pcmk__scan_ll(crm_element_value(last_failure,
                                             PCMK_XA_LAST_RC_CHANGE),
                           &failure_change, 0LL) != pcmk_rc_ok)
         || (failure_change < this_change)) {
         return false; // Failure is not known to be newer
     }
 
     return true;
 }
 
 /*!
  * \internal
  * \brief Update a resource's role etc. for a pending action
  *
  * \param[in,out] history       Parsed history entry for pending action
  * \param[in]     last_failure  Resource's last_failure entry, if known
  */
 static void
 process_pending_action(struct action_history *history,
                        const xmlNode *last_failure)
 {
     /* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
      * and there might be a RSC_monitor_INTERVAL entry with the last successful
      * or pending result.
      *
      * If last_failure contains the failure of the pending recurring monitor
      * we're processing here, and is newer, the action is no longer pending.
      * (Pending results have call ID -1, which sorts last, so the last failure
      * if any should be known.)
      */
     if (failure_is_newer(history, last_failure)) {
         return;
     }
 
     if (strcmp(history->task, PCMK_ACTION_START) == 0) {
         pcmk__set_rsc_flags(history->rsc, pcmk__rsc_start_pending);
         set_active(history->rsc);
 
     } else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
         history->rsc->priv->orig_role = pcmk_role_promoted;
 
     } else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
                && history->node->details->unclean) {
         /* A migrate_to action is pending on a unclean source, so force a stop
          * on the target.
          */
         const char *migrate_target = NULL;
         pcmk_node_t *target = NULL;
 
         migrate_target = crm_element_value(history->xml,
                                            PCMK__META_MIGRATE_TARGET);
         target = pcmk_find_node(history->rsc->priv->scheduler,
                                 migrate_target);
         if (target != NULL) {
             stop_action(history->rsc, target, FALSE);
         }
     }
 
     if (history->rsc->priv->pending_action != NULL) {
         /* There should never be multiple pending actions, but as a failsafe,
          * just remember the first one processed for display purposes.
          */
         return;
     }
 
     if (pcmk_is_probe(history->task, history->interval_ms)) {
         /* Pending probes are currently never displayed, even if pending
          * operations are requested. If we ever want to change that,
          * enable the below and the corresponding part of
          * native.c:native_pending_action().
          */
 #if 0
         history->rsc->private->pending_action = strdup("probe");
         history->rsc->private->pending_node = history->node;
 #endif
     } else {
         history->rsc->priv->pending_action = strdup(history->task);
         history->rsc->priv->pending_node = history->node;
     }
 }
 
 static void
 unpack_rsc_op(pcmk_resource_t *rsc, pcmk_node_t *node, xmlNode *xml_op,
               xmlNode **last_failure, enum pcmk__on_fail *on_fail)
 {
     int old_rc = 0;
     bool expired = false;
     pcmk_resource_t *parent = rsc;
     enum rsc_role_e fail_role = pcmk_role_unknown;
     enum pcmk__on_fail failure_strategy = pcmk__on_fail_restart;
 
     struct action_history history = {
         .rsc = rsc,
         .node = node,
         .xml = xml_op,
         .execution_status = PCMK_EXEC_UNKNOWN,
     };
 
     CRM_CHECK(rsc && node && xml_op, return);
 
     history.id = pcmk__xe_id(xml_op);
     if (history.id == NULL) {
         pcmk__config_err("Ignoring resource history entry for %s on %s "
                          "without ID", rsc->id, pcmk__node_name(node));
         return;
     }
 
     // Task and interval
     history.task = crm_element_value(xml_op, PCMK_XA_OPERATION);
     if (history.task == NULL) {
         pcmk__config_err("Ignoring resource history entry %s for %s on %s "
                          "without " PCMK_XA_OPERATION,
                          history.id, rsc->id, pcmk__node_name(node));
         return;
     }
     crm_element_value_ms(xml_op, PCMK_META_INTERVAL, &(history.interval_ms));
     if (!can_affect_state(&history)) {
         pcmk__rsc_trace(rsc,
                         "Ignoring resource history entry %s for %s on %s "
                         "with irrelevant action '%s'",
                         history.id, rsc->id, pcmk__node_name(node),
                         history.task);
         return;
     }
 
     if (unpack_action_result(&history) != pcmk_rc_ok) {
         return; // Error already logged
     }
 
     history.expected_exit_status = pe__target_rc_from_xml(xml_op);
     history.key = pcmk__xe_history_key(xml_op);
     crm_element_value_int(xml_op, PCMK__XA_CALL_ID, &(history.call_id));
 
     pcmk__rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
                     history.id, history.task, history.call_id,
                     pcmk__node_name(node),
                     pcmk_exec_status_str(history.execution_status),
                     crm_exit_str(history.exit_status));
 
     if (node->details->unclean) {
         pcmk__rsc_trace(rsc,
                         "%s is running on %s, which is unclean (further action "
                         "depends on value of stop's on-fail attribute)",
                         rsc->id, pcmk__node_name(node));
     }
 
     expired = check_operation_expiry(&history);
     old_rc = history.exit_status;
 
     remap_operation(&history, on_fail, expired);
 
     if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
         goto done;
     }
 
     if (!pcmk__is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
         mask_probe_failure(&history, old_rc, *last_failure, on_fail);
         goto done;
     }
 
     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
         parent = uber_parent(rsc);
     }
 
     switch (history.execution_status) {
         case PCMK_EXEC_PENDING:
             process_pending_action(&history, *last_failure);
             goto done;
 
         case PCMK_EXEC_DONE:
             update_resource_state(&history, history.exit_status, *last_failure,
                                   on_fail);
             goto done;
 
         case PCMK_EXEC_NOT_INSTALLED:
             unpack_failure_handling(&history, &failure_strategy, &fail_role);
             if (failure_strategy == pcmk__on_fail_ignore) {
                 crm_warn("Cannot ignore failed %s of %s on %s: "
                          "Resource agent doesn't exist "
                          QB_XS " status=%d rc=%d id=%s",
                          history.task, rsc->id, pcmk__node_name(node),
                          history.execution_status, history.exit_status,
                          history.id);
                 /* Also for printing it as "FAILED" by marking it as
                  * pcmk__rsc_failed later
                  */
                 *on_fail = pcmk__on_fail_ban;
             }
             resource_location(parent, node, -PCMK_SCORE_INFINITY,
                               "hard-error", rsc->priv->scheduler);
             unpack_rsc_op_failure(&history, failure_strategy, fail_role,
                                   last_failure, on_fail);
             goto done;
 
         case PCMK_EXEC_NOT_CONNECTED:
             if (pcmk__is_pacemaker_remote_node(node)
                 && pcmk_is_set(node->priv->remote->flags,
                                pcmk__rsc_managed)) {
                 /* We should never get into a situation where a managed remote
                  * connection resource is considered OK but a resource action
                  * behind the connection gets a "not connected" status. But as a
                  * fail-safe in case a bug or unusual circumstances do lead to
                  * that, ensure the remote connection is considered failed.
                  */
                 pcmk__set_rsc_flags(node->priv->remote,
                                     pcmk__rsc_failed|pcmk__rsc_stop_if_failed);
             }
             break; // Not done, do error handling
 
         case PCMK_EXEC_ERROR:
         case PCMK_EXEC_ERROR_HARD:
         case PCMK_EXEC_ERROR_FATAL:
         case PCMK_EXEC_TIMEOUT:
         case PCMK_EXEC_NOT_SUPPORTED:
         case PCMK_EXEC_INVALID:
             break; // Not done, do error handling
 
         default: // No other value should be possible at this point
             break;
     }
 
     unpack_failure_handling(&history, &failure_strategy, &fail_role);
     if ((failure_strategy == pcmk__on_fail_ignore)
         || ((failure_strategy == pcmk__on_fail_restart_container)
             && (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {
 
         char *last_change_s = last_change_str(xml_op);
 
         crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
                  QB_XS " %s",
                  history.task, crm_exit_str(history.exit_status),
                  (pcmk__str_empty(history.exit_reason)? "" : ": "),
                  pcmk__s(history.exit_reason, ""), rsc->id,
                  pcmk__node_name(node), last_change_s, history.id);
         free(last_change_s);
 
         update_resource_state(&history, history.expected_exit_status,
                               *last_failure, on_fail);
         crm_xml_add(xml_op, PCMK_XA_UNAME, node->priv->name);
         pcmk__set_rsc_flags(rsc, pcmk__rsc_ignore_failure);
 
         record_failed_op(&history);
 
         if ((failure_strategy == pcmk__on_fail_restart_container)
             && (*on_fail <= pcmk__on_fail_restart)) {
             *on_fail = failure_strategy;
         }
 
     } else {
         unpack_rsc_op_failure(&history, failure_strategy, fail_role,
                               last_failure, on_fail);
 
         if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
             uint8_t log_level = LOG_ERR;
 
             if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
                 log_level = LOG_NOTICE;
             }
             do_crm_log(log_level,
                        "Preventing %s from restarting on %s because "
                        "of hard failure (%s%s%s) " QB_XS " %s",
                        parent->id, pcmk__node_name(node),
                        crm_exit_str(history.exit_status),
                        (pcmk__str_empty(history.exit_reason)? "" : ": "),
                        pcmk__s(history.exit_reason, ""), history.id);
             resource_location(parent, node, -PCMK_SCORE_INFINITY,
                               "hard-error", rsc->priv->scheduler);
 
         } else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
             pcmk__sched_err(rsc->priv->scheduler,
                             "Preventing %s from restarting anywhere because "
                             "of fatal failure (%s%s%s) " QB_XS " %s",
                             parent->id, crm_exit_str(history.exit_status),
                             (pcmk__str_empty(history.exit_reason)? "" : ": "),
                             pcmk__s(history.exit_reason, ""), history.id);
             resource_location(parent, NULL, -PCMK_SCORE_INFINITY,
                               "fatal-error", rsc->priv->scheduler);
         }
     }
 
 done:
     pcmk__rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
                     rsc->id, pcmk__node_name(node), history.id,
                     pcmk_role_text(rsc->priv->orig_role),
                     pcmk_role_text(rsc->priv->next_role));
 }
 
 /*!
  * \internal
  * \brief Insert a node attribute with value into a \c GHashTable
  *
  * \param[in,out] key        Key to insert (either freed or owned by
  *                           \p user_data upon return)
  * \param[in]     value      Value to insert (owned by \p user_data upon return)
  * \param[in]     user_data  \c GHashTable to insert into
  */
 static gboolean
 insert_attr(gpointer key, gpointer value, gpointer user_data)
 {
     GHashTable *table = user_data;
 
     g_hash_table_insert(table, key, value);
     return TRUE;
 }
 
 static void
 add_node_attrs(const xmlNode *xml_obj, pcmk_node_t *node, bool overwrite,
                pcmk_scheduler_t *scheduler)
 {
     const char *cluster_name = NULL;
     const char *dc_id = crm_element_value(scheduler->input, PCMK_XA_DC_UUID);
     const pcmk_rule_input_t rule_input = {
         .now = scheduler->priv->now,
     };
 
     pcmk__insert_dup(node->priv->attrs,
                      CRM_ATTR_UNAME, node->priv->name);
 
     pcmk__insert_dup(node->priv->attrs, CRM_ATTR_ID, node->priv->id);
 
     if ((scheduler->dc_node == NULL)
         && pcmk__str_eq(node->priv->id, dc_id, pcmk__str_casei)) {
 
         scheduler->dc_node = node;
         pcmk__insert_dup(node->priv->attrs,
                          CRM_ATTR_IS_DC, PCMK_VALUE_TRUE);
 
     } else if (!pcmk__same_node(node, scheduler->dc_node)) {
         pcmk__insert_dup(node->priv->attrs,
                          CRM_ATTR_IS_DC, PCMK_VALUE_FALSE);
     }
 
     cluster_name = g_hash_table_lookup(scheduler->priv->options,
                                        PCMK_OPT_CLUSTER_NAME);
     if (cluster_name) {
         pcmk__insert_dup(node->priv->attrs, CRM_ATTR_CLUSTER_NAME,
                          cluster_name);
     }
 
     if (overwrite) {
         /* @TODO Try to reorder some unpacking so that we don't need the
          * overwrite argument or to unpack into a temporary table
          */
         GHashTable *unpacked = pcmk__strkey_table(free, free);
 
         pe__unpack_dataset_nvpairs(xml_obj, PCMK_XE_INSTANCE_ATTRIBUTES,
                                    &rule_input, unpacked, NULL, scheduler);
         g_hash_table_foreach_steal(unpacked, insert_attr, node->priv->attrs);
         g_hash_table_destroy(unpacked);
 
     } else {
         pe__unpack_dataset_nvpairs(xml_obj, PCMK_XE_INSTANCE_ATTRIBUTES,
                                    &rule_input, node->priv->attrs, NULL,
                                    scheduler);
     }
 
     pe__unpack_dataset_nvpairs(xml_obj, PCMK_XE_UTILIZATION, &rule_input,
                                node->priv->utilization, NULL, scheduler);
 
     if (pcmk__node_attr(node, CRM_ATTR_SITE_NAME, NULL,
                         pcmk__rsc_node_current) == NULL) {
         const char *site_name = pcmk__node_attr(node, "site-name", NULL,
                                                 pcmk__rsc_node_current);
 
         if (site_name) {
             pcmk__insert_dup(node->priv->attrs,
                              CRM_ATTR_SITE_NAME, site_name);
 
         } else if (cluster_name) {
             /* Default to cluster-name if unset */
             pcmk__insert_dup(node->priv->attrs,
                              CRM_ATTR_SITE_NAME, cluster_name);
         }
     }
 }
 
 static GList *
 extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
 {
     int counter = -1;
     int stop_index = -1;
     int start_index = -1;
 
     xmlNode *rsc_op = NULL;
 
     GList *gIter = NULL;
     GList *op_list = NULL;
     GList *sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = pcmk__xe_first_child(rsc_entry, PCMK__XE_LRM_RSC_OP, NULL,
                                        NULL);
          rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op, PCMK__XE_LRM_RSC_OP)) {
 
         crm_xml_add(rsc_op, PCMK_XA_RESOURCE, rsc);
         crm_xml_add(rsc_op, PCMK_XA_UNAME, node);
         op_list = g_list_prepend(op_list, rsc_op);
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* create active recurring operations as optional */
     if (active_filter == FALSE) {
         return sorted_op_list;
     }
 
     op_list = NULL;
 
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         if (start_index < stop_index) {
             crm_trace("Skipping %s: not active", pcmk__xe_id(rsc_entry));
             break;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s: old", pcmk__xe_id(rsc_op));
             continue;
         }
         op_list = g_list_append(op_list, rsc_op);
     }
 
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GList *
 find_operations(const char *rsc, const char *node, gboolean active_filter,
                 pcmk_scheduler_t *scheduler)
 {
     GList *output = NULL;
     GList *intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = pcmk__xe_first_child(scheduler->input, PCMK_XE_STATUS,
                                            NULL, NULL);
 
     pcmk_node_t *this_node = NULL;
 
     xmlNode *node_state = NULL;
 
     CRM_CHECK(status != NULL, return NULL);
 
     for (node_state = pcmk__xe_first_child(status, PCMK__XE_NODE_STATE, NULL,
                                            NULL);
          node_state != NULL;
          node_state = pcmk__xe_next(node_state, PCMK__XE_NODE_STATE)) {
 
         const char *uname = crm_element_value(node_state, PCMK_XA_UNAME);
 
         if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
             continue;
         }
 
         this_node = pcmk_find_node(scheduler, uname);
         if(this_node == NULL) {
             CRM_LOG_ASSERT(this_node != NULL);
             continue;
 
         } else if (pcmk__is_pacemaker_remote_node(this_node)) {
             determine_remote_online_status(scheduler, this_node);
 
         } else {
             determine_online_status(node_state, this_node, scheduler);
         }
 
         if (this_node->details->online
             || pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
             /* offline nodes run no resources...
              * unless stonith is enabled in which case we need to
              *   make sure rsc start events happen after the stonith
              */
             xmlNode *lrm_rsc = NULL;
 
             tmp = pcmk__xe_first_child(node_state, PCMK__XE_LRM, NULL,
                                        NULL);
             tmp = pcmk__xe_first_child(tmp, PCMK__XE_LRM_RESOURCES, NULL,
                                        NULL);
 
             for (lrm_rsc = pcmk__xe_first_child(tmp, PCMK__XE_LRM_RESOURCE,
                                                 NULL, NULL);
                  lrm_rsc != NULL;
                  lrm_rsc = pcmk__xe_next(lrm_rsc, PCMK__XE_LRM_RESOURCE)) {
 
                 const char *rsc_id = crm_element_value(lrm_rsc, PCMK_XA_ID);
 
                 if ((rsc != NULL)
                     && !pcmk__str_eq(rsc_id, rsc, pcmk__str_none)) {
                     continue;
                 }
 
                 intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
                 output = g_list_concat(output, intermediate);
             }
         }
     }
 
     return output;
 }
diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c
index ff4763a20c..2f8a46d195 100644
--- a/lib/services/services_linux.c
+++ b/lib/services/services_linux.c
@@ -1,1478 +1,1482 @@
 /*
  * Copyright 2010-2024 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <errno.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <grp.h>
 #include <string.h>
 #include <sys/time.h>
 #include <sys/resource.h>
 
 #include "crm/crm.h"
 #include "crm/common/mainloop.h"
 #include "crm/services.h"
 #include "crm/services_internal.h"
 
 #include "services_private.h"
 
 static void close_pipe(int fildes[]);
 
 /* We have two alternative ways of handling SIGCHLD when synchronously waiting
  * for spawned processes to complete. Both rely on polling a file descriptor to
  * discover SIGCHLD events.
  *
  * If sys/signalfd.h is available (e.g. on Linux), we call signalfd() to
  * generate the file descriptor. Otherwise, we use the "self-pipe trick"
  * (opening a pipe and writing a byte to it when SIGCHLD is received).
  */
 #ifdef HAVE_SYS_SIGNALFD_H
 
 // signalfd() implementation
 
 #include <sys/signalfd.h>
 
 // Everything needed to manage SIGCHLD handling
 struct sigchld_data_s {
     sigset_t mask;      // Signals to block now (including SIGCHLD)
     sigset_t old_mask;  // Previous set of blocked signals
     bool ignored;       // If SIGCHLD for another child has been ignored
 };
 
 // Initialize SIGCHLD data and prepare for use
 static bool
 sigchld_setup(struct sigchld_data_s *data)
 {
     sigemptyset(&(data->mask));
     sigaddset(&(data->mask), SIGCHLD);
 
     sigemptyset(&(data->old_mask));
 
     // Block SIGCHLD (saving previous set of blocked signals to restore later)
     if (sigprocmask(SIG_BLOCK, &(data->mask), &(data->old_mask)) < 0) {
         crm_info("Wait for child process completion failed: %s "
                  QB_XS " source=sigprocmask", pcmk_rc_str(errno));
         return false;
     }
 
     data->ignored = false;
 
     return true;
 }
 
 // Get a file descriptor suitable for polling for SIGCHLD events
 static int
 sigchld_open(struct sigchld_data_s *data)
 {
     int fd;
 
     CRM_CHECK(data != NULL, return -1);
 
     fd = signalfd(-1, &(data->mask), SFD_NONBLOCK);
     if (fd < 0) {
         crm_info("Wait for child process completion failed: %s "
                  QB_XS " source=signalfd", pcmk_rc_str(errno));
     }
     return fd;
 }
 
 // Close a file descriptor returned by sigchld_open()
 static void
 sigchld_close(int fd)
 {
     if (fd > 0) {
         close(fd);
     }
 }
 
 // Return true if SIGCHLD was received from polled fd
 static bool
 sigchld_received(int fd, int pid, struct sigchld_data_s *data)
 {
     struct signalfd_siginfo fdsi;
     ssize_t s;
 
     if (fd < 0) {
         return false;
     }
     s = read(fd, &fdsi, sizeof(struct signalfd_siginfo));
     if (s != sizeof(struct signalfd_siginfo)) {
         crm_info("Wait for child process completion failed: %s "
                  QB_XS " source=read", pcmk_rc_str(errno));
 
     } else if (fdsi.ssi_signo == SIGCHLD) {
         if (fdsi.ssi_pid == pid) {
             return true;
 
         } else {
             /* This SIGCHLD is for another child. We have to ignore it here but
              * will still need to resend it after this synchronous action has
              * completed and SIGCHLD has been restored to be handled by the
              * previous SIGCHLD handler, so that it will be handled.
              */
             data->ignored = true;
             return false;
         }
     }
     return false;
 }
 
 // Do anything needed after done waiting for SIGCHLD
 static void
 sigchld_cleanup(struct sigchld_data_s *data)
 {
     // Restore the original set of blocked signals
     if ((sigismember(&(data->old_mask), SIGCHLD) == 0)
         && (sigprocmask(SIG_UNBLOCK, &(data->mask), NULL) < 0)) {
         crm_warn("Could not clean up after child process completion: %s",
                  pcmk_rc_str(errno));
     }
 
     // Resend any ignored SIGCHLD for other children so that they'll be handled.
     if (data->ignored && kill(getpid(), SIGCHLD) != 0) {
         crm_warn("Could not resend ignored SIGCHLD to ourselves: %s",
                  pcmk_rc_str(errno));
     }
 }
 
 #else // HAVE_SYS_SIGNALFD_H not defined
 
 // Self-pipe implementation (see above for function descriptions)
 
 struct sigchld_data_s {
     int pipe_fd[2];             // Pipe file descriptors
     struct sigaction sa;        // Signal handling info (with SIGCHLD)
     struct sigaction old_sa;    // Previous signal handling info
     bool ignored;               // If SIGCHLD for another child has been ignored
 };
 
 // We need a global to use in the signal handler
 volatile struct sigchld_data_s *last_sigchld_data = NULL;
 
 static void
 sigchld_handler(void)
 {
     // We received a SIGCHLD, so trigger pipe polling
     if ((last_sigchld_data != NULL)
         && (last_sigchld_data->pipe_fd[1] >= 0)
         && (write(last_sigchld_data->pipe_fd[1], "", 1) == -1)) {
         crm_info("Wait for child process completion failed: %s "
                  QB_XS " source=write", pcmk_rc_str(errno));
     }
 }
 
 static bool
 sigchld_setup(struct sigchld_data_s *data)
 {
     int rc;
 
     data->pipe_fd[0] = data->pipe_fd[1] = -1;
 
     if (pipe(data->pipe_fd) == -1) {
         crm_info("Wait for child process completion failed: %s "
                  QB_XS " source=pipe", pcmk_rc_str(errno));
         return false;
     }
 
     rc = pcmk__set_nonblocking(data->pipe_fd[0]);
     if (rc != pcmk_rc_ok) {
         crm_info("Could not set pipe input non-blocking: %s " QB_XS " rc=%d",
                  pcmk_rc_str(rc), rc);
     }
     rc = pcmk__set_nonblocking(data->pipe_fd[1]);
     if (rc != pcmk_rc_ok) {
         crm_info("Could not set pipe output non-blocking: %s " QB_XS " rc=%d",
                  pcmk_rc_str(rc), rc);
     }
 
     // Set SIGCHLD handler
     data->sa.sa_handler = (sighandler_t) sigchld_handler;
     data->sa.sa_flags = 0;
     sigemptyset(&(data->sa.sa_mask));
     if (sigaction(SIGCHLD, &(data->sa), &(data->old_sa)) < 0) {
         crm_info("Wait for child process completion failed: %s "
                  QB_XS " source=sigaction", pcmk_rc_str(errno));
     }
 
     data->ignored = false;
 
     // Remember data for use in signal handler
     last_sigchld_data = data;
     return true;
 }
 
 static int
 sigchld_open(struct sigchld_data_s *data)
 {
     CRM_CHECK(data != NULL, return -1);
     return data->pipe_fd[0];
 }
 
 static void
 sigchld_close(int fd)
 {
     // Pipe will be closed in sigchld_cleanup()
     return;
 }
 
 static bool
 sigchld_received(int fd, int pid, struct sigchld_data_s *data)
 {
     char ch;
 
     if (fd < 0) {
         return false;
     }
 
     // Clear out the self-pipe
     while (read(fd, &ch, 1) == 1) /*omit*/;
     return true;
 }
 
 static void
 sigchld_cleanup(struct sigchld_data_s *data)
 {
     // Restore the previous SIGCHLD handler
     if (sigaction(SIGCHLD, &(data->old_sa), NULL) < 0) {
         crm_warn("Could not clean up after child process completion: %s",
                  pcmk_rc_str(errno));
     }
 
     close_pipe(data->pipe_fd);
 
     // Resend any ignored SIGCHLD for other children so that they'll be handled.
     if (data->ignored && kill(getpid(), SIGCHLD) != 0) {
         crm_warn("Could not resend ignored SIGCHLD to ourselves: %s",
                  pcmk_rc_str(errno));
     }
 }
 
 #endif
 
 /*!
  * \internal
  * \brief Close the two file descriptors of a pipe
  *
  * \param[in,out] fildes  Array of file descriptors opened by pipe()
  */
 static void
 close_pipe(int fildes[])
 {
     if (fildes[0] >= 0) {
         close(fildes[0]);
         fildes[0] = -1;
     }
     if (fildes[1] >= 0) {
         close(fildes[1]);
         fildes[1] = -1;
     }
 }
 
 #define out_type(is_stderr) ((is_stderr)? "stderr" : "stdout")
 
 // Maximum number of bytes of stdout or stderr we'll accept
 #define MAX_OUTPUT (10 * 1024 * 1024)
 
 static gboolean
 svc_read_output(int fd, svc_action_t * op, bool is_stderr)
 {
     char *data = NULL;
     ssize_t rc = 0;
     size_t len = 0;
     size_t discarded = 0;
     char buf[500];
     static const size_t buf_read_len = sizeof(buf) - 1;
 
     if (fd < 0) {
         crm_trace("No fd for %s", op->id);
         return FALSE;
     }
 
     if (is_stderr && op->stderr_data) {
         len = strlen(op->stderr_data);
         data = op->stderr_data;
         crm_trace("Reading %s stderr into offset %lld",
                   op->id, (long long) len);
 
     } else if (is_stderr == FALSE && op->stdout_data) {
         len = strlen(op->stdout_data);
         data = op->stdout_data;
         crm_trace("Reading %s stdout into offset %lld",
                   op->id, (long long) len);
 
     } else {
         crm_trace("Reading %s %s", op->id, out_type(is_stderr));
     }
 
     do {
         errno = 0;
         rc = read(fd, buf, buf_read_len);
         if (rc > 0) {
             if (len < MAX_OUTPUT) {
                 buf[rc] = 0;
                 crm_trace("Received %lld bytes of %s %s: %.80s",
                           (long long) rc, op->id, out_type(is_stderr), buf);
                 data = pcmk__realloc(data, len + rc + 1);
                 strcpy(data + len, buf);
                 len += rc;
             } else {
                 discarded += rc;
             }
 
         } else if (errno != EINTR) { // Fatal error or EOF
             rc = 0;
             break;
         }
     } while ((rc == buf_read_len) || (rc < 0));
 
     if (discarded > 0) {
         crm_warn("Truncated %s %s to %lld bytes (discarded %lld)",
                  op->id, out_type(is_stderr), (long long) len,
                  (long long) discarded);
     }
 
     if (is_stderr) {
         op->stderr_data = data;
     } else {
         op->stdout_data = data;
     }
 
     return rc != 0;
 }
 
 static int
 dispatch_stdout(gpointer userdata)
 {
     svc_action_t *op = (svc_action_t *) userdata;
 
     return svc_read_output(op->opaque->stdout_fd, op, FALSE);
 }
 
 static int
 dispatch_stderr(gpointer userdata)
 {
     svc_action_t *op = (svc_action_t *) userdata;
 
     return svc_read_output(op->opaque->stderr_fd, op, TRUE);
 }
 
 static void
 pipe_out_done(gpointer user_data)
 {
     svc_action_t *op = (svc_action_t *) user_data;
 
     crm_trace("%p", op);
 
     op->opaque->stdout_gsource = NULL;
     if (op->opaque->stdout_fd > STDOUT_FILENO) {
         close(op->opaque->stdout_fd);
     }
     op->opaque->stdout_fd = -1;
 }
 
 static void
 pipe_err_done(gpointer user_data)
 {
     svc_action_t *op = (svc_action_t *) user_data;
 
     op->opaque->stderr_gsource = NULL;
     if (op->opaque->stderr_fd > STDERR_FILENO) {
         close(op->opaque->stderr_fd);
     }
     op->opaque->stderr_fd = -1;
 }
 
 static struct mainloop_fd_callbacks stdout_callbacks = {
     .dispatch = dispatch_stdout,
     .destroy = pipe_out_done,
 };
 
 static struct mainloop_fd_callbacks stderr_callbacks = {
     .dispatch = dispatch_stderr,
     .destroy = pipe_err_done,
 };
 
 static void
 set_ocf_env(const char *key, const char *value, gpointer user_data)
 {
     if (setenv(key, value, 1) != 0) {
         crm_perror(LOG_ERR, "setenv failed for key:%s and value:%s", key, value);
     }
 }
 
 static void
 set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data)
 {
     char buffer[500];
 
     snprintf(buffer, sizeof(buffer), strcmp(key, "OCF_CHECK_LEVEL") != 0 ? "OCF_RESKEY_%s" : "%s", (char *)key);
     set_ocf_env(buffer, value, user_data);
 }
 
 static void
 set_alert_env(gpointer key, gpointer value, gpointer user_data)
 {
     int rc;
 
     if (value != NULL) {
         rc = setenv(key, value, 1);
     } else {
         rc = unsetenv(key);
     }
 
     if (rc < 0) {
         crm_perror(LOG_ERR, "setenv %s=%s",
                   (char*)key, (value? (char*)value : ""));
     } else {
         crm_trace("setenv %s=%s", (char*)key, (value? (char*)value : ""));
     }
 }
 
 /*!
  * \internal
  * \brief Add environment variables suitable for an action
  *
  * \param[in] op  Action to use
  */
 static void
 add_action_env_vars(const svc_action_t *op)
 {
     void (*env_setter)(gpointer, gpointer, gpointer) = NULL;
     if (op->agent == NULL) {
         env_setter = set_alert_env;  /* we deal with alert handler */
 
     } else if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
         env_setter = set_ocf_env_with_prefix;
     }
 
     if (env_setter != NULL && op->params != NULL) {
         g_hash_table_foreach(op->params, env_setter, NULL);
     }
 
     if (env_setter == NULL || env_setter == set_alert_env) {
         return;
     }
 
     set_ocf_env("OCF_RA_VERSION_MAJOR", PCMK_OCF_MAJOR_VERSION, NULL);
     set_ocf_env("OCF_RA_VERSION_MINOR", PCMK_OCF_MINOR_VERSION, NULL);
     set_ocf_env("OCF_ROOT", PCMK_OCF_ROOT, NULL);
     set_ocf_env("OCF_EXIT_REASON_PREFIX", PCMK_OCF_REASON_PREFIX, NULL);
 
     if (op->rsc) {
         set_ocf_env("OCF_RESOURCE_INSTANCE", op->rsc, NULL);
     }
 
     if (op->agent != NULL) {
         set_ocf_env("OCF_RESOURCE_TYPE", op->agent, NULL);
     }
 
     /* Notes: this is not added to specification yet. Sept 10,2004 */
     if (op->provider != NULL) {
         set_ocf_env("OCF_RESOURCE_PROVIDER", op->provider, NULL);
     }
 }
 
 static void
 pipe_in_single_parameter(gpointer key, gpointer value, gpointer user_data)
 {
     svc_action_t *op = user_data;
     char *buffer = crm_strdup_printf("%s=%s\n", (char *)key, (char *) value);
     size_t len = strlen(buffer);
     size_t total = 0;
     ssize_t ret = 0;
 
     do {
         errno = 0;
         ret = write(op->opaque->stdin_fd, buffer + total, len - total);
         if (ret > 0) {
             total += ret;
         }
     } while ((errno == EINTR) && (total < len));
     free(buffer);
 }
 
 /*!
  * \internal
  * \brief Pipe parameters in via stdin for action
  *
  * \param[in] op  Action to use
  */
 static void
 pipe_in_action_stdin_parameters(const svc_action_t *op)
 {
     if (op->params) {
         g_hash_table_foreach(op->params, pipe_in_single_parameter, (gpointer) op);
     }
 }
 
 gboolean
 recurring_action_timer(gpointer data)
 {
     svc_action_t *op = data;
 
     crm_debug("Scheduling another invocation of %s", op->id);
 
     /* Clean out the old result */
     free(op->stdout_data);
     op->stdout_data = NULL;
     free(op->stderr_data);
     op->stderr_data = NULL;
     op->opaque->repeat_timer = 0;
 
     services_action_async(op, NULL);
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Finalize handling of an asynchronous operation
  *
  * Given a completed asynchronous operation, cancel or reschedule it as
  * appropriate if recurring, call its callback if registered, stop tracking it,
  * and clean it up.
  *
  * \param[in,out] op  Operation to finalize
  *
  * \return Standard Pacemaker return code
  * \retval EINVAL      Caller supplied NULL or invalid \p op
  * \retval EBUSY       Uncanceled recurring action has only been cleaned up
  * \retval pcmk_rc_ok  Action has been freed
  *
  * \note If the return value is not pcmk_rc_ok, the caller is responsible for
  *       freeing the action.
  */
 int
 services__finalize_async_op(svc_action_t *op)
 {
     CRM_CHECK((op != NULL) && !(op->synchronous), return EINVAL);
 
     if (op->interval_ms != 0) {
         // Recurring operations must be either cancelled or rescheduled
         if (op->cancel) {
             services__set_cancelled(op);
             cancel_recurring_action(op);
         } else {
             op->opaque->repeat_timer = pcmk__create_timer(op->interval_ms,
                                                           recurring_action_timer,
                                                           op);
         }
     }
 
     if (op->opaque->callback != NULL) {
         op->opaque->callback(op);
     }
 
     // Stop tracking the operation (as in-flight or blocked)
     op->pid = 0;
     services_untrack_op(op);
 
     if ((op->interval_ms != 0) && !(op->cancel)) {
         // Do not free recurring actions (they will get freed when cancelled)
         services_action_cleanup(op);
         return EBUSY;
     }
 
     services_action_free(op);
     return pcmk_rc_ok;
 }
 
 static void
 close_op_input(svc_action_t *op)
 {
     if (op->opaque->stdin_fd >= 0) {
         close(op->opaque->stdin_fd);
     }
 }
 
 static void
 finish_op_output(svc_action_t *op, bool is_stderr)
 {
     mainloop_io_t **source;
     int fd;
 
     if (is_stderr) {
         source = &(op->opaque->stderr_gsource);
         fd = op->opaque->stderr_fd;
     } else {
         source = &(op->opaque->stdout_gsource);
         fd = op->opaque->stdout_fd;
     }
 
     if (op->synchronous || *source) {
         crm_trace("Finish reading %s[%d] %s",
                   op->id, op->pid, (is_stderr? "stderr" : "stdout"));
         svc_read_output(fd, op, is_stderr);
         if (op->synchronous) {
             close(fd);
         } else {
             mainloop_del_fd(*source);
             *source = NULL;
         }
     }
 }
 
 // Log an operation's stdout and stderr
 static void
 log_op_output(svc_action_t *op)
 {
     char *prefix = crm_strdup_printf("%s[%d] error output", op->id, op->pid);
 
     /* The library caller has better context to know how important the output
      * is, so log it at info and debug severity here. They can log it again at
      * higher severity if appropriate.
      */
     crm_log_output(LOG_INFO, prefix, op->stderr_data);
     strcpy(prefix + strlen(prefix) - strlen("error output"), "output");
     crm_log_output(LOG_DEBUG, prefix, op->stdout_data);
     free(prefix);
 }
 
 // Truncate exit reasons at this many characters
 #define EXIT_REASON_MAX_LEN 128
 
 static void
 parse_exit_reason_from_stderr(svc_action_t *op)
 {
     const char *reason_start = NULL;
     const char *reason_end = NULL;
     const int prefix_len = strlen(PCMK_OCF_REASON_PREFIX);
 
     if ((op->stderr_data == NULL) ||
         // Only OCF agents have exit reasons in stderr
         !pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_none)) {
         return;
     }
 
     // Find the last occurrence of the magic string indicating an exit reason
     for (const char *cur = strstr(op->stderr_data, PCMK_OCF_REASON_PREFIX);
          cur != NULL; cur = strstr(cur, PCMK_OCF_REASON_PREFIX)) {
 
         cur += prefix_len; // Skip over magic string
         reason_start = cur;
     }
 
     if ((reason_start == NULL) || (reason_start[0] == '\n')
         || (reason_start[0] == '\0')) {
         return; // No or empty exit reason
     }
 
     // Exit reason goes to end of line (or end of output)
     reason_end = strchr(reason_start, '\n');
     if (reason_end == NULL) {
         reason_end = reason_start + strlen(reason_start);
     }
 
     // Limit size of exit reason to something reasonable
     if (reason_end > (reason_start + EXIT_REASON_MAX_LEN)) {
         reason_end = reason_start + EXIT_REASON_MAX_LEN;
     }
 
     free(op->opaque->exit_reason);
     op->opaque->exit_reason = strndup(reason_start, reason_end - reason_start);
 }
 
 /*!
  * \internal
  * \brief Process the completion of an asynchronous child process
  *
  * \param[in,out] p         Child process that completed
  * \param[in]     pid       Process ID of child
  * \param[in]     core      (Unused)
  * \param[in]     signo     Signal that interrupted child, if any
  * \param[in]     exitcode  Exit status of child process
  */
 static void
 async_action_complete(mainloop_child_t *p, pid_t pid, int core, int signo,
                       int exitcode)
 {
     svc_action_t *op = mainloop_child_userdata(p);
 
     mainloop_clear_child_userdata(p);
     CRM_CHECK(op->pid == pid,
               services__set_result(op, services__generic_error(op),
                                    PCMK_EXEC_ERROR, "Bug in mainloop handling");
               return);
 
     /* Depending on the priority the mainloop gives the stdout and stderr
      * file descriptors, this function could be called before everything has
      * been read from them, so force a final read now.
      */
     finish_op_output(op, true);
     finish_op_output(op, false);
 
     close_op_input(op);
 
     if (signo == 0) {
         crm_debug("%s[%d] exited with status %d", op->id, op->pid, exitcode);
         services__set_result(op, exitcode, PCMK_EXEC_DONE, NULL);
         log_op_output(op);
         parse_exit_reason_from_stderr(op);
 
     } else if (mainloop_child_timeout(p)) {
         const char *kind = services__action_kind(op);
 
         crm_info("%s %s[%d] timed out after %s",
                  kind, op->id, op->pid, pcmk__readable_interval(op->timeout));
         services__format_result(op, services__generic_error(op),
                                 PCMK_EXEC_TIMEOUT,
                                 "%s did not complete within %s",
                                 kind, pcmk__readable_interval(op->timeout));
 
     } else if (op->cancel) {
         /* If an in-flight recurring operation was killed because it was
          * cancelled, don't treat that as a failure.
          */
         crm_info("%s[%d] terminated with signal %d (%s)",
                  op->id, op->pid, signo, strsignal(signo));
         services__set_result(op, PCMK_OCF_OK, PCMK_EXEC_CANCELLED, NULL);
 
     } else {
         crm_info("%s[%d] terminated with signal %d (%s)",
                  op->id, op->pid, signo, strsignal(signo));
         services__format_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                                 "%s interrupted by %s signal",
                                 services__action_kind(op), strsignal(signo));
     }
 
     services__finalize_async_op(op);
 }
 
 /*!
  * \internal
  * \brief Return agent standard's exit status for "generic error"
  *
  * When returning an internal error for an action, a value that is appropriate
  * to the action's agent standard must be used. This function returns a value
  * appropriate for errors in general.
  *
  * \param[in] op  Action that error is for
  *
  * \return Exit status appropriate to agent standard
  * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
  */
 int
 services__generic_error(const svc_action_t *op)
 {
     if ((op == NULL) || (op->standard == NULL)) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
 #if PCMK__ENABLE_LSB
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
         && pcmk__str_eq(op->action, PCMK_ACTION_STATUS, pcmk__str_casei)) {
 
         return PCMK_LSB_STATUS_UNKNOWN;
     }
 #endif
 
     return PCMK_OCF_UNKNOWN_ERROR;
 }
 
 /*!
  * \internal
  * \brief Return agent standard's exit status for "not installed"
  *
  * When returning an internal error for an action, a value that is appropriate
  * to the action's agent standard must be used. This function returns a value
  * appropriate for "not installed" errors.
  *
  * \param[in] op  Action that error is for
  *
  * \return Exit status appropriate to agent standard
  * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
  */
 int
 services__not_installed_error(const svc_action_t *op)
 {
     if ((op == NULL) || (op->standard == NULL)) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
 #if PCMK__ENABLE_LSB
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
         && pcmk__str_eq(op->action, PCMK_ACTION_STATUS, pcmk__str_casei)) {
 
         return PCMK_LSB_STATUS_NOT_INSTALLED;
     }
 #endif
 
     return PCMK_OCF_NOT_INSTALLED;
 }
 
 /*!
  * \internal
  * \brief Return agent standard's exit status for "insufficient privileges"
  *
  * When returning an internal error for an action, a value that is appropriate
  * to the action's agent standard must be used. This function returns a value
  * appropriate for "insufficient privileges" errors.
  *
  * \param[in] op  Action that error is for
  *
  * \return Exit status appropriate to agent standard
  * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
  */
 int
 services__authorization_error(const svc_action_t *op)
 {
     if ((op == NULL) || (op->standard == NULL)) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
 #if PCMK__ENABLE_LSB
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
         && pcmk__str_eq(op->action, PCMK_ACTION_STATUS, pcmk__str_casei)) {
 
         return PCMK_LSB_STATUS_INSUFFICIENT_PRIV;
     }
 #endif
 
     return PCMK_OCF_INSUFFICIENT_PRIV;
 }
 
 /*!
  * \internal
  * \brief Return agent standard's exit status for "not configured"
  *
  * When returning an internal error for an action, a value that is appropriate
  * to the action's agent standard must be used. This function returns a value
  * appropriate for "not configured" errors.
  *
  * \param[in] op        Action that error is for
  * \param[in] is_fatal  Whether problem is cluster-wide instead of only local
  *
  * \return Exit status appropriate to agent standard
  * \note Actions without a standard will get PCMK_OCF_UNKNOWN_ERROR.
  */
 int
 services__configuration_error(const svc_action_t *op, bool is_fatal)
 {
     if ((op == NULL) || (op->standard == NULL)) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
 #if PCMK__ENABLE_LSB
     if (pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB, pcmk__str_casei)
         && pcmk__str_eq(op->action, PCMK_ACTION_STATUS, pcmk__str_casei)) {
 
         return PCMK_LSB_NOT_CONFIGURED;
     }
 #endif
 
     return is_fatal? PCMK_OCF_NOT_CONFIGURED : PCMK_OCF_INVALID_PARAM;
 }
 
 
 /*!
  * \internal
  * \brief Set operation rc and status per errno from stat(), fork() or execvp()
  *
  * \param[in,out] op     Operation to set rc and status for
  * \param[in]     error  Value of errno after system call
  *
  * \return void
  */
 void
 services__handle_exec_error(svc_action_t * op, int error)
 {
     const char *name = op->opaque->exec;
 
     if (name == NULL) {
         name = op->agent;
         if (name == NULL) {
             name = op->id;
         }
     }
 
     switch (error) {   /* see execve(2), stat(2) and fork(2) */
         case ENOENT:   /* No such file or directory */
         case EISDIR:   /* Is a directory */
         case ENOTDIR:  /* Path component is not a directory */
         case EINVAL:   /* Invalid executable format */
         case ENOEXEC:  /* Invalid executable format */
             services__format_result(op, services__not_installed_error(op),
                                     PCMK_EXEC_NOT_INSTALLED, "%s: %s",
                                     name, pcmk_rc_str(error));
             break;
         case EACCES:   /* permission denied (various errors) */
         case EPERM:    /* permission denied (various errors) */
             services__format_result(op, services__authorization_error(op),
                                     PCMK_EXEC_ERROR, "%s: %s",
                                     name, pcmk_rc_str(error));
             break;
         default:
             services__set_result(op, services__generic_error(op),
                                  PCMK_EXEC_ERROR, pcmk_rc_str(error));
     }
 }
 
 /*!
  * \internal
  * \brief Exit a child process that failed before executing agent
  *
  * \param[in] op           Action that failed
  * \param[in] exit_status  Exit status code to use
  * \param[in] exit_reason  Exit reason to output if for OCF agent
  */
 static void
 exit_child(const svc_action_t *op, int exit_status, const char *exit_reason)
 {
     if ((op != NULL) && (exit_reason != NULL)
         && pcmk__str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF,
                         pcmk__str_none)) {
         fprintf(stderr, PCMK_OCF_REASON_PREFIX "%s\n", exit_reason);
     }
     pcmk_common_cleanup();
     _exit(exit_status);
 }
 
 static void
 action_launch_child(svc_action_t *op)
 {
     int rc;
 
     /* SIGPIPE is ignored (which is different from signal blocking) by the gnutls library.
      * Depending on the libqb version in use, libqb may set SIGPIPE to be ignored as well. 
      * We do not want this to be inherited by the child process. By resetting this the signal
      * to the default behavior, we avoid some potential odd problems that occur during OCF
      * scripts when SIGPIPE is ignored by the environment. */
     signal(SIGPIPE, SIG_DFL);
 
     if (sched_getscheduler(0) != SCHED_OTHER) {
         struct sched_param sp;
 
         memset(&sp, 0, sizeof(sp));
         sp.sched_priority = 0;
 
         if (sched_setscheduler(0, SCHED_OTHER, &sp) == -1) {
             crm_info("Could not reset scheduling policy for %s", op->id);
         }
     }
 
     if (setpriority(PRIO_PROCESS, 0, 0) == -1) {
         crm_info("Could not reset process priority for %s", op->id);
     }
 
     /* Man: The call setpgrp() is equivalent to setpgid(0,0)
      * _and_ compiles on BSD variants too
      * need to investigate if it works the same too.
      */
     setpgid(0, 0);
 
     pcmk__close_fds_in_child(false);
 
     /* It would be nice if errors in this function could be reported as
      * execution status (for example, PCMK_EXEC_NO_SECRETS for the secrets error
      * below) instead of exit status. However, we've already forked, so
      * exit status is all we have. At least for OCF actions, we can output an
      * exit reason for the parent to parse.
+     *
+     * @TODO It might be better to substitute secrets in the parent before
+     * forking, so that if it fails, we can give a better message and result,
+     * and avoid the fork.
      */
 
 #if PCMK__ENABLE_CIBSECRETS
     rc = pcmk__substitute_secrets(op->rsc, op->params);
     if (rc != pcmk_rc_ok) {
         if (pcmk__str_eq(op->action, PCMK_ACTION_STOP, pcmk__str_casei)) {
             crm_info("Proceeding with stop operation for %s "
                      "despite being unable to load CIB secrets (%s)",
                      op->rsc, pcmk_rc_str(rc));
         } else {
             crm_err("Considering %s unconfigured "
                     "because unable to load CIB secrets: %s",
                     op->rsc, pcmk_rc_str(rc));
             exit_child(op, services__configuration_error(op, false),
                        "Unable to load CIB secrets");
         }
     }
 #endif
 
     add_action_env_vars(op);
 
     /* Become the desired user */
     if (op->opaque->uid && (geteuid() == 0)) {
 
         // If requested, set effective group
         if (op->opaque->gid && (setgid(op->opaque->gid) < 0)) {
             crm_err("Considering %s unauthorized because could not set "
                     "child group to %d: %s",
                     op->id, op->opaque->gid, strerror(errno));
             exit_child(op, services__authorization_error(op),
                        "Could not set group for child process");
         }
 
         // Erase supplementary group list
         // (We could do initgroups() if we kept a copy of the username)
         if (setgroups(0, NULL) < 0) {
             crm_err("Considering %s unauthorized because could not "
                     "clear supplementary groups: %s", op->id, strerror(errno));
             exit_child(op, services__authorization_error(op),
                        "Could not clear supplementary groups for child process");
         }
 
         // Set effective user
         if (setuid(op->opaque->uid) < 0) {
             crm_err("Considering %s unauthorized because could not set user "
                     "to %d: %s", op->id, op->opaque->uid, strerror(errno));
             exit_child(op, services__authorization_error(op),
                        "Could not set user for child process");
         }
     }
 
     // Execute the agent (doesn't return if successful)
     execvp(op->opaque->exec, op->opaque->args);
 
     // An earlier stat() should have avoided most possible errors
     rc = errno;
     services__handle_exec_error(op, rc);
     crm_err("Unable to execute %s: %s", op->id, strerror(rc));
     exit_child(op, op->rc, "Child process was unable to execute file");
 }
 
 /*!
  * \internal
  * \brief Wait for synchronous action to complete, and set its result
  *
  * \param[in,out] op    Action to wait for
  * \param[in,out] data  Child signal data
  */
 static void
 wait_for_sync_result(svc_action_t *op, struct sigchld_data_s *data)
 {
     int status = 0;
     int timeout = op->timeout;
     time_t start = time(NULL);
     struct pollfd fds[3];
     int wait_rc = 0;
     const char *wait_reason = NULL;
 
     fds[0].fd = op->opaque->stdout_fd;
     fds[0].events = POLLIN;
     fds[0].revents = 0;
 
     fds[1].fd = op->opaque->stderr_fd;
     fds[1].events = POLLIN;
     fds[1].revents = 0;
 
     fds[2].fd = sigchld_open(data);
     fds[2].events = POLLIN;
     fds[2].revents = 0;
 
     crm_trace("Waiting for %s[%d]", op->id, op->pid);
     do {
         int poll_rc = poll(fds, 3, timeout);
 
         wait_reason = NULL;
 
         if (poll_rc > 0) {
             if (fds[0].revents & POLLIN) {
                 svc_read_output(op->opaque->stdout_fd, op, FALSE);
             }
 
             if (fds[1].revents & POLLIN) {
                 svc_read_output(op->opaque->stderr_fd, op, TRUE);
             }
 
             if ((fds[2].revents & POLLIN)
                 && sigchld_received(fds[2].fd, op->pid, data)) {
                 wait_rc = waitpid(op->pid, &status, WNOHANG);
 
                 if ((wait_rc > 0) || ((wait_rc < 0) && (errno == ECHILD))) {
                     // Child process exited or doesn't exist
                     break;
 
                 } else if (wait_rc < 0) {
                     wait_reason = pcmk_rc_str(errno);
                     crm_info("Wait for completion of %s[%d] failed: %s "
                              QB_XS " source=waitpid",
                              op->id, op->pid, wait_reason);
                     wait_rc = 0; // Act as if process is still running
 
 #ifndef HAVE_SYS_SIGNALFD_H
                 } else {
                    /* The child hasn't exited, so this SIGCHLD could be for
                     * another child. We have to ignore it here but will still
                     * need to resend it after this synchronous action has
                     * completed and SIGCHLD has been restored to be handled by
                     * the previous handler, so that it will be handled.
                     */
                     data->ignored = true;
 #endif
                 }
             }
 
         } else if (poll_rc == 0) {
             // Poll timed out with no descriptors ready
             timeout = 0;
             break;
 
         } else if ((poll_rc < 0) && (errno != EINTR)) {
             wait_reason = pcmk_rc_str(errno);
             crm_info("Wait for completion of %s[%d] failed: %s "
                      QB_XS " source=poll", op->id, op->pid, wait_reason);
             break;
         }
 
         timeout = op->timeout - (time(NULL) - start) * 1000;
 
     } while ((op->timeout < 0 || timeout > 0));
 
     crm_trace("Stopped waiting for %s[%d]", op->id, op->pid);
     finish_op_output(op, true);
     finish_op_output(op, false);
     close_op_input(op);
     sigchld_close(fds[2].fd);
 
     if (wait_rc <= 0) {
 
         if ((op->timeout > 0) && (timeout <= 0)) {
             services__format_result(op, services__generic_error(op),
                                     PCMK_EXEC_TIMEOUT,
                                     "%s did not exit within specified timeout",
                                     services__action_kind(op));
             crm_info("%s[%d] timed out after %dms",
                      op->id, op->pid, op->timeout);
 
         } else {
             services__set_result(op, services__generic_error(op),
                                  PCMK_EXEC_ERROR, wait_reason);
         }
 
         /* If only child hasn't been successfully waited for, yet.
            This is to limit killing wrong target a bit more. */
         if ((wait_rc == 0) && (waitpid(op->pid, &status, WNOHANG) == 0)) {
             if (kill(op->pid, SIGKILL)) {
                 crm_warn("Could not kill rogue child %s[%d]: %s",
                          op->id, op->pid, pcmk_rc_str(errno));
             }
             /* Safe to skip WNOHANG here as we sent non-ignorable signal. */
             while ((waitpid(op->pid, &status, 0) == (pid_t) -1)
                    && (errno == EINTR)) {
                 /* keep waiting */;
             }
         }
 
     } else if (WIFEXITED(status)) {
         services__set_result(op, WEXITSTATUS(status), PCMK_EXEC_DONE, NULL);
         parse_exit_reason_from_stderr(op);
         crm_info("%s[%d] exited with status %d", op->id, op->pid, op->rc);
 
     } else if (WIFSIGNALED(status)) {
         int signo = WTERMSIG(status);
 
         services__format_result(op, services__generic_error(op),
                                 PCMK_EXEC_ERROR, "%s interrupted by %s signal",
                                 services__action_kind(op), strsignal(signo));
         crm_info("%s[%d] terminated with signal %d (%s)",
                  op->id, op->pid, signo, strsignal(signo));
 
 #ifdef WCOREDUMP
         if (WCOREDUMP(status)) {
             crm_warn("%s[%d] dumped core", op->id, op->pid);
         }
 #endif
 
     } else {
         // Shouldn't be possible to get here
         services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
                              "Unable to wait for child to complete");
     }
 }
 
 /*!
  * \internal
  * \brief Execute an action whose standard uses executable files
  *
  * \param[in,out] op  Action to execute
  *
  * \return Standard Pacemaker return value
  * \retval EBUSY          Recurring operation could not be initiated
  * \retval pcmk_rc_error  Synchronous action failed
  * \retval pcmk_rc_ok     Synchronous action succeeded, or asynchronous action
  *                        should not be freed (because it's pending or because
  *                        it failed to execute and was already freed)
  *
  * \note If the return value for an asynchronous action is not pcmk_rc_ok, the
  *       caller is responsible for freeing the action.
  */
 int
 services__execute_file(svc_action_t *op)
 {
     int stdout_fd[2];
     int stderr_fd[2];
     int stdin_fd[2] = {-1, -1};
     int rc;
     struct stat st;
     struct sigchld_data_s data = { .ignored = false };
 
     // Catch common failure conditions early
     if (stat(op->opaque->exec, &st) != 0) {
         rc = errno;
         crm_info("Cannot execute '%s': %s " QB_XS " stat rc=%d",
                  op->opaque->exec, pcmk_rc_str(rc), rc);
         services__handle_exec_error(op, rc);
         goto done;
     }
 
     if (pipe(stdout_fd) < 0) {
         rc = errno;
         crm_info("Cannot execute '%s': %s " QB_XS " pipe(stdout) rc=%d",
                  op->opaque->exec, pcmk_rc_str(rc), rc);
         services__handle_exec_error(op, rc);
         goto done;
     }
 
     if (pipe(stderr_fd) < 0) {
         rc = errno;
 
         close_pipe(stdout_fd);
 
         crm_info("Cannot execute '%s': %s " QB_XS " pipe(stderr) rc=%d",
                  op->opaque->exec, pcmk_rc_str(rc), rc);
         services__handle_exec_error(op, rc);
         goto done;
     }
 
     if (pcmk_is_set(pcmk_get_ra_caps(op->standard), pcmk_ra_cap_stdin)) {
         if (pipe(stdin_fd) < 0) {
             rc = errno;
 
             close_pipe(stdout_fd);
             close_pipe(stderr_fd);
 
             crm_info("Cannot execute '%s': %s " QB_XS " pipe(stdin) rc=%d",
                      op->opaque->exec, pcmk_rc_str(rc), rc);
             services__handle_exec_error(op, rc);
             goto done;
         }
     }
 
     if (op->synchronous && !sigchld_setup(&data)) {
         close_pipe(stdin_fd);
         close_pipe(stdout_fd);
         close_pipe(stderr_fd);
         sigchld_cleanup(&data);
         services__set_result(op, services__generic_error(op), PCMK_EXEC_ERROR,
                              "Could not manage signals for child process");
         goto done;
     }
 
     op->pid = fork();
     switch (op->pid) {
         case -1:
             rc = errno;
             close_pipe(stdin_fd);
             close_pipe(stdout_fd);
             close_pipe(stderr_fd);
 
             crm_info("Cannot execute '%s': %s " QB_XS " fork rc=%d",
                      op->opaque->exec, pcmk_rc_str(rc), rc);
             services__handle_exec_error(op, rc);
             if (op->synchronous) {
                 sigchld_cleanup(&data);
             }
             goto done;
             break;
 
         case 0:                /* Child */
             close(stdout_fd[0]);
             close(stderr_fd[0]);
             if (stdin_fd[1] >= 0) {
                 close(stdin_fd[1]);
             }
             if (STDOUT_FILENO != stdout_fd[1]) {
                 if (dup2(stdout_fd[1], STDOUT_FILENO) != STDOUT_FILENO) {
                     crm_warn("Can't redirect output from '%s': %s "
                              QB_XS " errno=%d",
                              op->opaque->exec, pcmk_rc_str(errno), errno);
                 }
                 close(stdout_fd[1]);
             }
             if (STDERR_FILENO != stderr_fd[1]) {
                 if (dup2(stderr_fd[1], STDERR_FILENO) != STDERR_FILENO) {
                     crm_warn("Can't redirect error output from '%s': %s "
                              QB_XS " errno=%d",
                              op->opaque->exec, pcmk_rc_str(errno), errno);
                 }
                 close(stderr_fd[1]);
             }
             if ((stdin_fd[0] >= 0) &&
                 (STDIN_FILENO != stdin_fd[0])) {
                 if (dup2(stdin_fd[0], STDIN_FILENO) != STDIN_FILENO) {
                     crm_warn("Can't redirect input to '%s': %s "
                              QB_XS " errno=%d",
                              op->opaque->exec, pcmk_rc_str(errno), errno);
                 }
                 close(stdin_fd[0]);
             }
 
             if (op->synchronous) {
                 sigchld_cleanup(&data);
             }
 
             action_launch_child(op);
             pcmk__assert(false); // action_launch_child() should not return
     }
 
     /* Only the parent reaches here */
     close(stdout_fd[1]);
     close(stderr_fd[1]);
     if (stdin_fd[0] >= 0) {
         close(stdin_fd[0]);
     }
 
     op->opaque->stdout_fd = stdout_fd[0];
     rc = pcmk__set_nonblocking(op->opaque->stdout_fd);
     if (rc != pcmk_rc_ok) {
         crm_info("Could not set '%s' output non-blocking: %s "
                  QB_XS " rc=%d",
                  op->opaque->exec, pcmk_rc_str(rc), rc);
     }
 
     op->opaque->stderr_fd = stderr_fd[0];
     rc = pcmk__set_nonblocking(op->opaque->stderr_fd);
     if (rc != pcmk_rc_ok) {
         crm_info("Could not set '%s' error output non-blocking: %s "
                  QB_XS " rc=%d",
                  op->opaque->exec, pcmk_rc_str(rc), rc);
     }
 
     op->opaque->stdin_fd = stdin_fd[1];
     if (op->opaque->stdin_fd >= 0) {
         // using buffer behind non-blocking-fd here - that could be improved
         // as long as no other standard uses stdin_fd assume stonith
         rc = pcmk__set_nonblocking(op->opaque->stdin_fd);
         if (rc != pcmk_rc_ok) {
             crm_info("Could not set '%s' input non-blocking: %s "
                     QB_XS " fd=%d,rc=%d", op->opaque->exec,
                     pcmk_rc_str(rc), op->opaque->stdin_fd, rc);
         }
         pipe_in_action_stdin_parameters(op);
         // as long as we are handling parameters directly in here just close
         close(op->opaque->stdin_fd);
         op->opaque->stdin_fd = -1;
     }
 
     // after fds are setup properly and before we plug anything into mainloop
     if (op->opaque->fork_callback) {
         op->opaque->fork_callback(op);
     }
 
     if (op->synchronous) {
         wait_for_sync_result(op, &data);
         sigchld_cleanup(&data);
         goto done;
     }
 
     crm_trace("Waiting async for '%s'[%d]", op->opaque->exec, op->pid);
     mainloop_child_add_with_flags(op->pid, op->timeout, op->id, op,
                                   pcmk_is_set(op->flags, SVC_ACTION_LEAVE_GROUP)? mainloop_leave_pid_group : 0,
                                   async_action_complete);
 
     op->opaque->stdout_gsource = mainloop_add_fd(op->id,
                                                  G_PRIORITY_LOW,
                                                  op->opaque->stdout_fd, op,
                                                  &stdout_callbacks);
     op->opaque->stderr_gsource = mainloop_add_fd(op->id,
                                                  G_PRIORITY_LOW,
                                                  op->opaque->stderr_fd, op,
                                                  &stderr_callbacks);
     services_add_inflight_op(op);
     return pcmk_rc_ok;
 
 done:
     if (op->synchronous) {
         return (op->rc == PCMK_OCF_OK)? pcmk_rc_ok : pcmk_rc_error;
     } else {
         return services__finalize_async_op(op);
     }
 }
 
 GList *
 services_os_get_single_directory_list(const char *root, gboolean files, gboolean executable)
 {
     GList *list = NULL;
     struct dirent **namelist;
     int entries = 0, lpc = 0;
     char buffer[PATH_MAX];
 
     entries = scandir(root, &namelist, NULL, alphasort);
     if (entries <= 0) {
         return list;
     }
 
     for (lpc = 0; lpc < entries; lpc++) {
         struct stat sb;
 
         if ('.' == namelist[lpc]->d_name[0]) {
             free(namelist[lpc]);
             continue;
         }
 
         snprintf(buffer, sizeof(buffer), "%s/%s", root, namelist[lpc]->d_name);
 
         if (stat(buffer, &sb)) {
             continue;
         }
 
         if (S_ISDIR(sb.st_mode)) {
             if (files) {
                 free(namelist[lpc]);
                 continue;
             }
 
         } else if (S_ISREG(sb.st_mode)) {
             if (files == FALSE) {
                 free(namelist[lpc]);
                 continue;
 
             } else if (executable
                        && (sb.st_mode & S_IXUSR) == 0
                        && (sb.st_mode & S_IXGRP) == 0 && (sb.st_mode & S_IXOTH) == 0) {
                 free(namelist[lpc]);
                 continue;
             }
         }
 
         list = g_list_append(list, strdup(namelist[lpc]->d_name));
 
         free(namelist[lpc]);
     }
 
     free(namelist);
     return list;
 }
 
 GList *
 services_os_get_directory_list(const char *root, gboolean files, gboolean executable)
 {
     GList *result = NULL;
     char *dirs = strdup(root);
     char *dir = NULL;
 
     if (pcmk__str_empty(dirs)) {
         free(dirs);
         return result;
     }
 
     for (dir = strtok(dirs, ":"); dir != NULL; dir = strtok(NULL, ":")) {
         GList *tmp = services_os_get_single_directory_list(dir, files, executable);
 
         if (tmp) {
             result = g_list_concat(result, tmp);
         }
     }
 
     free(dirs);
 
     return result;
 }